diff options
9 files changed, 99 insertions, 154 deletions
diff --git a/kubernetes/common/common/templates/_dmaapProvisioning.tpl b/kubernetes/common/common/templates/_dmaapProvisioning.tpl index fae16ff7eb..3cf5bf1207 100644 --- a/kubernetes/common/common/templates/_dmaapProvisioning.tpl +++ b/kubernetes/common/common/templates/_dmaapProvisioning.tpl @@ -1,7 +1,7 @@ {{/* ################################################################################ # Copyright (C) 2021 Nordix Foundation. # -# Copyright (c) 2022 J. F. Lucas. All rights reserved. # +# Copyright (c) 2022-2023 J. F. Lucas. All rights reserved. # # # # Licensed under the Apache License, Version 2.0 (the "License"); # # you may not use this file except in compliance with the License. # @@ -19,51 +19,54 @@ {{/* This template generates a Kubernetes init containers common template to enable applications to provision - DMaaP feeds (on Data Router), with associated authorization. - DMaap Bus Controller endpoints are used to provision: - - - Feed on DR, with associated user authentication. + DMaaP feeds (on Data Router) for DCAE microservices, with associated authorization. + DMaap Data Router (DR) endpoints are used to provision: + - Feeds on DR, with associated user authentication. + - Subscribers to feeds on DR, to provide DR with username, password, and URL needed to deliver + files to subscribers. common.dmaap.provisioning.initContainer: - This template make use of Dmaap Bus Controller docker image to create resources on Dmaap Data Router - microservice, with the help of dbc-client.sh script it makes use of Bus Controller API to create Feeds. - If the resource creation is successful via script response is logged back at particular location with - appropriate naming convention. - - More details can be found at : - (https://wiki.onap.org/pages/viewpage.action?pageId=103417564) + This template creates an initContainer with some associated volumes. The initContainer + (oom/kubernetes/dmaap-datarouter/drprov-client) runs a script (drprov-client.sh) that uses the + DR provisioning API to create the feeds and subscribers needed by a microservice. The script + updates the microservice's configuration to supply information needed to access the feeds. The + configuration information comes from two volumes that are created by the dcaegen2-services-common + templates. + - app-config-input: comes from a configMap generated from the microservice's values.yaml file. + It may contain references to environment variables as placeholders for feed information that + will become available after feeds are provisioned. + - app-config: this template will copy the configuration file from the app-config-input volume, + replaced the environment variable references with the actual values for feed information, based + on data returned by the DR provisioning API. The template directly references data in .Values, and indirectly (through its use of templates from the ONAP "common" collection) references data in .Release. - Parameter for _dmaapProvisioning to be defined in values.yaml + Parameters for _dmaapProvisioning to be defined in values.yaml: + # DataRouter Feed Configuration + # (Note that DR configures publishers as part of the feed.) drFeedConfig: - feedName: bulk_pm_feed - owner: dcaecm feedVersion: 0.0 - asprClassification: unclassified + classification: unclassified feedDescription: DFC Feed Creation - - # DataRouter Publisher Configuration - drPubConfig: - - feedName: bulk_pm_feed - dcaeLocationName: loc00 + publisher: + username: xyz + password: xyz # DataRouter Subscriber Configuration drSubConfig: - feedName: bulk_pm_feed + feedVersion: 0.0 decompress: True - dcaeLocationName: loc00 privilegedSubscriber: True deliveryURL: https://dcae-pm-mapper:8443/delivery - # ConfigMap Configuration for DR Feed, Dr_Publisher, Dr_Subscriber + # ConfigMap Configuration for DR Feed, Dr_Subscriber volumes: - name: feeds-config path: /opt/app/config/feeds - - name: drpub-config - path: /opt/app/config/dr_pubs - name: drsub-config path: /opt/app/config/dr_subs @@ -76,8 +79,10 @@ {{- define "common.dmaap.provisioning._volumeMounts" -}} {{- $dot := default . .dot -}} -- mountPath: /opt/app/config/cache - name: dbc-response-cache +- mountPath: /config-input + name: app-config-input +- mountPath: /config + name: app-config {{- range $name, $volume := $dot.Values.volumes }} - name: {{ $volume.name }} mountPath: {{ $volume.path }} @@ -86,8 +91,6 @@ {{- define "common.dmaap.provisioning._volumes" -}} {{- $dot := default . .dot -}} -- name: dbc-response-cache - emptyDir: {} {{- range $name, $volume := $dot.Values.volumes }} - name: {{ $volume.name }} configMap: @@ -98,20 +101,14 @@ {{- define "common.dmaap.provisioning.initContainer" -}} {{- $dot := default . .dot -}} -{{- $drFeedConfig := default $dot.Values.drFeedConfig .drFeedConfig -}} -{{- if $drFeedConfig -}} +{{- $drNeedProvisioning := or $dot.Values.drFeedConfig $dot.Values.drSubConfig -}} +{{- if $drNeedProvisioning -}} - name: {{ include "common.name" $dot }}-init-dmaap-provisioning - image: {{ include "repositoryGenerator.image.dbcClient" $dot }} + image: {{ include "repositoryGenerator.image.drProvClient" $dot }} imagePullPolicy: {{ $dot.Values.global.pullPolicy | default $dot.Values.pullPolicy }} env: - - name: PROTO - value: "http" - - name: PORT - value: "8080" - - name: RESP_CACHE - value: /opt/app/config/cache - - name: REQUESTID - value: "{{ include "common.name" $dot }}-dmaap-provisioning" + - name: ONBEHALFHDR + value: "X-DMAAP-DR-ON-BEHALF-OF: drprovcl" {{- range $cred := $dot.Values.credentials }} - name: {{ $cred.name }} {{- include "common.secret.envFromSecretFast" (dict "global" $dot "uid" $cred.uid "key" $cred.key) | nindent 4 }} @@ -119,56 +116,5 @@ volumeMounts: {{- include "common.dmaap.provisioning._volumeMounts" $dot | trim | nindent 2 }} resources: {{ include "common.resources" $dot | nindent 4 }} -- name: {{ include "common.name" $dot }}-init-merge-config - image: {{ include "repositoryGenerator.image.envsubst" $dot }} - imagePullPolicy: {{ $dot.Values.global.pullPolicy | default $dot.Values.pullPolicy }} - command: - - /bin/sh - args: - - -c - - | - set -uex -o pipefail - if [ -d /opt/app/config/cache ]; then - cd /opt/app/config/cache - for file in $(ls feed*); do - NUM=$(echo "$file" | sed 's/feedConfig-\([0-9]\+\)-resp.json/\1/') - export DR_LOG_URL_"$NUM"="$(grep -o '"logURL":"[^"]*' "$file" | grep -w "feedlog" | cut -d '"' -f4)" - export DR_FILES_PUBLISHER_URL_"$NUM"="$(grep -o '"publishURL":"[^"]*' "$file" | cut -d '"' -f4)" - done - for file in $(ls drpub*); do - NUM=$(echo "$file" | sed 's/drpubConfig-\([0-9]\+\)-resp.json/\1/') - export DR_FILES_PUBLISHER_ID_"$NUM"="$(grep -o '"pubId":"[^"]*' "$file" | cut -d '"' -f4)" - done - for file in $(ls drsub*); do - NUM=$(echo "$file" | sed 's/drsubConfig-\([0-9]\+\)-resp.json/\1/') - export DR_FILES_SUBSCRIBER_ID_"$NUM"="$(grep -o '"subId":"[^"]*' "$file" | cut -d '"' -f4)" - done - for file in $(ls topics*); do - NUM=$(echo "$file" | sed 's/topicsConfig-\([0-9]\+\)-resp.json/\1/') - export MR_FILES_PUBLISHER_CLIENT_ID_"$NUM"="$(grep -o '"mrClientId":"[^"]*' "$file" | cut -d '"' -f4)" - done - else - echo "No Response logged for Dmaap BusController Http POST Request..!" - fi - cd /config-input && for PFILE in `ls -1`; do envsubst <${PFILE} >/config/${PFILE}; done - env: - {{- range $cred := $dot.Values.credentials }} - - name: {{ $cred.name }} - {{- include "common.secret.envFromSecretFast" (dict "global" $dot "uid" $cred.uid "key" $cred.key) | nindent 4 }} - {{- end }} - volumeMounts: - - mountPath: /opt/app/config/cache - name: dbc-response-cache - - mountPath: /config-input - name: app-config-input - - mountPath: /config - name: app-config - resources: - limits: - cpu: 200m - memory: 250Mi - requests: - cpu: 100m - memory: 200Mi {{- end -}} {{- end -}}
\ No newline at end of file diff --git a/kubernetes/common/repositoryGenerator/templates/_repository.tpl b/kubernetes/common/repositoryGenerator/templates/_repository.tpl index 09a799e713..1da838a5b9 100644 --- a/kubernetes/common/repositoryGenerator/templates/_repository.tpl +++ b/kubernetes/common/repositoryGenerator/templates/_repository.tpl @@ -147,8 +147,8 @@ {{- include "repositoryGenerator.image._helper" (merge (dict "image" "readinessImage") .) }} {{- end -}} -{{- define "repositoryGenerator.image.dbcClient" -}} - {{- include "repositoryGenerator.image._helper" (merge (dict "image" "dbcClientImage") .) }} +{{- define "repositoryGenerator.image.drProvClient" -}} + {{- include "repositoryGenerator.image._helper" (merge (dict "image" "drProvClientImage") .) }} {{- end -}} {{- define "repositoryGenerator.image.quitQuit" -}} diff --git a/kubernetes/common/repositoryGenerator/values.yaml b/kubernetes/common/repositoryGenerator/values.yaml index b89c2b2bf6..2c94c29b15 100644 --- a/kubernetes/common/repositoryGenerator/values.yaml +++ b/kubernetes/common/repositoryGenerator/values.yaml @@ -41,7 +41,7 @@ global: postgresImage: crunchydata/crunchy-postgres:centos8-13.2-4.6.1 readinessImage: onap/oom/readiness:3.0.1 dcaePolicySyncImage: onap/org.onap.dcaegen2.deployments.dcae-services-policy-sync:1.0.1 - dbcClientImage: onap/dmaap/dbc-client:2.0.11 + drProvClientImage: onap/dmaap/datarouter-prov-client:2.1.14 quitQuitImage: onap/oom/readiness:4.1.0 # Default credentials @@ -75,5 +75,5 @@ imageRepoMapping: postgresImage: dockerHubRepository readinessImage: repository dcaePolicySyncImage: repository - dbcClientImage: repository + drProvClientImage: repository quitQuitImage: repository diff --git a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl index afd3c38f31..bb35e80848 100644 --- a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl +++ b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl @@ -1,7 +1,7 @@ {{/* # Copyright © 2017 Amdocs, Bell Canada # Modifications Copyright © 2019 AT&T -# Copyright (c) 2021-2022 J. F. Lucas. All rights reserved. +# Copyright (c) 2021-2023 J. F. Lucas. All rights reserved. # Copyright (c) 2021 Nordix Foundation. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,14 +33,10 @@ The template always produces a configMap containing the microservice's initial configuration data. (See the documentation for dcaegen2-services-common.microserviceDeployment for more details.) -If the microservice is using one or more Data Router (DR) feeds, the +If the microservice is publishing to one or more Data Router (DR) feeds, the template produces a configMap containing the information needed to provision the feed(s). An init container performs the provisioning. -If the microservice acts as a DR publisher for one or more feeds, the -template produces a configMap containing the information needed to -provision the publisher(s). An init container performs the provisioning. - If the microservice acts as a DR subscriber for one or more feeds, the template produces a configMap containing the information needed to provision the subscribeer(s). An init container performs the provisioning. @@ -70,22 +66,22 @@ metadata: data: {{- range $i, $feed := .Values.drFeedConfig }} feedConfig-{{$i}}.json: |- - {{ $feed | toJson | indent 2 }} - {{- end }} -{{- end }} - -{{- if .Values.drPubConfig }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "common.fullname" . }}-drpub-config - namespace: {{ include "common.namespace" . }} - labels: {{ include "common.labels" . | nindent 6 }} -data: - {{- range $i, $drpub := .Values.drPubConfig }} - drpubConfig-{{$i}}.json: |- - {{ $drpub | toJson | indent 2 }} + { + "name": {{ $feed.feedName | quote }}, + "version": {{ $feed.feedVersion | quote }}, + "description": {{ $feed.feedDescription | default "None" | quote }}, + "authorization": { + "classification": {{ $feed.classification | quote }}, + "endpoint_addrs": [ + ], + "endpoint_ids": [ + { + "id": {{ $feed.publisher.username | quote }}, + "password": {{ $feed.publisher.password | quote }} + } + ] + } + } {{- end }} {{- end }} @@ -100,7 +96,23 @@ metadata: data: {{- range $i, $drsub := .Values.drSubConfig }} drsubConfig-{{$i}}.json: |- - {{ $drsub | toJson | indent 2 }} + { + "feed": { + "name": {{ $drsub.feedName | quote }}, + "version": {{ $drsub.feedVersion | quote }} + }, + "delivery": { + "url": {{ $drsub.deliveryURL | quote }}, + "user": {{ $drsub.username | quote }}, + "password": {{ $drsub.userpwd | quote }}, + "use100": {{ $drsub.use100 | default false }} + }, + "metadataOnly": {{ $drsub.metadataOnly | default false }}, + "groupid": {{ $drsub.groupId | default 0 }}, + "follow_redirect": {{ $drsub.followRedirect | default true }}, + "privileged_subscriber": {{ $drsub.privilegedSubscriber | default false }}, + "decompress": {{ $drsub.decompress | default false }} + } {{- end }} {{- end }} {{- end }} diff --git a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl index e7f9627a3d..b0c76a9b3a 100644 --- a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl +++ b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl @@ -240,7 +240,7 @@ post-processing. {{- $commonRelease := print (include "common.release" .) -}} {{- $policy := default dict .Values.policies -}} {{- $policyRls := default $commonRelease $policy.policyRelease -}} -{{- $drFeedConfig := default "" .Values.drFeedConfig -}} +{{- $drNeedProvisioning := or .Values.drFeedConfig .Values.drSubConfig -}} {{- $dcaeName := print (include "common.fullname" .) }} {{- $dcaeLabel := (dict "dcaeMicroserviceName" $dcaeName) -}} {{- $dot := . -}} @@ -310,7 +310,7 @@ spec: resources: {{ include "common.resources" . | nindent 10 }} volumeMounts: - mountPath: /app-config - name: {{ ternary "app-config-input" "app-config" (not $drFeedConfig) }} + name: {{ ternary "app-config-input" "app-config" (not $drNeedProvisioning) }} - mountPath: /app-config-input name: app-config-input {{- if $logDir }} diff --git a/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml b/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml index b9e7a39ecb..26bbd37490 100644 --- a/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml @@ -89,11 +89,12 @@ certificates: create: true # Dependencies +# Waiting for dmaap-dr-node (which depends on dmaap-dr-prov) +# to be sure that we can provision the DR feed that's needed readinessCheck: wait_for: containers: - - dmaap-bc - - dmaap-provisioning-job + - dmaap-dr-node - message-router # Probe Configuration @@ -166,10 +167,10 @@ applicationConfig: streams_publishes: PM_MEAS_FILES: dmaap_info: - publisher_id: ${DR_FILES_PUBLISHER_ID_0} + publisher_id: "dummy_id" location: loc00 - log_url: ${DR_LOG_URL_0} - publish_url: ${DR_FILES_PUBLISHER_URL_0} + log_url: ${DR_FEED_LOGURL_0} + publish_url: ${DR_FEED_PUBURL_0} username: ${DR_USERNAME} password: ${DR_PASSWORD} type: data_router @@ -187,24 +188,17 @@ applicationEnv: # DataRouter Feed Configuration drFeedConfig: - feedName: bulk_pm_feed - owner: dcaecm feedVersion: "0.0" - asprClassification: unclassified + classification: unclassified feedDescription: DFC Feed Creation - -# DataRouter Publisher Configuration -drPubConfig: - - feedName: bulk_pm_feed - username: ${DR_USERNAME} - userpwd: ${DR_PASSWORD} - dcaeLocationName: loc00 + publisher: + username: ${DR_USERNAME} + password: ${DR_PASSWORD} # ConfigMap Configuration for Feed, Dr_Publisher volumes: - name: feeds-config path: /opt/app/config/feeds - - name: drpub-config - path: /opt/app/config/dr_pubs # Resource Limit Flavor -By Default Using Small flavor: small diff --git a/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml b/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml index 1aee8c9a1b..ab23b54e06 100644 --- a/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml @@ -58,13 +58,16 @@ log: logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Dependencies +# Depend on the datafile-collector, which guarantees that +# the DR feed that pm-mapper susbscribes to will be created +# already by the datafile-collector DMaaP provisioning init +# container. Also guarantees that DR provisioning will be +# available for pm-mapper initContainter to create the +# subscription to the feed. readinessCheck: wait_for: containers: - - dmaap-bc - - dmaap-provisioning-job - dcae-datafile-collector - - message-router # Probe Configuration readiness: @@ -120,7 +123,7 @@ applicationConfig: dmaap_publisher: type: message_router dmaap_info: - client_id: ${MR_FILES_PUBLISHER_CLIENT_ID_0} + client_id: "dummy_id" location: san-francisco client_role: org.onap.dcae.pmPublisher topic_url: http://message-router:3904/events/unauthenticated.PERFORMANCE_MEASUREMENTS @@ -128,7 +131,7 @@ applicationConfig: dmaap_subscriber: type: data_router dmaap_info: - subscriber_id: ${DR_FILES_SUBSCRIBER_ID_0} + subscriber_id: "dummy_id" decompress: true privileged: true username: ${DR_USERNAME} @@ -141,28 +144,18 @@ applicationEnv: #Temporary Dummy CBS Port Value until internal SDK library is updated CONFIG_BINDING_SERVICE_SERVICE_PORT: '0000' -# DataRouter Feed Configuration -drFeedConfig: - - feedName: bulk_pm_feed - owner: dcaecm - feedVersion: "0.0" - asprClassification: unclassified - feedDescription: DFC Feed Creation - # DataRouter Subscriber Configuration drSubConfig: - feedName: bulk_pm_feed + feedVersion: "0.0" decompress: true username: ${DR_USERNAME} userpwd: ${DR_PASSWORD} - dcaeLocationName: loc00 privilegedSubscriber: true deliveryURL: http://dcae-pm-mapper:8081/delivery -# ConfigMap Configuration for Dr Feed, Subscriber, MR Topics +# ConfigMap Configuration for DR Subscriber volumes: - - name: feeds-config - path: /opt/app/config/feeds - name: drsub-config path: /opt/app/config/dr_subs diff --git a/kubernetes/dmaap/components/dmaap-dr-node/values.yaml b/kubernetes/dmaap/components/dmaap-dr-node/values.yaml index eef6755385..d6815fdee1 100644 --- a/kubernetes/dmaap/components/dmaap-dr-node/values.yaml +++ b/kubernetes/dmaap/components/dmaap-dr-node/values.yaml @@ -23,7 +23,7 @@ global: # Application configuration defaults. ################################################################# # application image -image: onap/dmaap/datarouter-node:2.1.13 +image: onap/dmaap/datarouter-node:2.1.14 pullPolicy: Always # default number of instances diff --git a/kubernetes/dmaap/components/dmaap-dr-prov/values.yaml b/kubernetes/dmaap/components/dmaap-dr-prov/values.yaml index af855c7640..23188a62c2 100644 --- a/kubernetes/dmaap/components/dmaap-dr-prov/values.yaml +++ b/kubernetes/dmaap/components/dmaap-dr-prov/values.yaml @@ -38,7 +38,7 @@ secrets: # Application configuration defaults. ################################################################# # application image -image: onap/dmaap/datarouter-prov:2.1.13 +image: onap/dmaap/datarouter-prov:2.1.14 pullPolicy: Always # default number of instances |