From 838ce7ea096c3dd52436803776764b2dd0b05971 Mon Sep 17 00:00:00 2001 From: Jack Lucas Date: Tue, 18 Jul 2023 17:52:13 -0400 Subject: [DMAAP] Introduce drprov-client Introduce the drprov-client (replaces dbc-client) and use it to provision data router feeds and subscriptions for DCAE datafile collector and pm-mapper. - Update the common dmaap provisioning template to use drprov-client instead of dbc-client. - Add drprov-client image to the repository generator. - Update the dcaegen2-services-common configMap template to generate JSON files suitable for provisioning data router directly. - Update the dcaegen2-services-common deployment template to use drprov-client as an init container instead of dbc-client. - Update dcae-datafile-collector and pm-mapper values.yaml to format feed and subscription data properly for drprov-client. - Update versions for datarouter-node and datarouter-prov - Update dependencies for DCAE datafile-collector and pm-mapper, which no longer depend on DMAAP bus controller. Issue-ID: DMAAP-1893 Signed-off-by: Jack Lucas Change-Id: Ia090365428362ff6aa94cfb4096ee4c6b554b480 --- .../common/common/templates/_dmaapProvisioning.tpl | 126 ++++++--------------- .../repositoryGenerator/templates/_repository.tpl | 4 +- kubernetes/common/repositoryGenerator/values.yaml | 4 +- 3 files changed, 40 insertions(+), 94 deletions(-) (limited to 'kubernetes/common') diff --git a/kubernetes/common/common/templates/_dmaapProvisioning.tpl b/kubernetes/common/common/templates/_dmaapProvisioning.tpl index fae16ff7eb..3cf5bf1207 100644 --- a/kubernetes/common/common/templates/_dmaapProvisioning.tpl +++ b/kubernetes/common/common/templates/_dmaapProvisioning.tpl @@ -1,7 +1,7 @@ {{/* ################################################################################ # Copyright (C) 2021 Nordix Foundation. # -# Copyright (c) 2022 J. F. Lucas. All rights reserved. # +# Copyright (c) 2022-2023 J. F. Lucas. All rights reserved. # # # # Licensed under the Apache License, Version 2.0 (the "License"); # # you may not use this file except in compliance with the License. # @@ -19,51 +19,54 @@ {{/* This template generates a Kubernetes init containers common template to enable applications to provision - DMaaP feeds (on Data Router), with associated authorization. - DMaap Bus Controller endpoints are used to provision: - - - Feed on DR, with associated user authentication. + DMaaP feeds (on Data Router) for DCAE microservices, with associated authorization. + DMaap Data Router (DR) endpoints are used to provision: + - Feeds on DR, with associated user authentication. + - Subscribers to feeds on DR, to provide DR with username, password, and URL needed to deliver + files to subscribers. common.dmaap.provisioning.initContainer: - This template make use of Dmaap Bus Controller docker image to create resources on Dmaap Data Router - microservice, with the help of dbc-client.sh script it makes use of Bus Controller API to create Feeds. - If the resource creation is successful via script response is logged back at particular location with - appropriate naming convention. - - More details can be found at : - (https://wiki.onap.org/pages/viewpage.action?pageId=103417564) + This template creates an initContainer with some associated volumes. The initContainer + (oom/kubernetes/dmaap-datarouter/drprov-client) runs a script (drprov-client.sh) that uses the + DR provisioning API to create the feeds and subscribers needed by a microservice. The script + updates the microservice's configuration to supply information needed to access the feeds. The + configuration information comes from two volumes that are created by the dcaegen2-services-common + templates. + - app-config-input: comes from a configMap generated from the microservice's values.yaml file. + It may contain references to environment variables as placeholders for feed information that + will become available after feeds are provisioned. + - app-config: this template will copy the configuration file from the app-config-input volume, + replaced the environment variable references with the actual values for feed information, based + on data returned by the DR provisioning API. The template directly references data in .Values, and indirectly (through its use of templates from the ONAP "common" collection) references data in .Release. - Parameter for _dmaapProvisioning to be defined in values.yaml + Parameters for _dmaapProvisioning to be defined in values.yaml: + # DataRouter Feed Configuration + # (Note that DR configures publishers as part of the feed.) drFeedConfig: - feedName: bulk_pm_feed - owner: dcaecm feedVersion: 0.0 - asprClassification: unclassified + classification: unclassified feedDescription: DFC Feed Creation - - # DataRouter Publisher Configuration - drPubConfig: - - feedName: bulk_pm_feed - dcaeLocationName: loc00 + publisher: + username: xyz + password: xyz # DataRouter Subscriber Configuration drSubConfig: - feedName: bulk_pm_feed + feedVersion: 0.0 decompress: True - dcaeLocationName: loc00 privilegedSubscriber: True deliveryURL: https://dcae-pm-mapper:8443/delivery - # ConfigMap Configuration for DR Feed, Dr_Publisher, Dr_Subscriber + # ConfigMap Configuration for DR Feed, Dr_Subscriber volumes: - name: feeds-config path: /opt/app/config/feeds - - name: drpub-config - path: /opt/app/config/dr_pubs - name: drsub-config path: /opt/app/config/dr_subs @@ -76,8 +79,10 @@ {{- define "common.dmaap.provisioning._volumeMounts" -}} {{- $dot := default . .dot -}} -- mountPath: /opt/app/config/cache - name: dbc-response-cache +- mountPath: /config-input + name: app-config-input +- mountPath: /config + name: app-config {{- range $name, $volume := $dot.Values.volumes }} - name: {{ $volume.name }} mountPath: {{ $volume.path }} @@ -86,8 +91,6 @@ {{- define "common.dmaap.provisioning._volumes" -}} {{- $dot := default . .dot -}} -- name: dbc-response-cache - emptyDir: {} {{- range $name, $volume := $dot.Values.volumes }} - name: {{ $volume.name }} configMap: @@ -98,20 +101,14 @@ {{- define "common.dmaap.provisioning.initContainer" -}} {{- $dot := default . .dot -}} -{{- $drFeedConfig := default $dot.Values.drFeedConfig .drFeedConfig -}} -{{- if $drFeedConfig -}} +{{- $drNeedProvisioning := or $dot.Values.drFeedConfig $dot.Values.drSubConfig -}} +{{- if $drNeedProvisioning -}} - name: {{ include "common.name" $dot }}-init-dmaap-provisioning - image: {{ include "repositoryGenerator.image.dbcClient" $dot }} + image: {{ include "repositoryGenerator.image.drProvClient" $dot }} imagePullPolicy: {{ $dot.Values.global.pullPolicy | default $dot.Values.pullPolicy }} env: - - name: PROTO - value: "http" - - name: PORT - value: "8080" - - name: RESP_CACHE - value: /opt/app/config/cache - - name: REQUESTID - value: "{{ include "common.name" $dot }}-dmaap-provisioning" + - name: ONBEHALFHDR + value: "X-DMAAP-DR-ON-BEHALF-OF: drprovcl" {{- range $cred := $dot.Values.credentials }} - name: {{ $cred.name }} {{- include "common.secret.envFromSecretFast" (dict "global" $dot "uid" $cred.uid "key" $cred.key) | nindent 4 }} @@ -119,56 +116,5 @@ volumeMounts: {{- include "common.dmaap.provisioning._volumeMounts" $dot | trim | nindent 2 }} resources: {{ include "common.resources" $dot | nindent 4 }} -- name: {{ include "common.name" $dot }}-init-merge-config - image: {{ include "repositoryGenerator.image.envsubst" $dot }} - imagePullPolicy: {{ $dot.Values.global.pullPolicy | default $dot.Values.pullPolicy }} - command: - - /bin/sh - args: - - -c - - | - set -uex -o pipefail - if [ -d /opt/app/config/cache ]; then - cd /opt/app/config/cache - for file in $(ls feed*); do - NUM=$(echo "$file" | sed 's/feedConfig-\([0-9]\+\)-resp.json/\1/') - export DR_LOG_URL_"$NUM"="$(grep -o '"logURL":"[^"]*' "$file" | grep -w "feedlog" | cut -d '"' -f4)" - export DR_FILES_PUBLISHER_URL_"$NUM"="$(grep -o '"publishURL":"[^"]*' "$file" | cut -d '"' -f4)" - done - for file in $(ls drpub*); do - NUM=$(echo "$file" | sed 's/drpubConfig-\([0-9]\+\)-resp.json/\1/') - export DR_FILES_PUBLISHER_ID_"$NUM"="$(grep -o '"pubId":"[^"]*' "$file" | cut -d '"' -f4)" - done - for file in $(ls drsub*); do - NUM=$(echo "$file" | sed 's/drsubConfig-\([0-9]\+\)-resp.json/\1/') - export DR_FILES_SUBSCRIBER_ID_"$NUM"="$(grep -o '"subId":"[^"]*' "$file" | cut -d '"' -f4)" - done - for file in $(ls topics*); do - NUM=$(echo "$file" | sed 's/topicsConfig-\([0-9]\+\)-resp.json/\1/') - export MR_FILES_PUBLISHER_CLIENT_ID_"$NUM"="$(grep -o '"mrClientId":"[^"]*' "$file" | cut -d '"' -f4)" - done - else - echo "No Response logged for Dmaap BusController Http POST Request..!" - fi - cd /config-input && for PFILE in `ls -1`; do envsubst <${PFILE} >/config/${PFILE}; done - env: - {{- range $cred := $dot.Values.credentials }} - - name: {{ $cred.name }} - {{- include "common.secret.envFromSecretFast" (dict "global" $dot "uid" $cred.uid "key" $cred.key) | nindent 4 }} - {{- end }} - volumeMounts: - - mountPath: /opt/app/config/cache - name: dbc-response-cache - - mountPath: /config-input - name: app-config-input - - mountPath: /config - name: app-config - resources: - limits: - cpu: 200m - memory: 250Mi - requests: - cpu: 100m - memory: 200Mi {{- end -}} {{- end -}} \ No newline at end of file diff --git a/kubernetes/common/repositoryGenerator/templates/_repository.tpl b/kubernetes/common/repositoryGenerator/templates/_repository.tpl index 09a799e713..1da838a5b9 100644 --- a/kubernetes/common/repositoryGenerator/templates/_repository.tpl +++ b/kubernetes/common/repositoryGenerator/templates/_repository.tpl @@ -147,8 +147,8 @@ {{- include "repositoryGenerator.image._helper" (merge (dict "image" "readinessImage") .) }} {{- end -}} -{{- define "repositoryGenerator.image.dbcClient" -}} - {{- include "repositoryGenerator.image._helper" (merge (dict "image" "dbcClientImage") .) }} +{{- define "repositoryGenerator.image.drProvClient" -}} + {{- include "repositoryGenerator.image._helper" (merge (dict "image" "drProvClientImage") .) }} {{- end -}} {{- define "repositoryGenerator.image.quitQuit" -}} diff --git a/kubernetes/common/repositoryGenerator/values.yaml b/kubernetes/common/repositoryGenerator/values.yaml index b89c2b2bf6..2c94c29b15 100644 --- a/kubernetes/common/repositoryGenerator/values.yaml +++ b/kubernetes/common/repositoryGenerator/values.yaml @@ -41,7 +41,7 @@ global: postgresImage: crunchydata/crunchy-postgres:centos8-13.2-4.6.1 readinessImage: onap/oom/readiness:3.0.1 dcaePolicySyncImage: onap/org.onap.dcaegen2.deployments.dcae-services-policy-sync:1.0.1 - dbcClientImage: onap/dmaap/dbc-client:2.0.11 + drProvClientImage: onap/dmaap/datarouter-prov-client:2.1.14 quitQuitImage: onap/oom/readiness:4.1.0 # Default credentials @@ -75,5 +75,5 @@ imageRepoMapping: postgresImage: dockerHubRepository readinessImage: repository dcaePolicySyncImage: repository - dbcClientImage: repository + drProvClientImage: repository quitQuitImage: repository -- cgit 1.2.3-korg