Merge "[DMAAP] Introduce drprov-client"

9 files changed, 99 insertions, 154 deletions

diff --git a/kubernetes/common/common/templates/_dmaapProvisioning.tpl b/kubernetes/common/common/templates/_dmaapProvisioning.tpl
index fae16ff7eb..3cf5bf1207 100644
--- a/kubernetes/common/common/templates/_dmaapProvisioning.tpl
+++ b/kubernetes/common/common/templates/_dmaapProvisioning.tpl
@@ -1,7 +1,7 @@
 {{/*
 ################################################################################
 #   Copyright (C) 2021 Nordix Foundation.                                      #
-#   Copyright (c) 2022 J. F. Lucas.  All rights reserved.                      #
+#   Copyright (c) 2022-2023 J. F. Lucas.  All rights reserved.                      #
 #                                                                              #
 #   Licensed under the Apache License, Version 2.0 (the "License");            #
 #   you may not use this file except in compliance with the License.           #
@@ -19,51 +19,54 @@
 
 {{/*
   This template generates a Kubernetes init containers common template to enable applications to provision
-  DMaaP feeds (on Data Router), with associated authorization.
-  DMaap Bus Controller endpoints are used to provision:
-
-  - Feed on DR, with associated user authentication.
+  DMaaP feeds (on Data Router) for DCAE microservices, with associated authorization.
+  DMaap Data Router (DR) endpoints are used to provision:
+  - Feeds on DR, with associated user authentication.
+  - Subscribers to feeds on DR, to provide DR with username, password, and URL needed to deliver
+    files to subscribers.
 
   common.dmaap.provisioning.initContainer:
-  This template make use of Dmaap Bus Controller docker image to create resources on Dmaap Data Router
-  microservice, with the help of dbc-client.sh script it makes use of Bus Controller API to create Feeds.
-  If the resource creation is successful via script response is logged back at particular location with
-  appropriate naming convention.
-
-  More details can be found at :
-  (https://wiki.onap.org/pages/viewpage.action?pageId=103417564)
+  This template creates an initContainer with some associated volumes.  The initContainer
+  (oom/kubernetes/dmaap-datarouter/drprov-client) runs a script (drprov-client.sh) that uses the
+  DR provisioning API to create the feeds and subscribers needed by a microservice.  The script
+  updates the microservice's configuration to supply information needed to access the feeds. The
+  configuration information comes from two volumes that are created by the dcaegen2-services-common
+  templates.
+  - app-config-input: comes from a configMap generated from the microservice's values.yaml file.
+    It may contain references to environment variables as placeholders for feed information that
+    will become available after feeds are provisioned.
+  - app-config: this template will copy the configuration file from the app-config-input volume,
+    replaced the environment variable references with the actual values for feed information, based
+    on data returned by the DR provisioning API.
 
   The template directly references data in .Values, and indirectly (through its
   use of templates from the ONAP "common" collection) references data in .Release.
 
-  Parameter for _dmaapProvisioning to be defined in values.yaml
+  Parameters for _dmaapProvisioning to be defined in values.yaml:
+
   # DataRouter Feed Configuration
+  # (Note that DR configures publishers as part of the feed.)
   drFeedConfig:
     - feedName: bulk_pm_feed
-      owner: dcaecm
       feedVersion: 0.0
-      asprClassification: unclassified
+      classification: unclassified
       feedDescription: DFC Feed Creation
-
-  # DataRouter Publisher Configuration
-  drPubConfig:
-    - feedName: bulk_pm_feed
-      dcaeLocationName: loc00
+      publisher:
+        username: xyz
+        password: xyz
 
   # DataRouter Subscriber Configuration
   drSubConfig:
     - feedName: bulk_pm_feed
+      feedVersion: 0.0
       decompress: True
-      dcaeLocationName: loc00
       privilegedSubscriber: True
       deliveryURL: https://dcae-pm-mapper:8443/delivery
 
-  # ConfigMap Configuration for DR Feed, Dr_Publisher, Dr_Subscriber
+  # ConfigMap Configuration for DR Feed, Dr_Subscriber
   volumes:
     - name: feeds-config
       path: /opt/app/config/feeds
-    - name: drpub-config
-      path: /opt/app/config/dr_pubs
     - name: drsub-config
       path: /opt/app/config/dr_subs
 
@@ -76,8 +79,10 @@
 
 {{- define "common.dmaap.provisioning._volumeMounts" -}}
 {{- $dot := default . .dot -}}
-- mountPath: /opt/app/config/cache
-  name: dbc-response-cache
+- mountPath: /config-input
+  name: app-config-input
+- mountPath: /config
+  name: app-config
 {{- range $name, $volume := $dot.Values.volumes }}
 - name: {{ $volume.name }}
   mountPath: {{ $volume.path }}
@@ -86,8 +91,6 @@
 
 {{- define "common.dmaap.provisioning._volumes" -}}
 {{- $dot := default . .dot -}}
-- name: dbc-response-cache
-  emptyDir: {}
 {{- range $name, $volume := $dot.Values.volumes }}
 - name: {{ $volume.name }}
   configMap:
@@ -98,20 +101,14 @@
 
 {{- define "common.dmaap.provisioning.initContainer" -}}
 {{- $dot := default . .dot -}}
-{{- $drFeedConfig := default $dot.Values.drFeedConfig .drFeedConfig -}}
-{{- if $drFeedConfig -}}
+{{- $drNeedProvisioning := or $dot.Values.drFeedConfig $dot.Values.drSubConfig -}}
+{{- if $drNeedProvisioning -}}
 - name: {{ include "common.name" $dot }}-init-dmaap-provisioning
-  image: {{ include "repositoryGenerator.image.dbcClient" $dot }}
+  image: {{ include "repositoryGenerator.image.drProvClient" $dot }}
   imagePullPolicy: {{ $dot.Values.global.pullPolicy | default $dot.Values.pullPolicy }}
   env:
-  - name: PROTO
-    value: "http"
-  - name: PORT
-    value: "8080"
-  - name: RESP_CACHE
-    value: /opt/app/config/cache
-  - name: REQUESTID
-    value: "{{ include "common.name" $dot }}-dmaap-provisioning"
+  - name: ONBEHALFHDR
+    value: "X-DMAAP-DR-ON-BEHALF-OF: drprovcl"
   {{- range $cred := $dot.Values.credentials }}
   - name: {{ $cred.name }}
     {{- include "common.secret.envFromSecretFast" (dict "global" $dot "uid" $cred.uid "key" $cred.key) | nindent 4 }}
@@ -119,56 +116,5 @@
   volumeMounts:
   {{- include "common.dmaap.provisioning._volumeMounts" $dot | trim | nindent 2 }}
   resources: {{ include "common.resources" $dot | nindent 4 }}
-- name: {{ include "common.name" $dot }}-init-merge-config
-  image: {{ include "repositoryGenerator.image.envsubst" $dot }}
-  imagePullPolicy: {{ $dot.Values.global.pullPolicy | default $dot.Values.pullPolicy }}
-  command:
-  - /bin/sh
-  args:
-  - -c
-  - |
-    set -uex -o pipefail
-    if [ -d /opt/app/config/cache ]; then
-      cd /opt/app/config/cache
-      for file in $(ls feed*); do
-        NUM=$(echo "$file" | sed 's/feedConfig-\([0-9]\+\)-resp.json/\1/')
-        export DR_LOG_URL_"$NUM"="$(grep -o '"logURL":"[^"]*' "$file" | grep -w "feedlog" | cut -d '"' -f4)"
-        export DR_FILES_PUBLISHER_URL_"$NUM"="$(grep -o '"publishURL":"[^"]*' "$file" | cut -d '"' -f4)"
-      done
-      for file in $(ls drpub*); do
-        NUM=$(echo "$file" | sed 's/drpubConfig-\([0-9]\+\)-resp.json/\1/')
-        export DR_FILES_PUBLISHER_ID_"$NUM"="$(grep -o '"pubId":"[^"]*' "$file" | cut -d '"' -f4)"
-      done
-      for file in $(ls drsub*); do
-        NUM=$(echo "$file" | sed 's/drsubConfig-\([0-9]\+\)-resp.json/\1/')
-        export DR_FILES_SUBSCRIBER_ID_"$NUM"="$(grep -o '"subId":"[^"]*' "$file" | cut -d '"' -f4)"
-      done
-      for file in $(ls topics*); do
-        NUM=$(echo "$file" | sed 's/topicsConfig-\([0-9]\+\)-resp.json/\1/')
-        export MR_FILES_PUBLISHER_CLIENT_ID_"$NUM"="$(grep -o '"mrClientId":"[^"]*' "$file" | cut -d '"' -f4)"
-      done
-    else
-      echo "No Response logged for Dmaap BusController Http POST Request..!"
-    fi
-    cd /config-input && for PFILE in `ls -1`; do envsubst <${PFILE} >/config/${PFILE}; done
-  env:
-  {{- range $cred := $dot.Values.credentials }}
-  - name: {{ $cred.name }}
-    {{- include "common.secret.envFromSecretFast" (dict "global" $dot "uid" $cred.uid "key" $cred.key) | nindent 4 }}
-  {{- end }}
-  volumeMounts:
-  - mountPath: /opt/app/config/cache
-    name: dbc-response-cache
-  - mountPath: /config-input
-    name: app-config-input
-  - mountPath: /config
-    name: app-config
-  resources:
-    limits:
-      cpu: 200m
-      memory: 250Mi
-    requests:
-      cpu: 100m
-      memory: 200Mi
 {{- end -}}
 {{- end -}}
\ No newline at end of file
diff --git a/kubernetes/common/repositoryGenerator/templates/_repository.tpl b/kubernetes/common/repositoryGenerator/templates/_repository.tpl
index 09a799e713..1da838a5b9 100644
--- a/kubernetes/common/repositoryGenerator/templates/_repository.tpl
+++ b/kubernetes/common/repositoryGenerator/templates/_repository.tpl
@@ -147,8 +147,8 @@
   {{- include "repositoryGenerator.image._helper" (merge (dict "image" "readinessImage") .) }}
 {{- end -}}
 
-{{- define "repositoryGenerator.image.dbcClient" -}}
-  {{- include "repositoryGenerator.image._helper" (merge (dict "image" "dbcClientImage") .) }}
+{{- define "repositoryGenerator.image.drProvClient" -}}
+  {{- include "repositoryGenerator.image._helper" (merge (dict "image" "drProvClientImage") .) }}
 {{- end -}}
 
 {{- define "repositoryGenerator.image.quitQuit" -}}
diff --git a/kubernetes/common/repositoryGenerator/values.yaml b/kubernetes/common/repositoryGenerator/values.yaml
index b89c2b2bf6..2c94c29b15 100644
--- a/kubernetes/common/repositoryGenerator/values.yaml
+++ b/kubernetes/common/repositoryGenerator/values.yaml
@@ -41,7 +41,7 @@ global:
   postgresImage: crunchydata/crunchy-postgres:centos8-13.2-4.6.1
   readinessImage: onap/oom/readiness:3.0.1
   dcaePolicySyncImage: onap/org.onap.dcaegen2.deployments.dcae-services-policy-sync:1.0.1
-  dbcClientImage: onap/dmaap/dbc-client:2.0.11
+  drProvClientImage: onap/dmaap/datarouter-prov-client:2.1.14
   quitQuitImage: onap/oom/readiness:4.1.0
 
   # Default credentials
@@ -75,5 +75,5 @@ imageRepoMapping:
   postgresImage: dockerHubRepository
   readinessImage: repository
   dcaePolicySyncImage: repository
-  dbcClientImage: repository
+  drProvClientImage: repository
   quitQuitImage: repository
diff --git a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl
index afd3c38f31..bb35e80848 100644
--- a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl
+++ b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl
@@ -1,7 +1,7 @@
 {{/*
 # Copyright © 2017 Amdocs, Bell Canada
 # Modifications Copyright © 2019 AT&T
-# Copyright (c) 2021-2022 J. F. Lucas.  All rights reserved.
+# Copyright (c) 2021-2023 J. F. Lucas.  All rights reserved.
 # Copyright (c) 2021 Nordix Foundation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,14 +33,10 @@ The template always produces a configMap containing the microservice's
 initial configuration data.  (See the documentation for
 dcaegen2-services-common.microserviceDeployment for more details.)
 
-If the microservice is using one or more Data Router (DR) feeds, the
+If the microservice is publishing to one or more Data Router (DR) feeds, the
 template produces a configMap containing the information needed to
 provision the feed(s).  An init container performs the provisioning.
 
-If the microservice acts as a DR publisher for one or more feeds, the
-template produces a configMap containing the information needed to
-provision the publisher(s).  An init container performs the provisioning.
-
 If the microservice acts as a DR subscriber for one or more feeds, the
 template produces a configMap containing the information needed to
 provision the subscribeer(s).  An init container performs the provisioning.
@@ -70,22 +66,22 @@ metadata:
 data:
   {{- range $i, $feed := .Values.drFeedConfig }}
   feedConfig-{{$i}}.json: |-
-  {{ $feed | toJson | indent 2 }}
-  {{- end }}
-{{- end }}
-
-{{- if .Values.drPubConfig }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "common.fullname" . }}-drpub-config
-  namespace: {{ include "common.namespace" . }}
-  labels: {{ include "common.labels" . | nindent 6 }}
-data:
-  {{- range $i, $drpub := .Values.drPubConfig }}
-  drpubConfig-{{$i}}.json: |-
-  {{ $drpub | toJson | indent 2 }}
+    {
+      "name": {{ $feed.feedName | quote }},
+      "version": {{ $feed.feedVersion | quote }},
+      "description": {{ $feed.feedDescription | default "None" | quote }},
+      "authorization": {
+        "classification": {{ $feed.classification | quote }},
+        "endpoint_addrs": [
+        ],
+        "endpoint_ids": [
+          {
+            "id": {{ $feed.publisher.username | quote }},
+            "password": {{ $feed.publisher.password | quote }}
+          }
+        ]
+      }
+    }
   {{- end }}
 {{- end }}
 
@@ -100,7 +96,23 @@ metadata:
 data:
   {{- range $i, $drsub := .Values.drSubConfig }}
   drsubConfig-{{$i}}.json: |-
-  {{ $drsub | toJson | indent 2 }}
+    {
+      "feed": {
+        "name": {{ $drsub.feedName | quote }},
+        "version": {{ $drsub.feedVersion | quote }}
+      },
+      "delivery": {
+        "url": {{ $drsub.deliveryURL | quote }},
+        "user": {{ $drsub.username | quote }},
+        "password": {{ $drsub.userpwd | quote }},
+        "use100": {{ $drsub.use100 | default false }}
+      },
+      "metadataOnly": {{ $drsub.metadataOnly | default false }},
+      "groupid": {{ $drsub.groupId | default 0 }},
+      "follow_redirect": {{ $drsub.followRedirect | default true }},
+      "privileged_subscriber": {{ $drsub.privilegedSubscriber | default false }},
+      "decompress": {{ $drsub.decompress | default false }}
+    }
   {{- end }}
 {{- end }}
 {{- end }}
diff --git a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl
index e7f9627a3d..b0c76a9b3a 100644
--- a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl
+++ b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl
@@ -240,7 +240,7 @@ post-processing.
 {{- $commonRelease :=  print (include "common.release" .) -}}
 {{- $policy := default dict .Values.policies -}}
 {{- $policyRls := default $commonRelease $policy.policyRelease -}}
-{{- $drFeedConfig := default "" .Values.drFeedConfig -}}
+{{- $drNeedProvisioning := or .Values.drFeedConfig .Values.drSubConfig -}}
 {{- $dcaeName := print (include "common.fullname" .) }}
 {{- $dcaeLabel := (dict "dcaeMicroserviceName" $dcaeName) -}}
 {{- $dot := . -}}
@@ -310,7 +310,7 @@ spec:
         resources: {{ include "common.resources" . | nindent 10 }}
         volumeMounts:
         - mountPath: /app-config
-          name: {{ ternary "app-config-input" "app-config" (not $drFeedConfig) }}
+          name: {{ ternary "app-config-input" "app-config" (not $drNeedProvisioning) }}
         - mountPath: /app-config-input
           name: app-config-input
         {{- if $logDir }}
diff --git a/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml b/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml
index b9e7a39ecb..26bbd37490 100644
--- a/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml
+++ b/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml
@@ -89,11 +89,12 @@ certificates:
         create: true
 
 # Dependencies
+# Waiting for dmaap-dr-node (which depends on dmaap-dr-prov)
+# to be sure that we can provision the DR feed that's needed
 readinessCheck:
   wait_for:
     containers:
-      - dmaap-bc
-      - dmaap-provisioning-job
+      - dmaap-dr-node
       - message-router
 
 # Probe Configuration
@@ -166,10 +167,10 @@ applicationConfig:
   streams_publishes:
     PM_MEAS_FILES:
       dmaap_info:
-        publisher_id: ${DR_FILES_PUBLISHER_ID_0}
+        publisher_id: "dummy_id"
         location: loc00
-        log_url: ${DR_LOG_URL_0}
-        publish_url: ${DR_FILES_PUBLISHER_URL_0}
+        log_url: ${DR_FEED_LOGURL_0}
+        publish_url: ${DR_FEED_PUBURL_0}
         username: ${DR_USERNAME}
         password: ${DR_PASSWORD}
       type: data_router
@@ -187,24 +188,17 @@ applicationEnv:
 # DataRouter Feed Configuration
 drFeedConfig:
   - feedName: bulk_pm_feed
-    owner: dcaecm
     feedVersion: "0.0"
-    asprClassification: unclassified
+    classification: unclassified
     feedDescription: DFC Feed Creation
-
-# DataRouter Publisher Configuration
-drPubConfig:
-  - feedName: bulk_pm_feed
-    username: ${DR_USERNAME}
-    userpwd: ${DR_PASSWORD}
-    dcaeLocationName: loc00
+    publisher:
+      username: ${DR_USERNAME}
+      password: ${DR_PASSWORD}
 
 # ConfigMap Configuration for Feed, Dr_Publisher
 volumes:
   - name: feeds-config
     path: /opt/app/config/feeds
-  - name: drpub-config
-    path: /opt/app/config/dr_pubs
 
 # Resource Limit Flavor -By Default Using Small
 flavor: small
diff --git a/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml b/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml
index 1aee8c9a1b..ab23b54e06 100644
--- a/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml
+++ b/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml
@@ -58,13 +58,16 @@ log:
 logConfigMapNamePrefix: '{{ include "common.fullname" . }}'
 
 # Dependencies
+# Depend on the datafile-collector, which guarantees that
+# the DR feed that pm-mapper susbscribes to will be created
+# already by the datafile-collector DMaaP provisioning init
+# container.  Also guarantees that DR provisioning will be
+# available for pm-mapper initContainter to create the
+# subscription to the feed.
 readinessCheck:
   wait_for:
     containers:
-      - dmaap-bc
-      - dmaap-provisioning-job
       - dcae-datafile-collector
-      - message-router
 
 # Probe Configuration
 readiness:
@@ -120,7 +123,7 @@ applicationConfig:
     dmaap_publisher:
       type: message_router
       dmaap_info:
-        client_id: ${MR_FILES_PUBLISHER_CLIENT_ID_0}
+        client_id: "dummy_id"
         location: san-francisco
         client_role: org.onap.dcae.pmPublisher
         topic_url: http://message-router:3904/events/unauthenticated.PERFORMANCE_MEASUREMENTS
@@ -128,7 +131,7 @@ applicationConfig:
     dmaap_subscriber:
       type: data_router
       dmaap_info:
-        subscriber_id: ${DR_FILES_SUBSCRIBER_ID_0}
+        subscriber_id: "dummy_id"
         decompress: true
         privileged: true
         username: ${DR_USERNAME}
@@ -141,28 +144,18 @@ applicationEnv:
   #Temporary Dummy CBS Port Value until internal SDK library is updated
   CONFIG_BINDING_SERVICE_SERVICE_PORT: '0000'
 
-# DataRouter Feed Configuration
-drFeedConfig:
-  - feedName: bulk_pm_feed
-    owner: dcaecm
-    feedVersion: "0.0"
-    asprClassification: unclassified
-    feedDescription: DFC Feed Creation
-
 # DataRouter Subscriber Configuration
 drSubConfig:
   - feedName: bulk_pm_feed
+    feedVersion: "0.0"
     decompress: true
     username: ${DR_USERNAME}
     userpwd: ${DR_PASSWORD}
-    dcaeLocationName: loc00
     privilegedSubscriber: true
     deliveryURL: http://dcae-pm-mapper:8081/delivery
 
-# ConfigMap Configuration for Dr Feed, Subscriber, MR Topics
+# ConfigMap Configuration for DR Subscriber
 volumes:
-  - name: feeds-config
-    path: /opt/app/config/feeds
   - name: drsub-config
     path: /opt/app/config/dr_subs
 
diff --git a/kubernetes/dmaap/components/dmaap-dr-node/values.yaml b/kubernetes/dmaap/components/dmaap-dr-node/values.yaml
index eef6755385..d6815fdee1 100644
--- a/kubernetes/dmaap/components/dmaap-dr-node/values.yaml
+++ b/kubernetes/dmaap/components/dmaap-dr-node/values.yaml
@@ -23,7 +23,7 @@ global:
 # Application configuration defaults.
 #################################################################
 # application image
-image: onap/dmaap/datarouter-node:2.1.13
+image: onap/dmaap/datarouter-node:2.1.14
 pullPolicy: Always
 
 # default number of instances
diff --git a/kubernetes/dmaap/components/dmaap-dr-prov/values.yaml b/kubernetes/dmaap/components/dmaap-dr-prov/values.yaml
index af855c7640..23188a62c2 100644
--- a/kubernetes/dmaap/components/dmaap-dr-prov/values.yaml
+++ b/kubernetes/dmaap/components/dmaap-dr-prov/values.yaml
@@ -38,7 +38,7 @@ secrets:
 # Application configuration defaults.
 #################################################################
 # application image
-image: onap/dmaap/datarouter-prov:2.1.13
+image: onap/dmaap/datarouter-prov:2.1.14
 pullPolicy: Always
 
 # default number of instances