Merge "[CDS-STRIMZI] Migrate cds to use strimzi kafka"

5 files changed, 231 insertions, 23 deletions

diff --git a/kubernetes/cds/components/cds-blueprints-processor/resources/config/application.properties b/kubernetes/cds/components/cds-blueprints-processor/resources/config/application.properties
index ea799e2119..0beaf4a42a 100755
--- a/kubernetes/cds/components/cds-blueprints-processor/resources/config/application.properties
+++ b/kubernetes/cds/components/cds-blueprints-processor/resources/config/application.properties
@@ -1,6 +1,6 @@
 {{/*
 #
-#  Copyright (c) 2017-2019 AT&T, IBM, Bell Canada, Nordix Foundation.
+#  Copyright (c) 2017-2022 AT&T, IBM, Bell Canada, Nordix Foundation.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -109,19 +109,70 @@ blueprintsprocessor.restclient.aai-data.additionalHeaders.X-FromAppId=cds-app-id
 blueprintsprocessor.restclient.aai-data.additionalHeaders.Accept=application/json
 
 # Self Service Request Kafka Message Consumer
-blueprintsprocessor.messageconsumer.self-service-api.kafkaEnable=false
-blueprintsprocessor.messageconsumer.self-service-api.type=kafka-basic-auth
-blueprintsprocessor.messageconsumer.self-service-api.bootstrapServers=message-router-kafka:9092
-blueprintsprocessor.messageconsumer.self-service-api.groupId=cds-consumer-group
-blueprintsprocessor.messageconsumer.self-service-api.topic=cds-consumer
-blueprintsprocessor.messageconsumer.self-service-api.clientId=cds-client
-blueprintsprocessor.messageconsumer.self-service-api.pollMillSec=1000
+blueprintsprocessor.messageconsumer.self-service-api.kafkaEnable={{ .Values.kafkaRequestConsumer.enabled  }}
+blueprintsprocessor.messageconsumer.self-service-api.type={{ .Values.kafkaRequestConsumer.type  }}
+{{- if eq .Values.useStrimziKafka true }}
+blueprintsprocessor.messageconsumer.self-service-api.bootstrapServers={{ include "common.release" . }}-strimzi-kafka-bootstrap:9092
+{{- else -}}
+blueprintsprocessor.messageconsumer.self-service-api.bootstrapServers={{ .Values.kafkaRequestConsumer.bootstrapServers  }}
+{{- end }}
+blueprintsprocessor.messageconsumer.self-service-api.groupId={{ .Values.kafkaRequestConsumer.groupId  }}
+blueprintsprocessor.messageconsumer.self-service-api.topic={{ .Values.kafkaRequestConsumer.topic  }}
+blueprintsprocessor.messageconsumer.self-service-api.clientId={{ .Values.kafkaRequestConsumer.clientId  }}
+blueprintsprocessor.messageconsumer.self-service-api.pollMillSec={{ .Values.kafkaRequestConsumer.pollMillSec  }}
+{{- if and (eq .Values.kafkaRequestConsumer.type "kafka-scram-plain-text-auth") (eq .Values.useStrimziKafka true) }}
+# SCRAM
+blueprintsprocessor.messageconsumer.self-service-api.scramUsername={{ include "common.release" . }}-{{ .Values.cdsKafkaUser }}
+blueprintsprocessor.messageconsumer.self-service-api.scramPassword=${JAAS_PASS}
+{{ end }}
 
 # Self Service Response Kafka Message Producer
-blueprintsprocessor.messageproducer.self-service-api.bootstrapServers=message-router-kafka:9092
-
-# Kafka Audit Service Configurations
-blueprintsprocessor.messageproducer.self-service-api.audit.kafkaEnable=false
+blueprintsprocessor.messageproducer.self-service-api.type={{ .Values.kafkaRequestProducer.type  }}
+{{- if eq .Values.useStrimziKafka true }}
+blueprintsprocessor.messageproducer.self-service-api.bootstrapServers={{ include "common.release" . }}-strimzi-kafka-bootstrap:9092
+{{- else -}}
+blueprintsprocessor.messageproducer.self-service-api.bootstrapServers={{ .Values.kafkaRequestProducer.bootstrapServers  }}
+{{- end }}
+blueprintsprocessor.messageproducer.self-service-api.clientId={{ .Values.kafkaRequestProducer.clientId }}
+blueprintsprocessor.messageproducer.self-service-api.topic={{ .Values.kafkaRequestProducer.topic }}
+{{- if and (eq .Values.kafkaRequestConsumer.type "kafka-scram-plain-text-auth") (eq .Values.useStrimziKafka true) }}
+# SCRAM
+blueprintsprocessor.messageproducer.self-service-api.scramUsername={{ include "common.release" . }}-{{ .Values.cdsKafkaUser }}
+blueprintsprocessor.messageproducer.self-service-api.scramPassword=${JAAS_PASS}
+{{ end }}
+
+# AUDIT KAFKA FEATURE CONFIGURATION
+# Audit feature dumps CDS request to a topic as well as a truncated response message to another topic.
+## Audit request
+blueprintsprocessor.messageproducer.self-service-api.audit.kafkaEnable={{ .Values.kafkaAuditRequest.enabled }}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.type={{ .Values.kafkaAuditRequest.type }}
+{{- if eq .Values.useStrimziKafka true }}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.bootstrapServers={{ include "common.release" . }}-strimzi-kafka-bootstrap:9092
+{{- else -}}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.bootstrapServers={{ .Values.kafkaAuditRequest.bootstrapServers  }}
+{{- end }}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.clientId={{ .Values.kafkaAuditRequest.clientId }}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.topic={{ .Values.kafkaAuditRequest.topic }}
+{{- if and (eq .Values.kafkaRequestConsumer.type "kafka-scram-plain-text-auth") (eq .Values.useStrimziKafka true) }}
+# SCRAM
+blueprintsprocessor.messageproducer.self-service-api.audit.request.scramUsername={{ include "common.release" . }}-{{ .Values.cdsKafkaUser }}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.scramPassword=${JAAS_PASS}
+{{ end }}
+
+## Audit response
+blueprintsprocessor.messageproducer.self-service-api.audit.response.type={{ .Values.kafkaAuditResponse.type }}
+{{- if eq .Values.useStrimziKafka true }}
+blueprintsprocessor.messageproducer.self-service-api.audit.response.bootstrapServers={{ include "common.release" . }}-strimzi-kafka-bootstrap:9092
+{{- else -}}
+blueprintsprocessor.messageproducer.self-service-api.audit.response.bootstrapServers={{ .Values.kafkaAuditRequest.bootstrapServers  }}
+{{- end }}
+blueprintsprocessor.messageproducer.self-service-api.audit.response.clientId={{ .Values.kafkaAuditResponse.clientId }}
+blueprintsprocessor.messageproducer.self-service-api.audit.response.topic={{ .Values.kafkaAuditResponse.topic }}
+{{- if and (eq .Values.kafkaRequestConsumer.type "kafka-scram-plain-text-auth") (eq .Values.useStrimziKafka true) }}
+# SCRAM
+blueprintsprocessor.messageproducer.self-service-api.audit.response.scramUsername={{ include "common.release" . }}-{{ .Values.cdsKafkaUser }}
+blueprintsprocessor.messageproducer.self-service-api.audit.response.scramPassword=${JAAS_PASS}
+{{ end }}
 
 # Executor Options
 blueprintsprocessor.resourceResolution.enabled=true
@@ -132,10 +183,10 @@ blueprintsprocessor.remoteScriptCommand.enabled=true
 ## Enable py-executor
 blueprintsprocessor.streamingRemoteExecution.enabled=true
 
-# Used in Health Check
-blueprintsprocessor.messageproducer.self-service-api.type=kafka-basic-auth
-blueprintsprocessor.messageproducer.self-service-api.clientId=cds-client
-blueprintsprocessor.messageproducer.self-service-api.topic=cds-producer
+## Used in Health Check
+#blueprintsprocessor.messageproducer.self-service-api.type=kafka-basic-auth
+#blueprintsprocessor.messageproducer.self-service-api.clientId=cds-client
+#blueprintsprocessor.messageproducer.self-service-api.topic=cds-producer
 
 
 #Encrypted username and password for health check service
diff --git a/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-topics.yaml b/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-topics.yaml
new file mode 100644
index 0000000000..555f4d4e60
--- /dev/null
+++ b/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-topics.yaml
@@ -0,0 +1,68 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{ if eq .Values.useStrimziKafka true }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: {{ .Values.kafkaRequestConsumer.topic  }}
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  partitions: 10
+  replicas: 2
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: {{ .Values.kafkaRequestProducer.topic }}
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  partitions: 10
+  replicas: 2
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: {{ .Values.kafkaAuditRequest.topic }}
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  partitions: 10
+  replicas: 2
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: {{ .Values.kafkaAuditResponse.topic }}
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  partitions: 10
+  replicas: 2
+  config:
+    retention.ms: 7200000
+    segment.bytes: 1073741824
+{{ end }}
\ No newline at end of file
diff --git a/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-user.yaml b/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-user.yaml
new file mode 100644
index 0000000000..65ee1d2a96
--- /dev/null
+++ b/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-user.yaml
@@ -0,0 +1,49 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{ if eq .Values.useStrimziKafka true }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaUser
+metadata:
+  name: {{ include "common.release" . }}-{{ .Values.cdsKafkaUser }}
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  authentication:
+    type: scram-sha-512
+  authorization:
+    type: simple
+    acls:
+    - resource:
+        type: group
+        name: {{ .Values.kafkaRequestConsumer.groupId  }}
+      operation: All
+    - resource:
+        type: topic
+        name: {{ .Values.kafkaRequestConsumer.topic  }}
+      operation: All
+    - resource:
+        type: topic
+        name: {{ .Values.kafkaRequestProducer.topic }}
+      operation: All
+    - resource:
+        type: topic
+        name: {{ .Values.kafkaAuditRequest.topic }}
+      operation: All
+    - resource:
+        type: topic
+        name: {{ .Values.kafkaAuditResponse.topic }}
+      operation: All
+{{ end }}
\ No newline at end of file
diff --git a/kubernetes/cds/components/cds-blueprints-processor/templates/deployment.yaml b/kubernetes/cds/components/cds-blueprints-processor/templates/deployment.yaml
index d92f09a4c8..d68e900222 100755
--- a/kubernetes/cds/components/cds-blueprints-processor/templates/deployment.yaml
+++ b/kubernetes/cds/components/cds-blueprints-processor/templates/deployment.yaml
@@ -1,6 +1,7 @@
 {{/*
 # Copyright (c) 2019 IBM, Bell Canada
 # Copyright (c) 2020 Samsung Electronics
+# Modification Copyright © 2022 Nordix Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -78,10 +79,6 @@ spec:
         args:
         - --container-name
         - cds-db
-        {{- if .Values.dmaapEnabled  }}
-        - --container-name
-        - message-router
-        {{ end }}
         env:
         - name: NAMESPACE
           valueFrom:
@@ -121,6 +118,10 @@ spec:
                 fieldPath: metadata.name
           - name: CLUSTER_CONFIG_FILE
             value: {{ .Values.config.appConfigDir }}/hazelcast.yaml
+          {{ if .Values.useStrimziKafka }}
+          - name: JAAS_PASS
+            value: {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "cds-kafka-secret" "key" "password") | indent 12 }}
+          {{ end }}
           ports:
           - containerPort: {{ .Values.service.http.internalPort }}
           - containerPort: {{ .Values.service.grpc.internalPort }}
diff --git a/kubernetes/cds/components/cds-blueprints-processor/values.yaml b/kubernetes/cds/components/cds-blueprints-processor/values.yaml
index a5180c53c6..af9482b663 100755
--- a/kubernetes/cds/components/cds-blueprints-processor/values.yaml
+++ b/kubernetes/cds/components/cds-blueprints-processor/values.yaml
@@ -1,5 +1,6 @@
 # Copyright (c) 2019 IBM, Bell Canada
 # Copyright (c) 2020 Samsung Electronics
+# Modification Copyright © 2022 Nordix Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -57,6 +58,13 @@ secrets:
     externalSecret: '{{ tpl (default "" .Values.config.sdncDB.dbRootPassExternalSecret) . }}'
     password: '{{ .Values.config.sdncDB.dbRootPass }}'
     passwordPolicy: required
+  - uid: cds-kafka-secret
+    externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}'
+    type: genericKV
+    envs:
+      - name: password
+        value: '{{ .Values.config.someConfig }}'
+        policy: generate
 
 #################################################################
 # AAF part
@@ -111,6 +119,7 @@ config:
     # dbCredsExternalSecret: <some secret name>
     # dbRootPassword: password
     # dbRootPassExternalSecret
+  someConfig: blah
 
 # default number of instances
 replicaCount: 1
@@ -119,10 +128,40 @@ nodeSelector: {}
 
 affinity: {}
 
-# flag for kafka-listener dependency. Set to true if you are using message-router otherwise set to false if you are using
-# custom kafka cluster.
-dmaapEnabled: true
+# If useStrimziKafka is true, the following also applies:
+# strimzi will create an associated kafka user and the topics defined for Request and Audit elements below.
+# The connection type must be kafka-scram-plain-text-auth
+# The bootstrapServers will target the strimzi kafka cluster by default
+useStrimziKafka: false
+cdsKafkaUser: cds-kafka-user
 
+kafkaRequestConsumer:
+  enabled: false
+  type: kafka-scram-plain-text-auth
+  bootstrapServers: host:port
+  groupId: cds-consumer
+  topic: cds.blueprint-processor.self-service-api.request
+  clientId: request-receiver-client-id
+  pollMillSec: 1000
+kafkaRequestProducer:
+  type: kafka-scram-plain-text-auth
+  bootstrapServers: host:port
+  clientId: request-producer-client-id
+  topic: cds.blueprint-processor.self-service-api.response
+  enableIdempotence: false
+kafkaAuditRequest:
+  enabled: false
+  type: kafka-scram-plain-text-auth
+  bootstrapServers: host:port
+  clientId: audit-request-producer-client-id
+  topic: cds.blueprint-processor.self-service-api.audit.request
+  enableIdempotence: false
+kafkaAuditResponse:
+  type: kafka-scram-plain-text-auth
+  bootstrapServers: host:port
+  clientId: audit-response-producer-client-id
+  topic: cds.blueprint-processor.self-service-api.audit.response
+  enableIdempotence: false
 
 # probe configuration parameters
 startup: