[SDC] Add strimzi kafka setup on sdc-be

Add SDC KafkaTopics and KafkaUser

Signed-off-by: efiacor <fiachra.corcoran@est.tech>
Change-Id: I78542539567d2e3b67376b154ad069df3d1dad55
Issue-ID: DMAAP-1744

13 files changed, 161 insertions, 7 deletions

diff --git a/kubernetes/sdc/components/sdc-be/templates/configmap.yaml b/kubernetes/sdc/components/sdc-be/templates/configmap.yaml
index aa632f33f4..332cd74661 100644
--- a/kubernetes/sdc/components/sdc-be/templates/configmap.yaml
+++ b/kubernetes/sdc/components/sdc-be/templates/configmap.yaml
@@ -27,3 +27,5 @@ metadata:
     heritage: {{ .Release.Service }}
 data:
 {{ tpl (.Files.Glob "resources/config/logging/*").AsConfig . | indent 2 }}
+
+
diff --git a/kubernetes/sdc/components/sdc-be/templates/deployment.yaml b/kubernetes/sdc/components/sdc-be/templates/deployment.yaml
index 24f169a5b9..16fc57aa92 100644
--- a/kubernetes/sdc/components/sdc-be/templates/deployment.yaml
+++ b/kubernetes/sdc/components/sdc-be/templates/deployment.yaml
@@ -31,8 +31,10 @@ spec:
         args:
         - --container-name
         - "sdc-onboarding-be"
+        {{- if not .Values.global.kafka.useKafka }}
         - --container-name
         - "message-router"
+        {{- end }}
         env:
         - name: NAMESPACE
           valueFrom:
@@ -159,6 +161,12 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: status.podIP
+          {{- if .Values.global.kafka.useKafka }}
+          - name: SASL_JAAS_CONFIG
+            {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "sdc-be-kafka-secret" "key" "sasl.jaas.config") | indent 12 }}
+          - name: USE_KAFKA
+            value: {{ .Values.global.kafka.useKafka | quote }}
+          {{- end }}
           volumeMounts:
           - name: sdc-environments
             mountPath: /app/jetty/chef-solo/environments/
diff --git a/kubernetes/sdc/components/sdc-be/templates/sdc-be-kafka-user.yaml b/kubernetes/sdc/components/sdc-be/templates/sdc-be-kafka-user.yaml
new file mode 100644
index 0000000000..6cd7f93c5a
--- /dev/null
+++ b/kubernetes/sdc/components/sdc-be/templates/sdc-be-kafka-user.yaml
@@ -0,0 +1,39 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+
+{{- if .Values.global.kafka.useKafka }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaUser
+metadata:
+  name: {{ include "common.release" . }}-{{ .Values.global.kafka.sdcBeKafkaUser }}
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  authentication:
+    type: {{ .Values.config.kafka.saslMech }}
+  authorization:
+    type: {{ .Values.config.kafka.authType }}
+    acls:
+    - resource:
+        type: group
+        name: {{ .Values.config.kafka.topicConsumer.groupId }}-{{ .Values.env.name }}
+      operation: Read
+    - resource:
+        type: topic
+        patternType: prefix
+        name: {{ .Values.config.kafka.topicConsumer.pattern }}
+      operation: All
+{{- end }}
diff --git a/kubernetes/sdc/components/sdc-be/templates/sdc-distro-topics.yaml b/kubernetes/sdc/components/sdc-be/templates/sdc-distro-topics.yaml
new file mode 100644
index 0000000000..9a6f7579e8
--- /dev/null
+++ b/kubernetes/sdc/components/sdc-be/templates/sdc-distro-topics.yaml
@@ -0,0 +1,40 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.global.kafka.useKafka }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: sdc-distro-notif-topic
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  topicName: {{ .Values.global.kafka.topics.sdcDistNotifTopic }}-{{ .Values.env.name }}
+  config:
+    retention.ms: {{ .Values.config.kafka.topicRetentionMs }}
+    segment.bytes: {{ .Values.config.kafka.topicSegmentBytes }}
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: sdc-distro-status-topic
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  topicName: {{ .Values.global.kafka.topics.sdcDistStatusTopic }}-{{ .Values.env.name }}
+  config:
+    retention.ms: {{ .Values.config.kafka.topicRetentionMs }}
+    segment.bytes: {{ .Values.config.kafka.topicSegmentBytes }}
+{{- end }}
\ No newline at end of file
diff --git a/kubernetes/sdc/components/sdc-be/templates/secret.yaml b/kubernetes/sdc/components/sdc-be/templates/secret.yaml
new file mode 100644
index 0000000000..bb5091f01a
--- /dev/null
+++ b/kubernetes/sdc/components/sdc-be/templates/secret.yaml
@@ -0,0 +1,17 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+
+{{ include "common.secretFast" . }}
diff --git a/kubernetes/sdc/components/sdc-be/values.yaml b/kubernetes/sdc/components/sdc-be/values.yaml
index 9f216352c0..faf46e5549 100644
--- a/kubernetes/sdc/components/sdc-be/values.yaml
+++ b/kubernetes/sdc/components/sdc-be/values.yaml
@@ -1,5 +1,6 @@
 # Copyright © 2017 Amdocs, Bell Canada
 # Modifications Copyright © 2018 AT&T, ZTE
+# Modifications Copyright © 2022 Nordix Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,13 +31,20 @@ global:
     replicaCount: 3
     clusterName: cassandra
     dataCenter: Pod
+  # Strimzi kafka config
+  kafka:
+    useKafka: overridden-from-parent-values-yaml
+    sdcBeKafkaUser: overridden-from-parent-values-yaml
+    topics:
+      sdcDistNotifTopic: overridden-from-parent-values-yaml
+      sdcDistStatusTopic: overridden-from-parent-values-yaml
 
 #################################################################
 # Application configuration defaults.
 #################################################################
 # application image
-image: onap/sdc-backend-all-plugins:1.11.9
-backendInitImage: onap/sdc-backend-init:1.11.9
+image: onap/sdc-backend-all-plugins:1.12.0
+backendInitImage: onap/sdc-backend-init:1.12.0
 
 pullPolicy: Always
 
@@ -70,9 +78,29 @@ certInitializer:
 #################################################################
 # SDC Config part
 #################################################################
+
+secrets:
+  - uid: sdc-be-kafka-secret
+    externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}'
+    type: genericKV
+    envs:
+      - name: sasl.jaas.config
+        value: '{{ .Values.config.someConfig }}'
+        policy: generate
+
 config:
   javaOptions: "-Xmx1536m -Xms1536m"
   cassandraSslEnabled: "false"
+  # Strimzi kafka config
+  kafka:
+    saslMech: scram-sha-512
+    securityProtocol: SASL_PLAINTEXT
+    authType: simple
+    topicRetentionMs: 7200000
+    topicSegmentBytes: 1073741824
+    topicConsumer:
+      pattern: SDC-DIST
+      groupId: sdc
 
 # default number of instances
 replicaCount: 1
diff --git a/kubernetes/sdc/components/sdc-cs/values.yaml b/kubernetes/sdc/components/sdc-cs/values.yaml
index 46208a21c6..b3dfbfa0ba 100644
--- a/kubernetes/sdc/components/sdc-cs/values.yaml
+++ b/kubernetes/sdc/components/sdc-cs/values.yaml
@@ -51,8 +51,8 @@ cassandra:
 
 # application image
 repository: nexus3.onap.org:10001
-image: onap/sdc-cassandra:1.11.9
-cassandraInitImage: onap/sdc-cassandra-init:1.11.9
+image: onap/sdc-cassandra:1.12.0
+cassandraInitImage: onap/sdc-cassandra-init:1.12.0
 pullPolicy: Always
 
 config:
diff --git a/kubernetes/sdc/components/sdc-fe/values.yaml b/kubernetes/sdc/components/sdc-fe/values.yaml
index 76c9e13767..abac356d40 100644
--- a/kubernetes/sdc/components/sdc-fe/values.yaml
+++ b/kubernetes/sdc/components/sdc-fe/values.yaml
@@ -47,7 +47,7 @@ certInitializer:
 # Application configuration defaults.
 #################################################################
 # application image
-image: onap/sdc-frontend:1.11.9
+image: onap/sdc-frontend:1.12.0
 pullPolicy: Always
 
 config:
diff --git a/kubernetes/sdc/components/sdc-onboarding-be/values.yaml b/kubernetes/sdc/components/sdc-onboarding-be/values.yaml
index 45cbcd617a..1db5af98d0 100644
--- a/kubernetes/sdc/components/sdc-onboarding-be/values.yaml
+++ b/kubernetes/sdc/components/sdc-onboarding-be/values.yaml
@@ -59,8 +59,8 @@ certInitializer:
 # Application configuration defaults.
 #################################################################
 # application image
-image: onap/sdc-onboard-backend:1.11.9
-onboardingInitImage: onap/sdc-onboard-cassandra-init:1.11.9
+image: onap/sdc-onboard-backend:1.12.0
+onboardingInitImage: onap/sdc-onboard-cassandra-init:1.12.0
 pullPolicy: Always
 
 # flag to enable debugging - application support required
diff --git a/kubernetes/sdc/resources/config/environments/AUTO.json b/kubernetes/sdc/resources/config/environments/AUTO.json
index c9535592d8..aee666f4af 100755
--- a/kubernetes/sdc/resources/config/environments/AUTO.json
+++ b/kubernetes/sdc/resources/config/environments/AUTO.json
@@ -35,6 +35,13 @@
                 "message-router.{{include "common.namespace" .}}"
             ]
         },
+        "Kafka": {
+             "bootstrap": "{{ include "common.release" . }}-{{ .Values.global.kafka.kafkaBootstrap }}"
+        },
+        "DistributionTopics": {
+             "notificationTopicName": "{{ .Values.global.kafka.topics.sdcDistNotifTopic }}",
+             "statusTopicName": "{{ .Values.global.kafka.topics.sdcDistStatusTopic }}"
+        },
         "Nodes": {
             "CS": [
                 "{{.Values.global.sdc_cassandra.serviceName}}.{{include "common.namespace" .}}"
diff --git a/kubernetes/sdc/values.yaml b/kubernetes/sdc/values.yaml
index e7643069bf..60a361eae6 100644
--- a/kubernetes/sdc/values.yaml
+++ b/kubernetes/sdc/values.yaml
@@ -1,5 +1,6 @@
 # Copyright © 2017 Amdocs, Bell Canada
 # Modifications Copyright © 2018 AT&T, ZTE
+# Modifications Copyright © 2021 Nordix Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -41,9 +42,19 @@ global:
    clusterName: cassandra
    dataCenter: Pod
   centralizedLoggingEnabled: true
+  # Kafka config
+  kafka:
+    useKafka: true
+    sdcBeKafkaUser: sdc-be-kafka-user
+    kafkaBootstrap: strimzi-kafka-bootstrap:9092
+    topics:
+      sdcDistNotifTopic: SDC-DISTR-NOTIF-TOPIC
+      sdcDistStatusTopic: SDC-DISTR-STATUS-TOPIC
 
 sdc-be:
   logConfigMapNamePrefix: '{{ include "common.release" . }}-sdc'
+  config:
+    jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.kafka.sdcBeKafkaUser }}'
 sdc-fe:
   logConfigMapNamePrefix: '{{ include "common.release" . }}-sdc'
 sdc-onboarding-be:
diff --git a/kubernetes/strimzi/templates/strimzi-kafka.yaml b/kubernetes/strimzi/templates/strimzi-kafka.yaml
index c7dc2aff08..03ee56a7a4 100644
--- a/kubernetes/strimzi/templates/strimzi-kafka.yaml
+++ b/kubernetes/strimzi/templates/strimzi-kafka.yaml
@@ -70,6 +70,7 @@ spec:
       min.insync.replicas: {{ .Values.replicaCount }}
       offsets.topic.replication.factor: {{ .Values.replicaCount }}
       transaction.state.log.replication.factor: {{ .Values.replicaCount }}
+      num.partitions: {{ .Values.numPartitions }}
       transaction.state.log.min.isr: {{ .Values.replicaCount }}
       log.message.format.version: {{ .Values.version }}
       inter.broker.protocol.version: {{ .Values.version }}
diff --git a/kubernetes/strimzi/values.yaml b/kubernetes/strimzi/values.yaml
index dcfa913604..99ccde5040 100644
--- a/kubernetes/strimzi/values.yaml
+++ b/kubernetes/strimzi/values.yaml
@@ -24,6 +24,7 @@ global:
 # Application configuration defaults.
 #################################################################
 replicaCount: 3
+numPartitions: 10
 kafkaInternalPort: 9092
 saslMechanism: scram-sha-512
 version: 3.2.3