Merge "[CPS-STRIMZI] Migrate cps to use strimzi kafka"

10 files changed, 182 insertions, 33 deletions

diff --git a/kubernetes/cps/components/cps-core/resources/config/application-helm.yml b/kubernetes/cps/components/cps-core/resources/config/application-helm.yml
index e9958f1114..e295a37b45 100644
--- a/kubernetes/cps/components/cps-core/resources/config/application-helm.yml
+++ b/kubernetes/cps/components/cps-core/resources/config/application-helm.yml
@@ -1,7 +1,7 @@
 {{/*
 #  Copyright (C) 2021 Pantheon.tech
 #  Modifications Copyright (C) 2020 Bell Canada.
-#  Modifications Copyright (C) 2021 Nordix Foundation.
+#  Modifications Copyright (C) 2021-2022 Nordix Foundation.
 #  Modifications Copyright (C) 2021 Orange
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,17 +28,21 @@ spring:
     password: ${DB_PASSWORD}
     driverClassName: org.postgresql.Driver
     initialization-mode: always
-
   liquibase:
     change-log: classpath:changelog/changelog-master.yaml
     labels: {{ .Values.config.liquibaseLabels }}
 
+  kafka:
+    producer:
+      client-id: cps-core
+
 security:
-    # comma-separated uri patterns which do not require authorization
-    permit-uri: /manage/**,/swagger-ui/**,/swagger-resources/**,/api-docs
-    auth:
-        username: ${CPS_USERNAME}
-        password: ${CPS_PASSWORD}
+  # comma-separated uri patterns which do not require authorization
+  permit-uri: /manage/**,/swagger-ui/**,/swagger-resources/**,/api-docs
+  auth:
+    username: ${CPS_USERNAME}
+    password: ${CPS_PASSWORD}
+
 logging:
   level:
     org:
@@ -49,12 +53,18 @@ dmi:
     username: ${DMI_USERNAME}
     password: ${DMI_PASSWORD}
 
-{{- if .Values.config.eventPublisher }}
+{{- if .Values.config.useStrimziKafka }}
+spring.kafka.bootstrap-servers: {{ include "common.release" . }}-{{ .Values.config.kafkaBootstrap }}:9092
+spring.kafka.security.protocol: SASL_PLAINTEXT
+spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
+spring.kafka.properties.sasl.jaas.config: ${JAASLOGIN}
+{{ else }}
 {{ toYaml .Values.config.eventPublisher | nindent 2 }}
 {{- end }}
 
 {{- if .Values.config.additional }}
 {{ toYaml .Values.config.additional | nindent 2 }}
 {{- end }}
+
 # Last empty line is required otherwise the last property will be missing from application.yml file in the pod.
 
diff --git a/kubernetes/cps/components/cps-core/templates/deployment.yaml b/kubernetes/cps/components/cps-core/templates/deployment.yaml
index e6ee161feb..54e2cc6cdf 100644
--- a/kubernetes/cps/components/cps-core/templates/deployment.yaml
+++ b/kubernetes/cps/components/cps-core/templates/deployment.yaml
@@ -1,7 +1,7 @@
 {{/*
 # Copyright (C) 2021 Pantheon.tech, Orange
 # Modifications Copyright (C) 2021 Bell Canada.
-# Modifications Copyright (C) 2021 Nordix Foundation.
+# Modifications Copyright (C) 2021-2022 Nordix Foundation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -57,7 +57,10 @@ spec:
             {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "dmi-plugin-user-creds" "key" "login") | indent 12 }}
           - name: DMI_PASSWORD
             {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "dmi-plugin-user-creds" "key" "password") | indent 12 }}
-
+          {{- if .Values.config.useStrimziKafka }}
+          - name: JAASLOGIN
+            {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "cps-kafka-user" "key" "sasl.jaas.config") | indent 12 }}
+          {{- end }}
         volumeMounts:
           - mountPath: /config-input
             name: init-data-input
diff --git a/kubernetes/cps/components/cps-core/values.yaml b/kubernetes/cps/components/cps-core/values.yaml
index d65924e90f..2afc1fd6f4 100644
--- a/kubernetes/cps/components/cps-core/values.yaml
+++ b/kubernetes/cps/components/cps-core/values.yaml
@@ -1,5 +1,6 @@
 # Copyright (C) 2021 Pantheon.tech, Orange, Bell Canada.
 # Modifications Copyright (C) 2022 Bell Canada
+# Modifications Copyright © 2022 Nordix Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -42,6 +43,13 @@ secrets:
     login: '{{ .Values.config.dmiPluginUserName }}'
     password: '{{ .Values.config.dmiPluginUserPassword }}'
     passwordPolicy: generate
+  - uid: cps-kafka-user
+    externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}'
+    type: genericKV
+    envs:
+      - name: sasl.jaas.config
+        value: '{{ .Values.config.someConfig }}'
+        policy: generate
 
 #################################################################
 # Global configuration defaults.
@@ -170,17 +178,23 @@ config:
   #appUserPassword:
   dmiPluginUserName: dmiuser
 # Any new property can be added in the env by setting in overrides in the format mentioned below
-# All the added properties must be in "key: value" format insead of yaml.
+# All the added properties must be in "key: value" format instead of yaml.
 #  additional:
 #    spring.config.max-size: 200
 #    spring.config.min-size: 10
 
-  eventPublisher:
-    spring.kafka.bootstrap-servers: message-router-kafka:9092
-    spring.kafka.security.protocol: SASL_PLAINTEXT
-    spring.kafka.properties.sasl.mechanism: PLAIN
-    spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username=admin password=admin_secret;
-    spring.kafka.producer.client-id: cps-core
+# kafka config
+  useStrimziKafka: true
+  kafkaBootstrap: strimzi-kafka-bootstrap
+# If targeting a custom kafka cluster, ie useStrimziKakfa: false
+# uncomment below config and target your kafka bootstrap servers,
+# along with any other security config.
+
+#  eventPublisher:
+#    spring.kafka.bootstrap-servers: <kafka-bootstrap>:9092
+#    spring.kafka.security.protocol: SASL_PLAINTEXT
+#    spring.kafka.properties.sasl.mechanism: PLAIN
+#    spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username=admin password=admin_secret;
 
   additional:
     notification.data-updated.enabled: true
diff --git a/kubernetes/cps/components/cps-temporal/resources/config/application-helm.yml b/kubernetes/cps/components/cps-temporal/resources/config/application-helm.yml
index 32ae51b51a..6e80843949 100644
--- a/kubernetes/cps/components/cps-temporal/resources/config/application-helm.yml
+++ b/kubernetes/cps/components/cps-temporal/resources/config/application-helm.yml
@@ -1,6 +1,7 @@
 {{/*
 # ============LICENSE_START=======================================================
 #  Copyright (c) 2021 Bell Canada.
+# Modifications Copyright © 2022 Nordix Foundation
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,19 +25,31 @@ spring:
     username: ${DB_USERNAME}
     password: ${DB_PASSWORD}
 
-security:
-  auth:
-    username: ${APP_USERNAME}
-    password: ${APP_PASSWORD}
+  kafka:
+    consumer:
+      group-id: {{ .Values.config.kafka.consumer.groupId }}
+
+app:
+  listener:
+    data-updated:
+      topic: {{ .Values.config.app.listener.dataUpdatedTopic }}
 
-# Event consumption properties (kafka)
-{{- if .Values.config.eventConsumption }}
-{{ toYaml .Values.config.eventConsumption | nindent 2 }}
+{{- if .Values.config.useStrimziKafka }}
+spring.kafka.bootstrap-servers: {{ include "common.release" . }}-{{ .Values.config.kafkaBootstrap }}:9092
+spring.kafka.security.protocol: SASL_PLAINTEXT
+spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
+spring.kafka.properties.sasl.jaas.config: ${JAASLOGIN}
+{{ else }}
+{{ toYaml .Values.config.eventPublisher | nindent 2 }}
 {{- end }}
 
-# Additional properties
 {{- if .Values.config.additional }}
 {{ toYaml .Values.config.additional | nindent 2 }}
 {{- end }}
 
+security:
+  auth:
+    username: ${APP_USERNAME}
+    password: ${APP_PASSWORD}
+
 # Last empty line is required otherwise the last property will be missing from application.yml file in the pod.
diff --git a/kubernetes/cps/components/cps-temporal/templates/deployment.yaml b/kubernetes/cps/components/cps-temporal/templates/deployment.yaml
index 806e65a865..71ff37193b 100644
--- a/kubernetes/cps/components/cps-temporal/templates/deployment.yaml
+++ b/kubernetes/cps/components/cps-temporal/templates/deployment.yaml
@@ -1,6 +1,7 @@
 {{/*
 # ============LICENSE_START=======================================================
 #  Copyright (c) 2021 Bell Canada.
+# Modifications Copyright © 2022 Nordix Foundation
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -53,6 +54,10 @@ spec:
             {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "app-user-creds" "key" "login") | indent 12 }}
           - name: APP_PASSWORD
             {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "app-user-creds" "key" "password") | indent 12 }}
+          {{- if .Values.config.useStrimziKafka }}
+          - name: JAASLOGIN
+            {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "cps-kafka-user" "key" "sasl.jaas.config") | indent 12 }}
+          {{- end }}
         volumeMounts:
           - mountPath: /config-input
             name: init-data-input
diff --git a/kubernetes/cps/components/cps-temporal/values.yaml b/kubernetes/cps/components/cps-temporal/values.yaml
index 68bc2a7e8a..a92791e019 100644
--- a/kubernetes/cps/components/cps-temporal/values.yaml
+++ b/kubernetes/cps/components/cps-temporal/values.yaml
@@ -1,5 +1,6 @@
 # ============LICENSE_START=======================================================
 #  Copyright (c) 2021 Bell Canada.
+# Modifications Copyright © 2022 Nordix Foundation
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -39,6 +40,13 @@ secrets:
     login: '{{ .Values.config.appUserName }}'
     password: '{{ .Values.config.appUserPassword }}'
     passwordPolicy: generate
+  - uid: cps-kafka-user
+    externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}'
+    type: genericKV
+    envs:
+      - name: sasl.jaas.config
+        value: '{{ .Values.config.someConfig }}'
+        policy: generate
 
 image: onap/cps-temporal:1.1.0
 containerPort: &svc_port 8080
@@ -139,13 +147,23 @@ config:
     profile: helm
   #appUserPassword:
 
-  # Event consumption (kafka) properties
-  # All Kafka properties must be in "key: value" format instead of yaml.
-  eventConsumption:
-    spring.kafka.bootstrap-servers: message-router-kafka:9092
-    spring.kafka.security.protocol: PLAINTEXT
-    spring.kafka.consumer.group-id: cps-temporal-group
-    app.listener.data-updated.topic: cps.data-updated-events
+# Event consumption (kafka) properties
+  useStrimziKafka: true
+  kafkaBootstrap: strimzi-kafka-bootstrap
+  kafka:
+    consumer:
+      groupId: cps-temporal-group
+  app:
+    listener:
+      dataUpdatedTopic: cps.data-updated-events
+# If targeting a custom kafka cluster, ie useStrimziKakfa: false
+# uncomment below config and target your kafka bootstrap servers,
+# along with any other security config.
+
+#  eventConsumption:
+#    spring.kafka.bootstrap-servers: <kafka-bootstrap>:9092
+#    spring.kafka.security.protocol: PLAINTEXT
+#    spring.kafka.consumer.group-id: cps-temporal-group
 
 # Any new property can be added in the env by setting in overrides in the format mentioned below
 # All the added properties must be in "key: value" format instead of yaml.
diff --git a/kubernetes/cps/templates/cps-kafka-topic.yaml b/kubernetes/cps/templates/cps-kafka-topic.yaml
new file mode 100644
index 0000000000..1a23ddfc9b
--- /dev/null
+++ b/kubernetes/cps/templates/cps-kafka-topic.yaml
@@ -0,0 +1,28 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.config.useStrimziKafka }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: {{ .Values.config.dataUpdatedTopic.name }}
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  partitions: {{ .Values.config.dataUpdatedTopic.partitions }}
+  config:
+    retention.ms: {{ .Values.config.dataUpdatedTopic.retentionMs }}
+    segment.bytes: {{ .Values.config.dataUpdatedTopic.segmentBytes }}
+{{- end }}
\ No newline at end of file
diff --git a/kubernetes/cps/templates/cps-kafka-user.yaml b/kubernetes/cps/templates/cps-kafka-user.yaml
new file mode 100644
index 0000000000..b3136d7f04
--- /dev/null
+++ b/kubernetes/cps/templates/cps-kafka-user.yaml
@@ -0,0 +1,41 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.config.useStrimziKafka }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaUser
+metadata:
+  name: {{ include "common.release" . }}-{{ .Values.global.cpsKafkaUser }}
+  labels:
+    strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+  authentication:
+    type: scram-sha-512
+  authorization:
+    type: simple
+    acls:
+    - resource:
+        type: group
+        name: {{ .Values.config.dataUpdatedTopic.consumer.groupId }}
+      operation: Read
+    - resource:
+        type: topic
+        name: {{ .Values.config.dataUpdatedTopic.name }}
+      operation: Read
+    - resource:
+        type: topic
+        name: {{ .Values.config.dataUpdatedTopic.name }}
+      operation: Write
+{{- end }}
\ No newline at end of file
diff --git a/kubernetes/cps/values.yaml b/kubernetes/cps/values.yaml
index 754b016fe8..700ad38844 100755
--- a/kubernetes/cps/values.yaml
+++ b/kubernetes/cps/values.yaml
@@ -1,4 +1,5 @@
 # Copyright (C) 2021 Bell Canada
+# Modifications Copyright © 2022 Nordix Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -40,9 +41,20 @@ global:
     virtualhost:
       baseurl: "simpledemo.onap.org"
 
+  kafkaBootstrap: strimzi-kafka-bootstrap
+  cpsKafkaUser: cps-kafka-user
+
 config:
   coreUserName: cpsuser
   dmiPluginUserName: dmiuser
+  useStrimziKafka: true
+  dataUpdatedTopic:
+    name: cps.data-updated-events
+    partitions: 10
+    retentionMs: 7200000
+    segmentBytes: 1073741824
+    consumer:
+      groupId: cps-temporal-group
 
 # Enable all CPS components by default
 cps-core:
@@ -50,9 +62,12 @@ cps-core:
   config:
     appUserExternalSecret: *core-creds-secret
     dmiPluginUserExternalSecret: *dmi-plugin-creds-secret
+    jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.cpsKafkaUser }}'
 
 cps-temporal:
   enabled: true
+  config:
+    jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.cpsKafkaUser }}'
 
 ncmp-dmi-plugin:
   enabled: true
diff --git a/kubernetes/strimzi/templates/strimzi-kafka.yaml b/kubernetes/strimzi/templates/strimzi-kafka.yaml
index df3afee96d..5f1e7303d9 100644
--- a/kubernetes/strimzi/templates/strimzi-kafka.yaml
+++ b/kubernetes/strimzi/templates/strimzi-kafka.yaml
@@ -64,9 +64,11 @@ spec:
           runAsUser: 0
           fsGroup: 0
     config:
+      default.replication.factor: {{ .Values.replicaCount }}
+      min.insync.replicas: {{ .Values.replicaCount }}
       offsets.topic.replication.factor: {{ .Values.replicaCount }}
       transaction.state.log.replication.factor: {{ .Values.replicaCount }}
-      transaction.state.log.min.isr: 2
+      transaction.state.log.min.isr: {{ .Values.replicaCount }}
       log.message.format.version: "3.0"
       inter.broker.protocol.version: "3.0"
     storage: