summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorefiacor <fiachra.corcoran@est.tech>2022-03-14 13:44:02 +0000
committerFiachra Corcoran <fiachra.corcoran@est.tech>2022-04-27 11:40:44 +0000
commit23361a3aedd49902f50792cf8ac1f659b0391567 (patch)
treef493fcad0c8970b53902c61a913b8e3e6d640bd0
parent7c2dbe2a9348e33db2dd8a32ddf9c5acb72567ef (diff)
[CPS-STRIMZI] Migrate cps to use strimzi kafka
Add strimzi kafka config Signed-off-by: efiacor <fiachra.corcoran@est.tech> Change-Id: Ib6af32bae9f85cbc2e3c9484b970229d63c299b1 Issue-ID: DMAAP-1681
-rw-r--r--kubernetes/cps/components/cps-core/resources/config/application-helm.yml26
-rw-r--r--kubernetes/cps/components/cps-core/templates/deployment.yaml7
-rw-r--r--kubernetes/cps/components/cps-core/values.yaml28
-rw-r--r--kubernetes/cps/components/cps-temporal/resources/config/application-helm.yml29
-rw-r--r--kubernetes/cps/components/cps-temporal/templates/deployment.yaml5
-rw-r--r--kubernetes/cps/components/cps-temporal/values.yaml32
-rw-r--r--kubernetes/cps/templates/cps-kafka-topic.yaml28
-rw-r--r--kubernetes/cps/templates/cps-kafka-user.yaml41
-rwxr-xr-xkubernetes/cps/values.yaml15
-rw-r--r--kubernetes/strimzi/templates/strimzi-kafka.yaml4
10 files changed, 182 insertions, 33 deletions
diff --git a/kubernetes/cps/components/cps-core/resources/config/application-helm.yml b/kubernetes/cps/components/cps-core/resources/config/application-helm.yml
index e9958f1114..e295a37b45 100644
--- a/kubernetes/cps/components/cps-core/resources/config/application-helm.yml
+++ b/kubernetes/cps/components/cps-core/resources/config/application-helm.yml
@@ -1,7 +1,7 @@
{{/*
# Copyright (C) 2021 Pantheon.tech
# Modifications Copyright (C) 2020 Bell Canada.
-# Modifications Copyright (C) 2021 Nordix Foundation.
+# Modifications Copyright (C) 2021-2022 Nordix Foundation.
# Modifications Copyright (C) 2021 Orange
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,17 +28,21 @@ spring:
password: ${DB_PASSWORD}
driverClassName: org.postgresql.Driver
initialization-mode: always
-
liquibase:
change-log: classpath:changelog/changelog-master.yaml
labels: {{ .Values.config.liquibaseLabels }}
+ kafka:
+ producer:
+ client-id: cps-core
+
security:
- # comma-separated uri patterns which do not require authorization
- permit-uri: /manage/**,/swagger-ui/**,/swagger-resources/**,/api-docs
- auth:
- username: ${CPS_USERNAME}
- password: ${CPS_PASSWORD}
+ # comma-separated uri patterns which do not require authorization
+ permit-uri: /manage/**,/swagger-ui/**,/swagger-resources/**,/api-docs
+ auth:
+ username: ${CPS_USERNAME}
+ password: ${CPS_PASSWORD}
+
logging:
level:
org:
@@ -49,12 +53,18 @@ dmi:
username: ${DMI_USERNAME}
password: ${DMI_PASSWORD}
-{{- if .Values.config.eventPublisher }}
+{{- if .Values.config.useStrimziKafka }}
+spring.kafka.bootstrap-servers: {{ include "common.release" . }}-{{ .Values.config.kafkaBootstrap }}:9092
+spring.kafka.security.protocol: SASL_PLAINTEXT
+spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
+spring.kafka.properties.sasl.jaas.config: ${JAASLOGIN}
+{{ else }}
{{ toYaml .Values.config.eventPublisher | nindent 2 }}
{{- end }}
{{- if .Values.config.additional }}
{{ toYaml .Values.config.additional | nindent 2 }}
{{- end }}
+
# Last empty line is required otherwise the last property will be missing from application.yml file in the pod.
diff --git a/kubernetes/cps/components/cps-core/templates/deployment.yaml b/kubernetes/cps/components/cps-core/templates/deployment.yaml
index e6ee161feb..54e2cc6cdf 100644
--- a/kubernetes/cps/components/cps-core/templates/deployment.yaml
+++ b/kubernetes/cps/components/cps-core/templates/deployment.yaml
@@ -1,7 +1,7 @@
{{/*
# Copyright (C) 2021 Pantheon.tech, Orange
# Modifications Copyright (C) 2021 Bell Canada.
-# Modifications Copyright (C) 2021 Nordix Foundation.
+# Modifications Copyright (C) 2021-2022 Nordix Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -57,7 +57,10 @@ spec:
{{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "dmi-plugin-user-creds" "key" "login") | indent 12 }}
- name: DMI_PASSWORD
{{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "dmi-plugin-user-creds" "key" "password") | indent 12 }}
-
+ {{- if .Values.config.useStrimziKafka }}
+ - name: JAASLOGIN
+ {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "cps-kafka-user" "key" "sasl.jaas.config") | indent 12 }}
+ {{- end }}
volumeMounts:
- mountPath: /config-input
name: init-data-input
diff --git a/kubernetes/cps/components/cps-core/values.yaml b/kubernetes/cps/components/cps-core/values.yaml
index d65924e90f..2afc1fd6f4 100644
--- a/kubernetes/cps/components/cps-core/values.yaml
+++ b/kubernetes/cps/components/cps-core/values.yaml
@@ -1,5 +1,6 @@
# Copyright (C) 2021 Pantheon.tech, Orange, Bell Canada.
# Modifications Copyright (C) 2022 Bell Canada
+# Modifications Copyright © 2022 Nordix Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -42,6 +43,13 @@ secrets:
login: '{{ .Values.config.dmiPluginUserName }}'
password: '{{ .Values.config.dmiPluginUserPassword }}'
passwordPolicy: generate
+ - uid: cps-kafka-user
+ externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}'
+ type: genericKV
+ envs:
+ - name: sasl.jaas.config
+ value: '{{ .Values.config.someConfig }}'
+ policy: generate
#################################################################
# Global configuration defaults.
@@ -170,17 +178,23 @@ config:
#appUserPassword:
dmiPluginUserName: dmiuser
# Any new property can be added in the env by setting in overrides in the format mentioned below
-# All the added properties must be in "key: value" format insead of yaml.
+# All the added properties must be in "key: value" format instead of yaml.
# additional:
# spring.config.max-size: 200
# spring.config.min-size: 10
- eventPublisher:
- spring.kafka.bootstrap-servers: message-router-kafka:9092
- spring.kafka.security.protocol: SASL_PLAINTEXT
- spring.kafka.properties.sasl.mechanism: PLAIN
- spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username=admin password=admin_secret;
- spring.kafka.producer.client-id: cps-core
+# kafka config
+ useStrimziKafka: true
+ kafkaBootstrap: strimzi-kafka-bootstrap
+# If targeting a custom kafka cluster, ie useStrimziKakfa: false
+# uncomment below config and target your kafka bootstrap servers,
+# along with any other security config.
+
+# eventPublisher:
+# spring.kafka.bootstrap-servers: <kafka-bootstrap>:9092
+# spring.kafka.security.protocol: SASL_PLAINTEXT
+# spring.kafka.properties.sasl.mechanism: PLAIN
+# spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username=admin password=admin_secret;
additional:
notification.data-updated.enabled: true
diff --git a/kubernetes/cps/components/cps-temporal/resources/config/application-helm.yml b/kubernetes/cps/components/cps-temporal/resources/config/application-helm.yml
index 32ae51b51a..6e80843949 100644
--- a/kubernetes/cps/components/cps-temporal/resources/config/application-helm.yml
+++ b/kubernetes/cps/components/cps-temporal/resources/config/application-helm.yml
@@ -1,6 +1,7 @@
{{/*
# ============LICENSE_START=======================================================
# Copyright (c) 2021 Bell Canada.
+# Modifications Copyright © 2022 Nordix Foundation
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -24,19 +25,31 @@ spring:
username: ${DB_USERNAME}
password: ${DB_PASSWORD}
-security:
- auth:
- username: ${APP_USERNAME}
- password: ${APP_PASSWORD}
+ kafka:
+ consumer:
+ group-id: {{ .Values.config.kafka.consumer.groupId }}
+
+app:
+ listener:
+ data-updated:
+ topic: {{ .Values.config.app.listener.dataUpdatedTopic }}
-# Event consumption properties (kafka)
-{{- if .Values.config.eventConsumption }}
-{{ toYaml .Values.config.eventConsumption | nindent 2 }}
+{{- if .Values.config.useStrimziKafka }}
+spring.kafka.bootstrap-servers: {{ include "common.release" . }}-{{ .Values.config.kafkaBootstrap }}:9092
+spring.kafka.security.protocol: SASL_PLAINTEXT
+spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
+spring.kafka.properties.sasl.jaas.config: ${JAASLOGIN}
+{{ else }}
+{{ toYaml .Values.config.eventPublisher | nindent 2 }}
{{- end }}
-# Additional properties
{{- if .Values.config.additional }}
{{ toYaml .Values.config.additional | nindent 2 }}
{{- end }}
+security:
+ auth:
+ username: ${APP_USERNAME}
+ password: ${APP_PASSWORD}
+
# Last empty line is required otherwise the last property will be missing from application.yml file in the pod.
diff --git a/kubernetes/cps/components/cps-temporal/templates/deployment.yaml b/kubernetes/cps/components/cps-temporal/templates/deployment.yaml
index 806e65a865..71ff37193b 100644
--- a/kubernetes/cps/components/cps-temporal/templates/deployment.yaml
+++ b/kubernetes/cps/components/cps-temporal/templates/deployment.yaml
@@ -1,6 +1,7 @@
{{/*
# ============LICENSE_START=======================================================
# Copyright (c) 2021 Bell Canada.
+# Modifications Copyright © 2022 Nordix Foundation
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -53,6 +54,10 @@ spec:
{{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "app-user-creds" "key" "login") | indent 12 }}
- name: APP_PASSWORD
{{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "app-user-creds" "key" "password") | indent 12 }}
+ {{- if .Values.config.useStrimziKafka }}
+ - name: JAASLOGIN
+ {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "cps-kafka-user" "key" "sasl.jaas.config") | indent 12 }}
+ {{- end }}
volumeMounts:
- mountPath: /config-input
name: init-data-input
diff --git a/kubernetes/cps/components/cps-temporal/values.yaml b/kubernetes/cps/components/cps-temporal/values.yaml
index 68bc2a7e8a..a92791e019 100644
--- a/kubernetes/cps/components/cps-temporal/values.yaml
+++ b/kubernetes/cps/components/cps-temporal/values.yaml
@@ -1,5 +1,6 @@
# ============LICENSE_START=======================================================
# Copyright (c) 2021 Bell Canada.
+# Modifications Copyright © 2022 Nordix Foundation
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -39,6 +40,13 @@ secrets:
login: '{{ .Values.config.appUserName }}'
password: '{{ .Values.config.appUserPassword }}'
passwordPolicy: generate
+ - uid: cps-kafka-user
+ externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}'
+ type: genericKV
+ envs:
+ - name: sasl.jaas.config
+ value: '{{ .Values.config.someConfig }}'
+ policy: generate
image: onap/cps-temporal:1.1.0
containerPort: &svc_port 8080
@@ -139,13 +147,23 @@ config:
profile: helm
#appUserPassword:
- # Event consumption (kafka) properties
- # All Kafka properties must be in "key: value" format instead of yaml.
- eventConsumption:
- spring.kafka.bootstrap-servers: message-router-kafka:9092
- spring.kafka.security.protocol: PLAINTEXT
- spring.kafka.consumer.group-id: cps-temporal-group
- app.listener.data-updated.topic: cps.data-updated-events
+# Event consumption (kafka) properties
+ useStrimziKafka: true
+ kafkaBootstrap: strimzi-kafka-bootstrap
+ kafka:
+ consumer:
+ groupId: cps-temporal-group
+ app:
+ listener:
+ dataUpdatedTopic: cps.data-updated-events
+# If targeting a custom kafka cluster, ie useStrimziKakfa: false
+# uncomment below config and target your kafka bootstrap servers,
+# along with any other security config.
+
+# eventConsumption:
+# spring.kafka.bootstrap-servers: <kafka-bootstrap>:9092
+# spring.kafka.security.protocol: PLAINTEXT
+# spring.kafka.consumer.group-id: cps-temporal-group
# Any new property can be added in the env by setting in overrides in the format mentioned below
# All the added properties must be in "key: value" format instead of yaml.
diff --git a/kubernetes/cps/templates/cps-kafka-topic.yaml b/kubernetes/cps/templates/cps-kafka-topic.yaml
new file mode 100644
index 0000000000..1a23ddfc9b
--- /dev/null
+++ b/kubernetes/cps/templates/cps-kafka-topic.yaml
@@ -0,0 +1,28 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.config.useStrimziKafka }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+ name: {{ .Values.config.dataUpdatedTopic.name }}
+ labels:
+ strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+ partitions: {{ .Values.config.dataUpdatedTopic.partitions }}
+ config:
+ retention.ms: {{ .Values.config.dataUpdatedTopic.retentionMs }}
+ segment.bytes: {{ .Values.config.dataUpdatedTopic.segmentBytes }}
+{{- end }} \ No newline at end of file
diff --git a/kubernetes/cps/templates/cps-kafka-user.yaml b/kubernetes/cps/templates/cps-kafka-user.yaml
new file mode 100644
index 0000000000..b3136d7f04
--- /dev/null
+++ b/kubernetes/cps/templates/cps-kafka-user.yaml
@@ -0,0 +1,41 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.config.useStrimziKafka }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaUser
+metadata:
+ name: {{ include "common.release" . }}-{{ .Values.global.cpsKafkaUser }}
+ labels:
+ strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+ authentication:
+ type: scram-sha-512
+ authorization:
+ type: simple
+ acls:
+ - resource:
+ type: group
+ name: {{ .Values.config.dataUpdatedTopic.consumer.groupId }}
+ operation: Read
+ - resource:
+ type: topic
+ name: {{ .Values.config.dataUpdatedTopic.name }}
+ operation: Read
+ - resource:
+ type: topic
+ name: {{ .Values.config.dataUpdatedTopic.name }}
+ operation: Write
+{{- end }} \ No newline at end of file
diff --git a/kubernetes/cps/values.yaml b/kubernetes/cps/values.yaml
index 754b016fe8..700ad38844 100755
--- a/kubernetes/cps/values.yaml
+++ b/kubernetes/cps/values.yaml
@@ -1,4 +1,5 @@
# Copyright (C) 2021 Bell Canada
+# Modifications Copyright © 2022 Nordix Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -40,9 +41,20 @@ global:
virtualhost:
baseurl: "simpledemo.onap.org"
+ kafkaBootstrap: strimzi-kafka-bootstrap
+ cpsKafkaUser: cps-kafka-user
+
config:
coreUserName: cpsuser
dmiPluginUserName: dmiuser
+ useStrimziKafka: true
+ dataUpdatedTopic:
+ name: cps.data-updated-events
+ partitions: 10
+ retentionMs: 7200000
+ segmentBytes: 1073741824
+ consumer:
+ groupId: cps-temporal-group
# Enable all CPS components by default
cps-core:
@@ -50,9 +62,12 @@ cps-core:
config:
appUserExternalSecret: *core-creds-secret
dmiPluginUserExternalSecret: *dmi-plugin-creds-secret
+ jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.cpsKafkaUser }}'
cps-temporal:
enabled: true
+ config:
+ jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.cpsKafkaUser }}'
ncmp-dmi-plugin:
enabled: true
diff --git a/kubernetes/strimzi/templates/strimzi-kafka.yaml b/kubernetes/strimzi/templates/strimzi-kafka.yaml
index a94879b8a0..1afb5fd6b9 100644
--- a/kubernetes/strimzi/templates/strimzi-kafka.yaml
+++ b/kubernetes/strimzi/templates/strimzi-kafka.yaml
@@ -64,9 +64,11 @@ spec:
runAsUser: 0
fsGroup: 0
config:
+ default.replication.factor: {{ .Values.replicaCount }}
+ min.insync.replicas: {{ .Values.replicaCount }}
offsets.topic.replication.factor: {{ .Values.replicaCount }}
transaction.state.log.replication.factor: {{ .Values.replicaCount }}
- transaction.state.log.min.isr: 2
+ transaction.state.log.min.isr: {{ .Values.replicaCount }}
log.message.format.version: "3.0"
inter.broker.protocol.version: "3.0"
storage: