diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | kubernetes/common/common/templates/_strimzikafka.tpl | 132 | ||||
| -rw-r--r-- | kubernetes/sdc/Chart.yaml | 3 | ||||
| -rw-r--r-- | kubernetes/sdc/components/sdc-be/Chart.yaml | 3 | ||||
| -rw-r--r-- | kubernetes/sdc/components/sdc-be/templates/deployment.yaml | 5 | ||||
| -rw-r--r-- | kubernetes/sdc/components/sdc-be/templates/kafkauser.yaml (renamed from kubernetes/sdc/components/sdc-be/templates/secret.yaml) | 5 | ||||
| -rw-r--r-- | kubernetes/sdc/components/sdc-be/templates/sdc-be-kafka-user.yaml | 39 | ||||
| -rw-r--r-- | kubernetes/sdc/components/sdc-be/templates/sdc-distro-topics.yaml | 40 | ||||
| -rw-r--r-- | kubernetes/sdc/components/sdc-be/values.yaml | 41 | ||||
| -rwxr-xr-x | kubernetes/sdc/resources/config/environments/AUTO.json | 4 | ||||
| -rw-r--r-- | kubernetes/sdc/templates/kafkatopic.yaml | 18 | ||||
| -rw-r--r-- | kubernetes/sdc/values.yaml | 25 |
12 files changed, 196 insertions, 120 deletions
diff --git a/.gitignore b/.gitignore index bb11f06df9..e7726292f1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ kubernetes/common/dist/* Chart.lock **/charts/*.tgz *.orig +*_build # AAI Schema **/schema/* diff --git a/kubernetes/common/common/templates/_strimzikafka.tpl b/kubernetes/common/common/templates/_strimzikafka.tpl new file mode 100644 index 0000000000..3fd46c7414 --- /dev/null +++ b/kubernetes/common/common/templates/_strimzikafka.tpl @@ -0,0 +1,132 @@ +{{/* +# Copyright © 2022 Nordix Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +{{/* + Create a Strimzi KafkaUser. + Usage: + include "common.kafkauser" . + + Strimzi kafka provides cluster access via its custom resource definition KafkaUser + which is deployed using its User Operator component. + See more info here - https://github.com/strimzi/strimzi-kafka-operator/blob/main/helm-charts/helm3/strimzi-kafka-operator/crds/044-Crd-kafkauser.yaml + This allows fine grained access control per user towards the kafka cluster. + See more info here - https://strimzi.io/docs/operators/latest/configuring.html#proc-configuring-kafka-user-str + + The kafka user definition is defined as part of .Values per component. + For general use by OOM components, the following list of acl types should suffice: + type: group (Used by the client app to be added to a particular kafka consumer group) + type: topic (1 or more kafka topics that the client needs to access. Commonly [Read,Write]) + + Note: The template will use the following default values. + + spec.authentication.type: scram-sha-512 (dictated by the available broker listeners on the kafka cluster) + spec.authorization.type: simple (Only type supported by strimzi at present) + spec.authorization.acls.resource.patternType: literal + + Example: + + kafkaUser: + acls: + - name: sdc (mandatory) + suffix: mysuffix (optional. Will be appended (with a hyphen) to the "name" entry. ie "sdc-mysuffix") + type: group (mandatory. Type "group" is used by the client as it's kafka consumer group) + operations: [Read] (mandatory. List of at least 1) + - name: SDC-DISTR + type: topic + patternType: prefix (optional. In this example, the user will be provided Read and Write access to all topics named "SDC-DISTR*") + operations: [Read, Write] +*/}} +{{- define "common.kafkauser" -}} +{{- $global := .global }} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaUser +metadata: + name: {{ include "common.name" . }}-ku + namespace: {{ include "common.namespace" $global }} + labels: + strimzi.io/cluster: {{ include "common.release" . }}-strimzi +spec: + authentication: + type: {{ .Values.kafkaUser.authenticationType | default "scram-sha-512" }} + authorization: + type: {{ .Values.kafkaUser.authorizationType | default "simple" }} + acls: + {{- range $acl := .Values.kafkaUser.acls }} + - resource: + type: {{ $acl.type }} + patternType: {{ $acl.patternType | default "literal" }} + name: {{ ternary (printf "%s-%s" $acl.name $acl.suffix) $acl.name (hasKey $acl "suffix") }} + operations: + {{- range $operation := $acl.operations }} + - {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* + Create a Strimzi KafkaTopic. + Usage: + include "common.kafkatopic" . + + Strimzi kafka provides kafka topic management via its custom resource definition KafkaTopic + which is deployed using its Topic Operator component. + See more info here - https://github.com/strimzi/strimzi-kafka-operator/blob/main/helm-charts/helm3/strimzi-kafka-operator/crds/043-Crd-kafkatopic.yaml + + Note: KafkaTopic names should adhere to kubernetes object naming conventions - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/ + maximum length of 253 characters and consist of lower case alphanumeric characters, -, and . + + Note: The template will use the following default values. + + spec.config.retention.ms: 7200000 (defaults to 2 hrs retention for kafka topic logs) + spec.config.segment.bytes: 1073741824 (defaults to 1gb) + spec.partitions: 6 (defaults to (2 * (default.replication.factor)) defined by the strimzi broker conf) + spec.replicas: 3 (defaults to default.replication.factor defined by the strimzi broker conf. Must be > 0 and <= (num of broker replicas)) + + The kafka topic definition is defined as part of .Values per component. + + Example: + + kafkaTopic: + - name: my-new-topic (mandatory) + retentionMs: 7200000 (optional. Defaults to 2hrs) + segmentBytes: 1073741824 (optional. Defaults to 1gb) + suffix: my-suffix (optional. Will be appended (with a hyphen) to the "name" value. ie "my-new-topic-my-suffix") + - name: my.other.topic + suffix: some.other-suffix +*/}} +{{- define "common.kafkatopic" -}} +{{- $global := .global }} +{{- range $topic := .Values.kafkaTopic }} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + name: {{ ($topic.name) | lower }}-kt + labels: + strimzi.io/cluster: {{ include "common.release" $ }}-strimzi +spec: + {{- if (hasKey $topic "partitions") }} + partitions: {{ $topic.partitions }} + {{- end }} + {{- if (hasKey $topic "replicas") }} + replicas: {{ $topic.replicas }} + {{- end }} + topicName: {{ ternary (printf "%s-%s" $topic.name $topic.suffix) $topic.name (hasKey $topic "suffix") }} + config: + retention.ms: {{ $topic.retentionMs | default "7200000" }} + segment.bytes: {{ $topic.segmentBytes | default "1073741824"}} +--- +{{- end }} +{{- end -}} diff --git a/kubernetes/sdc/Chart.yaml b/kubernetes/sdc/Chart.yaml index aaa3ac29b7..7c67f49da9 100644 --- a/kubernetes/sdc/Chart.yaml +++ b/kubernetes/sdc/Chart.yaml @@ -21,6 +21,9 @@ name: sdc version: 12.0.0 dependencies: + - name: common + version: ~12.x-0 + repository: '@local' - name: sdc-be version: ~12.x-0 repository: 'file://components/sdc-be' diff --git a/kubernetes/sdc/components/sdc-be/Chart.yaml b/kubernetes/sdc/components/sdc-be/Chart.yaml index e4052afc3d..fe9f39108b 100644 --- a/kubernetes/sdc/components/sdc-be/Chart.yaml +++ b/kubernetes/sdc/components/sdc-be/Chart.yaml @@ -21,6 +21,9 @@ name: sdc-be version: 12.0.0 dependencies: + - name: common + version: ~12.x-0 + repository: '@local' - name: certInitializer version: ~12.x-0 repository: '@local' diff --git a/kubernetes/sdc/components/sdc-be/templates/deployment.yaml b/kubernetes/sdc/components/sdc-be/templates/deployment.yaml index 16fc57aa92..d50a83b2fd 100644 --- a/kubernetes/sdc/components/sdc-be/templates/deployment.yaml +++ b/kubernetes/sdc/components/sdc-be/templates/deployment.yaml @@ -163,7 +163,10 @@ spec: fieldPath: status.podIP {{- if .Values.global.kafka.useKafka }} - name: SASL_JAAS_CONFIG - {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "sdc-be-kafka-secret" "key" "sasl.jaas.config") | indent 12 }} + valueFrom: + secretKeyRef: + name: {{ include "common.name" . }}-ku + key: sasl.jaas.config - name: USE_KAFKA value: {{ .Values.global.kafka.useKafka | quote }} {{- end }} diff --git a/kubernetes/sdc/components/sdc-be/templates/secret.yaml b/kubernetes/sdc/components/sdc-be/templates/kafkauser.yaml index bb5091f01a..5033d9d9aa 100644 --- a/kubernetes/sdc/components/sdc-be/templates/secret.yaml +++ b/kubernetes/sdc/components/sdc-be/templates/kafkauser.yaml @@ -13,5 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. */}} - -{{ include "common.secretFast" . }} +{{- if .Values.global.kafka.useKafka }} +{{ include "common.kafkauser" . }} +{{- end }} diff --git a/kubernetes/sdc/components/sdc-be/templates/sdc-be-kafka-user.yaml b/kubernetes/sdc/components/sdc-be/templates/sdc-be-kafka-user.yaml deleted file mode 100644 index 6cd7f93c5a..0000000000 --- a/kubernetes/sdc/components/sdc-be/templates/sdc-be-kafka-user.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{/* -# Copyright © 2022 Nordix Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -{{- if .Values.global.kafka.useKafka }} -apiVersion: kafka.strimzi.io/v1beta2 -kind: KafkaUser -metadata: - name: {{ include "common.release" . }}-{{ .Values.global.kafka.sdcBeKafkaUser }} - labels: - strimzi.io/cluster: {{ include "common.release" . }}-strimzi -spec: - authentication: - type: {{ .Values.config.kafka.saslMech }} - authorization: - type: {{ .Values.config.kafka.authType }} - acls: - - resource: - type: group - name: {{ .Values.config.kafka.topicConsumer.groupId }}-{{ .Values.env.name }} - operation: Read - - resource: - type: topic - patternType: prefix - name: {{ .Values.config.kafka.topicConsumer.pattern }} - operation: All -{{- end }} diff --git a/kubernetes/sdc/components/sdc-be/templates/sdc-distro-topics.yaml b/kubernetes/sdc/components/sdc-be/templates/sdc-distro-topics.yaml deleted file mode 100644 index 9a6f7579e8..0000000000 --- a/kubernetes/sdc/components/sdc-be/templates/sdc-distro-topics.yaml +++ /dev/null @@ -1,40 +0,0 @@ -{{/* -# Copyright © 2022 Nordix Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} -{{- if .Values.global.kafka.useKafka }} -apiVersion: kafka.strimzi.io/v1beta2 -kind: KafkaTopic -metadata: - name: sdc-distro-notif-topic - labels: - strimzi.io/cluster: {{ include "common.release" . }}-strimzi -spec: - topicName: {{ .Values.global.kafka.topics.sdcDistNotifTopic }}-{{ .Values.env.name }} - config: - retention.ms: {{ .Values.config.kafka.topicRetentionMs }} - segment.bytes: {{ .Values.config.kafka.topicSegmentBytes }} ---- -apiVersion: kafka.strimzi.io/v1beta2 -kind: KafkaTopic -metadata: - name: sdc-distro-status-topic - labels: - strimzi.io/cluster: {{ include "common.release" . }}-strimzi -spec: - topicName: {{ .Values.global.kafka.topics.sdcDistStatusTopic }}-{{ .Values.env.name }} - config: - retention.ms: {{ .Values.config.kafka.topicRetentionMs }} - segment.bytes: {{ .Values.config.kafka.topicSegmentBytes }} -{{- end }}
\ No newline at end of file diff --git a/kubernetes/sdc/components/sdc-be/values.yaml b/kubernetes/sdc/components/sdc-be/values.yaml index faf46e5549..b7b3acd909 100644 --- a/kubernetes/sdc/components/sdc-be/values.yaml +++ b/kubernetes/sdc/components/sdc-be/values.yaml @@ -31,13 +31,10 @@ global: replicaCount: 3 clusterName: cassandra dataCenter: Pod - # Strimzi kafka config + # Global Strimzi kafka config overridden + # from parent values.yaml kafka: useKafka: overridden-from-parent-values-yaml - sdcBeKafkaUser: overridden-from-parent-values-yaml - topics: - sdcDistNotifTopic: overridden-from-parent-values-yaml - sdcDistStatusTopic: overridden-from-parent-values-yaml ################################################################# # Application configuration defaults. @@ -53,7 +50,7 @@ debugEnabled: false #environment file env: - name: AUTO + name: &env AUTO certInitializer: nameOverride: sdc-be-cert-init @@ -78,29 +75,21 @@ certInitializer: ################################################################# # SDC Config part ################################################################# - -secrets: - - uid: sdc-be-kafka-secret - externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}' - type: genericKV - envs: - - name: sasl.jaas.config - value: '{{ .Values.config.someConfig }}' - policy: generate - config: javaOptions: "-Xmx1536m -Xms1536m" cassandraSslEnabled: "false" - # Strimzi kafka config - kafka: - saslMech: scram-sha-512 - securityProtocol: SASL_PLAINTEXT - authType: simple - topicRetentionMs: 7200000 - topicSegmentBytes: 1073741824 - topicConsumer: - pattern: SDC-DIST - groupId: sdc + +kafkaUser: + acls: + - name: sdc + suffix: *env + type: group + operations: [Read] + - name: SDC-DISTR + type: topic + patternType: prefix + operations: [Read, Write] + # default number of instances replicaCount: 1 diff --git a/kubernetes/sdc/resources/config/environments/AUTO.json b/kubernetes/sdc/resources/config/environments/AUTO.json index aee666f4af..065a756822 100755 --- a/kubernetes/sdc/resources/config/environments/AUTO.json +++ b/kubernetes/sdc/resources/config/environments/AUTO.json @@ -36,12 +36,14 @@ ] }, "Kafka": { - "bootstrap": "{{ include "common.release" . }}-{{ .Values.global.kafka.kafkaBootstrap }}" + "bootstrap": "{{ include "common.release" . }}-strimzi-kafka-bootstrap:9092" }, + {{- if .Values.global.kafka.useKafka }} "DistributionTopics": { "notificationTopicName": "{{ .Values.global.kafka.topics.sdcDistNotifTopic }}", "statusTopicName": "{{ .Values.global.kafka.topics.sdcDistStatusTopic }}" }, + {{- end }} "Nodes": { "CS": [ "{{.Values.global.sdc_cassandra.serviceName}}.{{include "common.namespace" .}}" diff --git a/kubernetes/sdc/templates/kafkatopic.yaml b/kubernetes/sdc/templates/kafkatopic.yaml new file mode 100644 index 0000000000..53352c4e56 --- /dev/null +++ b/kubernetes/sdc/templates/kafkatopic.yaml @@ -0,0 +1,18 @@ +{{/* +# Copyright © 2022 Nordix Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- if .Values.global.kafka.useKafka }} +{{ include "common.kafkatopic" . }} +{{- end }}
\ No newline at end of file diff --git a/kubernetes/sdc/values.yaml b/kubernetes/sdc/values.yaml index 60a361eae6..1e11ca6410 100644 --- a/kubernetes/sdc/values.yaml +++ b/kubernetes/sdc/values.yaml @@ -42,19 +42,26 @@ global: clusterName: cassandra dataCenter: Pod centralizedLoggingEnabled: true - # Kafka config + # global Kafka config passed to sdc-be chart kafka: + # If true, the following Strimzi KafkaTopics will be created useKafka: true - sdcBeKafkaUser: sdc-be-kafka-user - kafkaBootstrap: strimzi-kafka-bootstrap:9092 topics: - sdcDistNotifTopic: SDC-DISTR-NOTIF-TOPIC - sdcDistStatusTopic: SDC-DISTR-STATUS-TOPIC + sdcDistNotifTopic: ¬if-topic-name SDC-DISTR-NOTIF-TOPIC + sdcDistStatusTopic: &status-topic-name SDC-DISTR-STATUS-TOPIC + +# Environment file +env: + name: &env AUTO + +kafkaTopic: + - name: *notif-topic-name + suffix: *env + - name: *status-topic-name + suffix: *env sdc-be: logConfigMapNamePrefix: '{{ include "common.release" . }}-sdc' - config: - jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.kafka.sdcBeKafkaUser }}' sdc-fe: logConfigMapNamePrefix: '{{ include "common.release" . }}-sdc' sdc-onboarding-be: @@ -62,10 +69,6 @@ sdc-onboarding-be: sdc-wfd-fe: logConfigMapNamePrefix: '{{ include "common.release" . }}-sdc' -# Environment file -env: - name: AUTO - config: logstashServiceName: log-ls logstashPort: 5044 |