From fce4a5ad66779ba36f17d7bd9689a4384b40bbab Mon Sep 17 00:00:00 2001 From: Marat Salakhutdinov Date: Tue, 27 Oct 2020 11:40:03 -0400 Subject: [DMAAP] Fix scaling logic for message router kafka and zookeeper Currently if we want to scale message router kafka and zookeeper we need to do manual changes in charts to make it work. With this patch all can be done with override files. Issue-ID: OOM-2613 Signed-off-by: Marat Salakhutdinov Change-Id: I1782dca26f964f33c250520ee2e187619cee0e5e --- .../components/message-router-kafka/templates/pv.yaml | 4 ++-- .../components/message-router-kafka/templates/service.yaml | 4 ++-- .../components/message-router-kafka/templates/statefulset.yaml | 2 +- .../message-router/components/message-router-kafka/values.yaml | 1 + .../components/message-router-zookeeper/templates/pv.yaml | 4 ++-- .../message-router/resources/config/dmaap/MsgRtrApi.properties | 10 +++++----- 6 files changed, 13 insertions(+), 12 deletions(-) (limited to 'kubernetes/dmaap/components/message-router') diff --git a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/pv.yaml b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/pv.yaml index 263caf1059..c386163735 100644 --- a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/pv.yaml +++ b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/pv.yaml @@ -17,13 +17,13 @@ {{- $global := . -}} {{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}} {{- if eq "True" (include "common.needPV" .) -}} -{{ range $i, $e := until (atoi (quote $global.Values.replicaCount) | default 3) }} +{{ range $i, $e := until (int $global.Values.replicaCount) }} --- apiVersion: v1 kind: PersistentVolume metadata: name: {{ include "common.release" $global }}-{{ $global.Values.service.name }}-{{ $i }} - namespace: {{ $global.Release.Namespace }} + namespace: {{ include "common.namespace" $global }} labels: app: {{ $global.Values.service.name }} chart: {{ $global.Chart.Name }}-{{ $global.Chart.Version | replace "+" "_" }} diff --git a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/service.yaml b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/service.yaml index 88c83981bb..9a20f9c517 100644 --- a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/service.yaml +++ b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/service.yaml @@ -15,13 +15,13 @@ */}} {{- $root := . -}} -{{ range $i, $e := until (atoi (quote $root.Values.replicaCount) | default 3) }} +{{ range $i, $e := until (int $root.Values.replicaCount) }} --- apiVersion: v1 kind: Service metadata: name: {{ $root.Values.service.name }}-{{ $i }} - namespace: {{ $root.Release.Namespace }} + namespace: {{ include "common.namespace" $root }} labels: app: {{ $root.Values.service.name }} chart: {{ $root.Chart.Name }}-{{ $root.Chart.Version | replace "+" "_" }} diff --git a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/statefulset.yaml b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/statefulset.yaml index 1eabe3aad6..ba872a310a 100644 --- a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/statefulset.yaml +++ b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/statefulset.yaml @@ -164,7 +164,7 @@ spec: apiVersion: v1 fieldPath: status.hostIP - name: KAFKA_ZOOKEEPER_CONNECT - value: {{ include "common.release" . }}-{{.Values.zookeeper.name}}-0.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}},{{ include "common.release" . }}-{{.Values.zookeeper.name}}-1.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}},{{ include "common.release" . }}-{{.Values.zookeeper.name}}-2.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}} + value: "{{ include "common.kafkaNodes" (dict "dot" . "replicaCount" .Values.zookeeper.replicaCount "componentName" .Values.zookeeper.name "port" .Values.zookeeper.port ) }}" - name: KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE value: "{{ .Values.kafka.enableSupport }}" - name: KAFKA_OPTS diff --git a/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml b/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml index 6c3cbc385a..0c0f2a16c0 100644 --- a/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml +++ b/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml @@ -31,6 +31,7 @@ pullPolicy: Always zookeeper: name: message-router-zookeeper port: 2181 + replicaCount: 3 kafka: heapOptions: -Xmx5G -Xms1G diff --git a/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/templates/pv.yaml b/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/templates/pv.yaml index 263caf1059..c386163735 100644 --- a/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/templates/pv.yaml +++ b/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/templates/pv.yaml @@ -17,13 +17,13 @@ {{- $global := . -}} {{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}} {{- if eq "True" (include "common.needPV" .) -}} -{{ range $i, $e := until (atoi (quote $global.Values.replicaCount) | default 3) }} +{{ range $i, $e := until (int $global.Values.replicaCount) }} --- apiVersion: v1 kind: PersistentVolume metadata: name: {{ include "common.release" $global }}-{{ $global.Values.service.name }}-{{ $i }} - namespace: {{ $global.Release.Namespace }} + namespace: {{ include "common.namespace" $global }} labels: app: {{ $global.Values.service.name }} chart: {{ $global.Chart.Name }}-{{ $global.Chart.Version | replace "+" "_" }} diff --git a/kubernetes/dmaap/components/message-router/resources/config/dmaap/MsgRtrApi.properties b/kubernetes/dmaap/components/message-router/resources/config/dmaap/MsgRtrApi.properties index 8d79ccfc7e..4256b3b723 100755 --- a/kubernetes/dmaap/components/message-router/resources/config/dmaap/MsgRtrApi.properties +++ b/kubernetes/dmaap/components/message-router/resources/config/dmaap/MsgRtrApi.properties @@ -38,8 +38,7 @@ #config.zk.servers=172.18.1.1 #config.zk.servers={{.Values.zookeeper.name}}:{{.Values.zookeeper.port}} */}} -config.zk.servers={{include "common.release" .}}-{{.Values.zookeeper.name}}-0.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}},{{include "common.release" .}}-{{.Values.zookeeper.name}}-1.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}},{{include "common.release" .}}-{{.Values.zookeeper.name}}-2.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}} - +config.zk.servers={{ include "common.kafkaNodes" (dict "dot" . "replicaCount" (index .Values "message-router-zookeeper" "replicaCount") "componentName" .Values.zookeeper.name "port" .Values.zookeeper.port ) }} #config.zk.root=/fe3c/cambria/config @@ -52,7 +51,8 @@ config.zk.servers={{include "common.release" .}}-{{.Values.zookeeper.name}}-0.{{ ## if you want to change request.required.acks it can take this one value #kafka.metadata.broker.list=localhost:9092,localhost:9093 #kafka.metadata.broker.list={{.Values.kafka.name}}:{{.Values.kafka.port}} -kafka.metadata.broker.list={{include "common.release" .}}-{{.Values.kafka.name}}-0.{{.Values.kafka.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.kafka.port}},{{include "common.release" .}}-{{.Values.kafka.name}}-1.{{.Values.kafka.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.kafka.port}},{{include "common.release" .}}-{{.Values.kafka.name}}-2.{{.Values.kafka.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.kafka.port}} +kafka.metadata.broker.list={{ include "common.kafkaNodes" (dict "dot" . "replicaCount" (index .Values "message-router-kafka" "replicaCount") "componentName" .Values.kafka.name "port" .Values.kafka.port ) }} + ##kafka.request.required.acks=-1 #kafka.client.zookeeper=${config.zk.servers} consumer.timeout.ms=100 @@ -143,7 +143,7 @@ cambria.consumer.cache.touchFreqMs=120000 cambria.consumer.cache.zkBasePath=/fe3c/cambria/consumerCache consumer.timeout=17 default.partitions=3 -default.replicas=3 +default.replicas={{ index .Values "message-router-kafka" "replicaCount" }} ############################################################################## #100mb maxcontentlength=10000 @@ -171,4 +171,4 @@ msgRtr.mirrormaker.consumerid=1 kafka.max.poll.interval.ms=300000 kafka.heartbeat.interval.ms=60000 kafka.session.timeout.ms=240000 -kafka.max.poll.records=1000 \ No newline at end of file +kafka.max.poll.records=1000 -- cgit 1.2.3-korg