aboutsummaryrefslogtreecommitdiffstats
path: root/kubernetes/dmaap/components/message-router
diff options
context:
space:
mode:
authorMarat Salakhutdinov <marat.salakhutdinov@bell.ca>2020-10-27 11:40:03 -0400
committerMarat Salakhutdinov <marat.salakhutdinov@bell.ca>2021-03-31 15:21:15 -0400
commitfce4a5ad66779ba36f17d7bd9689a4384b40bbab (patch)
treec763e8a2aca473baf39a65e354dd1ef6d7b12822 /kubernetes/dmaap/components/message-router
parent48e2ca740dcc1fe91b93b54343f5d19f7d7e559a (diff)
[DMAAP] Fix scaling logic for message router kafka and zookeeper
Currently if we want to scale message router kafka and zookeeper we need to do manual changes in charts to make it work. With this patch all can be done with override files. Issue-ID: OOM-2613 Signed-off-by: Marat Salakhutdinov <marat.salakhutdinov@bell.ca> Change-Id: I1782dca26f964f33c250520ee2e187619cee0e5e
Diffstat (limited to 'kubernetes/dmaap/components/message-router')
-rw-r--r--kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/pv.yaml4
-rw-r--r--kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/service.yaml4
-rw-r--r--kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/statefulset.yaml2
-rw-r--r--kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml1
-rw-r--r--kubernetes/dmaap/components/message-router/components/message-router-zookeeper/templates/pv.yaml4
-rwxr-xr-xkubernetes/dmaap/components/message-router/resources/config/dmaap/MsgRtrApi.properties10
6 files changed, 13 insertions, 12 deletions
diff --git a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/pv.yaml b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/pv.yaml
index 263caf1059..c386163735 100644
--- a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/pv.yaml
+++ b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/pv.yaml
@@ -17,13 +17,13 @@
{{- $global := . -}}
{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}}
{{- if eq "True" (include "common.needPV" .) -}}
-{{ range $i, $e := until (atoi (quote $global.Values.replicaCount) | default 3) }}
+{{ range $i, $e := until (int $global.Values.replicaCount) }}
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: {{ include "common.release" $global }}-{{ $global.Values.service.name }}-{{ $i }}
- namespace: {{ $global.Release.Namespace }}
+ namespace: {{ include "common.namespace" $global }}
labels:
app: {{ $global.Values.service.name }}
chart: {{ $global.Chart.Name }}-{{ $global.Chart.Version | replace "+" "_" }}
diff --git a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/service.yaml b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/service.yaml
index 88c83981bb..9a20f9c517 100644
--- a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/service.yaml
+++ b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/service.yaml
@@ -15,13 +15,13 @@
*/}}
{{- $root := . -}}
-{{ range $i, $e := until (atoi (quote $root.Values.replicaCount) | default 3) }}
+{{ range $i, $e := until (int $root.Values.replicaCount) }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ $root.Values.service.name }}-{{ $i }}
- namespace: {{ $root.Release.Namespace }}
+ namespace: {{ include "common.namespace" $root }}
labels:
app: {{ $root.Values.service.name }}
chart: {{ $root.Chart.Name }}-{{ $root.Chart.Version | replace "+" "_" }}
diff --git a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/statefulset.yaml b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/statefulset.yaml
index 1eabe3aad6..ba872a310a 100644
--- a/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/statefulset.yaml
+++ b/kubernetes/dmaap/components/message-router/components/message-router-kafka/templates/statefulset.yaml
@@ -164,7 +164,7 @@ spec:
apiVersion: v1
fieldPath: status.hostIP
- name: KAFKA_ZOOKEEPER_CONNECT
- value: {{ include "common.release" . }}-{{.Values.zookeeper.name}}-0.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}},{{ include "common.release" . }}-{{.Values.zookeeper.name}}-1.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}},{{ include "common.release" . }}-{{.Values.zookeeper.name}}-2.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}}
+ value: "{{ include "common.kafkaNodes" (dict "dot" . "replicaCount" .Values.zookeeper.replicaCount "componentName" .Values.zookeeper.name "port" .Values.zookeeper.port ) }}"
- name: KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE
value: "{{ .Values.kafka.enableSupport }}"
- name: KAFKA_OPTS
diff --git a/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml b/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml
index 6c3cbc385a..0c0f2a16c0 100644
--- a/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml
+++ b/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml
@@ -31,6 +31,7 @@ pullPolicy: Always
zookeeper:
name: message-router-zookeeper
port: 2181
+ replicaCount: 3
kafka:
heapOptions: -Xmx5G -Xms1G
diff --git a/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/templates/pv.yaml b/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/templates/pv.yaml
index 263caf1059..c386163735 100644
--- a/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/templates/pv.yaml
+++ b/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/templates/pv.yaml
@@ -17,13 +17,13 @@
{{- $global := . -}}
{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}}
{{- if eq "True" (include "common.needPV" .) -}}
-{{ range $i, $e := until (atoi (quote $global.Values.replicaCount) | default 3) }}
+{{ range $i, $e := until (int $global.Values.replicaCount) }}
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: {{ include "common.release" $global }}-{{ $global.Values.service.name }}-{{ $i }}
- namespace: {{ $global.Release.Namespace }}
+ namespace: {{ include "common.namespace" $global }}
labels:
app: {{ $global.Values.service.name }}
chart: {{ $global.Chart.Name }}-{{ $global.Chart.Version | replace "+" "_" }}
diff --git a/kubernetes/dmaap/components/message-router/resources/config/dmaap/MsgRtrApi.properties b/kubernetes/dmaap/components/message-router/resources/config/dmaap/MsgRtrApi.properties
index 8d79ccfc7e..4256b3b723 100755
--- a/kubernetes/dmaap/components/message-router/resources/config/dmaap/MsgRtrApi.properties
+++ b/kubernetes/dmaap/components/message-router/resources/config/dmaap/MsgRtrApi.properties
@@ -38,8 +38,7 @@
#config.zk.servers=172.18.1.1
#config.zk.servers={{.Values.zookeeper.name}}:{{.Values.zookeeper.port}}
*/}}
-config.zk.servers={{include "common.release" .}}-{{.Values.zookeeper.name}}-0.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}},{{include "common.release" .}}-{{.Values.zookeeper.name}}-1.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}},{{include "common.release" .}}-{{.Values.zookeeper.name}}-2.{{.Values.zookeeper.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.zookeeper.port}}
-
+config.zk.servers={{ include "common.kafkaNodes" (dict "dot" . "replicaCount" (index .Values "message-router-zookeeper" "replicaCount") "componentName" .Values.zookeeper.name "port" .Values.zookeeper.port ) }}
#config.zk.root=/fe3c/cambria/config
@@ -52,7 +51,8 @@ config.zk.servers={{include "common.release" .}}-{{.Values.zookeeper.name}}-0.{{
## if you want to change request.required.acks it can take this one value
#kafka.metadata.broker.list=localhost:9092,localhost:9093
#kafka.metadata.broker.list={{.Values.kafka.name}}:{{.Values.kafka.port}}
-kafka.metadata.broker.list={{include "common.release" .}}-{{.Values.kafka.name}}-0.{{.Values.kafka.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.kafka.port}},{{include "common.release" .}}-{{.Values.kafka.name}}-1.{{.Values.kafka.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.kafka.port}},{{include "common.release" .}}-{{.Values.kafka.name}}-2.{{.Values.kafka.name}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.kafka.port}}
+kafka.metadata.broker.list={{ include "common.kafkaNodes" (dict "dot" . "replicaCount" (index .Values "message-router-kafka" "replicaCount") "componentName" .Values.kafka.name "port" .Values.kafka.port ) }}
+
##kafka.request.required.acks=-1
#kafka.client.zookeeper=${config.zk.servers}
consumer.timeout.ms=100
@@ -143,7 +143,7 @@ cambria.consumer.cache.touchFreqMs=120000
cambria.consumer.cache.zkBasePath=/fe3c/cambria/consumerCache
consumer.timeout=17
default.partitions=3
-default.replicas=3
+default.replicas={{ index .Values "message-router-kafka" "replicaCount" }}
##############################################################################
#100mb
maxcontentlength=10000
@@ -171,4 +171,4 @@ msgRtr.mirrormaker.consumerid=1
kafka.max.poll.interval.ms=300000
kafka.heartbeat.interval.ms=60000
kafka.session.timeout.ms=240000
-kafka.max.poll.records=1000 \ No newline at end of file
+kafka.max.poll.records=1000