diff options
Diffstat (limited to 'kubernetes/dmaap/components/message-router/charts/message-router-zookeeper')
6 files changed, 238 insertions, 33 deletions
diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/resources/config/jmx-zookeeper-prometheus.yml b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/resources/config/jmx-zookeeper-prometheus.yml new file mode 100644 index 0000000000..a75b644c5f --- /dev/null +++ b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/resources/config/jmx-zookeeper-prometheus.yml @@ -0,0 +1,21 @@ +jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:{{ .Values.jmx.port }}/jmxrmi +lowercaseOutputName: true +lowercaseOutputLabelNames: true +ssl: false +rules: +- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+)><>(\\w+)" + name: "message-router-zookeeper_$2" +- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+)><>(\\w+)" + name: "message-router-zookeeper_$3" + labels: + replicaId: "$2" +- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+)><>(\\w+)" + name: "message-router-zookeeper_$4" + labels: + replicaId: "$2" + memberType: "$3" +- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d+), name1=replica.(\\d+), name2=(\\w+), name3=(\\w+)><>(\\w+)" + name: "message-router-zookeeper_$4_$5" + labels: + replicaId: "$2" + memberType: "$3" diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/resources/config/zk_server_jaas.conf b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/resources/config/zk_server_jaas.conf new file mode 100644 index 0000000000..8266f6b2c6 --- /dev/null +++ b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/resources/config/zk_server_jaas.conf @@ -0,0 +1,4 @@ +Server { + org.apache.zookeeper.server.auth.DigestLoginModule required + user_${ZK_ADMIN}="${ZK_PSWD}"; +};
\ No newline at end of file diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/templates/configmap.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/templates/configmap.yaml new file mode 100644 index 0000000000..907111d898 --- /dev/null +++ b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/templates/configmap.yaml @@ -0,0 +1,42 @@ +# Copyright © 2020 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.prometheus.jmx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }}-prometheus-configmap + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +data: +{{ (.Files.Glob "resources/config/jmx-zookeeper-prometheus.yml").AsConfig | indent 2 }} +--- +{{ end }} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }}-jaas-configmap + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +data: +{{ (.Files.Glob "resources/config/zk_server_jaas.conf").AsConfig | indent 2 }}
\ No newline at end of file diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/templates/secrets.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/templates/secrets.yaml new file mode 100644 index 0000000000..729cad4cac --- /dev/null +++ b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/templates/secrets.yaml @@ -0,0 +1,15 @@ +# Copyright © 2020 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{ include "common.secret" . }} diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/templates/statefulset.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/templates/statefulset.yaml index af4c1719fb..e98e614d93 100644 --- a/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/templates/statefulset.yaml +++ b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/templates/statefulset.yaml @@ -36,6 +36,11 @@ spec: labels: app: {{ include "common.name" . }} release: {{ include "common.release" . }} + {{- if .Values.prometheus.jmx.enabled }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.prometheus.jmx.port | quote }} + {{- end }} spec: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -47,6 +52,10 @@ spec: operator: In values: - {{ include "common.name" . }} + - key: "release" + operator: In + values: + - {{ include "common.release" . }} topologyKey: "kubernetes.io/hostname" {{- if .Values.nodeAffinity }} nodeAffinity: @@ -58,13 +67,53 @@ spec: - sh - -exec - > - chown -R 1000:1000 /tmp/zookeeper/apikeys; + chown -R 1000:0 /tmp/zookeeper/apikeys; image: "{{ .Values.busyBoxRepository }}/{{ .Values.busyBoxImage }}" imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} volumeMounts: - mountPath: /tmp/zookeeper/apikeys name: zookeeper-data + - command: + - sh + args: + - -c + - "cd /config-input && for PFILE in `ls -1 .`; do envsubst <${PFILE} >/etc/zookeeper/secrets/jaas/${PFILE}; done" + env: + - name: ZK_ADMIN + {{- include "common.secret.envFromSecret" (dict "global" . "uid" "zk-admin" "key" "login") | indent 10 }} + - name: ZK_PSWD + {{- include "common.secret.envFromSecret" (dict "global" . "uid" "zk-admin" "key" "password") | indent 10 }} + volumeMounts: + - mountPath: /etc/zookeeper/secrets/jaas + name: jaas-config + - mountPath: /config-input + name: jaas + image: "{{ .Values.global.envsubstImage }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + name: {{ include "common.name" . }}-update-config containers: + {{- if .Values.prometheus.jmx.enabled }} + - name: prometheus-jmx-exporter + image: "{{ .Values.prometheus.jmx.imageRepository }}/{{ .Values.prometheus.jmx.image }}:{{ .Values.prometheus.jmx.imageTag }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - {{ .Values.prometheus.jmx.port | quote }} + - /etc/jmx-zookeeper/jmx-zookeeper-prometheus.yml + ports: + - containerPort: {{ .Values.prometheus.jmx.port }} + resources: +{{ toYaml .Values.prometheus.jmx.resources | indent 10 }} + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-zookeeper + {{- end }} - name: {{ include "common.name" . }} image: "{{ include "common.repository" . }}/{{ .Values.image }}" imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} @@ -77,39 +126,71 @@ spec: name: {{ .Values.service.serverPortName }} - containerPort: {{ .Values.service.leaderElectionPort }} name: {{ .Values.service.leaderElectionPortName }} + {{- if .Values.prometheus.jmx.enabled }} + - containerPort: {{ .Values.jmx.port }} + name: jmx + {{- end }} {{ if eq .Values.liveness.enabled true }} livenessProbe: exec: - command: - - sh - - -c - - "zookeeper-ready.sh 2181" + command: ['/bin/bash', '-c', 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok'] initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.liveness.periodSeconds }} timeoutSeconds: {{ .Values.liveness.timeoutSeconds }} {{ end }} readinessProbe: exec: - command: - - sh - - -c - - "zookeeper-ready.sh 2181" + command: ['/bin/bash', '-c', 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok'] initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.readiness.periodSeconds }} timeoutSeconds: {{ .Values.readiness.timeoutSeconds }} + resources: +{{ toYaml .Values.resources | indent 10 }} env: - - name: ZK_REPLICAS + - name : KAFKA_HEAP_OPTS + value: "{{ .Values.zkConfig.heapOptions }}" + {{- if .Values.jmx.port }} + - name : KAFKA_JMX_PORT + value: "{{ .Values.jmx.port }}" + {{- end }} + - name : ZOOKEEPER_REPLICAS value: "{{ .Values.replicaCount }}" - - name: ZK_INIT_LIMIT - value: "{{ .Values.zk.initLimit }}" - - name: ZK_SYNC_LIMIT - value: "{{ .Values.zk.syncLimit }}" + - name : ZOOKEEPER_TICK_TIME + value: "{{ .Values.zkConfig.tickTime }}" + - name : ZOOKEEPER_SYNC_LIMIT + value: "{{ .Values.zkConfig.syncLimit }}" + - name : ZOOKEEPER_INIT_LIMIT + value: "{{ .Values.zkConfig.initLimit }}" + - name : ZOOKEEPER_MAX_CLIENT_CNXNS + value: "{{ .Values.zkConfig.maxClientCnxns }}" + - name : ZOOKEEPER_AUTOPURGE_SNAP_RETAIN_COUNT + value: "{{ .Values.zkConfig.autoPurgeSnapRetainCount}}" + - name : ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL + value: "{{ .Values.zkConfig.autoPurgePurgeInterval}}" + - name: ZOOKEEPER_CLIENT_PORT + value: "{{ .Values.zkConfig.clientPort }}" + - name: KAFKA_OPTS + value: "{{ .Values.zkConfig.kafkaOpts }}" + - name: EXTRA_ARGS + value: "{{ .Values.zkConfig.extraArgs }}" + - name: ZOOKEEPER_SERVER_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + command: + - "bash" + - "-c" + - | + ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-}+1)) \ + /etc/confluent/docker/run volumeMounts: - mountPath: /etc/localtime name: localtime readOnly: true - mountPath: /var/lib/zookeeper/data name: zookeeper-data + - name: jaas-config + mountPath: /etc/zookeeper/secrets/jaas {{- if .Values.tolerations }} tolerations: {{ toYaml .Values.tolerations | indent 10 }} @@ -118,15 +199,29 @@ spec: - name: localtime hostPath: path: /etc/localtime + - name: jaas-config + emptyDir: + medium: Memory + - name: docker-socket + hostPath: + path: /var/run/docker.sock + - name: jaas + configMap: + name: {{ include "common.fullname" . }}-jaas-configmap + {{- if .Values.prometheus.jmx.enabled }} + - name: jmx-config + configMap: + name: {{ include "common.fullname" . }}-prometheus-configmap + {{- end }} {{ if not .Values.persistence.enabled }} - name: zookeeper-data emptyDir: {} {{ else }} volumeClaimTemplates: - metadata: - name: zookeeper-data + name: zookeeper-data labels: - app: {{ include "common.fullname" . }} + app: {{ include "common.fullname" . }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" release: "{{ include "common.release" . }}" heritage: "{{ .Release.Service }}" @@ -139,4 +234,4 @@ spec: storage: {{ .Values.persistence.size | quote }} {{ end }} imagePullSecrets: - - name: "{{ include "common.namespace" . }}-docker-registry-key" + - name: "{{ include "common.namespace" . }}-docker-registry-key"
\ No newline at end of file diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/values.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/values.yaml index eeb77ba7fd..4abb6e3c4a 100644 --- a/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/values.yaml +++ b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/values.yaml @@ -24,13 +24,14 @@ global: loggingImage: beats/filebeat:5.5.0 ubuntuInitRepository: registry.hub.docker.com persistence: {} + envsubstImage: dibi/envsubst ################################################################# # Application configuration defaults. ################################################################# # application image repository: nexus3.onap.org:10001 -image: onap/dmaap/zookeeper:6.0.0 +image: onap/dmaap/zookeeper:6.0.2 pullPolicy: Always ubuntuInitImage: oomk8s/ubuntu-init:2.0.0 busyBoxImage: busybox:1.30 @@ -39,12 +40,6 @@ busyBoxRepository: docker.io # flag to enable debugging - application support required debugEnabled: false -# application configuration -config: - # gerrit branch where the latest code is checked in - gerritBranch: master - # gerrit project where the latest code is checked in - gerritProject: http://gerrit.onap.org/r/dmaap/messagerouter/messageservice.git # default number of instances replicaCount: 3 @@ -59,22 +54,55 @@ tolerations: {} # probe configuration parameters liveness: - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 1 + initialDelaySeconds: 40 + periodSeconds: 20 + timeoutSeconds: 10 # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container enabled: true readiness: - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 1 + initialDelaySeconds: 40 + periodSeconds: 20 + timeoutSeconds: 10 #Zookeeper properties -zk: - initLimit: 5 - syncLimit: 2 +zkConfig: + tickTime: 2000 + syncLimit: 5 + initLimit: 10 + maxClientCnxns: 200 + autoPurgeSnapRetainCount: 3 + autoPurgePurgeInterval: 24 + heapOptions: -Xmx2G -Xms2G + kafkaOpts: -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider + extraArgs: -Djava.security.auth.login.config=/etc/zookeeper/secrets/jaas/zk_server_jaas.conf + clientPort: 2181 + +jmx: + port: 5555 + +prometheus: + jmx: + enabled: false + image: solsson/kafka-prometheus-jmx-exporter@sha256 + imageTag: 6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 + imageRepository: docker.io + port: 5556 + +jaas: + config: + zkAdminUser: kafka + zkAdminPassword: kafka_secret + #zkAdminPasswordExternal= some password + +secrets: + - uid: zk-admin + type: basicAuth + externalSecret: '{{ .Values.jaas.config.zkAdminPasswordExternal}}' + login: '{{ .Values.jaas.config.zkAdminUser }}' + password: '{{ .Values.jaas.config.zkAdminPassword }}' + passwordPolicy: required ## Persist data to a persitent volume persistence: |