DMAAP changes

add Mirror Maker
MR AAF integration
Fix BC images and minor BC bugs
Kafka upgrade to 1.1.1

Issue-ID: DMAAP-823
Change-Id: Ia7a1321a6fbfa9bfc65f90bca40ef236c2dbadef
Signed-off-by: sunil.unnava <sunil.unnava@att.com>

9 files changed, 354 insertions, 36 deletions

diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-kafka/templates/service-hs.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-kafka/templates/service-hs.yaml
new file mode 100644
index 0000000000..0dabe55e10
--- /dev/null
+++ b/kubernetes/dmaap/components/message-router/charts/message-router-kafka/templates/service-hs.yaml
@@ -0,0 +1,34 @@
+# Copyright © 2017 Amdocs, Bell Canada
+# Modifications Copyright © 2018 AT&T
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.service.name }}
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  ports:
+    - port: {{ .Values.service.internalPort }}
+      name: {{ .Values.service.portName }}
+  clusterIP: None
+  selector:
+    app: {{ include "common.name" . }}
+    release: {{ .Release.Name }}
+
diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-kafka/templates/service.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-kafka/templates/service.yaml
index 88de96b2d5..8200eda6b1 100644
--- a/kubernetes/dmaap/components/message-router/charts/message-router-kafka/templates/service.yaml
+++ b/kubernetes/dmaap/components/message-router/charts/message-router-kafka/templates/service.yaml
@@ -1,5 +1,4 @@
-# Copyright © 2017 Amdocs, Bell Canada
-# Modifications Copyright © 2018 AT&T
+# Copyright © 2019 AT&T
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,24 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+{{- $root := . -}}
+{{ range $i, $e := until (atoi (quote $root.Values.replicaCount) | default 3) }}
+---
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ .Values.service.name }}
-  namespace: {{ include "common.namespace" . }}
+  name: {{ $root.Values.service.name }}-{{ $i }}
+  namespace: {{ $root.Release.Namespace }}
   labels:
-    app: {{ include "common.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
+    app: {{ $root.Values.service.name }}
+    chart: {{ $root.Chart.Name }}-{{ $root.Chart.Version | replace "+" "_" }}
+    release: {{ $root.Release.Name }}
+    heritage: {{ $root.Release.Service }}
+
 spec:
-  type: {{ .Values.service.type }}
-  ports:
-    - port: {{ .Values.service.externalPort }}
-      targetPort: {{ .Values.service.internalPort }}
-      name: {{ .Values.service.portName }}
-  clusterIP: None
+  type: {{ $root.Values.service.type }} 
+  externalTrafficPolicy: Local
   selector:
-    app: {{ include "common.name" . }}
-    release: {{ .Release.Name }}
-
+   statefulset.kubernetes.io/pod-name: {{ $root.Release.Name }}-{{ $root.Values.service.name }}-{{ $i }}
+  ports:
+    - port: {{ $root.Values.service.externalPort }}
+      targetPort: {{ $root.Values.service.externalPort }}
+      nodePort: {{ $root.Values.service.baseNodePort  | add $i }}
+      name: {{ $root.Values.service.name }}-{{ $i }}
+{{ end }}
diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-kafka/templates/statefulset.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-kafka/templates/statefulset.yaml
index f4583a16ea..cc8fd5a93c 100644
--- a/kubernetes/dmaap/components/message-router/charts/message-router-kafka/templates/statefulset.yaml
+++ b/kubernetes/dmaap/components/message-router/charts/message-router-kafka/templates/statefulset.yaml
@@ -34,7 +34,11 @@ spec:
         release: {{ .Release.Name }}
     spec:
       podAntiAffinity:
+         {{if eq .Values.podAntiAffinityType "hard" -}}
+         requiredDuringSchedulingIgnoredDuringExecution:
+         {{- else -}}
          preferredDuringSchedulingIgnoredDuringExecution:
+         {{- end}}
          - weight: 1
            podAffinityTerm:
              labelSelector:
@@ -70,10 +74,19 @@ spec:
       - name: {{ include "common.name" .  }}
         image: "{{ include "common.repository" . }}/{{ .Values.image }}"
         imagePullPolicy:  {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+        command:
+        - sh
+        - -exc
+        - |
+          export KAFKA_BROKER_ID=${HOSTNAME##*-} && \
+          export ENDPOINT_PORT=$(( $KAFKA_BROKER_ID + {{ .Values.service.baseNodePort }} )) && \
+          export KAFKA_ADVERTISED_LISTENERS=EXTERNAL_SASL_PLAINTEXT://$(HOST_IP):$(( $KAFKA_BROKER_ID + {{ .Values.service.baseNodePort }} )),INTERNAL_SASL_PLAINTEXT://:{{ .Values.service.internalPort }} && \
+          exec start-kafka.sh
         resources:
 {{ include "common.resources" . | indent 12 }}
         ports:
         - containerPort: {{ .Values.service.internalPort }}
+        - containerPort: {{ .Values.service.externalPort }}
        {{ if eq .Values.liveness.enabled true }}
         livenessProbe:
           tcpSocket:
@@ -87,30 +100,31 @@ spec:
           initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
           periodSeconds: {{ .Values.readiness.periodSeconds }}
         env:
-        - name: HOST_NAME
+        - name: HOST_IP
           valueFrom:
             fieldRef:
               apiVersion: v1
-              fieldPath: metadata.name
-        - name: HOST_NAMESPACE
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: metadata.namespace
+              fieldPath: status.hostIP
         - name: KAFKA_ZOOKEEPER_CONNECT
           value: "{{.Values.zookeeper.name}}:{{.Values.zookeeper.port}}"
         - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
-          value: "INTERNAL_PLAINTEXT:PLAINTEXT,EXTERNAL_PLAINTEXT:PLAINTEXT"
-        - name: KAFKA_ADVERTISED_LISTENERS
-          value: "INTERNAL_PLAINTEXT://$(HOST_NAME).{{ .Values.service.name }}.$(HOST_NAMESPACE).svc.cluster.local:{{ .Values.service.internalPort}}"
+          value: "INTERNAL_SASL_PLAINTEXT:SASL_PLAINTEXT,EXTERNAL_SASL_PLAINTEXT:SASL_PLAINTEXT"
         - name: KAFKA_LISTENERS
-          value: "INTERNAL_PLAINTEXT://0.0.0.0:{{.Values.service.internalPort}}"
+          value: "EXTERNAL_SASL_PLAINTEXT://0.0.0.0:{{ .Values.service.externalPort }},INTERNAL_SASL_PLAINTEXT://0.0.0.0:{{ .Values.service.internalPort }}"
         - name: KAFKA_INTER_BROKER_LISTENER_NAME
-          value: "INTERNAL_PLAINTEXT"
+          value: "INTERNAL_SASL_PLAINTEXT"
+        - name: KAFKA_SASL_ENABLED_MECHANISMS
+          value: "PLAIN"
+        - name: KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL
+          value: "PLAIN"
+        - name: KAFKA_AUTHORIZER_CLASS_NAME
+          value: "{{ .Values.kafkaCustomAuthorizer }}"
+        - name: KAFKA_DELETE_TOPIC_ENABLE
+          value: "{{ .Values.deleteTopicEnable }}"
+        - name: aaf_locate_url
+          value: "https://aaf-locate:8095"
         - name: KAFKA_LOG_DIRS
           value: "kafka/logs"
-        - name: BROKER_ID_COMMAND
-          value: "hostname | awk -F '-' '{print $NF}'"
         - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
           value: "{{ .Values.replicaCount }}"
         - name: KAFKA_DEFAULT_REPLICATION_FACTOR
diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-kafka/values.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-kafka/values.yaml
index e719cb1ad4..5b83bb0a55 100644
--- a/kubernetes/dmaap/components/message-router/charts/message-router-kafka/values.yaml
+++ b/kubernetes/dmaap/components/message-router/charts/message-router-kafka/values.yaml
@@ -30,7 +30,7 @@ global:
 #################################################################
 # application image
 repository: nexus3.onap.org:10001
-image: onap/dmaap/kafka01101:0.0.1
+image: onap/dmaap/kafka111:0.0.5
 pullPolicy: Always
 ubuntuInitImage: oomk8s/ubuntu-init:2.0.0
 
@@ -44,6 +44,13 @@ debugEnabled: false
 # default number of instances
 replicaCount: 3
 
+#Kafka custom authorizer class name
+kafkaCustomAuthorizer: org.onap.dmaap.kafkaAuthorize.KafkaCustomAuthorizer
+deleteTopicEnable: true
+
+# To access Kafka outside cluster, this value must be set to hard and the number of nodes in K8S cluster must be equal or greater then replica count
+podAntiAffinityType: soft
+
 # defult partitions
 defaultpartitions: 3
 
@@ -55,6 +62,8 @@ affinity: {}
 
 tolerations: {}
 
+
+
 # probe configuration parameters
 liveness:
   initialDelaySeconds: 60
@@ -91,11 +100,14 @@ persistence:
   mountSubPath: message-router/data-kafka
 
 service:
-  type: ClusterIP
+  type: NodePort
   name: message-router-kafka
   portName: message-router-kafka
   internalPort: 9092
-  externalPort: 9092
+  externalPort: 9093
+  baseNodePort: 30490
+
+  
 
 ingress:
   enabled: false
diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/Chart.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/Chart.yaml
new file mode 100644
index 0000000000..1d0b7bea22
--- /dev/null
+++ b/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/Chart.yaml
@@ -0,0 +1,20 @@
+# Copyright © 2017 Amdocs, Bell Canada
+# Modifications Copyright © 2018 AT&T
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+description: ONAP Message Router Kafka Service
+name: message-router-mirrormaker
+version: 4.0.0
+
diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/templates/NOTES.txt b/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/templates/NOTES.txt
new file mode 100644
index 0000000000..a44d0f76ee
--- /dev/null
+++ b/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/templates/NOTES.txt
@@ -0,0 +1,34 @@
+# Copyright © 2018  AT&T Intellectual Property.  All rights reserved.
+# Modifications Copyright © 2018 Amdocs, Bell Canada
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range .Values.ingress.hosts }}
+  http://{{ . }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ include "common.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "common.name" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ include "common.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get svc -w {{ include "common.name" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ include "common.namespace" . }} {{ include "common.name" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.service.externalPort }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ include "common.namespace" . }} -l "app={{ include "common.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl port-forward $POD_NAME 8080:{{ .Values.service.internalPort }}
+{{- end }}
diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/templates/deployment.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/templates/deployment.yaml
new file mode 100644
index 0000000000..51c4149094
--- /dev/null
+++ b/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/templates/deployment.yaml
@@ -0,0 +1,103 @@
+# Copyright © 2017 Amdocs, Bell Canada
+# Modifications Copyright © 2018 AT&T
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: {{ include "common.fullname" . }}
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  serviceName: {{ .Values.service.name }}
+  replicas: {{ .Values.replicaCount }}
+  template:
+    metadata:
+      labels:
+        app:  {{ include "common.name" . }}
+        release: {{ .Release.Name }}
+    spec:
+      initContainers:
+      - name: {{ include "common.name" . }}-initcontainer
+        image: "{{ .Values.global.ubuntuInitRepository }}/{{ .Values.ubuntuInitImage }}"
+        imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+      - command:
+        - /root/ready.py
+        args:
+        - --container-name
+        - {{ .Values.messagerouter.container }}
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+        imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+        name: {{ include "common.name" . }}-readiness
+      containers:
+      - name: {{ include "common.name" .  }}
+        image: "{{ include "common.repository" . }}/{{ .Values.image }}"
+        imagePullPolicy:  {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+        command:
+        - sh
+        - -exc
+        - |
+          exec start-mirrormaker.sh
+        resources:
+{{ include "common.resources" . | indent 12 }}
+        ports:
+        - containerPort: {{ .Values.service.internalPort }}
+        {{ if eq .Values.liveness.enabled true }}
+        livenessProbe:
+          exec:
+            command:
+            - sh
+            - -c
+            - "touch /tmp/lprobe.txt"
+            - "rm /tmp/lprobe.txt"
+          initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }}
+          periodSeconds: {{ .Values.liveness.periodSeconds }}
+        {{ end }}
+        readinessProbe:
+          exec:
+            command:
+            - sh
+            - -c
+            - "touch /tmp/rprobe.txt"
+            - "rm /tmp/rprobe.txt"
+          initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
+          periodSeconds: {{ .Values.readiness.periodSeconds }}
+        env:
+        - name: KAFKA_HEAP_OPTS
+          value: "{{ .Values.kafkaHeapOptions }}"
+        volumeMounts:
+        - mountPath: /etc/localtime
+          name: localtime
+          readOnly: true
+        - mountPath: /var/run/docker.sock
+          name: docker-socket
+      volumes:
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      - name: docker-socket
+        hostPath:
+          path: /var/run/docker.sock
+      imagePullSecrets:
+      - name: "{{ include "common.namespace" . }}-docker-registry-key"
diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/values.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/values.yaml
new file mode 100644
index 0000000000..2fbd884ab8
--- /dev/null
+++ b/kubernetes/dmaap/components/message-router/charts/message-router-mirrormaker/values.yaml
@@ -0,0 +1,98 @@
+# Copyright © 2017 Amdocs, Bell Canada
+# Modifications Copyright © 2018 AT&T
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#################################################################
+# Global configuration defaults.
+#################################################################
+global:
+  nodePortPrefix: 302
+  readinessRepository: oomk8s
+  readinessImage: readiness-check:2.0.0
+  loggingRepository: docker.elastic.co
+  loggingImage: beats/filebeat:5.5.0
+  persistence: {}
+  ubuntuInitRepository: registry.hub.docker.com
+
+#################################################################
+# Application configuration defaults.
+#################################################################
+# application image
+repository: nexus3.onap.org:10001
+image: onap/dmaap/kafka111:0.0.5
+pullPolicy: Always
+ubuntuInitImage: oomk8s/ubuntu-init:2.0.0
+
+zookeeper:
+  name: message-router-zookeeper
+  port: 2181
+
+messagerouter:
+  container: message-router 
+
+# flag to enable debugging - application support required
+debugEnabled: false
+
+# default number of instances
+replicaCount: 1
+kafkaHeapOptions: -Xmx4G -Xms2G
+
+nodeSelector: {}
+
+affinity: {}
+
+# probe configuration parameters
+liveness:
+  initialDelaySeconds: 60
+  periodSeconds: 20
+  # necessary to disable liveness probe when setting breakpoints
+  # in debugger so K8s doesn't restart unresponsive container
+  enabled: true
+
+readiness:
+  initialDelaySeconds: 60
+  periodSeconds: 20
+
+
+service:
+  type: NodePort
+  name: message-router-mirrormaker
+  portName: message-router-mirrormaker
+  internalPort: 9092
+  
+ 
+
+ingress:
+  enabled: false
+
+# Resource Limit flavor -By Default using small
+flavor: large
+# Segregation for Different environment (Small and Large)
+resources:
+  small:
+    limits:
+      cpu: 2000m
+      memory: 4Gi
+    requests:
+      cpu: 500m
+      memory: 1Gi
+  large:
+    limits:
+      cpu: 4000m
+      memory: 8Gi
+    requests:
+      cpu: 1000m
+      memory: 2Gi
+  unlimited: {}
+
diff --git a/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/values.yaml b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/values.yaml
index c0a7450d5e..37a1babe17 100644
--- a/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/values.yaml
+++ b/kubernetes/dmaap/components/message-router/charts/message-router-zookeeper/values.yaml
@@ -30,7 +30,7 @@ global:
 #################################################################
 # application image
 repository: nexus3.onap.org:10001
-image: onap/dmaap/zookeeper:2.0.0
+image: onap/dmaap/zookeeper:3.0.0
 pullPolicy: Always
 ubuntuInitImage: oomk8s/ubuntu-init:2.0.0
 
@@ -130,4 +130,4 @@ resources:
     requests:
       cpu: 1000m
       memory: 2Gi
-  unlimited: {}
\ No newline at end of file
+  unlimited: {}