[STRIMZI] Adding strimzi project

Deploy a 2 node replica strimzi kafka cluster
Reduce MR kafka to 1 replica
Add kafka ready check to cps-temporal
Update readthedocs with strimzi prerequisite
Modify deploy.sh to deploy the strimzi kafka in advance

Signed-off-by: efiacor <fiachra.corcoran@est.tech>
Change-Id: I87e54ad69a174174cb86f096c07d58878e3ab14e
Issue-ID: DMAAP-1621

16 files changed, 320 insertions, 4 deletions

diff --git a/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml b/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml
index be0de969c1..c998e9ec67 100644
--- a/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml
+++ b/kubernetes/dmaap/components/message-router/components/message-router-kafka/values.yaml
@@ -60,7 +60,7 @@ pullPolicy: Always
 zookeeper:
   name: message-router-zookeeper
   port: 2181
-  replicaCount: 3
+  replicaCount: 1
 
 kafka:
   heapOptions: -Xmx5G -Xms1G
@@ -129,7 +129,7 @@ secrets:
 debugEnabled: false
 
 # default number of instances
-replicaCount: 3
+replicaCount: 1
 
 
 # To access Kafka outside cluster, this value must be set to hard and the number of nodes in K8S cluster must be equal or greater then replica count
diff --git a/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/values.yaml b/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/values.yaml
index fae361a4c8..79ced4dde2 100644
--- a/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/values.yaml
+++ b/kubernetes/dmaap/components/message-router/components/message-router-zookeeper/values.yaml
@@ -32,9 +32,9 @@ debugEnabled: false
 
 
 # default number of instances
-replicaCount: 3
+replicaCount: 1
 
-zookeeperServers: 3
+zookeeperServers: 1
 
 nodeSelector: {}
 
diff --git a/kubernetes/helm/plugins/deploy/deploy.sh b/kubernetes/helm/plugins/deploy/deploy.sh
index 13b0ed86e2..81bc4135e9 100755
--- a/kubernetes/helm/plugins/deploy/deploy.sh
+++ b/kubernetes/helm/plugins/deploy/deploy.sh
@@ -87,6 +87,20 @@ resolve_deploy_flags() {
   echo "$DEPLOY_FLAGS"
 }
 
+
+check_for_dep() {
+    try=0
+    retries=30
+    until (kubectl get deployment -n $RELEASE | grep -P "\b$2\b") &>/dev/null; do
+        (( ++try > retries )) && exit 1
+        echo "$1 not found. Retry $try/$retries"
+        sleep 5
+    done
+    echo "$1 found. Waiting for pod intialisation"
+    sleep 15
+}
+
+
 deploy() {
   # validate params
   if [ -z "$1" ] || [ -z "$2" ]; then
@@ -219,6 +233,13 @@ deploy() {
   #“helm ls” is an expensive command in that it can take a long time to execute.
   #So cache the results to prevent repeated execution.
   ALL_HELM_RELEASES=$(helm ls -q)
+
+  #Deploy the srtimzi-kafka chart in advance. Dependent charts require the entity-operator
+  #for management of the strimzi crds
+  helm upgrade -i "${RELEASE}-strimzi" $CACHE_SUBCHART_DIR/strimzi
+  echo "waiting for ${RELEASE}-strimzi-entity-operator to be deployed"
+  check_for_dep ${RELEASE}-strimzi-entity-operator
+
   for subchart in * ; do
     SUBCHART_OVERRIDES=$CACHE_SUBCHART_DIR/$subchart/subchart-overrides.yaml
 
diff --git a/kubernetes/onap/Chart.yaml b/kubernetes/onap/Chart.yaml
index 4754f9fd2f..371bc7167e 100644
--- a/kubernetes/onap/Chart.yaml
+++ b/kubernetes/onap/Chart.yaml
@@ -143,6 +143,10 @@ dependencies:
     version: ~10.x-0
     repository: '@local'
     condition: so.enabled
+  - name: strimzi
+    version: ~10.x-0
+    repository: '@local'
+    condition: strimzi.enabled
   - name: uui
     version: ~10.x-0
     repository: '@local'
diff --git a/kubernetes/onap/resources/overrides/onap-all-ingress-nginx-vhost.yaml b/kubernetes/onap/resources/overrides/onap-all-ingress-nginx-vhost.yaml
index 528cd2c687..a56126a093 100644
--- a/kubernetes/onap/resources/overrides/onap-all-ingress-nginx-vhost.yaml
+++ b/kubernetes/onap/resources/overrides/onap-all-ingress-nginx-vhost.yaml
@@ -74,6 +74,8 @@ so:
   so-vnfm-adapter:
     ingress:
       enabled: true
+strimzi:
+  enabled: true
 uui:
   enabled: true
 vfc:
diff --git a/kubernetes/onap/resources/overrides/onap-all.yaml b/kubernetes/onap/resources/overrides/onap-all.yaml
index 9306985d33..6d08d01139 100644
--- a/kubernetes/onap/resources/overrides/onap-all.yaml
+++ b/kubernetes/onap/resources/overrides/onap-all.yaml
@@ -82,6 +82,8 @@ sdnc:
   enabled: true
 so:
   enabled: true
+strimzi:
+  enabled: true
 uui:
   enabled: true
 vfc:
diff --git a/kubernetes/onap/resources/overrides/onap-vfw.yaml b/kubernetes/onap/resources/overrides/onap-vfw.yaml
index 181a1d18e4..f9da136043 100644
--- a/kubernetes/onap/resources/overrides/onap-vfw.yaml
+++ b/kubernetes/onap/resources/overrides/onap-vfw.yaml
@@ -57,5 +57,7 @@ sdnc:
   enabled: true
 so:
   enabled: true
+strimzi:
+  enabled: true
 vid:
   enabled: true
diff --git a/kubernetes/onap/resources/overrides/sm-onap.yaml b/kubernetes/onap/resources/overrides/sm-onap.yaml
index b07b6d3aa2..37080b8290 100644
--- a/kubernetes/onap/resources/overrides/sm-onap.yaml
+++ b/kubernetes/onap/resources/overrides/sm-onap.yaml
@@ -134,6 +134,8 @@ so:
     openStackKeyStoneUrl: "$OPENSTACK_KEYSTONE_URL"
     openStackServiceTenantName: "$OPENSTACK_TENANT_NAME"
     openStackEncryptedPasswordHere: "$OPENSTACK_ENCRYPTED_PASSWORD"
+strimzi:
+  enabled: false
 uui:
   enabled: false
 vid:
diff --git a/kubernetes/onap/values.yaml b/kubernetes/onap/values.yaml
index f5b5c8ed7d..0e8dd21994 100755
--- a/kubernetes/onap/values.yaml
+++ b/kubernetes/onap/values.yaml
@@ -372,6 +372,8 @@ so:
   #   server:
   #     monitoring:
   #       password: demo123456!
+strimzi:
+  enabled: false
 uui:
   enabled: false
 vfc:
diff --git a/kubernetes/strimzi/.helmignore b/kubernetes/strimzi/.helmignore
new file mode 100644
index 0000000000..0f976e9ff3
--- /dev/null
+++ b/kubernetes/strimzi/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+Chart.lock
diff --git a/kubernetes/strimzi/Chart.yaml b/kubernetes/strimzi/Chart.yaml
new file mode 100644
index 0000000000..6ce866bdeb
--- /dev/null
+++ b/kubernetes/strimzi/Chart.yaml
@@ -0,0 +1,33 @@
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v2
+description: ONAP Strimzi kafka
+name: strimzi
+version: 10.0.0
+
+dependencies:
+  - name: common
+    version: ~10.x-0
+    # local reference to common chart, as it is
+    # a part of this chart's package and will not
+    # be published independently to a repo (at this point)
+    repository: '@local'
+  - name: repositoryGenerator
+    version: ~10.x-0
+    repository: '@local'
+  - name: serviceAccount
+    version: ~10.x-0
+    repository: '@local'
+
diff --git a/kubernetes/strimzi/Makefile b/kubernetes/strimzi/Makefile
new file mode 100644
index 0000000000..51d7de122c
--- /dev/null
+++ b/kubernetes/strimzi/Makefile
@@ -0,0 +1,51 @@
+# Copyright © 2020 Samsung Electronics
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+OUTPUT_DIR := $(ROOT_DIR)/../dist
+PACKAGE_DIR := $(OUTPUT_DIR)/packages
+SECRET_DIR := $(OUTPUT_DIR)/secrets
+
+EXCLUDES := dist resources templates charts docker
+HELM_BIN := helm
+HELM_CHARTS := $(filter-out $(EXCLUDES), $(sort $(patsubst %/.,%,$(wildcard */.))))
+
+.PHONY: $(EXCLUDES) $(HELM_CHARTS)
+
+all: $(HELM_CHARTS)
+
+$(HELM_CHARTS):
+	@echo "\n[$@]"
+	@make package-$@
+
+make-%:
+	@if [ -f $*/Makefile ]; then make -C $*; fi
+
+dep-%: make-%
+	@if [ -f $*/Chart.yaml ]; then $(HELM_BIN) dep up $*; fi
+
+lint-%: dep-%
+	@if [ -f $*/Chart.yaml ]; then $(HELM_BIN) lint $*; fi
+
+package-%: lint-%
+	@mkdir -p $(PACKAGE_DIR)
+	@if [ -f $*/Chart.yaml ]; then $(HELM_BIN) package -d $(PACKAGE_DIR) $*; fi
+	@$(HELM_BIN) repo index $(PACKAGE_DIR)
+
+clean:
+	@rm -f */Chart.lock
+	@rm -f *tgz */charts/*tgz
+	@rm -rf $(PACKAGE_DIR)
+%:
+	@:
diff --git a/kubernetes/strimzi/templates/pv-kafka.yaml b/kubernetes/strimzi/templates/pv-kafka.yaml
new file mode 100644
index 0000000000..616f03e788
--- /dev/null
+++ b/kubernetes/strimzi/templates/pv-kafka.yaml
@@ -0,0 +1,16 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{ include "common.replicaPV" (dict "dot" . "suffix" "kafka" "persistenceInfos" .Values.persistenceKafka) }}
diff --git a/kubernetes/strimzi/templates/pv-zk.yaml b/kubernetes/strimzi/templates/pv-zk.yaml
new file mode 100644
index 0000000000..60f4ca6e79
--- /dev/null
+++ b/kubernetes/strimzi/templates/pv-zk.yaml
@@ -0,0 +1,17 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+
+{{ include "common.replicaPV" (dict "dot" . "suffix" "zk" "persistenceInfos" .Values.persistenceZk) }}
diff --git a/kubernetes/strimzi/templates/strimzi-kafka.yaml b/kubernetes/strimzi/templates/strimzi-kafka.yaml
new file mode 100644
index 0000000000..4ca53a2913
--- /dev/null
+++ b/kubernetes/strimzi/templates/strimzi-kafka.yaml
@@ -0,0 +1,90 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: {{ include "common.fullname" . }}
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ include "common.release" . }}
+    heritage: {{ .Release.Service }}
+spec:
+  kafka:
+    version: {{ .Values.version }}
+    replicas: {{ .Values.replicaCount }}
+    listeners:
+      - name: plain
+        port: {{ .Values.kafkaInternalPort }}
+        type: internal
+        tls: false
+        authentication:
+          type: {{ .Values.saslMechanism }}
+      - name: tls
+        port: 9093
+        type: internal
+        tls: true
+        authentication:
+          type: tls
+      - name: external
+        port: 9094
+        type: nodeport
+        tls: true
+        authentication:
+          type: tls
+    authorization:
+      type: simple
+      superUsers:
+        - {{ include "common.release" . }}-{{ .Values.kafkaStrimziAdminUser }}
+    template:
+      pod:
+        securityContext:
+          runAsUser: 0
+          fsGroup: 0
+    config:
+      offsets.topic.replication.factor: {{ .Values.replicaCount }}
+      transaction.state.log.replication.factor: {{ .Values.replicaCount }}
+      transaction.state.log.min.isr: 2
+      log.message.format.version: "3.0"
+      inter.broker.protocol.version: "3.0"
+    storage:
+      type: jbod
+      class: {{ include "common.storageClass" (dict "dot" . "suffix" "kafka" "persistenceInfos" .Values.persistenceKafka) }}
+      volumes:
+      - id: 0
+        type: persistent-claim
+        size: {{ .Values.persistenceKafka.size }}
+        deleteClaim: false
+        class: {{ include "common.storageClass" (dict "dot" . "suffix" "kafka" "persistenceInfos" .Values.persistenceKafka) }}
+  zookeeper:
+    template:
+      pod:
+        securityContext:
+          runAsUser: 0
+          fsGroup: 0
+    replicas: {{ .Values.replicaCount }}
+    config:
+      ssl.hostnameVerification: false
+    storage:
+      type: persistent-claim
+      size: {{ .Values.persistenceZk.size }}
+      deleteClaim: false
+      class: {{ include "common.storageClass" (dict "dot" . "suffix" "zk" "persistenceInfos" .Values.persistenceZk) }}
+  entityOperator:
+    topicOperator: {}
+    userOperator: {}
+
diff --git a/kubernetes/strimzi/values.yaml b/kubernetes/strimzi/values.yaml
new file mode 100644
index 0000000000..e8db803f94
--- /dev/null
+++ b/kubernetes/strimzi/values.yaml
@@ -0,0 +1,52 @@
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#################################################################
+# Global configuration defaults.
+#################################################################
+global:
+  nodePortPrefix: 302
+  persistence:
+    mountPath: /dockerdata-nfs
+
+#################################################################
+# Application configuration defaults.
+#################################################################
+replicaCount: 2
+kafkaInternalPort: 9092
+saslMechanism: scram-sha-512
+version: 3.0.0
+kafkaStrimziAdminUser: strimzi-kafka-admin
+persistence: {}
+
+persistenceKafka:
+  enabled: true
+  size: 2Gi
+  volumeReclaimPolicy: Retain
+  accessMode: ReadWriteOnce
+  mountPath: /dockerdata-nfs
+  mountSubPath: strimzi-kafka/kafka
+persistenceZk:
+  enabled: true
+  size: 2Gi
+  volumeReclaimPolicy: Retain
+  accessMode: ReadWriteOnce
+  mountPath: /dockerdata-nfs
+  mountSubPath: strimzi-kafka/zk
+
+#Pods Service Account
+serviceAccount:
+  nameOverride: strimzi-kafka
+  roles:
+    - read