[COMMON] add pre upgrade script for mariadb-galera

When upgrading from a version to another, it may be impossible to do it
"simply" because of changes in immutable properties of statefulsets. We
change that here by creating a temporary deployment which will hold the
whole databases during the time the old statefulset gets destroyed and
the new one gets created.

Issue-ID: OOM-2316
Signed-off-by: tringuyen <tri.nguyen@tatacommunications.com>
Signed-off-by: Sylvain Desbureaux <sylvain.desbureaux@orange.com>
Change-Id: I318d72830d5002f50597e23e0753e292f8b47c53

6 files changed, 323 insertions, 3 deletions

diff --git a/kubernetes/common/mariadb-galera/resources/create-deployment.yml b/kubernetes/common/mariadb-galera/resources/create-deployment.yml
new file mode 100644
index 0000000000..61bfc78945
--- /dev/null
+++ b/kubernetes/common/mariadb-galera/resources/create-deployment.yml
@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata: {{- include "common.resourceMetadata" (dict "suffix" "upgrade-deployment" "dot" .) | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app:  {{ include "common.fullname" . }}
+  template:
+    metadata:
+      labels:
+        app:  {{ include "common.fullname" . }}
+    spec:
+      containers:
+      - name: {{ include "common.name" . }}
+        image: "{{ include "common.repository" . }}/{{ .Values.image }}"
+        ports:
+        - containerPort: {{ .Values.service.internalPort }}
+          name: {{ .Values.service.portName }}
+        - containerPort: {{ .Values.service.sstPort }}
+          name: {{ .Values.service.sstPortName }}
+        - containerPort: {{ .Values.service.replicationPort }}
+          name: {{ .Values.service.replicationName }}
+        - containerPort: {{ .Values.service.istPort }}
+          name: {{ .Values.service.istPortName }}
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+        - name: MYSQL_USER
+          valueFrom:
+            secretKeyRef:
+              key: login
+              name: {{ include "common.fullname" . }}-temp-upgrade-usercred
+        - name: MYSQL_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: password
+              name: {{ include "common.fullname" . }}-temp-upgrade-usercred
+        - name: MYSQL_DATABASE
+          value: {{ default "" .Values.config.mysqlDatabase | quote }}
+        - name: MYSQL_ROOT_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: password
+              name: {{ include "common.fullname" . }}-temp-upgrade-root
+      subdomain: {{ .Values.service.name }}
+      hostname: {{ .Values.nameOverride }}-upgrade-deployment
\ No newline at end of file
diff --git a/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh b/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh
new file mode 100644
index 0000000000..132ac27ea2
--- /dev/null
+++ b/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+TEMP_POD=$(kubectl get pod -n $NAMESPACE_ENV --selector \
+  app='{{ include "common.fullname" . }}' -o \
+  jsonpath='{.items[?(@.metadata.ownerReferences[].kind=="ReplicaSet")].metadata.name}')
+
+tmp_MYSQL_PASSWORD=$(echo -n $(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- printenv \
+  MYSQL_PASSWORD) | base64)
+
+tmp_ROOT_PASSWORD=$(echo -n $(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- printenv \
+  MYSQL_ROOT_PASSWORD) | base64)
+
+FLAG_EX_ROOT_SEC='{{ include "common.secret.getSecretNameFast" (dict "global" . "uid" (include "common.mariadb.secret.rootPassUID" .)) }}'
+
+FLAG_EX_SEC='{{ include "common.secret.getSecretNameFast" (dict "global" . "uid" (include "common.mariadb.secret.userCredentialsUID" .)) }}'
+
+kubectl patch secret $FLAG_EX_ROOT_SEC -p \
+  '{"data":{"password":"'"$tmp_ROOT_PASSWORD"'"}}'
+
+kubectl patch secret $FLAG_EX_SEC -p \
+  '{"data":{"password":"'"$tmp_MYSQL_PASSWORD"'"}}'
+
+kubectl delete pod -n $NAMESPACE_ENV {{ include "common.fullname" . }}-0 --now
+kubectl delete deployment -n $NAMESPACE_ENV {{ include "common.fullname" . }}-upgrade-deployment
+kubectl delete secret -n $NAMESPACE_ENV {{ include "common.fullname" . }}-temp-upgrade-root
+kubectl delete secret -n $NAMESPACE_ENV {{ include "common.fullname" . }}-temp-upgrade-usercred
\ No newline at end of file
diff --git a/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh b/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh
new file mode 100644
index 0000000000..ff44606e23
--- /dev/null
+++ b/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+MYSQL_USER=$(kubectl exec -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }}-0 -- printenv MYSQL_USER)
+
+MYSQL_PASSWORD=$(kubectl exec -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }}-0 -- printenv MYSQL_PASSWORD)
+
+MYSQL_ROOT_PASSWORD=$(kubectl exec -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }}-0 -- printenv MYSQL_ROOT_PASSWORD)
+
+kubectl create secret generic \
+  '{{ include "common.fullname" . }}'-temp-upgrade-root \
+  --from-literal=password=$MYSQL_ROOT_PASSWORD
+
+kubectl create secret generic \
+  '{{ include "common.fullname" . }}'-temp-upgrade-usercred \
+  --from-literal=login=$MYSQL_USER --from-literal=password=$MYSQL_PASSWORD
+
+kubectl create -f /upgrade/create-deployment.yml
+
+TEMP_POD=$(kubectl get pod -n $NAMESPACE_ENV --selector \
+  app='{{ include "common.fullname" . }}' -o \
+  jsonpath='{.items[?(@.metadata.ownerReferences[].kind=="ReplicaSet")].metadata.name}')
+
+CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \
+  mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+  -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" | \
+  awk '{print $2}')
+
+CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \
+  mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+  -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \
+  | awk '{print $2}')
+
+STS_REPLICA=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+
+DEPLOYMENT_REPLICA=$(kubectl get deployment -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }}-upgrade-deployment -o \
+  jsonpath='{.status.replicas}')
+
+while [[ ! $CLUSTER_NO == $((STS_REPLICA+DEPLOYMENT_REPLICA)) ]] \
+   || [[ ! $CLUSTER_STATE == "Synced" ]]
+do
+    echo "$CLUSTER_NO and $CLUSTER_STATE"
+    CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \
+      --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+      -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" \
+      | awk '{print $2}')
+    CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \
+      --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+      -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \
+      | awk '{print $2}')
+    sleep 2
+    if [[ $CLUSTER_NO == $((STS_REPLICA+DEPLOYMENT_REPLICA)) ]] \
+       && [[ $CLUSTER_STATE == "Synced" ]]
+    then
+        echo "The cluster has $CLUSTER_NO members and  $CLUSTER_STATE state."
+        break
+    fi
+done
+
+MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysqladmin \
+  -uroot -p$MYSQL_ROOT_PASSWORD ping)
+
+while [[ ! $MYSQL_STATUS == "mysqld is alive" ]]
+do
+  echo "Mariadb deployment is not ready yet."
+  sleep 2
+  MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysqladmin \
+  -uroot -p$MYSQL_ROOT_PASSWORD ping)
+  if [[ $MYSQL_STATUS == "mysqld is alive" ]]
+  then
+    echo "Mariadb deployment is ready."
+    break
+  fi
+done
+
+kubectl scale statefulsets {{ include "common.fullname" . }} --replicas=0
+MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+echo "The the cluster has $MY_REPLICA_NUMBER replicas."
+
+while [[ ! $MY_REPLICA_NUMBER == "0" ]]
+do
+    echo "The cluster is not scaled to 0 yet. Please wait ..."
+    MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+      {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+    echo "The current status of the cluster is $MY_REPLICA_NUMBER"
+    sleep 2
+    if [[ $MY_REPLICA_NUMBER == "0" ]]
+    then
+        break
+    fi
+done
+
+for (( index=0; index<$STS_REPLICA; index+=1 ))
+do
+    kubectl delete pvc \
+    "{{ include "common.fullname" . }}-data-{{ include "common.fullname" . }}-$index"
+done
diff --git a/kubernetes/common/mariadb-galera/templates/configmap.yaml b/kubernetes/common/mariadb-galera/templates/configmap.yaml
index a7064d7ce4..685901fa95 100644
--- a/kubernetes/common/mariadb-galera/templates/configmap.yaml
+++ b/kubernetes/common/mariadb-galera/templates/configmap.yaml
@@ -1,6 +1,6 @@
 {{/*
 # Copyright © 2018 Amdocs, Bell Canada
-# Copyright © 2020 Samsung Electronics
+# Copyright © 2020 Samsung Electronics, and TATA Communications
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 */}}
-
 {{- if .Values.externalConfig }}
 apiVersion: v1
 kind: ConfigMap
@@ -43,3 +42,37 @@ metadata:
     heritage: {{ .Release.Service }}
 data:
 {{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "common.fullname" . }}-upgrade-deployment
+  annotations:
+    "helm.sh/hook": "pre-upgrade"
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": hook-succeeded
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+{{ tpl (.Files.Glob "resources/*").AsConfig . | indent 2 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "common.fullname" . }}-post-upgrade-deployment
+  annotations:
+    "helm.sh/hook": "post-upgrade"
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": hook-succeeded
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+{{ tpl (.Files.Glob "resources/post-upgrade-script.sh").AsConfig . | indent 2 }}
diff --git a/kubernetes/common/mariadb-galera/templates/job.yaml b/kubernetes/common/mariadb-galera/templates/job.yaml
new file mode 100644
index 0000000000..cc71bb855c
--- /dev/null
+++ b/kubernetes/common/mariadb-galera/templates/job.yaml
@@ -0,0 +1,107 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "common.fullname" . }}-pre-upgrade
+  annotations:
+    "helm.sh/hook": "pre-upgrade"
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    spec:
+      securityContext:
+        fsGroup: 1001
+        runAsUser: 1001
+      containers:
+      - name: mariadb-job-pre-upgrade
+        image: {{ .Values.global.kubectlImage}}
+        imagePullPolicy: IfNotPresent
+        env:
+          - name: NAMESPACE_ENV
+            valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+        command: ["/bin/bash", "-c", "--"]
+        args: ["/upgrade/upgrade-scripts.sh"]
+        volumeMounts:
+        - name: config-mariadb-upgrade
+          mountPath: /upgrade
+      volumes:
+      - name: config-mariadb-upgrade
+        configMap:
+            name: {{ include "common.fullname" . }}-upgrade-deployment
+            defaultMode: 0777
+      restartPolicy: OnFailure
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "common.fullname" . }}-post-upgrade
+  annotations:
+    "helm.sh/hook": "post-upgrade"
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    spec:
+      securityContext:
+        fsGroup: 1001
+        runAsUser: 0
+      initContainers:
+      - image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+        name: mariadb-galera-upgrade-readiness
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        command:
+        - /root/ready.py
+        args:
+        - --container-name
+        - mariadb-galera
+      containers:
+      - name: mariadb-job-post-upgrade
+        image: {{ .Values.global.kubectlImage}}
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: NAMESPACE_ENV
+          valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.namespace
+        command: ["/bin/bash", "-c", "--"]
+        args: ["/upgrade/post-upgrade-script.sh"]
+        volumeMounts:
+        - name: config-mariadb-upgrade
+          mountPath: /upgrade
+      volumes:
+      - name: config-mariadb-upgrade
+        configMap:
+            name: {{ include "common.fullname" . }}-post-upgrade-deployment
+            defaultMode: 0777
+      restartPolicy: OnFailure
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "common.fullname" . }}-post-delete
+  annotations:
+    "helm.sh/hook": "post-delete"
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    spec:
+      containers:
+      - name: mariadb-job-post-delete
+        image: {{ .Values.global.kubectlImage}}
+        imagePullPolicy: IfNotPresent
+        command: ["/bin/bash", "-c", "--"]
+        args:
+        - for ((index=0;index<{{ $.Values.replicaCount }};index+=1));
+          do kubectl delete pvc "{{ include "common.fullname" . }}-data-{{ include "common.fullname" . }}-$index";
+          done; kubectl delete deployment {{ include "common.fullname" . }}-upgrade-deployment;
+      restartPolicy: OnFailure
diff --git a/kubernetes/common/mariadb-galera/values.yaml b/kubernetes/common/mariadb-galera/values.yaml
index af08ea3d58..4ccb0e5c6e 100644
--- a/kubernetes/common/mariadb-galera/values.yaml
+++ b/kubernetes/common/mariadb-galera/values.yaml
@@ -42,7 +42,10 @@ global:
 
   readinessRepository: oomk8s
   readinessImage: readiness-check:2.0.2
-
+  busyboxImage: busybox:1.30
+  busyboxRepository: docker.io
+  # kubeclt image
+  kubectlImage: "bitnami/kubectl:1.15"
 
 #################################################################
 # Application configuration defaults.