[COMMON] fix logic of post upgrade hook script to handle upgrade run without any changes to charts that could cause loss of all data.

Issue-ID: OOM-2442
Signed-off-by: Marat Salakhutdinov <marat.salakhutdinov@bell.ca>
Change-Id: Ifd3bbc0f63e95b1c8797ecd09a891195faafc8c2

2 files changed, 120 insertions, 2 deletions

diff --git a/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh b/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh
index 132ac27ea2..ec09df3a86 100644
--- a/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh
+++ b/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh
@@ -20,7 +20,123 @@ kubectl patch secret $FLAG_EX_ROOT_SEC -p \
 kubectl patch secret $FLAG_EX_SEC -p \
   '{"data":{"password":"'"$tmp_MYSQL_PASSWORD"'"}}'
 
-kubectl delete pod -n $NAMESPACE_ENV {{ include "common.fullname" . }}-0 --now
+MYSQL_USER=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- printenv MYSQL_USER)
+
+MYSQL_PASSWORD=$(echo -n $(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- printenv MYSQL_PASSWORD))
+
+MYSQL_ROOT_PASSWORD=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- printenv MYSQL_ROOT_PASSWORD)
+
+CURRENT_STS_REPLICA=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+
+DEPLOYMENT_REPLICA=$(kubectl get deployment -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }}-upgrade-deployment -o \
+  jsonpath='{.status.replicas}')
+
+if [[ $CURRENT_STS_REPLICA == "0" ]]
+then
+  echo "Seems there was no upgrade of cluster and we will scale up cluster replicas back to $REPLICA_COUNT now"
+  kubectl scale statefulsets {{ include "common.fullname" . }} --replicas=$REPLICA_COUNT
+fi
+
+MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+
+while [[ ! $MY_REPLICA_NUMBER == $REPLICA_COUNT ]]
+do
+    echo "The cluster is not scaled up to $REPLICA_COUNT yet. Please wait ..."
+    MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+      {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+    echo "The current status of the cluster is $MY_REPLICA_NUMBER"
+    sleep 2
+    if [[ $MY_REPLICA_NUMBER == $REPLICA_COUNT ]]
+    then
+        break
+    fi
+done
+
+CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \
+  mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+  -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" | \
+  awk '{print $2}')
+
+CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \
+  mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+  -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \
+  | awk '{print $2}')
+
+while [[ ! $CLUSTER_NO == $((REPLICA_COUNT+DEPLOYMENT_REPLICA)) ]] \
+   || [[ ! $CLUSTER_STATE == "Synced" ]]
+do
+    echo "$CLUSTER_NO and $CLUSTER_STATE"
+    CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \
+      --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+      -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" \
+      | awk '{print $2}')
+    CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \
+      --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+      -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \
+      | awk '{print $2}')
+    sleep 2
+    if [[ $CLUSTER_NO == $((REPLICA_COUNT+DEPLOYMENT_REPLICA)) ]] \
+       && [[ $CLUSTER_STATE == "Synced" ]]
+    then
+        echo "The cluster has $CLUSTER_NO members and  $CLUSTER_STATE state."
+        break
+    fi
+done
+
+MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV {{ include "common.fullname" . }}-0 -- mysqladmin \
+  -uroot -p$MYSQL_ROOT_PASSWORD ping)
+
+while [[ ! $MYSQL_STATUS == "mysqld is alive" ]]
+do
+  echo "Mariadb deployment is not ready yet."
+  sleep 2
+  MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV {{ include "common.fullname" . }}-0 -- mysqladmin \
+  -uroot -p$MYSQL_ROOT_PASSWORD ping)
+  if [[ $MYSQL_STATUS == "mysqld is alive" ]]
+  then
+    echo "Mariadb deployment is ready and cluster size is $CLUSTER_NO"
+    break
+  fi
+done
+
+echo "Deleting upgrade deployment now"
+
 kubectl delete deployment -n $NAMESPACE_ENV {{ include "common.fullname" . }}-upgrade-deployment
 kubectl delete secret -n $NAMESPACE_ENV {{ include "common.fullname" . }}-temp-upgrade-root
-kubectl delete secret -n $NAMESPACE_ENV {{ include "common.fullname" . }}-temp-upgrade-usercred
\ No newline at end of file
+kubectl delete secret -n $NAMESPACE_ENV {{ include "common.fullname" . }}-temp-upgrade-usercred
+
+CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV {{ include "common.fullname" . }}-0 -- \
+  mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+  -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" | \
+  awk '{print $2}')
+
+CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV {{ include "common.fullname" . }}-0 -- \
+  mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+  -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \
+  | awk '{print $2}')
+
+while [[ ! $CLUSTER_NO == $REPLICA_COUNT ]] \
+   || [[ ! $CLUSTER_STATE == "Synced" ]]
+do
+    echo "$CLUSTER_NO and $CLUSTER_STATE"
+    CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV {{ include "common.fullname" . }}-0 -- mysql \
+      --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+      -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" \
+      | awk '{print $2}')
+    CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV {{ include "common.fullname" . }}-0 -- mysql \
+      --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+      -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \
+      | awk '{print $2}')
+    sleep 2
+    if [[ $CLUSTER_NO == $REPLICA_COUNT ]] \
+       && [[ $CLUSTER_STATE == "Synced" ]]
+    then
+        echo "The cluster has $CLUSTER_NO members and  $CLUSTER_STATE state."
+        break
+    fi
+done
+
+echo "The cluster upgrade is finished now"
diff --git a/kubernetes/common/mariadb-galera/templates/job.yaml b/kubernetes/common/mariadb-galera/templates/job.yaml
index cc71bb855c..db56f3e046 100644
--- a/kubernetes/common/mariadb-galera/templates/job.yaml
+++ b/kubernetes/common/mariadb-galera/templates/job.yaml
@@ -72,6 +72,8 @@ spec:
               fieldRef:
                 apiVersion: v1
                 fieldPath: metadata.namespace
+        - name: REPLICA_COUNT
+          value: "{{ .Values.replicaCount }}"
         command: ["/bin/bash", "-c", "--"]
         args: ["/upgrade/post-upgrade-script.sh"]
         volumeMounts: