[COMMON] add pre upgrade script for mariadb-galera

When upgrading from a version to another, it may be impossible to do it
"simply" because of changes in immutable properties of statefulsets. We
change that here by creating a temporary deployment which will hold the
whole databases during the time the old statefulset gets destroyed and
the new one gets created.

Issue-ID: OOM-2316
Signed-off-by: tringuyen <tri.nguyen@tatacommunications.com>
Signed-off-by: Sylvain Desbureaux <sylvain.desbureaux@orange.com>
Change-Id: I318d72830d5002f50597e23e0753e292f8b47c53

1 files changed, 101 insertions, 0 deletions

diff --git a/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh b/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh
new file mode 100644
index 0000000000..ff44606e23
--- /dev/null
+++ b/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+MYSQL_USER=$(kubectl exec -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }}-0 -- printenv MYSQL_USER)
+
+MYSQL_PASSWORD=$(kubectl exec -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }}-0 -- printenv MYSQL_PASSWORD)
+
+MYSQL_ROOT_PASSWORD=$(kubectl exec -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }}-0 -- printenv MYSQL_ROOT_PASSWORD)
+
+kubectl create secret generic \
+  '{{ include "common.fullname" . }}'-temp-upgrade-root \
+  --from-literal=password=$MYSQL_ROOT_PASSWORD
+
+kubectl create secret generic \
+  '{{ include "common.fullname" . }}'-temp-upgrade-usercred \
+  --from-literal=login=$MYSQL_USER --from-literal=password=$MYSQL_PASSWORD
+
+kubectl create -f /upgrade/create-deployment.yml
+
+TEMP_POD=$(kubectl get pod -n $NAMESPACE_ENV --selector \
+  app='{{ include "common.fullname" . }}' -o \
+  jsonpath='{.items[?(@.metadata.ownerReferences[].kind=="ReplicaSet")].metadata.name}')
+
+CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \
+  mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+  -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" | \
+  awk '{print $2}')
+
+CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \
+  mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+  -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \
+  | awk '{print $2}')
+
+STS_REPLICA=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+
+DEPLOYMENT_REPLICA=$(kubectl get deployment -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }}-upgrade-deployment -o \
+  jsonpath='{.status.replicas}')
+
+while [[ ! $CLUSTER_NO == $((STS_REPLICA+DEPLOYMENT_REPLICA)) ]] \
+   || [[ ! $CLUSTER_STATE == "Synced" ]]
+do
+    echo "$CLUSTER_NO and $CLUSTER_STATE"
+    CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \
+      --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+      -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" \
+      | awk '{print $2}')
+    CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \
+      --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+      -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \
+      | awk '{print $2}')
+    sleep 2
+    if [[ $CLUSTER_NO == $((STS_REPLICA+DEPLOYMENT_REPLICA)) ]] \
+       && [[ $CLUSTER_STATE == "Synced" ]]
+    then
+        echo "The cluster has $CLUSTER_NO members and  $CLUSTER_STATE state."
+        break
+    fi
+done
+
+MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysqladmin \
+  -uroot -p$MYSQL_ROOT_PASSWORD ping)
+
+while [[ ! $MYSQL_STATUS == "mysqld is alive" ]]
+do
+  echo "Mariadb deployment is not ready yet."
+  sleep 2
+  MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysqladmin \
+  -uroot -p$MYSQL_ROOT_PASSWORD ping)
+  if [[ $MYSQL_STATUS == "mysqld is alive" ]]
+  then
+    echo "Mariadb deployment is ready."
+    break
+  fi
+done
+
+kubectl scale statefulsets {{ include "common.fullname" . }} --replicas=0
+MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+  {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+echo "The the cluster has $MY_REPLICA_NUMBER replicas."
+
+while [[ ! $MY_REPLICA_NUMBER == "0" ]]
+do
+    echo "The cluster is not scaled to 0 yet. Please wait ..."
+    MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+      {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+    echo "The current status of the cluster is $MY_REPLICA_NUMBER"
+    sleep 2
+    if [[ $MY_REPLICA_NUMBER == "0" ]]
+    then
+        break
+    fi
+done
+
+for (( index=0; index<$STS_REPLICA; index+=1 ))
+do
+    kubectl delete pvc \
+    "{{ include "common.fullname" . }}-data-{{ include "common.fullname" . }}-$index"
+done