summaryrefslogtreecommitdiffstats
path: root/kubernetes/common/mariadb-galera/resources
diff options
context:
space:
mode:
authortringuyen <tri.nguyen@tatacommunications.com>2020-04-01 17:51:06 +0000
committerSylvain Desbureaux <sylvain.desbureaux@orange.com>2020-06-10 14:10:02 +0000
commitf5993cdff9c9d873966d24e7987094fee4d13f0d (patch)
tree09beb1733decc427e141b10c4a70a986b442fe76 /kubernetes/common/mariadb-galera/resources
parent1903b2c5c802e16d30207f6ec4ba58d6bc034280 (diff)
[COMMON] add pre upgrade script for mariadb-galera
When upgrading from a version to another, it may be impossible to do it "simply" because of changes in immutable properties of statefulsets. We change that here by creating a temporary deployment which will hold the whole databases during the time the old statefulset gets destroyed and the new one gets created. Issue-ID: OOM-2316 Signed-off-by: tringuyen <tri.nguyen@tatacommunications.com> Signed-off-by: Sylvain Desbureaux <sylvain.desbureaux@orange.com> Change-Id: I318d72830d5002f50597e23e0753e292f8b47c53 (cherry picked from commit eb9eb59171a43d25fb012aaad0a1d37ca86bc2bf)
Diffstat (limited to 'kubernetes/common/mariadb-galera/resources')
-rw-r--r--kubernetes/common/mariadb-galera/resources/create-deployment.yml50
-rw-r--r--kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh26
-rw-r--r--kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh101
3 files changed, 177 insertions, 0 deletions
diff --git a/kubernetes/common/mariadb-galera/resources/create-deployment.yml b/kubernetes/common/mariadb-galera/resources/create-deployment.yml
new file mode 100644
index 0000000000..61bfc78945
--- /dev/null
+++ b/kubernetes/common/mariadb-galera/resources/create-deployment.yml
@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata: {{- include "common.resourceMetadata" (dict "suffix" "upgrade-deployment" "dot" .) | nindent 4 }}
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: {{ include "common.fullname" . }}
+ template:
+ metadata:
+ labels:
+ app: {{ include "common.fullname" . }}
+ spec:
+ containers:
+ - name: {{ include "common.name" . }}
+ image: "{{ include "common.repository" . }}/{{ .Values.image }}"
+ ports:
+ - containerPort: {{ .Values.service.internalPort }}
+ name: {{ .Values.service.portName }}
+ - containerPort: {{ .Values.service.sstPort }}
+ name: {{ .Values.service.sstPortName }}
+ - containerPort: {{ .Values.service.replicationPort }}
+ name: {{ .Values.service.replicationName }}
+ - containerPort: {{ .Values.service.istPort }}
+ name: {{ .Values.service.istPortName }}
+ env:
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ - name: MYSQL_USER
+ valueFrom:
+ secretKeyRef:
+ key: login
+ name: {{ include "common.fullname" . }}-temp-upgrade-usercred
+ - name: MYSQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: password
+ name: {{ include "common.fullname" . }}-temp-upgrade-usercred
+ - name: MYSQL_DATABASE
+ value: {{ default "" .Values.config.mysqlDatabase | quote }}
+ - name: MYSQL_ROOT_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: password
+ name: {{ include "common.fullname" . }}-temp-upgrade-root
+ subdomain: {{ .Values.service.name }}
+ hostname: {{ .Values.nameOverride }}-upgrade-deployment \ No newline at end of file
diff --git a/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh b/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh
new file mode 100644
index 0000000000..132ac27ea2
--- /dev/null
+++ b/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+TEMP_POD=$(kubectl get pod -n $NAMESPACE_ENV --selector \
+ app='{{ include "common.fullname" . }}' -o \
+ jsonpath='{.items[?(@.metadata.ownerReferences[].kind=="ReplicaSet")].metadata.name}')
+
+tmp_MYSQL_PASSWORD=$(echo -n $(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- printenv \
+ MYSQL_PASSWORD) | base64)
+
+tmp_ROOT_PASSWORD=$(echo -n $(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- printenv \
+ MYSQL_ROOT_PASSWORD) | base64)
+
+FLAG_EX_ROOT_SEC='{{ include "common.secret.getSecretNameFast" (dict "global" . "uid" (include "common.mariadb.secret.rootPassUID" .)) }}'
+
+FLAG_EX_SEC='{{ include "common.secret.getSecretNameFast" (dict "global" . "uid" (include "common.mariadb.secret.userCredentialsUID" .)) }}'
+
+kubectl patch secret $FLAG_EX_ROOT_SEC -p \
+ '{"data":{"password":"'"$tmp_ROOT_PASSWORD"'"}}'
+
+kubectl patch secret $FLAG_EX_SEC -p \
+ '{"data":{"password":"'"$tmp_MYSQL_PASSWORD"'"}}'
+
+kubectl delete pod -n $NAMESPACE_ENV {{ include "common.fullname" . }}-0 --now
+kubectl delete deployment -n $NAMESPACE_ENV {{ include "common.fullname" . }}-upgrade-deployment
+kubectl delete secret -n $NAMESPACE_ENV {{ include "common.fullname" . }}-temp-upgrade-root
+kubectl delete secret -n $NAMESPACE_ENV {{ include "common.fullname" . }}-temp-upgrade-usercred \ No newline at end of file
diff --git a/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh b/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh
new file mode 100644
index 0000000000..ff44606e23
--- /dev/null
+++ b/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+MYSQL_USER=$(kubectl exec -n $NAMESPACE_ENV \
+ {{ include "common.fullname" . }}-0 -- printenv MYSQL_USER)
+
+MYSQL_PASSWORD=$(kubectl exec -n $NAMESPACE_ENV \
+ {{ include "common.fullname" . }}-0 -- printenv MYSQL_PASSWORD)
+
+MYSQL_ROOT_PASSWORD=$(kubectl exec -n $NAMESPACE_ENV \
+ {{ include "common.fullname" . }}-0 -- printenv MYSQL_ROOT_PASSWORD)
+
+kubectl create secret generic \
+ '{{ include "common.fullname" . }}'-temp-upgrade-root \
+ --from-literal=password=$MYSQL_ROOT_PASSWORD
+
+kubectl create secret generic \
+ '{{ include "common.fullname" . }}'-temp-upgrade-usercred \
+ --from-literal=login=$MYSQL_USER --from-literal=password=$MYSQL_PASSWORD
+
+kubectl create -f /upgrade/create-deployment.yml
+
+TEMP_POD=$(kubectl get pod -n $NAMESPACE_ENV --selector \
+ app='{{ include "common.fullname" . }}' -o \
+ jsonpath='{.items[?(@.metadata.ownerReferences[].kind=="ReplicaSet")].metadata.name}')
+
+CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \
+ mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+ -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" | \
+ awk '{print $2}')
+
+CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \
+ mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+ -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \
+ | awk '{print $2}')
+
+STS_REPLICA=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+ {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+
+DEPLOYMENT_REPLICA=$(kubectl get deployment -n $NAMESPACE_ENV \
+ {{ include "common.fullname" . }}-upgrade-deployment -o \
+ jsonpath='{.status.replicas}')
+
+while [[ ! $CLUSTER_NO == $((STS_REPLICA+DEPLOYMENT_REPLICA)) ]] \
+ || [[ ! $CLUSTER_STATE == "Synced" ]]
+do
+ echo "$CLUSTER_NO and $CLUSTER_STATE"
+ CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \
+ --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+ -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" \
+ | awk '{print $2}')
+ CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \
+ --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \
+ -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \
+ | awk '{print $2}')
+ sleep 2
+ if [[ $CLUSTER_NO == $((STS_REPLICA+DEPLOYMENT_REPLICA)) ]] \
+ && [[ $CLUSTER_STATE == "Synced" ]]
+ then
+ echo "The cluster has $CLUSTER_NO members and $CLUSTER_STATE state."
+ break
+ fi
+done
+
+MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysqladmin \
+ -uroot -p$MYSQL_ROOT_PASSWORD ping)
+
+while [[ ! $MYSQL_STATUS == "mysqld is alive" ]]
+do
+ echo "Mariadb deployment is not ready yet."
+ sleep 2
+ MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysqladmin \
+ -uroot -p$MYSQL_ROOT_PASSWORD ping)
+ if [[ $MYSQL_STATUS == "mysqld is alive" ]]
+ then
+ echo "Mariadb deployment is ready."
+ break
+ fi
+done
+
+kubectl scale statefulsets {{ include "common.fullname" . }} --replicas=0
+MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+ {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+echo "The the cluster has $MY_REPLICA_NUMBER replicas."
+
+while [[ ! $MY_REPLICA_NUMBER == "0" ]]
+do
+ echo "The cluster is not scaled to 0 yet. Please wait ..."
+ MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \
+ {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}')
+ echo "The current status of the cluster is $MY_REPLICA_NUMBER"
+ sleep 2
+ if [[ $MY_REPLICA_NUMBER == "0" ]]
+ then
+ break
+ fi
+done
+
+for (( index=0; index<$STS_REPLICA; index+=1 ))
+do
+ kubectl delete pvc \
+ "{{ include "common.fullname" . }}-data-{{ include "common.fullname" . }}-$index"
+done