From eb9eb59171a43d25fb012aaad0a1d37ca86bc2bf Mon Sep 17 00:00:00 2001 From: tringuyen Date: Wed, 1 Apr 2020 17:51:06 +0000 Subject: [COMMON] add pre upgrade script for mariadb-galera When upgrading from a version to another, it may be impossible to do it "simply" because of changes in immutable properties of statefulsets. We change that here by creating a temporary deployment which will hold the whole databases during the time the old statefulset gets destroyed and the new one gets created. Issue-ID: OOM-2316 Signed-off-by: tringuyen Signed-off-by: Sylvain Desbureaux Change-Id: I318d72830d5002f50597e23e0753e292f8b47c53 --- .../mariadb-galera/resources/create-deployment.yml | 50 ++++++++++ .../resources/post-upgrade-script.sh | 26 ++++++ .../mariadb-galera/resources/upgrade-scripts.sh | 101 +++++++++++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 kubernetes/common/mariadb-galera/resources/create-deployment.yml create mode 100644 kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh create mode 100644 kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh (limited to 'kubernetes/common/mariadb-galera/resources') diff --git a/kubernetes/common/mariadb-galera/resources/create-deployment.yml b/kubernetes/common/mariadb-galera/resources/create-deployment.yml new file mode 100644 index 0000000000..61bfc78945 --- /dev/null +++ b/kubernetes/common/mariadb-galera/resources/create-deployment.yml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: {{- include "common.resourceMetadata" (dict "suffix" "upgrade-deployment" "dot" .) | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "common.fullname" . }} + template: + metadata: + labels: + app: {{ include "common.fullname" . }} + spec: + containers: + - name: {{ include "common.name" . }} + image: "{{ include "common.repository" . }}/{{ .Values.image }}" + ports: + - containerPort: {{ .Values.service.internalPort }} + name: {{ .Values.service.portName }} + - containerPort: {{ .Values.service.sstPort }} + name: {{ .Values.service.sstPortName }} + - containerPort: {{ .Values.service.replicationPort }} + name: {{ .Values.service.replicationName }} + - containerPort: {{ .Values.service.istPort }} + name: {{ .Values.service.istPortName }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MYSQL_USER + valueFrom: + secretKeyRef: + key: login + name: {{ include "common.fullname" . }}-temp-upgrade-usercred + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: {{ include "common.fullname" . }}-temp-upgrade-usercred + - name: MYSQL_DATABASE + value: {{ default "" .Values.config.mysqlDatabase | quote }} + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: {{ include "common.fullname" . }}-temp-upgrade-root + subdomain: {{ .Values.service.name }} + hostname: {{ .Values.nameOverride }}-upgrade-deployment \ No newline at end of file diff --git a/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh b/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh new file mode 100644 index 0000000000..132ac27ea2 --- /dev/null +++ b/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +TEMP_POD=$(kubectl get pod -n $NAMESPACE_ENV --selector \ + app='{{ include "common.fullname" . }}' -o \ + jsonpath='{.items[?(@.metadata.ownerReferences[].kind=="ReplicaSet")].metadata.name}') + +tmp_MYSQL_PASSWORD=$(echo -n $(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- printenv \ + MYSQL_PASSWORD) | base64) + +tmp_ROOT_PASSWORD=$(echo -n $(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- printenv \ + MYSQL_ROOT_PASSWORD) | base64) + +FLAG_EX_ROOT_SEC='{{ include "common.secret.getSecretNameFast" (dict "global" . "uid" (include "common.mariadb.secret.rootPassUID" .)) }}' + +FLAG_EX_SEC='{{ include "common.secret.getSecretNameFast" (dict "global" . "uid" (include "common.mariadb.secret.userCredentialsUID" .)) }}' + +kubectl patch secret $FLAG_EX_ROOT_SEC -p \ + '{"data":{"password":"'"$tmp_ROOT_PASSWORD"'"}}' + +kubectl patch secret $FLAG_EX_SEC -p \ + '{"data":{"password":"'"$tmp_MYSQL_PASSWORD"'"}}' + +kubectl delete pod -n $NAMESPACE_ENV {{ include "common.fullname" . }}-0 --now +kubectl delete deployment -n $NAMESPACE_ENV {{ include "common.fullname" . }}-upgrade-deployment +kubectl delete secret -n $NAMESPACE_ENV {{ include "common.fullname" . }}-temp-upgrade-root +kubectl delete secret -n $NAMESPACE_ENV {{ include "common.fullname" . }}-temp-upgrade-usercred \ No newline at end of file diff --git a/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh b/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh new file mode 100644 index 0000000000..ff44606e23 --- /dev/null +++ b/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh @@ -0,0 +1,101 @@ +#!/bin/bash +MYSQL_USER=$(kubectl exec -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }}-0 -- printenv MYSQL_USER) + +MYSQL_PASSWORD=$(kubectl exec -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }}-0 -- printenv MYSQL_PASSWORD) + +MYSQL_ROOT_PASSWORD=$(kubectl exec -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }}-0 -- printenv MYSQL_ROOT_PASSWORD) + +kubectl create secret generic \ + '{{ include "common.fullname" . }}'-temp-upgrade-root \ + --from-literal=password=$MYSQL_ROOT_PASSWORD + +kubectl create secret generic \ + '{{ include "common.fullname" . }}'-temp-upgrade-usercred \ + --from-literal=login=$MYSQL_USER --from-literal=password=$MYSQL_PASSWORD + +kubectl create -f /upgrade/create-deployment.yml + +TEMP_POD=$(kubectl get pod -n $NAMESPACE_ENV --selector \ + app='{{ include "common.fullname" . }}' -o \ + jsonpath='{.items[?(@.metadata.ownerReferences[].kind=="ReplicaSet")].metadata.name}') + +CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \ + mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \ + -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" | \ + awk '{print $2}') + +CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \ + mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \ + -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \ + | awk '{print $2}') + +STS_REPLICA=$(kubectl get statefulsets -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}') + +DEPLOYMENT_REPLICA=$(kubectl get deployment -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }}-upgrade-deployment -o \ + jsonpath='{.status.replicas}') + +while [[ ! $CLUSTER_NO == $((STS_REPLICA+DEPLOYMENT_REPLICA)) ]] \ + || [[ ! $CLUSTER_STATE == "Synced" ]] +do + echo "$CLUSTER_NO and $CLUSTER_STATE" + CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \ + --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \ + -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" \ + | awk '{print $2}') + CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \ + --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \ + -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \ + | awk '{print $2}') + sleep 2 + if [[ $CLUSTER_NO == $((STS_REPLICA+DEPLOYMENT_REPLICA)) ]] \ + && [[ $CLUSTER_STATE == "Synced" ]] + then + echo "The cluster has $CLUSTER_NO members and $CLUSTER_STATE state." + break + fi +done + +MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysqladmin \ + -uroot -p$MYSQL_ROOT_PASSWORD ping) + +while [[ ! $MYSQL_STATUS == "mysqld is alive" ]] +do + echo "Mariadb deployment is not ready yet." + sleep 2 + MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysqladmin \ + -uroot -p$MYSQL_ROOT_PASSWORD ping) + if [[ $MYSQL_STATUS == "mysqld is alive" ]] + then + echo "Mariadb deployment is ready." + break + fi +done + +kubectl scale statefulsets {{ include "common.fullname" . }} --replicas=0 +MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}') +echo "The the cluster has $MY_REPLICA_NUMBER replicas." + +while [[ ! $MY_REPLICA_NUMBER == "0" ]] +do + echo "The cluster is not scaled to 0 yet. Please wait ..." + MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}') + echo "The current status of the cluster is $MY_REPLICA_NUMBER" + sleep 2 + if [[ $MY_REPLICA_NUMBER == "0" ]] + then + break + fi +done + +for (( index=0; index<$STS_REPLICA; index+=1 )) +do + kubectl delete pvc \ + "{{ include "common.fullname" . }}-data-{{ include "common.fullname" . }}-$index" +done -- cgit 1.2.3-korg