diff options
author | 2020-04-01 17:51:06 +0000 | |
---|---|---|
committer | 2020-06-10 14:10:02 +0000 | |
commit | f5993cdff9c9d873966d24e7987094fee4d13f0d (patch) | |
tree | 09beb1733decc427e141b10c4a70a986b442fe76 /kubernetes/common/mariadb-galera/resources | |
parent | 1903b2c5c802e16d30207f6ec4ba58d6bc034280 (diff) |
[COMMON] add pre upgrade script for mariadb-galera
When upgrading from a version to another, it may be impossible to do it
"simply" because of changes in immutable properties of statefulsets. We
change that here by creating a temporary deployment which will hold the
whole databases during the time the old statefulset gets destroyed and
the new one gets created.
Issue-ID: OOM-2316
Signed-off-by: tringuyen <tri.nguyen@tatacommunications.com>
Signed-off-by: Sylvain Desbureaux <sylvain.desbureaux@orange.com>
Change-Id: I318d72830d5002f50597e23e0753e292f8b47c53
(cherry picked from commit eb9eb59171a43d25fb012aaad0a1d37ca86bc2bf)
Diffstat (limited to 'kubernetes/common/mariadb-galera/resources')
3 files changed, 177 insertions, 0 deletions
diff --git a/kubernetes/common/mariadb-galera/resources/create-deployment.yml b/kubernetes/common/mariadb-galera/resources/create-deployment.yml new file mode 100644 index 0000000000..61bfc78945 --- /dev/null +++ b/kubernetes/common/mariadb-galera/resources/create-deployment.yml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: {{- include "common.resourceMetadata" (dict "suffix" "upgrade-deployment" "dot" .) | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "common.fullname" . }} + template: + metadata: + labels: + app: {{ include "common.fullname" . }} + spec: + containers: + - name: {{ include "common.name" . }} + image: "{{ include "common.repository" . }}/{{ .Values.image }}" + ports: + - containerPort: {{ .Values.service.internalPort }} + name: {{ .Values.service.portName }} + - containerPort: {{ .Values.service.sstPort }} + name: {{ .Values.service.sstPortName }} + - containerPort: {{ .Values.service.replicationPort }} + name: {{ .Values.service.replicationName }} + - containerPort: {{ .Values.service.istPort }} + name: {{ .Values.service.istPortName }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MYSQL_USER + valueFrom: + secretKeyRef: + key: login + name: {{ include "common.fullname" . }}-temp-upgrade-usercred + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: {{ include "common.fullname" . }}-temp-upgrade-usercred + - name: MYSQL_DATABASE + value: {{ default "" .Values.config.mysqlDatabase | quote }} + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: {{ include "common.fullname" . }}-temp-upgrade-root + subdomain: {{ .Values.service.name }} + hostname: {{ .Values.nameOverride }}-upgrade-deployment
\ No newline at end of file diff --git a/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh b/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh new file mode 100644 index 0000000000..132ac27ea2 --- /dev/null +++ b/kubernetes/common/mariadb-galera/resources/post-upgrade-script.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +TEMP_POD=$(kubectl get pod -n $NAMESPACE_ENV --selector \ + app='{{ include "common.fullname" . }}' -o \ + jsonpath='{.items[?(@.metadata.ownerReferences[].kind=="ReplicaSet")].metadata.name}') + +tmp_MYSQL_PASSWORD=$(echo -n $(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- printenv \ + MYSQL_PASSWORD) | base64) + +tmp_ROOT_PASSWORD=$(echo -n $(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- printenv \ + MYSQL_ROOT_PASSWORD) | base64) + +FLAG_EX_ROOT_SEC='{{ include "common.secret.getSecretNameFast" (dict "global" . "uid" (include "common.mariadb.secret.rootPassUID" .)) }}' + +FLAG_EX_SEC='{{ include "common.secret.getSecretNameFast" (dict "global" . "uid" (include "common.mariadb.secret.userCredentialsUID" .)) }}' + +kubectl patch secret $FLAG_EX_ROOT_SEC -p \ + '{"data":{"password":"'"$tmp_ROOT_PASSWORD"'"}}' + +kubectl patch secret $FLAG_EX_SEC -p \ + '{"data":{"password":"'"$tmp_MYSQL_PASSWORD"'"}}' + +kubectl delete pod -n $NAMESPACE_ENV {{ include "common.fullname" . }}-0 --now +kubectl delete deployment -n $NAMESPACE_ENV {{ include "common.fullname" . }}-upgrade-deployment +kubectl delete secret -n $NAMESPACE_ENV {{ include "common.fullname" . }}-temp-upgrade-root +kubectl delete secret -n $NAMESPACE_ENV {{ include "common.fullname" . }}-temp-upgrade-usercred
\ No newline at end of file diff --git a/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh b/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh new file mode 100644 index 0000000000..ff44606e23 --- /dev/null +++ b/kubernetes/common/mariadb-galera/resources/upgrade-scripts.sh @@ -0,0 +1,101 @@ +#!/bin/bash +MYSQL_USER=$(kubectl exec -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }}-0 -- printenv MYSQL_USER) + +MYSQL_PASSWORD=$(kubectl exec -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }}-0 -- printenv MYSQL_PASSWORD) + +MYSQL_ROOT_PASSWORD=$(kubectl exec -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }}-0 -- printenv MYSQL_ROOT_PASSWORD) + +kubectl create secret generic \ + '{{ include "common.fullname" . }}'-temp-upgrade-root \ + --from-literal=password=$MYSQL_ROOT_PASSWORD + +kubectl create secret generic \ + '{{ include "common.fullname" . }}'-temp-upgrade-usercred \ + --from-literal=login=$MYSQL_USER --from-literal=password=$MYSQL_PASSWORD + +kubectl create -f /upgrade/create-deployment.yml + +TEMP_POD=$(kubectl get pod -n $NAMESPACE_ENV --selector \ + app='{{ include "common.fullname" . }}' -o \ + jsonpath='{.items[?(@.metadata.ownerReferences[].kind=="ReplicaSet")].metadata.name}') + +CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \ + mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \ + -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" | \ + awk '{print $2}') + +CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- \ + mysql --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \ + -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \ + | awk '{print $2}') + +STS_REPLICA=$(kubectl get statefulsets -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}') + +DEPLOYMENT_REPLICA=$(kubectl get deployment -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }}-upgrade-deployment -o \ + jsonpath='{.status.replicas}') + +while [[ ! $CLUSTER_NO == $((STS_REPLICA+DEPLOYMENT_REPLICA)) ]] \ + || [[ ! $CLUSTER_STATE == "Synced" ]] +do + echo "$CLUSTER_NO and $CLUSTER_STATE" + CLUSTER_NO=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \ + --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \ + -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_cluster_size';" \ + | awk '{print $2}') + CLUSTER_STATE=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysql \ + --skip-column-names -h{{ $.Values.service.name }} -u$MYSQL_USER \ + -p$MYSQL_PASSWORD -e "SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';" \ + | awk '{print $2}') + sleep 2 + if [[ $CLUSTER_NO == $((STS_REPLICA+DEPLOYMENT_REPLICA)) ]] \ + && [[ $CLUSTER_STATE == "Synced" ]] + then + echo "The cluster has $CLUSTER_NO members and $CLUSTER_STATE state." + break + fi +done + +MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysqladmin \ + -uroot -p$MYSQL_ROOT_PASSWORD ping) + +while [[ ! $MYSQL_STATUS == "mysqld is alive" ]] +do + echo "Mariadb deployment is not ready yet." + sleep 2 + MYSQL_STATUS=$(kubectl exec -n $NAMESPACE_ENV $TEMP_POD -- mysqladmin \ + -uroot -p$MYSQL_ROOT_PASSWORD ping) + if [[ $MYSQL_STATUS == "mysqld is alive" ]] + then + echo "Mariadb deployment is ready." + break + fi +done + +kubectl scale statefulsets {{ include "common.fullname" . }} --replicas=0 +MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}') +echo "The the cluster has $MY_REPLICA_NUMBER replicas." + +while [[ ! $MY_REPLICA_NUMBER == "0" ]] +do + echo "The cluster is not scaled to 0 yet. Please wait ..." + MY_REPLICA_NUMBER=$(kubectl get statefulsets -n $NAMESPACE_ENV \ + {{ include "common.fullname" . }} -o jsonpath='{.status.replicas}') + echo "The current status of the cluster is $MY_REPLICA_NUMBER" + sleep 2 + if [[ $MY_REPLICA_NUMBER == "0" ]] + then + break + fi +done + +for (( index=0; index<$STS_REPLICA; index+=1 )) +do + kubectl delete pvc \ + "{{ include "common.fullname" . }}-data-{{ include "common.fullname" . }}-$index" +done |