Backup Charts for mariadb cluster

Issue-ID: OOM-2088

Signed-off-by: Akansha Dua <akansha.dua@amdocs.com>
Change-Id: Ie7f25c812984abc8631a86b2385dd14a365a8ea5

4 files changed, 334 insertions, 1 deletions

diff --git a/kubernetes/common/mariadb-galera/templates/backup/cronjob.yaml b/kubernetes/common/mariadb-galera/templates/backup/cronjob.yaml
new file mode 100644
index 0000000000..7d3ec75c00
--- /dev/null
+++ b/kubernetes/common/mariadb-galera/templates/backup/cronjob.yaml
@@ -0,0 +1,178 @@
+{{/*
+# Copyright © 2019 Amdocs, Bell Canada
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.backup.enabled }}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: {{ include "common.fullname" . }}-backup
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.fullname" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  schedule: {{ .Values.backup.cron | quote }}
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 120
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          initContainers:
+          - command:
+            - /root/ready.py
+            args:
+            - --container-name
+            - {{ include "common.name" . }}
+            env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+            imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+            name: {{ include "common.name" . }}-readiness
+          - name: mariadb-galera-backup-init
+            image: "{{ include "common.repository" . }}/{{ .Values.backupImage }}"
+            imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+            command:
+            - /bin/bash
+            - -c
+            - |
+              remove_dir(){
+                dirToRemove=$1
+                rm -rf $dirToRemove
+                echo "Failed" > /backup/backup.log
+                echo "Backup failed!!!"
+              }
+
+              target_dir=/backup/backup-`date +%s`
+              mkdir -p $target_dir
+
+              mysqlhost={{ include "common.fullname" . }}-{{ sub .Values.replicaCount 1 }}.{{ .Values.service.name }}
+
+              mariabackup --backup --target-dir=$target_dir --user=root --password=$DB_PASS --host=$mysqlhost
+
+              ret_code=$?
+              if [ $ret_code -ne 0 ]; then
+                remove_dir $target_dir
+                exit 0
+              fi
+
+              echo "Starting Backup Preparation!!!"
+              mariabackup --prepare --target-dir=$target_dir
+              ret_code=$?
+              if [ $ret_code -ne 0 ]; then
+                remove_dir $target_dir
+                exit 0
+              fi
+              echo "Success" > /backup/backup.log
+              echo "Backup Successful!!!"
+            env:
+            - name: DB_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "common.fullname" . }}
+                  key: db-root-password
+            volumeMounts:
+            - name: backup-data
+              mountPath: /backup
+            - name: db-data
+              mountPath: /var/lib/mysql
+          containers:
+          - name: mariadb-backup-validate
+            image: "{{ include "common.repository" . }}/{{ .Values.backupImage }}"
+            imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+            env:
+            - name: MYSQL_ROOT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "common.fullname" . }}
+                  key: db-root-password
+            command:
+            - /bin/bash
+            - -c
+            - |
+              remove_dir(){
+                dirToRemove=$1
+                rm -rf $dirToRemove
+                echo "Validation Failed!!!";
+              }
+
+              backup_result=`cat /backup/backup.log`
+              rm -rf /backup/backup.log
+
+              if [ "$backup_result" == "Failed" ]; then
+                echo "Backup Failed!!! So Validation Failed!!!";
+                exit 0
+              fi
+
+              target_dir=$(ls -td -- /backup/backup-* | head -n 1)
+              cp -Ra $target_dir/* /var/lib/mysql/
+
+              if [ ! "$(ls -A /var/lib/mysql)" ]; then
+                remove_dir $target_dir
+                exit 0
+              fi
+              
+              /docker-entrypoint.sh mysqld &
+
+              count=0
+              until mysql --user=root --password=$MYSQL_ROOT_PASSWORD  -e "SELECT 1";
+                do sleep 3;
+                count=`expr $count + 1`;
+                if [ $count -ge 30 ]; then
+                  remove_dir $target_dir
+                  exit 0;
+                fi;
+              done
+
+              mysqlcheck -A  --user=root --password=$MYSQL_ROOT_PASSWORD > /tmp/output.log
+              error_lines=`cat /tmp/output.log| grep -v "OK" | wc -l`
+
+              cat /tmp/output.log
+
+              if [ $error_lines -gt 1 ];then
+                remove_dir $target_dir
+              else
+                echo "Validation successful!!!"
+                cd /backup
+                totalFiles=`ls -t | grep "backup-" | wc -l`
+                if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then
+                  filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}`
+                  ls -tr | grep backup | head -$filestoDelete | xargs rm -rf
+                fi
+              fi
+            volumeMounts:
+            - mountPath: /etc/localtime
+              name: localtime
+              readOnly: true
+            - name: backup-data
+              mountPath: /backup
+          volumes:
+          - name: localtime
+            hostPath:
+              path: /etc/localtime
+          - name: db-data
+            persistentVolumeClaim:
+              claimName: {{ include "common.fullname" . }}-db-data
+          - name: backup-data
+            persistentVolumeClaim:
+              claimName: {{ include "common.fullname" . }}-backup 
+{{- end }}
diff --git a/kubernetes/common/mariadb-galera/templates/backup/pv.yaml b/kubernetes/common/mariadb-galera/templates/backup/pv.yaml
new file mode 100644
index 0000000000..2972191563
--- /dev/null
+++ b/kubernetes/common/mariadb-galera/templates/backup/pv.yaml
@@ -0,0 +1,59 @@
+{{/*
+# Copyright © 2019 Amdocs, Bell Canada
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.backup.enabled }}
+{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}}
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: {{ include "common.fullname" . }}-backup
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+    name: {{ include "common.fullname" . }}-backup
+spec:
+  capacity:
+    storage: {{ .Values.persistence.size}}
+  accessModes:
+  - {{ .Values.persistence.accessMode }}
+  persistentVolumeReclaimPolicy: {{ .Values.persistence.volumeReclaimPolicy }}
+  hostPath:
+    path: {{ .Values.global.persistence.backup.mountPath | default .Values.persistence.backup.mountPath }}/{{ include "common.namespace" . }}/{{include "common.name" . }}
+---
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: {{ include "common.fullname" . }}-db-data
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+    name: {{ include "common.fullname" . }}-db-data
+spec:
+  capacity:
+    storage: {{ .Values.persistence.size}}
+  accessModes:
+  - {{ .Values.persistence.accessMode }}
+  persistentVolumeReclaimPolicy: {{ .Values.persistence.volumeReclaimPolicy }}
+  hostPath:
+    path: {{ .Values.global.persistence.mountPath | default .Values.persistence.mountPath }}/{{ .Release.Name }}/{{ .Values.persistence.mountSubPath }}{{ sub .Values.replicaCount 1 }}
+{{- end -}}
+{{- end -}}
+
diff --git a/kubernetes/common/mariadb-galera/templates/backup/pvc.yaml b/kubernetes/common/mariadb-galera/templates/backup/pvc.yaml
new file mode 100644
index 0000000000..a983c8af98
--- /dev/null
+++ b/kubernetes/common/mariadb-galera/templates/backup/pvc.yaml
@@ -0,0 +1,81 @@
+{{/*
+# Copyright © 2019 Amdocs, Bell Canada
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.backup.enabled }}
+{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ include "common.fullname" . }}-backup
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}-backup
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+{{- if .Values.persistence.annotations }}
+  annotations:
+{{ toYaml .Values.persistence.annotations | indent 4 }}
+{{- end }}
+spec:
+  selector:
+    matchLabels:
+      name: {{ include "common.fullname" . }}-backup
+  accessModes:
+    - {{ .Values.persistence.accessMode }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+{{- if .Values.persistence.storageClass }}
+{{- if (eq "-" .Values.persistence.storageClass) }}
+  storageClassName: ""
+{{- else }}
+  storageClassName: "{{ .Values.persistence.storageClass }}"
+{{- end }}
+{{- end }}
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ include "common.fullname" . }}-db-data
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}-db-data
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+{{- if .Values.persistence.annotations }}
+  annotations:
+{{ toYaml .Values.persistence.annotations | indent 4 }}
+{{- end }}
+spec:
+  selector:
+    matchLabels:
+      name: {{ include "common.fullname" . }}-db-data
+  accessModes:
+    - {{ .Values.persistence.accessMode }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+{{- if .Values.persistence.storageClass }}
+{{- if (eq "-" .Values.persistence.storageClass) }}
+  storageClassName: ""
+{{- else }}
+  storageClassName: "{{ .Values.persistence.storageClass }}"
+{{- end }}
+{{- end }}
+{{- end -}}
+{{- end -}}
+
diff --git a/kubernetes/common/mariadb-galera/values.yaml b/kubernetes/common/mariadb-galera/values.yaml
index 6b1e186e8b..a662b1e04a 100644
--- a/kubernetes/common/mariadb-galera/values.yaml
+++ b/kubernetes/common/mariadb-galera/values.yaml
@@ -17,9 +17,16 @@
 #################################################################
 global:
   nodePortPrefix: 302
-  persistence: {}
+  persistence:
+    mountPath: /dockerdata-nfs
+    backup:
+      mountPath: /dockerdata-nfs/backup
+
   repository: nexus3.onap.org:10001
 
+  readinessRepository: oomk8s
+  readinessImage: readiness-check:2.0.2
+
 
 #################################################################
 # Application configuration defaults.
@@ -28,6 +35,7 @@ global:
 #repository: mysql
 repository: nexus3.onap.org:10001
 image: adfinissygroup/k8s-mariadb-galera-centos:v002
+backupImage: library/mariadb:10.1.38 
 imageInit: busybox
 pullPolicy: IfNotPresent
 
@@ -82,6 +90,8 @@ persistence:
   mountPath: /dockerdata-nfs
   mountSubPath: "mariadb-galera/data"
   mysqlPath: /var/lib/mysql
+  backup:
+    mountPath: /dockerdata-nfs/backup
 
 service:
   internalPort: 3306
@@ -139,3 +149,8 @@ nameOverride: mariadb-galera
 
 # DNS name for mariadb-galera cluster - should be unique accross all projects other clusters
 #dnsnameOverride: mariadb-galera
+
+backup:
+  enabled: false
+  cron: "00 00 * * *"
+  retentionPeriod: 3