[COMMON][MARIADB] Fix backup job

The backup job stopped working after upgrade to bitnami images.
Mariabackup was not designed to work remotely, it is supposed to run
on the database server. Because of this we need to mount the data pvc
into the backup job pod. It will however connect to the database daemon
using a hostname, so we need to connect to the first replica in the
cluster. Also had to set readOnlyRootFilesystem=false and add emptyDir
volumes to solve various permission issues.

Issue-ID: OOM-2932
Signed-off-by: Jozsef Csongvai <jozsef.csongvai@bell.ca>
Change-Id: I776903f9ec541f8dc5818b2ba4c1292226ec2bc6

2 files changed, 36 insertions, 10 deletions

diff --git a/kubernetes/common/mariadb-galera/templates/backup/cronjob.yaml b/kubernetes/common/mariadb-galera/templates/backup/cronjob.yaml
index 210fbd02ba..4248cfe85c 100644
--- a/kubernetes/common/mariadb-galera/templates/backup/cronjob.yaml
+++ b/kubernetes/common/mariadb-galera/templates/backup/cronjob.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 */}}
 
-{{- if .Values.backup.enabled }}
+{{- if and .Values.backup.enabled .Values.persistence.enabled }}
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
@@ -37,7 +37,10 @@ spec:
             - name: mariadb-galera-backup-init
               image: {{ include "repositoryGenerator.image.mariadb" . }}
               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
-              {{ include "common.containerSecurityContext" . | indent 14 | trim }}
+              securityContext:
+                allowPrivilegeEscalation: false
+                privileged: false
+                readOnlyRootFilesystem: false
               command:
                 - /bin/bash
                 - -c
@@ -52,7 +55,7 @@ spec:
                     target_dir=/backup/backup-`date +%s`
                     mkdir -p $target_dir
 
-                    mysqlhost={{ include "common.servicename" . }}.{{ include "common.namespace" . }}
+                    mysqlhost={{ include "common.fullname" . }}-0.{{ include "common.servicename" . }}-headless.{{ include "common.namespace" . }}
 
                     mariabackup --backup --target-dir=$target_dir --user=root --password=$DB_PASS --host=$mysqlhost
 
@@ -78,13 +81,18 @@ spec:
               volumeMounts:
                 - name: backup-dir
                   mountPath: /backup
+                - name: data
+                  mountPath: /bitnami/mariadb
           containers:
             - name: mariadb-backup-validate
               image: {{ include "repositoryGenerator.image.mariadb" . }}
               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
-              {{ include "common.containerSecurityContext" . | indent 14 | trim }}
+              securityContext:
+                allowPrivilegeEscalation: false
+                privileged: false
+                readOnlyRootFilesystem: false
               env:
-                - name: MYSQL_ROOT_PASSWORD
+                - name: MARIADB_ROOT_PASSWORD
                   {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" (include "common.mariadb.secret.rootPassUID" .) "key" "password") | indent 18 }}
               command:
                 - /bin/bash
@@ -105,17 +113,17 @@ spec:
                   fi
 
                   target_dir=$(ls -td -- /backup/backup-* | head -n 1)
-                  cp -Ra $target_dir/* /var/lib/mysql/
+                  cp -Ra $target_dir/* /bitnami/mariadb/data
 
-                  if [ ! "$(ls -A /var/lib/mysql)" ]; then
+                  if [ ! "$(ls -A /bitnami/mariadb/data)" ]; then
                     remove_dir $target_dir
                     exit 0
                   fi
 
-                  /docker-entrypoint.sh mysqld &
+                  /opt/bitnami/scripts/mariadb/entrypoint.sh /opt/bitnami/scripts/mariadb/run.sh &
 
                   count=0
-                  until mysql --user=root --password=$MYSQL_ROOT_PASSWORD  -e "SELECT 1";
+                  until mysql --user=root --password=$MARIADB_ROOT_PASSWORD  -e "SELECT 1";
                     do sleep 3;
                     count=`expr $count + 1`;
                     if [ $count -ge 30 ]; then
@@ -124,7 +132,7 @@ spec:
                     fi;
                   done
 
-                  mysqlcheck -A  --user=root --password=$MYSQL_ROOT_PASSWORD > /tmp/output.log
+                  mysqlcheck -A  --user=root --password=$MARIADB_ROOT_PASSWORD > /tmp/output.log
                   error_lines=`cat /tmp/output.log| grep -v "OK" | wc -l`
 
                   cat /tmp/output.log
@@ -142,6 +150,10 @@ spec:
                   fi
               resources: {{ include "common.resources" . | nindent 12 }}
               volumeMounts:
+                - mountPath: /bitnami/mariadb/data
+                  name: tmp-data
+                - mountPath: /opt/bitnami/mariadb/tmp
+                  name: tmp
                 - mountPath: /etc/localtime
                   name: localtime
                   readOnly: true
@@ -153,7 +165,18 @@ spec:
             - name: localtime
               hostPath:
                 path: /etc/localtime
+            - name: data
+              persistentVolumeClaim:
+            {{- if .Values.persistence.existingClaim }}
+                claimName: {{ .Values.persistence.existingClaim }}
+            {{- else }}
+                claimName: {{ include "common.fullname" . }}-{{ include "common.fullname" . }}-0
+            {{- end }}
             - name: backup-dir
               persistentVolumeClaim:
                 claimName: {{ include "common.fullname" . }}-backup-data
+            - name: tmp-data
+              emptyDir: {}
+            - name: tmp
+              emptyDir: {}
 {{- end }}
diff --git a/kubernetes/common/mariadb-galera/values.yaml b/kubernetes/common/mariadb-galera/values.yaml
index 9f7c882134..d65c4f7943 100644
--- a/kubernetes/common/mariadb-galera/values.yaml
+++ b/kubernetes/common/mariadb-galera/values.yaml
@@ -174,6 +174,8 @@ galera:
     # password:
     # externalSecret:
 
+## The backup job will mount the mariadb data pvc in order to run mariabackup.
+## For this reason the db data pvc needs to have accessMode: ReadWriteMany.
 backup:
   enabled: false
   cron: "00 00 * * *"
@@ -458,6 +460,7 @@ persistence:
   ##
   annotations:
   ## Persistent Volume Access Mode
+  ## Use ReadWriteMany if backup is enabled, see backup section.
   ##
   accessMode: ReadWriteOnce
   ## Persistent Volume size