backup charts for cassandra

Issue-ID: OOM-2089

Signed-off-by: Akansha Dua <akansha.dua@amdocs.com>
Change-Id: Ia1086deda37ba6e613afcc6bd064e7bf4ee692b9

1 files changed, 242 insertions, 0 deletions

diff --git a/kubernetes/common/cassandra/templates/backup/cronjob.yaml b/kubernetes/common/cassandra/templates/backup/cronjob.yaml
new file mode 100644
index 0000000000..630ac43ba3
--- /dev/null
+++ b/kubernetes/common/cassandra/templates/backup/cronjob.yaml
@@ -0,0 +1,242 @@
+{{/*
+# Copyright © 2019 Amdocs, Bell Canada
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.backup.enabled }}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: {{ include "common.fullname" . }}-backup
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.fullname" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  schedule: {{ .Values.backup.cron | quote }}
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 120
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          initContainers:
+            - command:
+              - /root/ready.py
+              args:
+              - --container-name
+              - {{ include "common.name" . }}
+              env:
+              - name: NAMESPACE
+                valueFrom:
+                  fieldRef:
+                    apiVersion: v1
+                    fieldPath: metadata.namespace
+              image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+              imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+              name: {{ include "common.name" . }}-readiness
+            - name: "cassandra-backup-init"
+              image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+              imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+              command:
+              - /bin/bash
+              - -c
+              - |
+                clearSnapshot(){
+                  curr_time=$1
+                  echo "Clearing snapshots!!!"
+                  command="nodetool clearsnapshot -t $curr_time"
+                  /root/exec.py -p "cassandra" -c "$command"
+                }
+                {{ $root := . }}
+                curr_time=`date +%s`
+                pids=""
+                set -x
+
+                echo "Copying data"
+                {{ range $i, $e := until (int .Values.replicaCount) }}
+                  target_dir=/backup/temp/cassandra-{{ $i }}
+                  mkdir -p $target_dir
+                  cp -Ra /onap-data/cassandra-{{ $i }}/data/ $target_dir/
+                {{- end }}
+
+                echo "Executing cleanup!!"
+                command="nodetool cleanup"
+                /root/exec.py -p "cassandra" -c "$command"                
+                echo "Cleaned Node!! Backing up database now!!!"
+
+                command="nodetool snapshot -t $curr_time"
+                /root/exec.py -p "cassandra" -c "$command"
+                retCode=$?
+                if [ $retCode -ne 0 ]; then
+                  echo "Backup Failed!!!"
+                  rm -rf /backup/temp
+                  clearSnapshot $curr_time
+                  echo "Failed" > /backup/backup.log
+                  exit 0
+                fi
+
+                backup_dir=/backup/temp
+                {{ range $i, $e := until (int .Values.replicaCount) }}
+                  for d in $backup_dir/cassandra-{{ $i }}/data/*/ ; do
+                    d=$(echo $d | sed 's:/*$::')
+                    keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
+                    if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
+                      /root/restore.sh -b $backup_dir/cassandra-{{ $i }}/data -s  /onap-data/cassandra-{{ $i }}/data/$keyspace_name -k $keyspace_name -t $curr_time &
+                      pids="$pids $!"
+                    fi                  
+                  done
+                {{- end }}
+
+                for p in $pids; do
+                  wait $p
+                  if [ $? -ne 0 ]; then
+                    rm -rf /backup/temp
+                    echo "Creation of Backup Failed!!!"
+                    clearSnapshot $curr_time
+                    echo "Failed" > /backup/backup.log
+                    exit 0
+                  fi
+                done
+
+                clearSnapshot $curr_time
+
+                exit_code=$?
+                if [ $exit_code -ne 0 ]; then
+                  rm -rf /backup/temp
+                  echo "Backup Failed!!!"
+                  echo "Failed" > /backup/backup.log
+                  exit 0
+                fi
+                
+                mv /backup/temp /backup/backup-${curr_time}
+                echo "Success" > /backup/backup.log
+                echo "Cassandra Backup Succeeded"
+              env:
+              - name: NAMESPACE
+                valueFrom:
+                  fieldRef:
+                    apiVersion: v1
+                    fieldPath: metadata.namespace
+              volumeMounts:
+              - mountPath: /etc/localtime
+                name: localtime
+                readOnly: true
+              - mountPath: /onap-data
+                name: data-dir
+              - mountPath: /backup
+                name: backup-dir
+              - name: scripts
+                mountPath: /root/restore.sh
+                subPath: restore.sh
+              - name: scripts
+                mountPath: /root/exec.py
+                subPath: exec.py           
+          containers:
+            - name: cassandra-backup-validate
+              image: "{{ .Values.image }}"
+              imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+              command:
+              - /bin/bash
+              - -c
+              - |
+                remove_dir(){
+                  dirToRemove=$1
+                  rm -rf $dirToRemove
+                }
+
+                backup_result=`cat /backup/backup.log`
+                rm -rf /backup/backup.log
+
+                if [ "$backup_result" == "Failed" ]; then
+                  echo "Backup Failed!!! So Validation Failed!!!";
+                  exit 0
+                fi
+
+                target_dir=$(ls -td -- /backup/*/ | head -n 1)
+                chown -R cassandra.cassandra $target_dir
+                {{- $root := . -}}
+                {{ range $i, $e := until (int .Values.replicaCount) }}
+                  dbSize=$(du -ks $target_dir/cassandra-{{ $i }}/data|awk -F " " '{ printf $1 }')
+                  minDbSize={{ (int $root.Values.backup.dbSize) }}
+                  if [ $dbSize -lt $minDbSize ]; then
+                    remove_dir $target_dir
+                    echo "Validation Failed!!! dbSize ($dbSize) is less than minimum size (1)!!!"
+                    exit 0
+                  fi
+                  rm -rf /var/lib/cassandra/*
+                  cp -Ra $target_dir/cassandra-{{ $i }}/data /var/lib/cassandra
+                  export CASSANDRA_LISTEN_ADDRESS="127.0.0.1"
+                  /docker-entrypoint.sh -Dcassandra.ignore_dc=true -Dcassandra.ignore_rack=true &
+                  CASS_PID=$!
+                  sleep 45
+
+                  for d in $target_dir/cassandra-{{ $i }}/data/*/; do
+                      d=$(echo $d | sed 's:/*$::')
+                      keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
+                      if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
+                        echo "Verifying the data for  $keyspace_name "
+                        nodetool verify -e $keyspace_name
+                        ret=$?
+                        if [ $ret -ne 0 ]; then
+                          remove_dir $target_dir
+                          echo "Validation Failed!!!"
+                          exit 0
+                        fi
+                      fi
+                  done
+                  kill -9 $CASS_PID
+                {{- end }}
+                echo "Validation Successful!!!" 
+                cd /backup
+                totalFiles=`ls -t | grep "backup-" | wc -l`
+                if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then
+                  filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}`
+                  ls -tr | grep backup | head -$filestoDelete | xargs rm -rf
+                fi
+              env:
+              - name: CASSANDRA_CLUSTER_NAME
+                value: {{ .Values.config.clusterName }}
+              - name: MAX_HEAP_SIZE
+                value: {{ .Values.config.heap.max }}
+              - name: HEAP_NEWSIZE
+                value: {{ .Values.config.heap.min }}
+              - name: HOST_IP
+                valueFrom:
+                  fieldRef:
+                    fieldPath: status.podIP
+              volumeMounts:
+              - name: backup-dir
+                mountPath: /backup
+              - name: localtime
+                mountPath: /etc/localtime
+                readOnly: true
+          volumes:
+          - name: localtime
+            hostPath:
+              path: /etc/localtime
+          - name: scripts
+            configMap:
+              name: {{ include "common.fullname" $ }}-configmap
+              defaultMode: 0755
+          - name: data-dir
+            persistentVolumeClaim:
+              claimName: {{ include "common.fullname" . }}-db-data
+          - name: backup-dir
+            persistentVolumeClaim:
+              claimName: {{ include "common.fullname" . }}-backup-data
+{{- end -}}
+