diff options
Diffstat (limited to 'kubernetes/common/cassandra/templates/backup/cronjob.yaml')
| -rw-r--r-- | kubernetes/common/cassandra/templates/backup/cronjob.yaml | 242 |
1 files changed, 242 insertions, 0 deletions
diff --git a/kubernetes/common/cassandra/templates/backup/cronjob.yaml b/kubernetes/common/cassandra/templates/backup/cronjob.yaml new file mode 100644 index 0000000000..630ac43ba3 --- /dev/null +++ b/kubernetes/common/cassandra/templates/backup/cronjob.yaml @@ -0,0 +1,242 @@ +{{/* +# Copyright © 2019 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- if .Values.backup.enabled }} +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: {{ include "common.fullname" . }}-backup + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.fullname" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + schedule: {{ .Values.backup.cron | quote }} + concurrencyPolicy: Forbid + startingDeadlineSeconds: 120 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + initContainers: + - command: + - /root/ready.py + args: + - --container-name + - {{ include "common.name" . }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + name: {{ include "common.name" . }}-readiness + - name: "cassandra-backup-init" + image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + command: + - /bin/bash + - -c + - | + clearSnapshot(){ + curr_time=$1 + echo "Clearing snapshots!!!" + command="nodetool clearsnapshot -t $curr_time" + /root/exec.py -p "cassandra" -c "$command" + } + {{ $root := . }} + curr_time=`date +%s` + pids="" + set -x + + echo "Copying data" + {{ range $i, $e := until (int .Values.replicaCount) }} + target_dir=/backup/temp/cassandra-{{ $i }} + mkdir -p $target_dir + cp -Ra /onap-data/cassandra-{{ $i }}/data/ $target_dir/ + {{- end }} + + echo "Executing cleanup!!" + command="nodetool cleanup" + /root/exec.py -p "cassandra" -c "$command" + echo "Cleaned Node!! Backing up database now!!!" + + command="nodetool snapshot -t $curr_time" + /root/exec.py -p "cassandra" -c "$command" + retCode=$? + if [ $retCode -ne 0 ]; then + echo "Backup Failed!!!" + rm -rf /backup/temp + clearSnapshot $curr_time + echo "Failed" > /backup/backup.log + exit 0 + fi + + backup_dir=/backup/temp + {{ range $i, $e := until (int .Values.replicaCount) }} + for d in $backup_dir/cassandra-{{ $i }}/data/*/ ; do + d=$(echo $d | sed 's:/*$::') + keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }') + if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then + /root/restore.sh -b $backup_dir/cassandra-{{ $i }}/data -s /onap-data/cassandra-{{ $i }}/data/$keyspace_name -k $keyspace_name -t $curr_time & + pids="$pids $!" + fi + done + {{- end }} + + for p in $pids; do + wait $p + if [ $? -ne 0 ]; then + rm -rf /backup/temp + echo "Creation of Backup Failed!!!" + clearSnapshot $curr_time + echo "Failed" > /backup/backup.log + exit 0 + fi + done + + clearSnapshot $curr_time + + exit_code=$? + if [ $exit_code -ne 0 ]; then + rm -rf /backup/temp + echo "Backup Failed!!!" + echo "Failed" > /backup/backup.log + exit 0 + fi + + mv /backup/temp /backup/backup-${curr_time} + echo "Success" > /backup/backup.log + echo "Cassandra Backup Succeeded" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - mountPath: /etc/localtime + name: localtime + readOnly: true + - mountPath: /onap-data + name: data-dir + - mountPath: /backup + name: backup-dir + - name: scripts + mountPath: /root/restore.sh + subPath: restore.sh + - name: scripts + mountPath: /root/exec.py + subPath: exec.py + containers: + - name: cassandra-backup-validate + image: "{{ .Values.image }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + command: + - /bin/bash + - -c + - | + remove_dir(){ + dirToRemove=$1 + rm -rf $dirToRemove + } + + backup_result=`cat /backup/backup.log` + rm -rf /backup/backup.log + + if [ "$backup_result" == "Failed" ]; then + echo "Backup Failed!!! So Validation Failed!!!"; + exit 0 + fi + + target_dir=$(ls -td -- /backup/*/ | head -n 1) + chown -R cassandra.cassandra $target_dir + {{- $root := . -}} + {{ range $i, $e := until (int .Values.replicaCount) }} + dbSize=$(du -ks $target_dir/cassandra-{{ $i }}/data|awk -F " " '{ printf $1 }') + minDbSize={{ (int $root.Values.backup.dbSize) }} + if [ $dbSize -lt $minDbSize ]; then + remove_dir $target_dir + echo "Validation Failed!!! dbSize ($dbSize) is less than minimum size (1)!!!" + exit 0 + fi + rm -rf /var/lib/cassandra/* + cp -Ra $target_dir/cassandra-{{ $i }}/data /var/lib/cassandra + export CASSANDRA_LISTEN_ADDRESS="127.0.0.1" + /docker-entrypoint.sh -Dcassandra.ignore_dc=true -Dcassandra.ignore_rack=true & + CASS_PID=$! + sleep 45 + + for d in $target_dir/cassandra-{{ $i }}/data/*/; do + d=$(echo $d | sed 's:/*$::') + keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }') + if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then + echo "Verifying the data for $keyspace_name " + nodetool verify -e $keyspace_name + ret=$? + if [ $ret -ne 0 ]; then + remove_dir $target_dir + echo "Validation Failed!!!" + exit 0 + fi + fi + done + kill -9 $CASS_PID + {{- end }} + echo "Validation Successful!!!" + cd /backup + totalFiles=`ls -t | grep "backup-" | wc -l` + if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then + filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}` + ls -tr | grep backup | head -$filestoDelete | xargs rm -rf + fi + env: + - name: CASSANDRA_CLUSTER_NAME + value: {{ .Values.config.clusterName }} + - name: MAX_HEAP_SIZE + value: {{ .Values.config.heap.max }} + - name: HEAP_NEWSIZE + value: {{ .Values.config.heap.min }} + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: backup-dir + mountPath: /backup + - name: localtime + mountPath: /etc/localtime + readOnly: true + volumes: + - name: localtime + hostPath: + path: /etc/localtime + - name: scripts + configMap: + name: {{ include "common.fullname" $ }}-configmap + defaultMode: 0755 + - name: data-dir + persistentVolumeClaim: + claimName: {{ include "common.fullname" . }}-db-data + - name: backup-dir + persistentVolumeClaim: + claimName: {{ include "common.fullname" . }}-backup-data +{{- end -}} + |