summaryrefslogtreecommitdiffstats
path: root/kubernetes/dmaap/Makefile
blob: 3a1931121af4129744a86adfc934c75bb0733d9f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# Copyright © 2018  AT&T Intellectual Property.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

make-dmaap: make-dmaap-bc make-message-router make-dmaap-dr-node make-dmaap-dr-prov

make-dmaap-bc:
	cd components && helm dep up dmaap-bc && helm lint dmaap-bc

make-message-router:
	cd components && helm dep up message-router && helm lint message-router

make-dmaap-dr-node:
	cd components && helm dep up dmaap-dr-node && helm lint dmaap-dr-node

make-dmaap-dr-prov:
	cd components && helm dep up dmaap-dr-prov && helm lint dmaap-dr-prov

clean:
	@find . -type f -name '*.tgz' -delete
	@find . -type f -name '*.lock' -delete
88888 } /* Comment.PreprocFile */ .highlight .c1 { color: #888888 } /* Comment.Single */ .highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */ .highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ .highlight .ge { font-style: italic } /* Generic.Emph */ .highlight .gr { color: #aa0000 } /* Generic.Error */ .highlight .gh { color: #333333 } /* Generic.Heading */ .highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ .highlight .go { color: #888888 } /* Generic.Output */ .highlight .gp { color: #555555 } /* Generic.Prompt */ .highlight .gs { font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #666666 } /* Generic.Subheading */ .highlight .gt { color: #aa0000 } /* Generic.Traceback */ .highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008800 } /* Keyword.Pseudo */ .highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */ .highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */ .highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */ .highlight .na { color: #336699 } /* Name.Attribute */ .highlight .nb { color: #003388 } /* Name.Builtin */ .highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */ .highlight .no { color: #003366; font-weight: bold } /* Name.Constant */ .highlight .nd { color: #555555 } /* Name.Decorator */ .highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */ .highlight .nl { color: #336699; font-style: italic } /* Name.Label */ .highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */ .highlight .py { color: #336699; font-weight: bold } /* Name.Property */ .highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #336699 } /* Name.Variable */ .highlight .ow { color: #008800 } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */ .highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */ .highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */ .highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ .highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */ .highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */ .highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */ .highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */ .highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */ .highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ }
{{/*
# Copyright © 2019 Amdocs, Bell Canada
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/}}
{{- if .Values.backup.enabled }}
apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: {{ include "common.fullname" . }}-backup
  namespace: {{ include "common.namespace" . }}
  labels:
    app: {{ include "common.fullname" . }}
    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
    release: {{ include "common.release" . }}
    heritage: {{ .Release.Service }}
spec:
  schedule: {{ .Values.backup.cron | quote }}
  concurrencyPolicy: Forbid
  startingDeadlineSeconds: 120
  jobTemplate:
    spec:
      template:
        spec:
          restartPolicy: Never
          initContainers:
            - command:
              - /root/ready.py
              args:
              - --container-name
              - {{ include "common.name" . }}
              env:
              - name: NAMESPACE
                valueFrom:
                  fieldRef:
                    apiVersion: v1
                    fieldPath: metadata.namespace
              image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
              imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
              name: {{ include "common.name" . }}-readiness
            - name: "cassandra-backup-init"
              image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
              imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
              command:
              - /bin/bash
              - -c
              - |
                clearSnapshot(){
                  curr_time=$1
                  echo "Clearing snapshots!!!"
                  command="nodetool clearsnapshot -t $curr_time"
                  /root/exec.py -p "cassandra" -c "$command"
                }
                {{ $root := . }}
                curr_time=`date +%s`
                pids=""
                set -x

                echo "Copying data"
                {{ range $i, $e := until (int .Values.replicaCount) }}
                  target_dir=/backup/temp/cassandra-{{ $i }}
                  mkdir -p $target_dir
                  cp -Ra /onap-data/cassandra-{{ $i }}/data/ $target_dir/
                {{- end }}

                echo "Executing cleanup!!"
                command="nodetool cleanup"
                /root/exec.py -p "cassandra" -c "$command"
                echo "Cleaned Node!! Backing up database now!!!"

                command="nodetool snapshot -t $curr_time"
                /root/exec.py -p "cassandra" -c "$command"
                retCode=$?
                if [ $retCode -ne 0 ]; then
                  echo "Backup Failed!!!"
                  rm -rf /backup/temp
                  clearSnapshot $curr_time
                  echo "Failed" > /backup/backup.log
                  exit 0
                fi

                backup_dir=/backup/temp
                {{ range $i, $e := until (int .Values.replicaCount) }}
                  for d in $backup_dir/cassandra-{{ $i }}/data/*/ ; do
                    d=$(echo $d | sed 's:/*$::')
                    keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
                    if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
                      /root/restore.sh -b $backup_dir/cassandra-{{ $i }}/data -s  /onap-data/cassandra-{{ $i }}/data/$keyspace_name -k $keyspace_name -t $curr_time &
                      pids="$pids $!"
                    fi
                  done
                {{- end }}

                for p in $pids; do
                  wait $p
                  if [ $? -ne 0 ]; then
                    rm -rf /backup/temp
                    echo "Creation of Backup Failed!!!"
                    clearSnapshot $curr_time
                    echo "Failed" > /backup/backup.log
                    exit 0
                  fi
                done

                clearSnapshot $curr_time

                exit_code=$?
                if [ $exit_code -ne 0 ]; then
                  rm -rf /backup/temp
                  echo "Backup Failed!!!"
                  echo "Failed" > /backup/backup.log
                  exit 0
                fi

                mv /backup/temp /backup/backup-${curr_time}
                echo "Success" > /backup/backup.log
                echo "Cassandra Backup Succeeded"
              env:
              - name: NAMESPACE
                valueFrom:
                  fieldRef:
                    apiVersion: v1
                    fieldPath: metadata.namespace
              volumeMounts:
              - mountPath: /etc/localtime
                name: localtime
                readOnly: true
              {{- range $i := until (int .Values.replicaCount)}}
              - mountPath: /onap-data/cassandra-{{ $i }}
                name: data-dir-{{ $i }}
              {{- end }}
              - mountPath: /backup
                name: backup-dir
              - name: scripts
                mountPath: /root/restore.sh
                subPath: restore.sh
              - name: scripts
                mountPath: /root/exec.py
                subPath: exec.py
          containers:
            - name: cassandra-backup-validate
              image: "{{ .Values.image }}"
              imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
              command:
              - /bin/bash
              - -c
              - |
                remove_dir(){
                  dirToRemove=$1
                  rm -rf $dirToRemove
                }

                backup_result=`cat /backup/backup.log`
                rm -rf /backup/backup.log

                if [ "$backup_result" == "Failed" ]; then
                  echo "Backup Failed!!! So Validation Failed!!!";
                  exit 0
                fi

                target_dir=$(ls -td -- /backup/*/ | head -n 1)
                chown -R cassandra.cassandra $target_dir
                {{- $root := . -}}
                {{ range $i, $e := until (int .Values.replicaCount) }}
                  dbSize=$(du -ks $target_dir/cassandra-{{ $i }}/data|awk -F " " '{ printf $1 }')
                  minDbSize={{ (int $root.Values.backup.dbSize) }}
                  if [ $dbSize -lt $minDbSize ]; then
                    remove_dir $target_dir
                    echo "Validation Failed!!! dbSize ($dbSize) is less than minimum size (1)!!!"
                    exit 0
                  fi
                  rm -rf /var/lib/cassandra/*
                  cp -Ra $target_dir/cassandra-{{ $i }}/data /var/lib/cassandra
                  export CASSANDRA_LISTEN_ADDRESS="127.0.0.1"
                  /docker-entrypoint.sh -Dcassandra.ignore_dc=true -Dcassandra.ignore_rack=true &
                  CASS_PID=$!
                  sleep 45

                  for d in $target_dir/cassandra-{{ $i }}/data/*/; do
                      d=$(echo $d | sed 's:/*$::')
                      keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
                      if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
                        echo "Verifying the data for  $keyspace_name "
                        nodetool verify -e $keyspace_name
                        ret=$?
                        if [ $ret -ne 0 ]; then
                          remove_dir $target_dir
                          echo "Validation Failed!!!"
                          exit 0
                        fi
                      fi
                  done
                  kill -9 $CASS_PID
                {{- end }}
                echo "Validation Successful!!!"
                cd /backup
                totalFiles=`ls -t | grep "backup-" | wc -l`
                if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then
                  filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}`
                  ls -tr | grep backup | head -$filestoDelete | xargs rm -rf
                fi
              env:
              - name: CASSANDRA_CLUSTER_NAME
                value: {{ .Values.config.clusterName }}
              - name: MAX_HEAP_SIZE
                value: {{ .Values.config.heap.max }}
              - name: HEAP_NEWSIZE
                value: {{ .Values.config.heap.min }}
              - name: HOST_IP
                valueFrom:
                  fieldRef:
                    fieldPath: status.podIP
              volumeMounts:
              - name: backup-dir
                mountPath: /backup
              - name: localtime
                mountPath: /etc/localtime
                readOnly: true
          volumes:
          - name: localtime
            hostPath:
              path: /etc/localtime
          - name: scripts
            configMap:
              name: {{ include "common.fullname" . }}-configmap
              defaultMode: 0755
          {{- range $i := until (int .Values.replicaCount)}}
          - name: data-dir-{{ $i }}
            persistentVolumeClaim:
              claimName: {{ include "common.fullname" . }}-data-{{ $i }}
          {{- end }}
          - name: backup-dir
            persistentVolumeClaim:
              claimName: {{ include "common.fullname" . }}-backup-data
{{- end -}}