diff options
| -rwxr-xr-x | tools/helm-healer.sh | 537 |
1 files changed, 537 insertions, 0 deletions
diff --git a/tools/helm-healer.sh b/tools/helm-healer.sh new file mode 100755 index 00000000..b030fcac --- /dev/null +++ b/tools/helm-healer.sh @@ -0,0 +1,537 @@ +#!/bin/sh + +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# +# globals and defaults +# + +NAMESPACE= +OVERRIDES= +HELM_CHART_RELEASE_NAME= +HELM_DELETE_ALL= +HELM_SKIP_DEPLOY= +VOLUME_STORAGE= +HELM_TIMEOUT=3600 +RELEASE_PREFIX=onap + +# +# control variables +# + +CMD=$(basename "$0") +COLOR_ON_RED='\033[0;31;1m' +COLOR_ON_GREEN='\033[0;32;1m' +COLOR_OFF='\033[0m' + + +# +# functions +# + +help() +{ +cat <<EOF +${CMD} - simple tool for fixing onap helm deployment + +DESCRIPTION + This script does nothing smart or special it just tries to + redeploy onap component. It can fix only problems related to + race conditions or timeouts. Nothing else. It will not fix + broken ONAP - there is no such ambition - that effort should + be directed in the upstream. + +USAGE + ${CMD} -h|--help + This help + + ${CMD} -n|--namespace <namespace> + (-f|--file <override>)... + (-s|--storage <directory>)|--no-storage-deletion + [-p|--release-prefix <release prefix>] + [-t|--timeout <secs>] + [(-c|--component <component release name>)...| + (-D|--delete-all)] + [-C|--clean-only] + + Usage 1 (simple heuristics - redeploy failed components): + ${CMD} -n onap -f /some/override1.yml -s /dockerdata-nfs + + Usage 2 (redeploy ONLY explicit listed components): + ${CMD} -n onap -f /some/override1.yml -s /dockerdata-nfs \ + -c onap-aaf -c onap-sdc -c onap-portal + + Usage 3 (delete EVERYTHING and redeploy): + ${CMD} -n onap -f /some/override1.yml -s /dockerdata-nfs \ + --delete-all + + Usage 4 (just clean - do not redeploy) + ${CMD} -n onap -f /some/override1.yml -s /dockerdata-nfs \ + --delete-all --clean-only + + Namespace argument and at least one override file are mandatory + for this script to execute. Also you must provide path to the + storage or explicitly request to not delete file storage of the + component. + + Storage should be directory where persistent volume resides. It + will work only if component created a persistent volume with the + same filename as its release name. Otherwise no effect. The + exception is when '--delete-all' is used - in that case all + content of the storage is deleted (because ONAP is not consistent + with the volume directory names - eg.: sdnc). + + CAUTION 1: filename of an override file cannot contain whitespace! + This is actually helm/onap deploy plugin issue which does not + handle such files. So I dropped the more complicated version of + this script when there is no reason to support something on what + will helm deploy choke anyway. + + '--prefix' option is helm release argument - it is actually prefix + when you list the helm releases - helm is little confusing here. + + CAUTION 2: By default release prefix is 'onap' - if you deployed + release 'onap' and now run this script with different prefix then + it will skip all 'onap-*' components and will deploy a new release + with new prefix - BEWARE TO USE PROPER RELEASE PREFIX! + + Timeout set the waiting time for helm deploy per component. + + '--component' references to release name of the chart which you + want to redeploy excplicitly - otherwise 'ALL FAILED' components + will be redeployed. You can target more than one component at once + - just use the argument multiple times. + + Component option is mutually exclusive with the '--delete-all' + which will delete all components - healthy or not. Actually it will + delete the whole NAMESPACE and everything in it. + + '--clean-only' can be used with any usage: heuristics, explicit + component list or with '--delete-all'. It basically just skips the + last step - the actual redeploy. +EOF +} + +msg() +{ + echo -e "${COLOR_ON_GREEN}INFO: $@ ${COLOR_OFF}" +} + +error() +{ + echo -e "${COLOR_ON_RED}ERROR: $@ ${COLOR_OFF}" +} + +# remove all successfully completed jobs +clean_jobs() +{ + kubectl get jobs -n ${NAMESPACE} \ + --ignore-not-found=true \ + --no-headers=true | \ + while read -r _job _completion _duration _age ; do + _done=$(echo ${_completion} | awk 'BEGIN {FS="/";} {print $1;}') + _desired=$(echo ${_completion} | awk 'BEGIN {FS="/";} {print $2;}') + if [ "$_desired" -eq "$_done" ] ; then + delete_job "$_job" + fi + done +} + +get_failed_labels() +{ + get_labels 'status.phase==Failed' +} + +# arg: [optional: selector] +get_labels() +{ + if [ -n "$1" ] ; then + _selector="--field-selector=${1}" + else + _selector= + fi + + kubectl get pods -n ${NAMESPACE} \ + --show-labels=true \ + --include-uninitialized=true \ + ${_selector} \ + --ignore-not-found=true \ + --no-headers=true | \ + while read -r _pod _ready _status _restart _age _labels ; do + [ -z "$_labels" ] && break + for _label in $(echo "$_labels" | tr ',' ' ') ; do + case "$_label" in + release=*) + _label=$(echo "$_label" | sed 's/release=//') + echo "$_label" + ;; + esac + done + done | sort -u +} + +# arg: <release name> +helm_undeploy() +{ + msg "Undeploy helm release name: ${1}" + helm undeploy ${1} --purge +} + +# arg: <job name> +delete_job() +{ + kubectl delete job -n ${NAMESPACE} \ + --cascade=true \ + --now=true \ + --include-uninitialized=true \ + --wait=true \ + ${1} + + # wait for job to be deleted + _output=start + while [ -n "$_output" ] && sleep 1 ; do + _output=$(kubectl get pods -n ${NAMESPACE} \ + --ignore-not-found=true \ + --no-headers=true \ + --selector="job-name=${1}") + done +} + +# arg: <resource> <release name> +delete_resource() +{ + _resource="$1" + _release="$2" + + msg "Delete ${_resource} for ${_release}..." + { + kubectl get ${_resource} -n ${NAMESPACE} \ + --ignore-not-found=true \ + --selector="release=${_release}" \ + --no-headers=true + + # this is due to missing "release" label in some pods + # grep for the rescue... + kubectl get ${_resource} -n ${NAMESPACE} \ + --no-headers=true | grep "^${_release}" + } | awk '{print $1}' | sort -u | while read -r _name _rest ; do + echo "Deleting '${_name}'" + kubectl delete ${_resource} -n ${NAMESPACE} \ + --cascade=true \ + --now=true \ + --include-uninitialized=true \ + --wait=true \ + ${_name} \ + 2>&1 | grep -iv 'not[[:space:]]*found' + + # wait for resource to be deleted + _output=start + while [ -n "$_output" ] && sleep 1 ; do + _output=$(kubectl get ${_resource} -n ${NAMESPACE} \ + --ignore-not-found=true \ + --no-headers=true \ + --field-selector="metadata.name=${_name}") + done + done +} + +delete_namespace() +{ + msg "Delete the whole namespace: ${NAMESPACE}" + kubectl delete namespace \ + --cascade=true \ + --now=true \ + --include-uninitialized=true \ + --wait=true \ + "$NAMESPACE" + + # wait for namespace to be deleted + _output=start + while [ -n "$_output" ] && sleep 1 ; do + _output=$(kubectl get all -n ${NAMESPACE} \ + --ignore-not-found=true \ + --no-headers=true) + done +} + +# arg: [optional: subdir] +delete_storage() +{ + _node=$(kubectl get nodes \ + --selector=node-role.kubernetes.io/worker \ + -o wide \ + --no-headers=true | \ + awk '{print $6}' | head -n 1) + + if [ -z "$_node" ] ; then + error "Could not list kubernetes nodes - SKIPPING DELETION" + else + if [ -n "$1" ] ; then + msg "Delete directory '${VOLUME_STORAGE}/${1}' on $_node" + ssh -T $_node <<EOF +rm -rf "${VOLUME_STORAGE}/${1}" +EOF + else + msg "Delete directories '${VOLUME_STORAGE}/*' on $_node" + ssh -T $_node <<EOF +find "${VOLUME_STORAGE}" -maxdepth 1 -mindepth 1 -exec rm -rf '{}' \; +EOF + fi + fi +} + +# arg: <release name> +redeploy_component() +{ + _chart=$(echo "$1" | sed 's/[^-]*-//') + helm_undeploy ${1} + # TODO: does deleted secret per component break something? + for x in jobs deployments pods pvc pv ; do + delete_resource ${x} ${1} + done + + if [ -n "$VOLUME_STORAGE" ] ; then + msg "Persistent volume data deletion in directory: ${VOLUME_STORAGE}/${1}" + delete_storage "$1" + fi + + # TODO: until I can verify that this does the same for this component as helm deploy + #msg "Redeployment of the component ${1}..." + #helm install "local/${_chart}" --name ${1} --namespace ${NAMESPACE} --wait --timeout ${HELM_TIMEOUT} +} + + +# +# arguments +# + +state=nil +arg_namespace= +arg_overrides= +arg_timeout= +arg_storage= +arg_nostorage= +arg_components= +arg_prefix= +arg_deleteall= +arg_cleanonly= +while [ -n "$1" ] ; do + case $state in + nil) + case "$1" in + -h|--help) + help + exit 0 + ;; + -n|--namespace) + state=namespace + ;; + -f|--file) + state=override + ;; + -t|--timeout) + state=timeout + ;; + -s|--storage) + state=storage + ;; + --no-storage-deletion) + if [ -n "$arg_storage" ] ; then + error "Usage of storage argument together with no storage deletion option!" + exit 1 + elif [ -z "$arg_nostorage" ] ; then + arg_nostorage=nostorage + else + error "Duplicit argument for no storage option! (IGNORING)" + fi + ;; + -c|--component) + if [ -n "$arg_deleteall" ] ; then + error "'Delete all components' used already - argument mismatch" + exit 1 + fi + state=component + ;; + -D|--delete-all) + if [ -n "$arg_components" ] ; then + error "Explicit component(s) provided already - argument mismatch" + exit 1 + elif [ -z "$arg_deleteall" ] ; then + arg_deleteall=deleteall + else + error "Duplicit argument for 'delete all' option! (IGNORING)" + fi + ;; + -p|--prefix) + state=prefix + ;; + -C|--clean-only) + if [ -z "$arg_cleanonly" ] ; then + arg_cleanonly=cleanonly + else + error "Duplicit argument for 'clean only' option! (IGNORING)" + fi + ;; + *) + error "Unknown parameter: $1" + exit 1 + ;; + esac + ;; + namespace) + if [ -z "$arg_namespace" ] ; then + arg_namespace="$1" + state=nil + else + error "Duplicit argument for namespace!" + exit 1 + fi + ;; + override) + if ! [ -f "$1" ] ; then + error "Wrong filename for override file: $1" + exit 1 + fi + arg_overrides="${arg_overrides} -f $1" + state=nil + ;; + component) + arg_components="${arg_components} $1" + state=nil + ;; + prefix) + if [ -z "$arg_prefix" ] ; then + arg_prefix="$1" + state=nil + else + error "Duplicit argument for release prefix!" + exit 1 + fi + ;; + timeout) + if [ -z "$arg_timeout" ] ; then + if ! echo "$1" | grep -q '^[0-9]\+$' ; then + error "Timeout must be an integer: $1" + exit 1 + fi + arg_timeout="$1" + state=nil + else + error "Duplicit argument for timeout!" + exit 1 + fi + ;; + storage) + if [ -n "$arg_nostorage" ] ; then + error "Usage of storage argument together with no storage deletion option!" + exit 1 + elif [ -z "$arg_storage" ] ; then + arg_storage="$1" + state=nil + else + error "Duplicit argument for storage!" + exit 1 + fi + ;; + esac + shift +done + +# sanity check +if [ -z "$arg_namespace" ] ; then + error "Missing namespace" + help + exit 1 +else + NAMESPACE="$arg_namespace" +fi + +if [ -z "$arg_overrides" ] ; then + error "Missing override file(s)" + help + exit 1 +else + OVERRIDES="$arg_overrides" +fi + +if [ -n "$arg_prefix" ] ; then + RELEASE_PREFIX="$arg_prefix" +fi + +if [ -n "$arg_timeout" ] ; then + HELM_TIMEOUT="$arg_timeout" +fi + +if [ -n "$arg_storage" ] ; then + VOLUME_STORAGE="$arg_storage" +elif [ -z "$arg_nostorage" ] ; then + error "Missing storage argument! If it is intended then use '--no-storage-deletion' option" + exit 1 +fi + +if [ -n "$arg_components" ] ; then + HELM_CHART_RELEASE_NAME="$arg_components" +fi + +if [ -n "$arg_deleteall" ] ; then + HELM_DELETE_ALL=yes +fi + +if [ -n "$arg_cleanonly" ] ; then + HELM_SKIP_DEPLOY=yes +fi + + +# +# main +# + +# if --delete-all is used then redeploy all components (the current namespace is deleted) +if [ -n "$HELM_DELETE_ALL" ] ; then + # undeploy helm release (prefix) + helm_undeploy "$RELEASE_PREFIX" + + # we will delete the whole namespace + delete_namespace + + if [ -n "$VOLUME_STORAGE" ] ; then + delete_storage + fi +# delete and redeploy explicit or failed components... +else + # if a helm chart release name was given then just redeploy said component and quit + if [ -n "$HELM_CHART_RELEASE_NAME" ] ; then + msg "Explicitly asked for component redeploy: ${HELM_CHART_RELEASE_NAME}" + _COMPONENTS="$HELM_CHART_RELEASE_NAME" + # simple heuristics: redeploy only failed components + else + msg "Delete successfully completed jobs..." + clean_jobs + + msg "Find failed components..." + _COMPONENTS=$(get_failed_labels) + fi + + for _component in ${_COMPONENTS} ; do + if echo "$_component" | grep -q "^${RELEASE_PREFIX}-" ; then + msg "Redeploy component: ${_component}" + redeploy_component ${_component} + else + error "Component release name '${_component}' does not match release prefix: ${RELEASE_PREFIX} (SKIP)" + fi + done +fi + +if [ -z "$HELM_SKIP_DEPLOY" ] ; then + # TODO: this is suboptimal - find a way how to deploy only the affected component... + msg "Redeploy onap..." + msg helm deploy ${RELEASE_PREFIX} local/onap --namespace ${NAMESPACE} ${OVERRIDES} --timeout ${HELM_TIMEOUT} + helm deploy ${RELEASE_PREFIX} local/onap --namespace ${NAMESPACE} ${OVERRIDES} --timeout ${HELM_TIMEOUT} +else + msg "Clean only option used: Skipping redeploy..." +fi + +msg DONE + +exit $? + |