#!/bin/bash ############################################################################# # # Copyright © 2019 Amdocs. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ############################################################################# # v20190115 # https://wiki.onap.org/display/DW/ONAP+on+Kubernetes # source from https://jira.onap.org/browse/OOM-320, 326, 321, 898, 925 # Michael O'Brien # usage() { cat < ${FAILED_PODS_LIMIT} at the ${COUNTER}th 15 sec interval" echo "" COUNTER=$((COUNTER + 1 )) MAX_WAIT_PERIODS=$((MAX_WAIT_PERIODS - 1)) if [ "$MAX_WAIT_PERIODS" -eq 0 ]; then FAILED_PODS_LIMIT=800 fi done echo "report on non-running containers" PENDING=$(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3') PENDING_COUNT=$(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3' | wc -l) PENDING_COUNT_AAI=$(kubectl get pods -n $ENVIRON | grep aai- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_AAI" -gt 0 ]; then echo "down-aai=${PENDING_COUNT_AAI}" fi # todo don't stop if aai is down PENDING_COUNT_APPC=$(kubectl get pods -n $ENVIRON | grep appc- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_APPC" -gt 0 ]; then echo "down-appc=${PENDING_COUNT_APPC}" fi PENDING_COUNT_MR=$(kubectl get pods -n $ENVIRON | grep message-router- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_MR" -gt 0 ]; then echo "down-mr=${PENDING_COUNT_MR}" fi PENDING_COUNT_SO=$(kubectl get pods -n $ENVIRON | grep so- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_SO" -gt 0 ]; then echo "down-so=${PENDING_COUNT_SO}" fi PENDING_COUNT_POLICY=$(kubectl get pods -n $ENVIRON | grep policy- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_POLICY" -gt 0 ]; then echo "down-policy=${PENDING_COUNT_POLICY}" fi PENDING_COUNT_PORTAL=$(kubectl get pods -n $ENVIRON | grep portal- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_PORTAL" -gt 0 ]; then echo "down-portal=${PENDING_COUNT_PORTAL}" fi PENDING_COUNT_LOG=$(kubectl get pods -n $ENVIRON | grep log- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_LOG" -gt 0 ]; then echo "down-log=${PENDING_COUNT_LOG}" fi PENDING_COUNT_ROBOT=$(kubectl get pods -n $ENVIRON | grep robot- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_ROBOT" -gt 0 ]; then echo "down-robot=${PENDING_COUNT_ROBOT}" fi PENDING_COUNT_SDC=$(kubectl get pods -n $ENVIRON | grep sdc- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_SDC" -gt 0 ]; then echo "down-sdc=${PENDING_COUNT_SDC}" fi PENDING_COUNT_SDNC=$(kubectl get pods -n $ENVIRON | grep sdnc- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_SDNC" -gt 0 ]; then echo "down-sdnc=${PENDING_COUNT_SDNC}" fi PENDING_COUNT_VID=$(kubectl get pods -n $ENVIRON | grep vid- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_VID" -gt 0 ]; then echo "down-vid=${PENDING_COUNT_VID}" fi PENDING_COUNT_AAF=$(kubectl get pods -n $ENVIRON | grep aaf- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_AAF" -gt 0 ]; then echo "down-aaf=${PENDING_COUNT_AAF}" fi PENDING_COUNT_CONSUL=$(kubectl get pods -n $ENVIRON | grep consul- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_CONSUL" -gt 0 ]; then echo "down-consul=${PENDING_COUNT_CONSUL}" fi PENDING_COUNT_MSB=$(kubectl get pods -n $ENVIRON | grep msb- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_MSB" -gt 0 ]; then echo "down-msb=${PENDING_COUNT_MSB}" fi PENDING_COUNT_DCAE=$(kubectl get pods -n $ENVIRON | grep dcaegen2- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_DCAE" -gt 0 ]; then echo "down-dcae=${PENDING_COUNT_DCAE}" fi PENDING_COUNT_CLI=$(kubectl get pods -n $ENVIRON | grep cli- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_CLI" -gt 0 ]; then echo "down-cli=${PENDING_COUNT_CLI}" fi PENDING_COUNT_MULTICLOUD=$(kubectl get pods -n $ENVIRON | grep multicloud- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_MULTICLOUD" -gt 0 ]; then echo "down-multicloud=${PENDING_COUNT_MULTICLOUD}" fi PENDING_COUNT_CLAMP=$(kubectl get pods -n $ENVIRON | grep clamp- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_CLAMP" -gt 0 ]; then echo "down-clamp=${PENDING_COUNT_CLAMP}" fi PENDING_COUNT_VNFSDK=$(kubectl get pods -n $ENVIRON | grep vnfsdk- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_VNFSDK" -gt 0 ]; then echo "down-vnfsdk=${PENDING_COUNT_VNFSDK}" fi PENDING_COUNT_UUI=$(kubectl get pods -n $ENVIRON | grep uui- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_UUI" -gt 0 ]; then echo "down-uui=${PENDING_COUNT_UUI}" fi PENDING_COUNT_VFC=$(kubectl get pods -n $ENVIRON | grep vfc- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_VFC" -gt 0 ]; then echo "down-vfc=${PENDING_COUNT_VFC}" fi PENDING_COUNT_KUBE2MSB=$(kubectl get pods -n $ENVIRON | grep kube2msb- | grep -E '0/|1/2|1/3|2/3' | wc -l) if [ "$PENDING_COUNT_KUBE2MSB" -gt 0 ]; then echo "down-kube2msb=${PENDING_COUNT_KUBE2MSB}" fi echo "pending containers=${PENDING_COUNT}" echo "${PENDING}" echo "check filebeat 2/2|3/3 count for ELK stack logging consumption" FILEBEAT=$(kubectl get pods --all-namespaces -a | grep -E '2/|3/') echo "${FILEBEAT}" echo "sleep 5 min - to allow rest frameworks to finish" sleep 300 echo "List of ONAP Modules" LIST_ALL=$(kubectl get pods --all-namespaces -a -o wide ) echo "${LIST_ALL}" echo "run healthcheck 2 times to warm caches and frameworks so rest endpoints report properly - see OOM-447" echo "curl with aai cert to cloud-region PUT" curl -X PUT https://127.0.0.1:30233/aai/v11/cloud-infrastructure/cloud-regions/cloud-region/CloudOwner/RegionOne --data "@aai-cloud-region-put.json" -H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" -H "X-TransactionId:jimmy-postman" -H "X-FromAppId:AAI" -H "Content-Type:application/json" -H "Accept:application/json" --cacert aaiapisimpledemoopenecomporg_20171003.crt -k echo "get the cloud region back" curl -X GET https://127.0.0.1:30233/aai/v11/cloud-infrastructure/cloud-regions/ -H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" -H "X-TransactionId:jimmy-postman" -H "X-FromAppId:AAI" -H "Content-Type:application/json" -H "Accept:application/json" --cacert aaiapisimpledemoopenecomporg_20171003.crt -k # OOM-484 - robot scripts moved cd oom/kubernetes/robot echo "run healthcheck prep 1" # OOM-722 adds namespace parameter if [ "$BRANCH" == "amsterdam" ]; then ./ete-k8s.sh health > ~/health1.out else ./ete-k8s.sh $ENVIRON health > ~/health1.out fi echo "sleep 5 min" sleep 300 echo "run healthcheck prep 2" if [ "$BRANCH" == "amsterdam" ]; then ./ete-k8s.sh health > ~/health2.out else ./ete-k8s.sh $ENVIRON health > ~/health2.out fi echo "run healthcheck for real - wait a further 5 min" sleep 300 if [ "$BRANCH" == "amsterdam" ]; then ./ete-k8s.sh health else ./ete-k8s.sh $ENVIRON health fi echo "run partial vFW" echo "report results" cd ../../../ echo "$(date)" #set +a } BRANCH= ENVIRON=onap FULL_MANAGED_DEPLOY=true APPLY_WORKAROUNDS=true DELETE_PREV_OOM=false REMOVE_OOM_AT_END=false CLONE_NEW_OOM=true SPLIT_DEPLOY_DELAY=600 DOCKER_PREPULL=false NEXUS3_AND_PORT=nexus3.onap.org:10001 while getopts ":u:b:e:p:n:s:f:c:d:w:r" PARAM; do case $PARAM in u) usage exit 1 ;; b) BRANCH=${OPTARG} ;; e) ENVIRON=${OPTARG} ;; p) DOCKER_PREPULL=${OPTARG} ;; n) NEXUS3_AND_PORT=${OPTARG} ;; s) SPLIT_DEPLOY_DELAY=${OPTARG} ;; f) FULL_MANAGED_DEPLOY=${OPTARG} ;; c) CLONE_NEW_OOM=${OPTARG} ;; d) DELETE_PREV_OOM=${OPTARG} ;; w) APPLY_WORKAROUNDS=${OPTARG} ;; r) REMOVE_OOM_AT_END=${OPTARG} ;; ?) usage exit ;; esac done if [[ -z $BRANCH ]]; then usage exit 1 fi deploy_onap $BRANCH $ENVIRON $DOCKER_PREPULL $NEXUS3_AND_PORT $SPLIT_DEPLOY_DELAY $FULL_MANAGED_DEPLOY $CLONE_NEW_OOM $DELETE_PREV_OOM $APPLY_WORKAROUNDS $REMOVE_OOM_AT_END printf "**** Done ****\n"