From 6f718d2cd042390c42c6e4b728131ef8161d1c7a Mon Sep 17 00:00:00 2001 From: Michael O'Brien Date: Mon, 14 May 2018 20:25:45 -0400 Subject: cd.sh oom deployment script Change-Id: Ie0a3959fee80fb9c01dfdc2f7e26efc3e7617a68 Signed-off-by: Michael O'Brien Issue-ID: LOG-326 --- deploy/cd.sh | 365 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100755 deploy/cd.sh diff --git a/deploy/cd.sh b/deploy/cd.sh new file mode 100755 index 0000000..5956b1c --- /dev/null +++ b/deploy/cd.sh @@ -0,0 +1,365 @@ +#!/bin/bash +############################################################################# +# +# Copyright © 2018 Amdocs, Bell. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################# +# v20180318 +# https://wiki.onap.org/display/DW/ONAP+on+Kubernetes +# source from https://jira.onap.org/browse/OOM-715, 716, 711 +# Michael O'Brien +# + +usage() { + cat < ${FAILED_PODS_LIMIT} at the ${COUNTER}th 15 sec interval" + echo "" + COUNTER=$((COUNTER + 1 )) + MAX_WAIT_PERIODS=$((MAX_WAIT_PERIODS - 1)) + if [ "$MAX_WAIT_PERIODS" -eq 0 ]; then + FAILED_PODS_LIMIT=800 + fi + done + + echo "report on non-running containers" + PENDING=$(kubectl get pods --all-namespaces | grep -E '0/|1/2') + PENDING_COUNT=$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l) + PENDING_COUNT_AAI=$(kubectl get pods -n $ENVIRON | grep aai- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_AAI" -gt 0 ]; then + echo "down-aai=${PENDING_COUNT_AAI}" + fi + + # todo don't stop if aai is down + PENDING_COUNT_APPC=$(kubectl get pods -n $ENVIRON | grep appc- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_APPC" -gt 0 ]; then + echo "down-appc=${PENDING_COUNT_APPC}" + fi + PENDING_COUNT_MR=$(kubectl get pods -n $ENVIRON | grep message-router- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_MR" -gt 0 ]; then + echo "down-mr=${PENDING_COUNT_MR}" + fi + PENDING_COUNT_SO=$(kubectl get pods -n $ENVIRON | grep so- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_SO" -gt 0 ]; then + echo "down-so=${PENDING_COUNT_SO}" + fi + PENDING_COUNT_POLICY=$(kubectl get pods -n $ENVIRON | grep policy- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_POLICY" -gt 0 ]; then + echo "down-policy=${PENDING_COUNT_POLICY}" + fi + PENDING_COUNT_PORTAL=$(kubectl get pods -n $ENVIRON | grep portal- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_PORTAL" -gt 0 ]; then + echo "down-portal=${PENDING_COUNT_PORTAL}" + fi + PENDING_COUNT_LOG=$(kubectl get pods -n $ENVIRON | grep log- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_LOG" -gt 0 ]; then + echo "down-log=${PENDING_COUNT_LOG}" + fi + PENDING_COUNT_ROBOT=$(kubectl get pods -n $ENVIRON | grep robot- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_ROBOT" -gt 0 ]; then + echo "down-robot=${PENDING_COUNT_ROBOT}" + fi + PENDING_COUNT_SDC=$(kubectl get pods -n $ENVIRON | grep sdc- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_SDC" -gt 0 ]; then + echo "down-sdc=${PENDING_COUNT_SDC}" + fi + PENDING_COUNT_SDNC=$(kubectl get pods -n $ENVIRON | grep sdnc- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_SDNC" -gt 0 ]; then + echo "down-sdnc=${PENDING_COUNT_SDNC}" + fi + PENDING_COUNT_VID=$(kubectl get pods -n $ENVIRON | grep vid- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_VID" -gt 0 ]; then + echo "down-vid=${PENDING_COUNT_VID}" + fi + + PENDING_COUNT_AAF=$(kubectl get pods -n $ENVIRON | grep aaf- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_AAF" -gt 0 ]; then + echo "down-aaf=${PENDING_COUNT_AAF}" + fi + PENDING_COUNT_CONSUL=$(kubectl get pods -n $ENVIRON | grep consul- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_CONSUL" -gt 0 ]; then + echo "down-consul=${PENDING_COUNT_CONSUL}" + fi + PENDING_COUNT_MSB=$(kubectl get pods -n $ENVIRON | grep msb- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_MSB" -gt 0 ]; then + echo "down-msb=${PENDING_COUNT_MSB}" + fi + PENDING_COUNT_DCAE=$(kubectl get pods -n $ENVIRON | grep dcaegen2- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_DCAE" -gt 0 ]; then + echo "down-dcae=${PENDING_COUNT_DCAE}" + fi + PENDING_COUNT_CLI=$(kubectl get pods -n $ENVIRON | grep cli- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_CLI" -gt 0 ]; then + echo "down-cli=${PENDING_COUNT_CLI}" + fi + PENDING_COUNT_MULTICLOUD=$(kubectl get pods -n $ENVIRON | grep multicloud- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_MULTICLOUD" -gt 0 ]; then + echo "down-multicloud=${PENDING_COUNT_MULTICLOUD}" + fi + PENDING_COUNT_CLAMP=$(kubectl get pods -n $ENVIRON | grep clamp- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_CLAMP" -gt 0 ]; then + echo "down-clamp=${PENDING_COUNT_CLAMP}" + fi + PENDING_COUNT_VNFSDK=$(kubectl get pods -n $ENVIRON | grep vnfsdk- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_VNFSDK" -gt 0 ]; then + echo "down-vnfsdk=${PENDING_COUNT_VNFSDK}" + fi + PENDING_COUNT_UUI=$(kubectl get pods -n $ENVIRON | grep uui- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_UUI" -gt 0 ]; then + echo "down-uui=${PENDING_COUNT_UUI}" + fi + PENDING_COUNT_VFC=$(kubectl get pods -n $ENVIRON | grep vfc- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_VFC" -gt 0 ]; then + echo "down-vfc=${PENDING_COUNT_VFC}" + fi + PENDING_COUNT_KUBE2MSB=$(kubectl get pods -n $ENVIRON | grep kube2msb- | grep -E '0/|1/2' | wc -l) + if [ "$PENDING_COUNT_KUBE2MSB" -gt 0 ]; then + echo "down-kube2msb=${PENDING_COUNT_KUBE2MSB}" + fi + echo "pending containers=${PENDING_COUNT}" + echo "${PENDING}" + + echo "check filebeat 2/2 count for ELK stack logging consumption" + FILEBEAT=$(kubectl get pods --all-namespaces -a | grep 2/) + echo "${FILEBEAT}" + echo "sleep 5 min - to allow rest frameworks to finish" + sleep 300 + echo "List of ONAP Modules" + LIST_ALL=$(kubectl get pods --all-namespaces -a --show-all ) + echo "${LIST_ALL}" + echo "run healthcheck 2 times to warm caches and frameworks so rest endpoints report properly - see OOM-447" + + echo "curl with aai cert to cloud-region PUT" + + curl -X PUT https://127.0.0.1:30233/aai/v11/cloud-infrastructure/cloud-regions/cloud-region/CloudOwner/RegionOne --data "@aai-cloud-region-put.json" -H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" -H "X-TransactionId:jimmy-postman" -H "X-FromAppId:AAI" -H "Content-Type:application/json" -H "Accept:application/json" --cacert aaiapisimpledemoopenecomporg_20171003.crt -k + + echo "get the cloud region back" + curl -X GET https://127.0.0.1:30233/aai/v11/cloud-infrastructure/cloud-regions/ -H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" -H "X-TransactionId:jimmy-postman" -H "X-FromAppId:AAI" -H "Content-Type:application/json" -H "Accept:application/json" --cacert aaiapisimpledemoopenecomporg_20171003.crt -k + + # OOM-484 - robot scripts moved + cd oom/kubernetes/robot + echo "run healthcheck prep 1" + # OOM-722 adds namespace parameter + if [ "$BRANCH" == "amsterdam" ]; then + ./ete-k8s.sh health > ~/health1.out + else + ./ete-k8s.sh $ENVIRON health > ~/health1.out + fi + echo "sleep 5 min" + sleep 300 + echo "run healthcheck prep 2" + if [ "$BRANCH" == "amsterdam" ]; then + ./ete-k8s.sh health > ~/health2.out + else + ./ete-k8s.sh $ENVIRON health > ~/health2.out + fi + echo "run healthcheck for real - wait a further 5 min" + sleep 300 + if [ "$BRANCH" == "amsterdam" ]; then + ./ete-k8s.sh health + else + ./ete-k8s.sh $ENVIRON health + fi + echo "run partial vFW" +# sudo chmod 777 /dockerdata-nfs/onap +# if [ "$BRANCH" == "amsterdam" ]; then +# ./demo-k8s.sh init_robot +# else +# ./demo-k8s.sh $ENVIRON init +# fi +# if [ "$BRANCH" == "amsterdam" ]; then +# ./demo-k8s.sh init +# else +# ./demo-k8s.sh $ENVIRON init +# fi + echo "report results" + cd ../../../ + + echo "$(date)" + #set +a +} + +BRANCH= +ENVIRON=onap +APPLY_WORKAROUNDS=true +DELETE_PREV_OOM=false +REMOVE_OOM_AT_END=false +CLONE_NEW_OOM=true + +while getopts ":u:b:e:c:d:w:r" PARAM; do + case $PARAM in + u) + usage + exit 1 + ;; + b) + BRANCH=${OPTARG} + ;; + e) + ENVIRON=${OPTARG} + ;; + c) + CLONE_NEW_OOM=${OPTARG} + ;; + d) + DELETE_PREV_OOM=${OPTARG} + ;; + w) + APPLY_WORKAROUNDS=${OPTARG} + ;; + r) + REMOVE_OOM_AT_END=${OPTARG} + ;; + ?) + usage + exit + ;; + esac +done + +if [[ -z $BRANCH ]]; then + usage + exit 1 +fi + +deploy_onap $BRANCH $ENVIRON $CLONE_NEW_OOM $DELETE_PREV_OOM $APPLY_WORKAROUNDS $REMOVE_OOM_AT_END + +printf "**** Done ****\n" -- cgit 1.2.3-korg