adjust cd.sh for managed seq deploy

Change-Id: I1d137b562c02b77bc2e0cb855b5953c59219e4e6
Issue-ID: LOG-898
Signed-off-by: Michael O'Brien <michael@obrienlabs.org>

1 files changed, 194 insertions, 79 deletions

diff --git a/deploy/cd.sh b/deploy/cd.sh
index 616ba81..a6b385d 100755
--- a/deploy/cd.sh
+++ b/deploy/cd.sh
@@ -15,9 +15,9 @@
 # limitations under the License.
 #
 #############################################################################
-# v20181226
+# v20100109
 # https://wiki.onap.org/display/DW/ONAP+on+Kubernetes
-# source from https://jira.onap.org/browse/OOM-320, 326, 321
+# source from https://jira.onap.org/browse/OOM-320, 326, 321, 898
 # Michael O'Brien
 #
 
@@ -27,12 +27,15 @@ Usage: $0 [PARAMs]
 example 
 ./cd.sh -b amsterdam -e onap (will rerun onap in the onap namespace, no new repo, no deletion of existing repo, no sdnc workaround, no onap removal at the end
 ./cd.sh -b master -e onap -s 500 -c true -d true -w true -r true (run as cd server, new oom, delete prev oom, run workarounds, clean onap at the end of the script
-./cd.sh -b master -e onap -s 600 -c true -d false -w true -r false (standard new server/dev environment - use this as the default)
+./cd.sh -b 3.0.0-ONAP -e onap -p true -n nexus3.onap.org:10001 -f true -s 600 -c true -d false -w true -r false (standard new server/dev environment - use this as the default)
 provide a dev0.yaml/dev1.yaml override set (0=platform, 1=rest of pods) - copy from https://git.onap.org/oom/tree/kubernetes/onap/resources/environments/dev.yaml
-
+ note: the managed deploy where -f is true - deploys all of ONAP - for a subset driven by the enabled flags in the dev yaml - use -f false - as the --set enabled flags override in the prior case
 -u                  : Display usage
 -b [branch]         : branch = master/beijing or amsterdam (required)
 -e [environment]    : use the default (onap)
+-p [true|false]     : docker prepull (default false)
+-n [nexus3 url:port]: nexus3.onap.org:10001 or proxy - used in prepull
+-f [true|false]     : managed deploy - time sequenced - defaults to a full deploy
 -s [seconds]        : delay between base and rest of onap dual-deployments based on dev0 and dev1.yaml
 -c [true|false]     : FLAG clone new oom repo (default: true)
 -d [true|false]     : FLAG delete prev oom - (cd build) (default: false)
@@ -41,16 +44,48 @@ provide a dev0.yaml/dev1.yaml override set (0=platform, 1=rest of pods) - copy f
 EOF
 }
 
+wait_for_pod() {
+      local RUNNING_PODS_LIMIT=$2
+      local MAX_WAIT_PERIODS=140 # 35 MIN
+      local COUNTER=1
+      local PENDING_PODS=0
+      local TARGET_POD_PREFIX=$1
+      local PENDING=0
+      while [  $PENDING -lt $RUNNING_PODS_LIMIT  ]; do
+        PENDING=$(kubectl get pods --all-namespaces | grep $TARGET_POD_PREFIX | grep -E '1/1|2/2|1/3|2/3' | wc -l)
+        PENDING_PODS=$PENDING
+        sleep 15
+        LIST_PENDING=$(kubectl get pods --all-namespaces -o wide | grep $TARGET_POD_PREFIX | grep -E '1/1|2/2|1/3|2/3' )
+        echo "${PENDING} running < ${RUNNING_PODS_LIMIT} at the ${COUNTER}th 15 sec interval for $TARGET_POD_PREFIX"
+        COUNTER=$((COUNTER + 1 ))
+        MAX_WAIT_PERIODS=$((MAX_WAIT_PERIODS - 1))
+        if [ "$MAX_WAIT_PERIODS" -eq 0 ]; then
+          PENDING=800
+        fi
+     done
+     dt="$(date +"%T")"
+     echo "$dt: ${PENDING} pods are up (1/1|2/2|3/3) for $TARGET_POD_PREFIX at the ${COUNTER}th 15 sec interval"
+     echo "deployments: note order: helm list"
+     sudo helm list
+     kubectl get pods --all-namespaces -o wide
+}
+
 deploy_onap() {
-  
   echo "$(date)"
-  echo "running with: -b $BRANCH -e $ENVIRON -s $SPLIT_DEPLOY_DELAY -c $CLONE_NEW_OOM -d $DELETE_PREV_OOM -w $APPLY_WORKAROUNDS -r $REMOVE_OOM_AT_END"
+  echo "running with: -b $BRANCH -e $ENVIRON -p $DOCKER_PREPULL -n $NEXUS3_AND_PORT -f $FULL_MANAGED_DEPLOY -s $SPLIT_DEPLOY_DELAY -c $CLONE_NEW_OOM -d $DELETE_PREV_OOM -w $APPLY_WORKAROUNDS -r $REMOVE_OOM_AT_END"
   echo "provide onap-parameters.yaml(amsterdam) or dev0.yaml+dev1.yaml (master) and aai-cloud-region-put.json"
   echo "provide a dev0.yaml and dev1.yaml override (0=platform, 1=rest of pods) - copy from https://git.onap.org/oom/tree/kubernetes/onap/resources/environments/dev.yaml"
+
+  if [[ "$BRANCH" == "beijing" ]]; then
+    echo "beijing install deployment no longer supported for a full install because of the configmap 1g limit  - use casablanca+ for helm deploy"
+    exit 1
+  fi
+
   # fix virtual memory for onap-log:elasticsearch under Rancher 1.6.11 - OOM-431
   sudo sysctl -w vm.max_map_count=262144
   if [[ "$DELETE_PREV_OOM" != false ]]; then
-    echo "remove existing oom"
+    echo "remove currently deployed pods"
+    sudo helm list
     # master/beijing only - not amsterdam
     if [ "$BRANCH" == "amsterdam" ]; then
       oom/kubernetes/oneclick/deleteAll.bash -n $ENVIRON
@@ -66,43 +101,57 @@ deploy_onap() {
     fi
 
     # verify
-    DELETED=$(kubectl get pods --all-namespaces -a | grep -E '0/|1/2' | wc -l)
-    echo "showing {$DELETED} undeleted pods"
+    DELETED=$(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3' | wc -l)
+    echo "showing $DELETED undeleted pods"
     echo "verify deletion is finished."
-    while [  $(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l) -gt 0 ]; do
+    # max number of cycles exits to --force block next
+    local MAX_DELETION_WAIT_PERIODS_BEFORE_RUNNING_FORCE=120 # 30 min
+    local DELETE_COUNTER=0
+    while [  $(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3' | wc -l) -gt 0 ]; do
       sleep 15
-      echo "waiting for deletions to complete"
+      echo "waiting for deletions to complete, iteration $DELETE_COUNTER of $MAX_DELETION_WAIT_PERIODS_BEFORE_RUNNING_FORCE"
+      # addressing rare occurrence on Terminating instances requiring scripted --force in next merge for LOG-914
+      COUNTER=$((COUNTER + 1 ))
+      MAX_DELETION_WAIT_PERIODS_BEFORE_RUNNING_FORCE=$((MAX_DELETION_WAIT_PERIODS_BEFORE_RUNNING_FORCE - 1))
+      if [ "$MAX_DELETION_WAIT_PERIODS_BEFORE_RUNNING_FORCE" -eq 0 ]; then
+        #https://wiki.onap.org/display/DW/ONAP+Development#ONAPDevelopment-WorkingwithJSONPath
+        export POD_NAMES=$(kubectl get pods --field-selector=status.phase!=Running --all-namespaces -o jsonpath="{.items[*].metadata.name}")
+        echo "--force delete on pods: $POD_NAMES"
+        for pod in $POD_NAMES; do
+          echo "running: kubectl delete pods $pod --grace-period=0 --force -n $ENVIRON"
+          kubectl delete pods $pod --grace-period=0 --force -n $ENVIRON
+        done
+      fi
     done
-    # wait for 0/1 before deleting
-    echo "sleeping 30 sec"
-    # delete potential hanging clustered pods
-    #kubectl delete pod $ENVIRON-aaf-sms-vault-0 -n $ENVIRON --grace-period=0 --force
+    echo "Pod deletions completed"
+    
     # specific to when there is no helm release
     kubectl delete pv --all
     kubectl delete pvc --all
     kubectl delete secrets --all
     kubectl delete clusterrolebinding --all
-    # replace with watch
     # keep jenkins 120 sec timeout happy with echos
     sleep 30
-    echo "List of ONAP Modules - look for terminating pods"
-    LIST_ALL=$(kubectl get pods --all-namespaces --show-all -o wide )
+    echo "List of ONAP Modules - look for terminating pods - should be none - only the kubernetes system"
+    LIST_ALL=$(kubectl get pods --all-namespaces -o wide )
     echo "${LIST_ALL}"
 
     # for use by continuous deployment only
-    echo " deleting /dockerdata-nfs/ all onap-* deployments"
+    echo " deleting /dockerdata-nfs/ all onap-* deployment directories - why: some pod config jobs will not run on a non-empty nfs subdir"
     sudo chmod -R 777 /dockerdata-nfs/*
     rm -rf /dockerdata-nfs/*
   fi
+
   # for use by continuous deployment only
   if [[ "$CLONE_NEW_OOM" != false ]]; then
     rm -rf oom
     echo "pull new oom"
     git clone -b $BRANCH http://gerrit.onap.org/r/oom
-    # https://wiki.onap.org/display/DW/OOM+Helm+%28un%29Deploy+plugins
-    sudo cp -R ~/oom/kubernetes/helm/plugins/ ~/.helm
   fi
 
+  # https://wiki.onap.org/display/DW/OOM+Helm+%28un%29Deploy+plugins
+  sudo cp -R ~/oom/kubernetes/helm/plugins/ ~/.helm
+
   if [ "$BRANCH" == "amsterdam" ]; then
     echo "start config pod"
     # still need to source docker variables
@@ -119,15 +168,18 @@ deploy_onap() {
       echo "waiting for config pod to complete"
     done
   else
-    echo "moving values.yaml to oom/kubernetes/"
-    #sudo cp values.yaml oom/kubernetes/onap
+    echo "using dev0|1.yaml in working dir"
   fi
 
   # usually the prepull takes up to 25-300 min - however hourly builds will finish the docker pulls before the config pod is finished
-  #echo "pre pull docker images - 35+ min"
-  #wget https://jira.onap.org/secure/attachment/11261/prepull_docker.sh
-  #chmod 777 prepull_docker.sh
-  #./prepull_docker.sh
+  if [[ "$DOCKER_PREPULL" != false ]]; then
+    echo "pre pull docker images - 40+ min for 75G - use a proxy"
+    sudo wget https://git.onap.org/logging-analytics/plain/deploy/docker_prepull.sh
+    sudo chmod 777 docker_prepull.sh
+    # run only on slave nodes vis cloudformation or heat template
+    sudo ./docker_prepull.sh -b $BRANCH -s $NEXUS3_AND_PORT -v true
+  fi
+
   echo "start onap pods"
   if [ "$BRANCH" == "amsterdam" ]; then
     cd oom/kubernetes/oneclick
@@ -138,30 +190,92 @@ deploy_onap() {
     sudo make clean
     sudo make all
     sudo make $ENVIRON
+   
+    local DISABLE_CHARTS_YAML=onap/resources/environments/disable-allcharts.yaml
+    local DEV0_YAML=~/dev0.yaml
+    local DEV1_YAML=~/dev1.yaml
     #sudo helm install local/onap -n onap --namespace $ENVIRON
+    dt="$(date +"%T")"
+    echo "$dt: starting ONAP install"
     # run an empty deploy first to get a round a random helm deploy failure on a release upgrade failure (deploy plugin runs as upgrade instead of install)
     echo "deploying empty onap deployment as base 1 of 3"
-    sudo helm deploy onap local/onap --namespace $ENVIRON -f onap/resources/environments/disable-allcharts.yaml --verbose
+    sudo helm deploy onap local/onap --namespace $ENVIRON -f $DISABLE_CHARTS_YAML --verbose
     # deploy platform pods first - dev0 and dev1 can be the same is required
-    echo "deploying base onap pods as base 2 of 3"
-    sudo helm deploy onap local/onap --namespace $ENVIRON -f onap/resources/environments/disable-allcharts.yaml -f ~/dev0.yaml --verbose
-    echo "sleep ${SPLIT_DEPLOY_DELAY} sec to allow base platform pods to complete - without a grep on 0/1|0/2| non-Complete jobs"
-    sleep $SPLIT_DEPLOY_DELAY
-    echo "deploying rest of onap pods as base 3 of 3"
-    sudo helm deploy onap local/onap --namespace $ENVIRON -f onap/resources/environments/disable-allcharts.yaml -f ~/dev1.yaml --verbose  
-  cd ../../
+    echo "deploying base onap pods as base 2 of 3 - sleep 30 between"
+    sleep 30
+    
+    if [[ "$FULL_MANAGED_DEPLOY" != true ]]; then
+      echo "deploying onap subset based on dev0.yaml - use -f true option to bring up all of onap in sequence"
+      sudo helm deploy onap local/onap --namespace $ENVIRON -f $DISABLE_CHARTS_YAML -f $DEV0_YAML --verbose
+      echo "sleep ${SPLIT_DEPLOY_DELAY} sec to allow base platform pods to complete - without a grep on 0/1|0/2|0/3 non-Complete jobs"
+      sleep $SPLIT_DEPLOY_DELAY
+      echo "deploying rest of onap pods as base 3 of 3"
+      sudo helm deploy onap local/onap --namespace $ENVIRON -f $DISABLE_CHARTS_YAML -f $DEV1_YAML --verbose  
+    else
+      # for now master and casablanca have the same pod structure
+      if [[ "$BRANCH" == "casablanca" ]] || [[ "$BRANCH" == "3.0.0-ONAP" ]] || [[ "$BRANCH" == "3.0.1-ONAP" ]] || [[ "$BRANCH" == "master" ]]; then 
+        if [[ "$BRANCH" == "master" ]]; then
+          DEPLOY_ORDER_POD_NAME_ARRAY=('robot consul aaf dmaap dcaegen2 msb aai esr multicloud oof so sdc sdnc vid policy portal log vfc uui vnfsdk appc clamp cli pomba vvp contrib sniro-emulator')
+          # don't count completed pods
+          DEPLOY_NUMBER_PODS_DESIRED_ARRAY=(1 4 13 11 13 5 15 2 6 17 10 12 11 2 8 6 3 18 2 5 5 5 1 11 11 3 1)
+          # account for podd that have varying deploy times or replicaset sizes
+          # don't count the 0/1 completed pods - and skip most of the ResultSet instances except 1
+          # dcae boostrap is problematic
+          DEPLOY_NUMBER_PODS_PARTIAL_ARRAY=(1 2 11 9 13 5 11 2 6 16 10 12 11 2 8 6 3 18 2 5 5 5 1 9 11 3 1)
+        else
+          # casablanca branches
+          DEPLOY_ORDER_POD_NAME_ARRAY=('robot consul aaf dmaap dcaegen2 msb aai esr multicloud oof so sdc sdnc vid policy portal log vfc uui vnfsdk appc clamp cli pomba vvp contrib sniro-emulator')
+          # don't count completed pods
+          DEPLOY_NUMBER_PODS_DESIRED_ARRAY=(1 4 13 11 13 5 15 2 6 17 10 12 11 2 8 6 3 18 2 5 5 5 1 11 11 3 1) 
+          # account for podd that have varying deploy times or replicaset sizes
+          # don't count the 0/1 completed pods - and skip most of the ResultSet instances except 1
+          # dcae boostrap is problematic
+          DEPLOY_NUMBER_PODS_PARTIAL_ARRAY=(1 2 11 9 13 5 11 2 6 16 10 12 11 2 8 6 3 18 2 5 5 5 1 9 11 3 1)
+        fi
+        echo "deploying for $BRANCH using profile $DEPLOY_ORDER_POD_NAME_ARRAY"
+      else
+        echo "branch $BRANCH not supported or unknown - check with LOG-326 or LOG-898"
+        exit 1
+      fi
+      
+      echo "deploying full onap system in dependency order - in sequence for staged use of hd/ram/network resources"
+      # http://tldp.org/HOWTO/Bash-Prog-Intro-HOWTO-7.html
+      # iterate over the multidimensional array and build up a deploy list as we deploy each pod
+      local DEPLOY_INDEX=0
+      local PODS_PARTIAL=0
+      local APPENDABLE_ENABLED_FLAGS=
+      for POD_NAME in $DEPLOY_ORDER_POD_NAME_ARRAY; do
+        PODS_PARTIAL=${DEPLOY_NUMBER_PODS_PARTIAL_ARRAY[$DEPLOY_INDEX]}
+        PODS_DESIRED=${DEPLOY_NUMBER_PODS_DESIRED_ARRAY[$DEPLOY_INDEX]}
+        echo "deploying $DEPLOY_INDEX for $POD_NAME - expecting $PODS_PARTIAL of a possible $PODS_DESIRED"
+        DEPLOY_INDEX=$((DEPLOY_INDEX + 1 ))
+        # append --set pod.enabled=true
+        APPENDABLE_ENABLED_FLAGS+=" --set "
+        APPENDABLE_ENABLED_FLAGS+=$POD_NAME
+        APPENDABLE_ENABLED_FLAGS+=".enabled=true"
+        echo $APPENDABLE_ENABLED_FLAGS
+        sudo helm deploy onap local/onap --namespace $ENVIRON -f $DISABLE_CHARTS_YAML  -f $DEV0_YAML $APPENDABLE_ENABLED_FLAGS --verbose
+        #sleep $SPLIT_DEPLOY_DELAY
+        # using name- as a match will bring in some extra pods like so-so-vfc and sdnc-sdnc-portal
+        # however using name-name will not work as some pods are named msb-kube2msb or oof-cmso not oof-oof
+        # using thenamespace-pod will work like onap-oof - keyed on passed in namespace
+        wait_for_pod $ENVIRON-$POD_NAME $PODS_PARTIAL  
+     done
+    fi
+    cd ../../
   fi
 
-  echo "wait for all pods up for 15-80 min"
+  dt="$(date +"%T")"
+  echo "$dt: wait for all pods up for 15-80 min"
   FAILED_PODS_LIMIT=0
   MAX_WAIT_PERIODS=480 # 120 MIN
   COUNTER=0
   PENDING_PODS=0
-  while [  $(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l) -gt $FAILED_PODS_LIMIT ]; do
-    PENDING=$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l)
+  while [  $(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3' | wc -l) -gt $FAILED_PODS_LIMIT ]; do
+    PENDING=$(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3' | wc -l)
     PENDING_PODS=$PENDING
     sleep 15
-    LIST_PENDING=$(kubectl get pods --all-namespaces -o wide | grep -E '0/|1/2' )
+    LIST_PENDING=$(kubectl get pods --all-namespaces -o wide | grep -E '0/|1/2|1/2|2/3' )
     echo "${LIST_PENDING}"
     echo "${PENDING} pending > ${FAILED_PODS_LIMIT} at the ${COUNTER}th 15 sec interval"
     echo ""
@@ -173,109 +287,109 @@ deploy_onap() {
   done
 
   echo "report on non-running containers"
-  PENDING=$(kubectl get pods --all-namespaces | grep -E '0/|1/2')
-  PENDING_COUNT=$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l)
-  PENDING_COUNT_AAI=$(kubectl get pods -n $ENVIRON | grep aai- | grep -E '0/|1/2' | wc -l)
+  PENDING=$(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3')
+  PENDING_COUNT=$(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3' | wc -l)
+  PENDING_COUNT_AAI=$(kubectl get pods -n $ENVIRON | grep aai- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_AAI" -gt 0 ]; then
     echo "down-aai=${PENDING_COUNT_AAI}"
   fi
 
   # todo don't stop if aai is down
-  PENDING_COUNT_APPC=$(kubectl get pods -n $ENVIRON | grep appc- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_APPC=$(kubectl get pods -n $ENVIRON | grep appc- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_APPC" -gt 0 ]; then
     echo "down-appc=${PENDING_COUNT_APPC}"
   fi
-  PENDING_COUNT_MR=$(kubectl get pods -n $ENVIRON | grep message-router- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_MR=$(kubectl get pods -n $ENVIRON | grep message-router- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_MR" -gt 0 ]; then
     echo "down-mr=${PENDING_COUNT_MR}"
   fi
-  PENDING_COUNT_SO=$(kubectl get pods -n $ENVIRON | grep so- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_SO=$(kubectl get pods -n $ENVIRON | grep so- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_SO" -gt 0 ]; then
     echo "down-so=${PENDING_COUNT_SO}"
   fi
-  PENDING_COUNT_POLICY=$(kubectl get pods -n $ENVIRON | grep policy- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_POLICY=$(kubectl get pods -n $ENVIRON | grep policy- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_POLICY" -gt 0 ]; then
     echo "down-policy=${PENDING_COUNT_POLICY}"
   fi
-  PENDING_COUNT_PORTAL=$(kubectl get pods -n $ENVIRON | grep portal- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_PORTAL=$(kubectl get pods -n $ENVIRON | grep portal- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_PORTAL" -gt 0 ]; then
     echo "down-portal=${PENDING_COUNT_PORTAL}"
   fi
-  PENDING_COUNT_LOG=$(kubectl get pods -n $ENVIRON | grep log- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_LOG=$(kubectl get pods -n $ENVIRON | grep log- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_LOG" -gt 0 ]; then
     echo "down-log=${PENDING_COUNT_LOG}"
   fi
-  PENDING_COUNT_ROBOT=$(kubectl get pods -n $ENVIRON | grep robot- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_ROBOT=$(kubectl get pods -n $ENVIRON | grep robot- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_ROBOT" -gt 0 ]; then
     echo "down-robot=${PENDING_COUNT_ROBOT}"
   fi
-  PENDING_COUNT_SDC=$(kubectl get pods -n $ENVIRON | grep sdc- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_SDC=$(kubectl get pods -n $ENVIRON | grep sdc- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_SDC" -gt 0 ]; then
     echo "down-sdc=${PENDING_COUNT_SDC}"
   fi
-  PENDING_COUNT_SDNC=$(kubectl get pods -n $ENVIRON | grep sdnc- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_SDNC=$(kubectl get pods -n $ENVIRON | grep sdnc- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_SDNC" -gt 0 ]; then
     echo "down-sdnc=${PENDING_COUNT_SDNC}"
   fi
-  PENDING_COUNT_VID=$(kubectl get pods -n $ENVIRON | grep vid- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_VID=$(kubectl get pods -n $ENVIRON | grep vid- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_VID" -gt 0 ]; then
     echo "down-vid=${PENDING_COUNT_VID}"
   fi
 
-  PENDING_COUNT_AAF=$(kubectl get pods -n $ENVIRON | grep aaf- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_AAF=$(kubectl get pods -n $ENVIRON | grep aaf- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_AAF" -gt 0 ]; then
     echo "down-aaf=${PENDING_COUNT_AAF}"
   fi
-  PENDING_COUNT_CONSUL=$(kubectl get pods -n $ENVIRON | grep consul- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_CONSUL=$(kubectl get pods -n $ENVIRON | grep consul- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_CONSUL" -gt 0 ]; then
     echo "down-consul=${PENDING_COUNT_CONSUL}"
   fi
-  PENDING_COUNT_MSB=$(kubectl get pods -n $ENVIRON | grep msb- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_MSB=$(kubectl get pods -n $ENVIRON | grep msb- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_MSB" -gt 0 ]; then
     echo "down-msb=${PENDING_COUNT_MSB}"
   fi
-  PENDING_COUNT_DCAE=$(kubectl get pods -n $ENVIRON | grep dcaegen2- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_DCAE=$(kubectl get pods -n $ENVIRON | grep dcaegen2- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_DCAE" -gt 0 ]; then
     echo "down-dcae=${PENDING_COUNT_DCAE}"
   fi
-  PENDING_COUNT_CLI=$(kubectl get pods -n $ENVIRON | grep cli- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_CLI=$(kubectl get pods -n $ENVIRON | grep cli- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_CLI" -gt 0 ]; then
     echo "down-cli=${PENDING_COUNT_CLI}"
   fi
-  PENDING_COUNT_MULTICLOUD=$(kubectl get pods -n $ENVIRON | grep multicloud- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_MULTICLOUD=$(kubectl get pods -n $ENVIRON | grep multicloud- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_MULTICLOUD" -gt 0 ]; then
     echo "down-multicloud=${PENDING_COUNT_MULTICLOUD}"
   fi
-  PENDING_COUNT_CLAMP=$(kubectl get pods -n $ENVIRON | grep clamp- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_CLAMP=$(kubectl get pods -n $ENVIRON | grep clamp- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_CLAMP" -gt 0 ]; then
     echo "down-clamp=${PENDING_COUNT_CLAMP}"
   fi
-  PENDING_COUNT_VNFSDK=$(kubectl get pods -n $ENVIRON | grep vnfsdk- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_VNFSDK=$(kubectl get pods -n $ENVIRON | grep vnfsdk- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_VNFSDK" -gt 0 ]; then
     echo "down-vnfsdk=${PENDING_COUNT_VNFSDK}"
   fi
-  PENDING_COUNT_UUI=$(kubectl get pods -n $ENVIRON | grep uui- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_UUI=$(kubectl get pods -n $ENVIRON | grep uui- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_UUI" -gt 0 ]; then
     echo "down-uui=${PENDING_COUNT_UUI}"
   fi
-  PENDING_COUNT_VFC=$(kubectl get pods -n $ENVIRON | grep vfc- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_VFC=$(kubectl get pods -n $ENVIRON | grep vfc- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_VFC" -gt 0 ]; then
     echo "down-vfc=${PENDING_COUNT_VFC}"
   fi
-  PENDING_COUNT_KUBE2MSB=$(kubectl get pods -n $ENVIRON | grep kube2msb- | grep -E '0/|1/2' | wc -l)
+  PENDING_COUNT_KUBE2MSB=$(kubectl get pods -n $ENVIRON | grep kube2msb- | grep -E '0/|1/2|1/3|2/3' | wc -l)
   if [ "$PENDING_COUNT_KUBE2MSB" -gt 0 ]; then
     echo "down-kube2msb=${PENDING_COUNT_KUBE2MSB}"
   fi
   echo "pending containers=${PENDING_COUNT}"
   echo "${PENDING}"
 
-  echo "check filebeat 2/2 count for ELK stack logging consumption"
-  FILEBEAT=$(kubectl get pods --all-namespaces -a | grep 2/)
+  echo "check filebeat 2/2|3/3 count for ELK stack logging consumption"
+  FILEBEAT=$(kubectl get pods --all-namespaces -a | grep -E '2/|3/')
   echo "${FILEBEAT}"
   echo "sleep 5 min - to allow rest frameworks to finish"
   sleep 300
   echo "List of ONAP Modules"
-  LIST_ALL=$(kubectl get pods --all-namespaces -a  --show-all )
+  LIST_ALL=$(kubectl get pods --all-namespaces -a -o wide )
   echo "${LIST_ALL}"
   echo "run healthcheck 2 times to warm caches and frameworks so rest endpoints report properly - see OOM-447"
 
@@ -311,17 +425,6 @@ deploy_onap() {
     ./ete-k8s.sh $ENVIRON health
   fi
   echo "run partial vFW"
-#  sudo chmod 777 /dockerdata-nfs/onap
-#  if [ "$BRANCH" == "amsterdam" ]; then
-#    ./demo-k8s.sh init_robot
-#  else
-#    ./demo-k8s.sh $ENVIRON init
-#  fi
-#  if [ "$BRANCH" == "amsterdam" ]; then
-#    ./demo-k8s.sh init
-#  else
-#    ./demo-k8s.sh $ENVIRON init
-#  fi
   echo "report results"
   cd ../../../
   
@@ -331,13 +434,16 @@ deploy_onap() {
 
 BRANCH=
 ENVIRON=onap
+FULL_MANAGED_DEPLOY=true
 APPLY_WORKAROUNDS=true
 DELETE_PREV_OOM=false
 REMOVE_OOM_AT_END=false
 CLONE_NEW_OOM=true
 SPLIT_DEPLOY_DELAY=600
+DOCKER_PREPULL=false
+NEXUS3_AND_PORT=nexus3.onap.org:10001
 
-while getopts ":u:b:e:s:c:d:w:r" PARAM; do
+while getopts ":u:b:e:p:n:s:f:c:d:w:r" PARAM; do
   case $PARAM in
     u)
       usage
@@ -349,9 +455,18 @@ while getopts ":u:b:e:s:c:d:w:r" PARAM; do
     e)
       ENVIRON=${OPTARG}
       ;;
+    p)
+      DOCKER_PREPULL=${OPTARG}
+      ;;
+    n)
+      NEXUS3_AND_PORT=${OPTARG}
+      ;;
     s)
       SPLIT_DEPLOY_DELAY=${OPTARG}
       ;;
+    f)
+      FULL_MANAGED_DEPLOY=${OPTARG}
+      ;;
     c)
       CLONE_NEW_OOM=${OPTARG}
       ;;
@@ -376,6 +491,6 @@ if [[ -z $BRANCH ]]; then
   exit 1
 fi
 
-deploy_onap  $BRANCH $ENVIRON $SPLIT_DEPLOY_DELAY $CLONE_NEW_OOM $DELETE_PREV_OOM $APPLY_WORKAROUNDS $REMOVE_OOM_AT_END
+deploy_onap  $BRANCH $ENVIRON $DOCKER_PREPULL $NEXUS3_AND_PORT $SPLIT_DEPLOY_DELAY $FULL_MANAGED_DEPLOY $CLONE_NEW_OOM $DELETE_PREV_OOM $APPLY_WORKAROUNDS $REMOVE_OOM_AT_END
 
 printf "**** Done ****\n"