aboutsummaryrefslogtreecommitdiffstats
path: root/deploy/cd.sh
blob: ae8e45a33e81c8431196ff1efc4bbcba20839f28 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
#!/bin/bash
#############################################################################
#
# Copyright © 2019 Amdocs.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#############################################################################
# v20190115
# https://wiki.onap.org/display/DW/ONAP+on+Kubernetes
# source from https://jira.onap.org/browse/OOM-320, 326, 321, 898, 925, 914
# Michael O'Brien
#

usage() {
  cat <<EOF
Usage: $0 [PARAMs]
example 
./cd.sh -b amsterdam -e onap (will rerun onap in the onap namespace, no new repo, no deletion of existing repo, no sdnc workaround, no onap removal at the end
./cd.sh -b master -e onap -s 500 -c true -d true -w true -r true (run as cd server, new oom, delete prev oom, run workarounds, clean onap at the end of the script
./cd.sh -b 3.0.0-ONAP -e onap -p true -n nexus3.onap.org:10001 -f true -s 600 -c true -d false -w true -r false (standard new server/dev environment - use this as the default)
provide a dev0.yaml/dev1.yaml override set (0=platform, 1=rest of pods) - copy from https://git.onap.org/oom/tree/kubernetes/onap/resources/environments/dev.yaml
 note: the managed deploy where -f is true - deploys all of ONAP - for a subset driven by the enabled flags in the dev yaml - use -f false - as the --set enabled flags override in the prior case
-u                  : Display usage
-b [branch]         : branch = master/beijing or amsterdam (required)
-e [environment]    : use the default (onap)
-p [true|false]     : docker prepull (default false)
-n [nexus3 url:port]: nexus3.onap.org:10001 or proxy - used in prepull
-f [true|false]     : managed deploy - time sequenced - defaults to a full deploy
-s [seconds]        : delay between base and rest of onap dual-deployments based on dev0 and dev1.yaml
-c [true|false]     : FLAG clone new oom repo (default: true)
-d [true|false]     : FLAG delete prev oom - (cd build) (default: false)
-w [true|false]     : FLAG apply workarounds  IE: sdnc (default: true)
-r [true|false]     : FLAG remove oom at end of script - for use by CD only (default: false)
EOF
}

wait_for_pod() {
      local RUNNING_PODS_LIMIT=$2
      local MAX_WAIT_PERIODS=140 # 35 MIN
      local COUNTER=1
      local PENDING_PODS=0
      local TARGET_POD_PREFIX=$1
      local PENDING=0
      while [  $PENDING -lt $RUNNING_PODS_LIMIT  ]; do
        PENDING=$(kubectl get pods --all-namespaces | grep $TARGET_POD_PREFIX | grep -E '1/1|2/2|1/3|2/3' | wc -l)
        PENDING_PODS=$PENDING
        sleep 15
        LIST_PENDING=$(kubectl get pods --all-namespaces -o wide | grep $TARGET_POD_PREFIX | grep -E '1/1|2/2|1/3|2/3' )
        echo "${PENDING} running < ${RUNNING_PODS_LIMIT} at the ${COUNTER}th 15 sec interval for $TARGET_POD_PREFIX"
        COUNTER=$((COUNTER + 1 ))
        MAX_WAIT_PERIODS=$((MAX_WAIT_PERIODS - 1))
        if [ "$MAX_WAIT_PERIODS" -eq 0 ]; then
          PENDING=800
        fi
     done
     dt="$(date +"%T")"
     echo "$dt: ${PENDING} pods are up (1/1|2/2|3/3) for $TARGET_POD_PREFIX at the ${COUNTER}th 15 sec interval"
     echo "deployments: note order: helm list"
     sudo helm list
     kubectl get pods --all-namespaces -o wide
}

deploy_onap() {
  echo "$(date)"
  echo "running with: -b $BRANCH -e $ENVIRON -p $DOCKER_PREPULL -n $NEXUS3_AND_PORT -f $FULL_MANAGED_DEPLOY -s $SPLIT_DEPLOY_DELAY -c $CLONE_NEW_OOM -d $DELETE_PREV_OOM -w $APPLY_WORKAROUNDS -r $REMOVE_OOM_AT_END"
  echo "provide onap-parameters.yaml(amsterdam) or dev0.yaml+dev1.yaml (master) and aai-cloud-region-put.json"
  echo "provide a dev0.yaml and dev1.yaml override (0=platform, 1=rest of pods) - copy from https://git.onap.org/oom/tree/kubernetes/onap/resources/environments/dev.yaml"

  if [[ "$BRANCH" == "beijing" ]]; then
    echo "beijing install deployment no longer supported for a full install because of the configmap 1g limit  - use casablanca+ for helm deploy"
    exit 1
  fi

  # fix virtual memory for onap-log:elasticsearch under Rancher 1.6.11 - OOM-431
  sudo sysctl -w vm.max_map_count=262144
  if [[ "$DELETE_PREV_OOM" != false ]]; then
    echo "remove currently deployed pods"
    sudo helm list
    # master/beijing only - not amsterdam
    if [ "$BRANCH" == "amsterdam" ]; then
      oom/kubernetes/oneclick/deleteAll.bash -n $ENVIRON
    else
      echo "kubectl delete namespace $ENVIRON"
      # workaround for secondary orchestration in dcae
      kubectl delete namespace $ENVIRON
      echo "sleep for 4 min to allow the delete to finish pod terminations before trying a helm delete"
      sleep 240
      sudo helm delete --purge $ENVIRON
    fi

    # verify
    DELETED=$(kubectl get pods --namespace $ENVIRON | grep -E '0/|1/2|1/3|2/3' | wc -l)
    echo "showing $DELETED undeleted pods"
    kubectl get pods --namespace $ENVIRON | grep -E '0/|1/2|1/3|2/3'
    echo "verify deletion is finished."

    # this block: for very infrequent rogue pods that are out-of-band of kubernetes - they dont delete without a force delete
    # max number of cycles exits to --force block next
    local MAX_DELETION_WAIT_PERIODS_BEFORE_RUNNING_FORCE=40 # 10 min
    while [  $(kubectl get pods --namespace $ENVIRON  | grep -E '0/|1/2|1/3|2/3' | wc -l) -gt 0 ]; do
      sleep 15
      echo "waiting for deletions to complete, iterations left: $MAX_DELETION_WAIT_PERIODS_BEFORE_RUNNING_FORCE"
      # addressing rare occurrence on Terminating instances requiring scripted --force in next merge for LOG-914
      MAX_DELETION_WAIT_PERIODS_BEFORE_RUNNING_FORCE=$((MAX_DELETION_WAIT_PERIODS_BEFORE_RUNNING_FORCE - 1))
      if [ "$MAX_DELETION_WAIT_PERIODS_BEFORE_RUNNING_FORCE" -eq 0 ]; then
        #https://wiki.onap.org/display/DW/ONAP+Development#ONAPDevelopment-WorkingwithJSONPath
        #export POD_NAMES=$(kubectl get pods --field-selector=status.phase!=Running --all-namespaces -o jsonpath="{.items[*].metadata.name}")
        export POD_NAMES=$(kubectl get pods --namespace $ENVIRON -o jsonpath="{.items[*].metadata.name}")
        echo "--force delete on pods: $POD_NAMES"
        for pod in $POD_NAMES; do
          echo "running: kubectl delete pods $pod --grace-period=0 --force -n $ENVIRON"
          kubectl delete pods $pod --grace-period=0 --force -n $ENVIRON
        done
      fi
    done

    echo "Pod deletions completed - running helm undeploy then kubectl delete pv,pvc,secrets,cluserrolebindings"
    sudo helm undeploy $ENVIRON --purge   
    # specific to when there is no helm release
    kubectl delete pv --all
    kubectl delete pvc --all
    kubectl delete secrets --all
    kubectl delete clusterrolebinding --all
    # keep jenkins 120 sec timeout happy with echos
    sleep 30
    echo "List of ONAP Modules - look for terminating pods - should be none - only the kubernetes system"
    LIST_ALL=$(kubectl get pods --all-namespaces -o wide )
    echo "${LIST_ALL}"

    # for use by continuous deployment only
    echo " deleting /dockerdata-nfs/ all onap-* deployment directories - why: some pod config jobs will not run on a non-empty nfs subdir"
    sudo chmod -R 777 /dockerdata-nfs/*
    rm -rf /dockerdata-nfs/*
  fi

  # for use by continuous deployment only
  if [[ "$CLONE_NEW_OOM" != false ]]; then
    rm -rf oom
    echo "pull new oom"
    git clone -b $BRANCH http://gerrit.onap.org/r/oom
  fi

  # https://wiki.onap.org/display/DW/OOM+Helm+%28un%29Deploy+plugins
  sudo cp -R ~/oom/kubernetes/helm/plugins/ ~/.helm

  if [ "$BRANCH" == "amsterdam" ]; then
    echo "start config pod"
    # still need to source docker variables
    source oom/kubernetes/oneclick/setenv.bash
    #echo "source setenv override"
    echo "moving onap-parameters.yaml to oom/kubernetes/config"
    cp onap-parameters.yaml oom/kubernetes/config
    cd oom/kubernetes/config
    ./createConfig.sh -n $ENVIRON
    cd ../../../
    echo "verify onap-config is 0/1 not 1/1 - as in completed - an error pod - means you are missing onap-parameters.yaml or values are not set in it."
    while [  $(kubectl get pods -n onap -a | grep config | grep 0/1 | grep Completed | wc -l) -eq 0 ]; do
      sleep 15
      echo "waiting for config pod to complete"
    done
  else
    echo "using dev0|1.yaml in working dir"
  fi

  # usually the prepull takes up to 25-300 min - however hourly builds will finish the docker pulls before the config pod is finished
  if [[ "$DOCKER_PREPULL" != false ]]; then
    echo "pre pull docker images - 40+ min for 75G - use a proxy"
    sudo wget https://git.onap.org/logging-analytics/plain/deploy/docker_prepull.sh
    sudo chmod 777 docker_prepull.sh
    # run only on slave nodes vis cloudformation or heat template
    sudo ./docker_prepull.sh -b $BRANCH -s $NEXUS3_AND_PORT -v true
  fi

  echo "start onap pods"
  if [ "$BRANCH" == "amsterdam" ]; then
    cd oom/kubernetes/oneclick
    ./createAll.bash -n $ENVIRON
    cd ../../../
  else
    cd oom/kubernetes/
    sudo make clean
    sudo make all
    sudo make $ENVIRON
   
    local DISABLE_CHARTS_YAML=onap/resources/environments/disable-allcharts.yaml
    local DEV0_YAML=~/dev0.yaml
    local DEV1_YAML=~/dev1.yaml
    #sudo helm install local/onap -n onap --namespace $ENVIRON
    dt="$(date +"%T")"
    echo "$dt: starting ONAP install"
    # run an empty deploy first to get a round a random helm deploy failure on a release upgrade failure (deploy plugin runs as upgrade instead of install)
    echo "deploying empty onap deployment as base 1 of 3"
    sudo helm deploy onap local/onap --namespace $ENVIRON -f $DISABLE_CHARTS_YAML --verbose
    # deploy platform pods first - dev0 and dev1 can be the same is required
    echo "deploying base onap pods as base 2 of 3 - sleep 30 between"
    sleep 30
    
    if [[ "$FULL_MANAGED_DEPLOY" != true ]]; then
      echo "deploying onap subset based on dev0.yaml - use -f true option to bring up all of onap in sequence"
      sudo helm deploy onap local/onap --namespace $ENVIRON -f $DISABLE_CHARTS_YAML -f $DEV0_YAML --verbose
      echo "sleep ${SPLIT_DEPLOY_DELAY} sec to allow base platform pods to complete - without a grep on 0/1|0/2|0/3 non-Complete jobs"
      sleep $SPLIT_DEPLOY_DELAY
      echo "deploying rest of onap pods as base 3 of 3"
      sudo helm deploy onap local/onap --namespace $ENVIRON -f $DISABLE_CHARTS_YAML -f $DEV1_YAML --verbose  
    else
      # for now master and casablanca have the same pod structure
      if [[ "$BRANCH" == "casablanca" ]] || [[ "$BRANCH" == "3.0.0-ONAP" ]] || [[ "$BRANCH" == "3.0.1-ONAP" ]] || [[ "$BRANCH" == "master" ]]; then 
        if [[ "$BRANCH" == "master" ]]; then
          # node DCAEGEN2 must deploy after consul, msb and dmaap but not any later than deploy 5
          DEPLOY_ORDER_POD_NAME_ARRAY=('consul msb dmaap dcaegen2 aaf robot aai esr multicloud oof so sdc sdnc vid policy portal log vfc uui vnfsdk appc clamp cli pomba vvp contrib sniro-emulator')
          # don't count completed pods
          DEPLOY_NUMBER_PODS_DESIRED_ARRAY=(4 5 11 11 13 1 15 2 6 17 10 12 11 2 8 6 3 18 2 5 5 5 1 11 11 3 1)
          # account for podd that have varying deploy times or replicaset sizes
          # don't count the 0/1 completed pods - and skip most of the ResultSet instances except 1
          # dcae boostrap is problematic
          DEPLOY_NUMBER_PODS_PARTIAL_ARRAY=(2 5 11 9 11 1 11 2 6 16 10 12 11 2 8 6 3 18 2 5 5 5 1 9 11 3 1)
        else
          # casablanca branches
          DEPLOY_ORDER_POD_NAME_ARRAY=('consul msb dmaap dcaegen2 aaf robot aai esr multicloud oof so sdc sdnc vid policy portal log vfc uui vnfsdk appc clamp cli pomba vvp contrib sniro-emulator')
          # don't count completed pods
          DEPLOY_NUMBER_PODS_DESIRED_ARRAY=(4 5 11 11 13 1 15 2 6 17 10 12 11 2 8 6 3 18 2 5 5 5 1 11 11 3 1) 
          # account for podd that have varying deploy times or replicaset sizes
          # don't count the 0/1 completed pods - and skip most of the ResultSet instances except 1
          # dcae boostrap is problematic
          DEPLOY_NUMBER_PODS_PARTIAL_ARRAY=(2 5 11 9 11 1 11 2 6 16 10 12 11 2 8 6 3 18 2 5 5 5 1 9 11 3 1)
        fi
        echo "deploying for $BRANCH using profile $DEPLOY_ORDER_POD_NAME_ARRAY"
      else
        echo "branch $BRANCH not supported or unknown - check with LOG-326 or LOG-898"
        exit 1
      fi
      
      echo "deploying full onap system in dependency order - in sequence for staged use of hd/ram/network resources"
      # http://tldp.org/HOWTO/Bash-Prog-Intro-HOWTO-7.html
      # iterate over the multidimensional array and build up a deploy list as we deploy each pod
      local DEPLOY_INDEX=0
      local PODS_PARTIAL=0
      local APPENDABLE_ENABLED_FLAGS=
      for POD_NAME in $DEPLOY_ORDER_POD_NAME_ARRAY; do
        PODS_PARTIAL=${DEPLOY_NUMBER_PODS_PARTIAL_ARRAY[$DEPLOY_INDEX]}
        PODS_DESIRED=${DEPLOY_NUMBER_PODS_DESIRED_ARRAY[$DEPLOY_INDEX]}
        echo "deploying $DEPLOY_INDEX for $POD_NAME - expecting $PODS_PARTIAL of a possible $PODS_DESIRED"
        DEPLOY_INDEX=$((DEPLOY_INDEX + 1 ))
        # append --set pod.enabled=true
        APPENDABLE_ENABLED_FLAGS+=" --set "
        APPENDABLE_ENABLED_FLAGS+=$POD_NAME
        APPENDABLE_ENABLED_FLAGS+=".enabled=true"
        echo $APPENDABLE_ENABLED_FLAGS
        sudo helm deploy onap local/onap --namespace $ENVIRON -f $DISABLE_CHARTS_YAML  -f $DEV0_YAML $APPENDABLE_ENABLED_FLAGS --verbose
        #sleep $SPLIT_DEPLOY_DELAY
        # using name- as a match will bring in some extra pods like so-so-vfc and sdnc-sdnc-portal
        # however using name-name will not work as some pods are named msb-kube2msb or oof-cmso not oof-oof
        # using thenamespace-pod will work like onap-oof - keyed on passed in namespace
        wait_for_pod $ENVIRON-$POD_NAME $PODS_PARTIAL  
     done
    fi
    cd ../../
  fi

  dt="$(date +"%T")"
  echo "$dt: wait for all pods up for 15-80 min"
  FAILED_PODS_LIMIT=0
  MAX_WAIT_PERIODS=480 # 120 MIN
  COUNTER=0
  PENDING_PODS=0
  while [  $(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3' | wc -l) -gt $FAILED_PODS_LIMIT ]; do
    PENDING=$(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3' | wc -l)
    PENDING_PODS=$PENDING
    sleep 15
    LIST_PENDING=$(kubectl get pods --all-namespaces -o wide | grep -E '0/|1/2|1/2|2/3' )
    echo "${LIST_PENDING}"
    echo "${PENDING} pending > ${FAILED_PODS_LIMIT} at the ${COUNTER}th 15 sec interval"
    echo ""
    COUNTER=$((COUNTER + 1 ))
    MAX_WAIT_PERIODS=$((MAX_WAIT_PERIODS - 1))
    if [ "$MAX_WAIT_PERIODS" -eq 0 ]; then
      FAILED_PODS_LIMIT=800
    fi
  done

  echo "report on non-running containers"
  PENDING=$(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3')
  PENDING_COUNT=$(kubectl get pods --all-namespaces | grep -E '0/|1/2|1/3|2/3' | wc -l)
  PENDING_COUNT_AAI=$(kubectl get pods -n $ENVIRON | grep aai- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_AAI" -gt 0 ]; then
    echo "down-aai=${PENDING_COUNT_AAI}"
  fi

  # todo don't stop if aai is down
  PENDING_COUNT_APPC=$(kubectl get pods -n $ENVIRON | grep appc- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_APPC" -gt 0 ]; then
    echo "down-appc=${PENDING_COUNT_APPC}"
  fi
  PENDING_COUNT_MR=$(kubectl get pods -n $ENVIRON | grep message-router- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_MR" -gt 0 ]; then
    echo "down-mr=${PENDING_COUNT_MR}"
  fi
  PENDING_COUNT_SO=$(kubectl get pods -n $ENVIRON | grep so- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_SO" -gt 0 ]; then
    echo "down-so=${PENDING_COUNT_SO}"
  fi
  PENDING_COUNT_POLICY=$(kubectl get pods -n $ENVIRON | grep policy- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_POLICY" -gt 0 ]; then
    echo "down-policy=${PENDING_COUNT_POLICY}"
  fi
  PENDING_COUNT_PORTAL=$(kubectl get pods -n $ENVIRON | grep portal- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_PORTAL" -gt 0 ]; then
    echo "down-portal=${PENDING_COUNT_PORTAL}"
  fi
  PENDING_COUNT_LOG=$(kubectl get pods -n $ENVIRON | grep log- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_LOG" -gt 0 ]; then
    echo "down-log=${PENDING_COUNT_LOG}"
  fi
  PENDING_COUNT_ROBOT=$(kubectl get pods -n $ENVIRON | grep robot- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_ROBOT" -gt 0 ]; then
    echo "down-robot=${PENDING_COUNT_ROBOT}"
  fi
  PENDING_COUNT_SDC=$(kubectl get pods -n $ENVIRON | grep sdc- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_SDC" -gt 0 ]; then
    echo "down-sdc=${PENDING_COUNT_SDC}"
  fi
  PENDING_COUNT_SDNC=$(kubectl get pods -n $ENVIRON | grep sdnc- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_SDNC" -gt 0 ]; then
    echo "down-sdnc=${PENDING_COUNT_SDNC}"
  fi
  PENDING_COUNT_VID=$(kubectl get pods -n $ENVIRON | grep vid- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_VID" -gt 0 ]; then
    echo "down-vid=${PENDING_COUNT_VID}"
  fi

  PENDING_COUNT_AAF=$(kubectl get pods -n $ENVIRON | grep aaf- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_AAF" -gt 0 ]; then
    echo "down-aaf=${PENDING_COUNT_AAF}"
  fi
  PENDING_COUNT_CONSUL=$(kubectl get pods -n $ENVIRON | grep consul- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_CONSUL" -gt 0 ]; then
    echo "down-consul=${PENDING_COUNT_CONSUL}"
  fi
  PENDING_COUNT_MSB=$(kubectl get pods -n $ENVIRON | grep msb- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_MSB" -gt 0 ]; then
    echo "down-msb=${PENDING_COUNT_MSB}"
  fi
  PENDING_COUNT_DCAE=$(kubectl get pods -n $ENVIRON | grep dcaegen2- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_DCAE" -gt 0 ]; then
    echo "down-dcae=${PENDING_COUNT_DCAE}"
  fi
  PENDING_COUNT_CLI=$(kubectl get pods -n $ENVIRON | grep cli- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_CLI" -gt 0 ]; then
    echo "down-cli=${PENDING_COUNT_CLI}"
  fi
  PENDING_COUNT_MULTICLOUD=$(kubectl get pods -n $ENVIRON | grep multicloud- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_MULTICLOUD" -gt 0 ]; then
    echo "down-multicloud=${PENDING_COUNT_MULTICLOUD}"
  fi
  PENDING_COUNT_CLAMP=$(kubectl get pods -n $ENVIRON | grep clamp- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_CLAMP" -gt 0 ]; then
    echo "down-clamp=${PENDING_COUNT_CLAMP}"
  fi
  PENDING_COUNT_VNFSDK=$(kubectl get pods -n $ENVIRON | grep vnfsdk- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_VNFSDK" -gt 0 ]; then
    echo "down-vnfsdk=${PENDING_COUNT_VNFSDK}"
  fi
  PENDING_COUNT_UUI=$(kubectl get pods -n $ENVIRON | grep uui- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_UUI" -gt 0 ]; then
    echo "down-uui=${PENDING_COUNT_UUI}"
  fi
  PENDING_COUNT_VFC=$(kubectl get pods -n $ENVIRON | grep vfc- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_VFC" -gt 0 ]; then
    echo "down-vfc=${PENDING_COUNT_VFC}"
  fi
  PENDING_COUNT_KUBE2MSB=$(kubectl get pods -n $ENVIRON | grep kube2msb- | grep -E '0/|1/2|1/3|2/3' | wc -l)
  if [ "$PENDING_COUNT_KUBE2MSB" -gt 0 ]; then
    echo "down-kube2msb=${PENDING_COUNT_KUBE2MSB}"
  fi
  echo "pending containers=${PENDING_COUNT}"
  echo "${PENDING}"

  echo "check filebeat 2/2|3/3 count for ELK stack logging consumption"
  FILEBEAT=$(kubectl get pods --all-namespaces -a | grep -E '2/|3/')
  echo "${FILEBEAT}"
  echo "sleep 5 min - to allow rest frameworks to finish"
  sleep 300
  echo "List of ONAP Modules"
  LIST_ALL=$(kubectl get pods --all-namespaces -a -o wide )
  echo "${LIST_ALL}"
  echo "run healthcheck 2 times to warm caches and frameworks so rest endpoints report properly - see OOM-447"

  echo "curl with aai cert to cloud-region PUT"

  curl -X PUT https://127.0.0.1:30233/aai/v11/cloud-infrastructure/cloud-regions/cloud-region/CloudOwner/RegionOne --data "@aai-cloud-region-put.json" -H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" -H "X-TransactionId:jimmy-postman" -H "X-FromAppId:AAI" -H "Content-Type:application/json" -H "Accept:application/json" --cacert aaiapisimpledemoopenecomporg_20171003.crt -k

  echo "get the cloud region back"
  curl -X GET https://127.0.0.1:30233/aai/v11/cloud-infrastructure/cloud-regions/ -H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" -H "X-TransactionId:jimmy-postman" -H "X-FromAppId:AAI" -H "Content-Type:application/json" -H "Accept:application/json" --cacert aaiapisimpledemoopenecomporg_20171003.crt -k

  # OOM-484 - robot scripts moved
  cd oom/kubernetes/robot
  echo "run healthcheck prep 1"
  # OOM-722 adds namespace parameter
  if [ "$BRANCH" == "amsterdam" ]; then
    ./ete-k8s.sh health > ~/health1.out
  else
    ./ete-k8s.sh $ENVIRON health > ~/health1.out
  fi
  echo "sleep 5 min"
  sleep 300
  echo "run healthcheck prep 2"
  if [ "$BRANCH" == "amsterdam" ]; then
    ./ete-k8s.sh health > ~/health2.out
  else
    ./ete-k8s.sh $ENVIRON health > ~/health2.out
  fi
  echo "run healthcheck for real - wait a further 5 min"
  sleep 300
  if [ "$BRANCH" == "amsterdam" ]; then
    ./ete-k8s.sh health
  else
    ./ete-k8s.sh $ENVIRON health
  fi
  echo "run partial vFW"
  echo "report results"
  cd ../../../
  
  echo "$(date)"
  #set +a
}

BRANCH=
ENVIRON=onap
FULL_MANAGED_DEPLOY=true
APPLY_WORKAROUNDS=true
DELETE_PREV_OOM=false
REMOVE_OOM_AT_END=false
CLONE_NEW_OOM=true
SPLIT_DEPLOY_DELAY=600
DOCKER_PREPULL=false
NEXUS3_AND_PORT=nexus3.onap.org:10001

while getopts ":u:b:e:p:n:s:f:c:d:w:r" PARAM; do
  case $PARAM in
    u)
      usage
      exit 1
      ;;
    b)
      BRANCH=${OPTARG}
      ;;
    e)
      ENVIRON=${OPTARG}
      ;;
    p)
      DOCKER_PREPULL=${OPTARG}
      ;;
    n)
      NEXUS3_AND_PORT=${OPTARG}
      ;;
    s)
      SPLIT_DEPLOY_DELAY=${OPTARG}
      ;;
    f)
      FULL_MANAGED_DEPLOY=${OPTARG}
      ;;
    c)
      CLONE_NEW_OOM=${OPTARG}
      ;;
    d)
      DELETE_PREV_OOM=${OPTARG}
      ;;
    w)
      APPLY_WORKAROUNDS=${OPTARG}
      ;;
    r)
      REMOVE_OOM_AT_END=${OPTARG}
      ;;
    ?)
      usage
      exit
      ;;
  esac
done

if [[ -z $BRANCH ]]; then
  usage
  exit 1
fi

deploy_onap  $BRANCH $ENVIRON $DOCKER_PREPULL $NEXUS3_AND_PORT $SPLIT_DEPLOY_DELAY $FULL_MANAGED_DEPLOY $CLONE_NEW_OOM $DELETE_PREV_OOM $APPLY_WORKAROUNDS $REMOVE_OOM_AT_END

printf "**** Done ****\n"