Reduce OOM k8s footprint further

Reduce OOM k8s footprint by using the Rancher VM
as the orchestration node.

Also cleaned up the rancher deployment script.

Change-Id: Ic5594eac2830d83d58d2b0a1fa5eee770d97063d
Issue-ID: INT-586
Signed-off-by: Gary Wu <gary.i.wu@huawei.com>

2 files changed, 25 insertions, 17 deletions

diff --git a/deployment/heat/onap-oom/scripts/deploy.sh b/deployment/heat/onap-oom/scripts/deploy.sh
index 459af34bc..c3a73c763 100755
--- a/deployment/heat/onap-oom/scripts/deploy.sh
+++ b/deployment/heat/onap-oom/scripts/deploy.sh
@@ -156,28 +156,36 @@ ssh-keygen -R $RANCHER_IP
 sleep 2m
 ssh -o StrictHostKeychecking=no -i $SSH_KEY ubuntu@$RANCHER_IP "sed -u '/Cloud-init.*finished/q' <(tail -n+0 -f /var/log/cloud-init-output.log)"
 
-for n in $(seq 1 8); do
-    echo "Wait count $n of 8"
-    sleep 15m
-    ssh -i $SSH_KEY ubuntu@$RANCHER_IP  'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap health"'
-    RESULT=$?
-    if [ $RESULT -eq 0 ]; then
-  	break
-    fi
+PREV_RESULT=0
+for n in $(seq 1 20); do
+  RESULT=$(ssh -i $SSH_KEY ubuntu@$RANCHER_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'Running|Complete|NAME' | wc -l)
+  if [[ $? -eq 0 && ( $RESULT -eq 0 || $RESULT -eq $PREV_RESULT ) ]]; then
+    break
+  fi
+  sleep 15m
+  PREV_RESULT=$RESULT
+done
+
+PREV_RESULT=0
+for n in $(seq 1 20); do
+  echo "Wait for HEALTHCHECK count $n of 10"
+  ROBOT_POD=$(ssh -i $SSH_KEY ubuntu@$RANCHER_IP 'sudo su -c "kubectl --namespace onap get pods"' | grep robot | sed 's/ .*//')
+  ssh -i $SSH_KEY ubuntu@$RANCHER_IP  'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap health"'
+  RESULT=$?
+  if [[ $RESULT -lt 10 && ( $RESULT -eq 0 || $RESULT -eq $PREV_RESULT ) ]]; then
+    break
+  fi
+  sleep 15m
+  PREV_RESULT=$RESULT
 done
-ROBOT_POD=$(ssh -i $SSH_KEY ubuntu@$RANCHER_IP 'sudo su -c "kubectl --namespace onap get pods"' | grep robot | sed 's/ .*//')
 if [ "$ROBOT_POD" == "" ]; then
-    exit 1
+  exit 1
 fi
 
 LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep health | head -1" | ssh -i $SSH_KEY ubuntu@$RANCHER_IP sudo su)
-if [ "$LOG_DIR" == "" ]; then
-    exit 1
-fi
-
 echo "kubectl cp -n onap $ROBOT_POD:share/logs/$LOG_DIR /tmp/robot/logs/$LOG_DIR" | ssh -i $SSH_KEY ubuntu@$RANCHER_IP sudo su
-rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$RANCHER_IP:/tmp/robot/logs/$LOG_DIR/ $WORKSPACE/archives/
-
 echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+mkdir -p $WORKSPACE/archives/healthcheck
+rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$RANCHER_IP:/tmp/robot/logs/$LOG_DIR/ $WORKSPACE/archives/healthcheck
 
 exit 0
diff --git a/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh b/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh
index 570011128..928142f5d 100755
--- a/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh
+++ b/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh
@@ -101,7 +101,7 @@ for VM_NUM in $(seq 3); do
     VM_TYPE=etcd HOST_LABEL=etcd VM_NUM=$VM_NUM envsubst < $PARTS_DIR/onap-oom-2.yaml
 done
 
-for VM_NUM in $(seq 1); do
+for VM_NUM in $(seq 0); do
     VM_TYPE=orch HOST_LABEL=orchestration VM_NUM=$VM_NUM envsubst < $PARTS_DIR/onap-oom-2.yaml
 done