Use kubectl wait in KuD wait_for_pod test function

This handles intermittent Err* Pod status such as ErrImagePull.
Additionally, remove the duplication of wait_for_pod among the tests.

Issue-ID: MULTICLOUD-1435
Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
Change-Id: I70c69e0427b80e31b2f97515ebdbc3496a91ced1

9 files changed, 19 insertions, 134 deletions

diff --git a/kud/tests/_functions.sh b/kud/tests/_functions.sh
index 367888e5..e2d92079 100755
--- a/kud/tests/_functions.sh
+++ b/kud/tests/_functions.sh
@@ -223,24 +223,13 @@ function wait_deployment {
     done
 }
 
-# wait_for_pod() - Wait until first pod matched by kubectl filters is in running status
+# wait_for_pod() - Wait until pods matched by kubectl filters is in running status
 function wait_for_pod {
     #Example usage:
     # wait_for_pods example_pod
     # wait_for_pods --namespace test different_pod
     # wait_for_pods -n test -l app=plugin_test
-
-    status_phase=""
-    while [[ "$status_phase" != "Running" ]]; do
-        new_phase="$(kubectl get pods -o 'go-template={{ index .items 0 "status" "phase" }}' "$@" )"
-        if [[ "$new_phase" != "$status_phase" ]]; then
-            echo "$(date +%H:%M:%S) - Filter=[$*] : $new_phase"
-            status_phase="$new_phase"
-        fi
-        if [[ "$new_phase" == "Err"* ]]; then
-            exit 1
-        fi
-    done
+    kubectl wait --for=condition=Ready --timeout=600s pod "$@"
 }
 
 # wait_for_deployment() - Wait until the deployment is ready
diff --git a/kud/tests/cmk.sh b/kud/tests/cmk.sh
index e8e77c6c..3b55fa83 100755
--- a/kud/tests/cmk.sh
+++ b/kud/tests/cmk.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
 source _common.sh
+source _functions.sh
 
 ENV=$(kubectl get nodes --all-namespaces | wc -l)
 if [[ $ENV -gt 2 ]]; then
@@ -17,23 +18,6 @@ CORE=2
 DIR=/tmp
 pod_name=cmk-test-pod
 
-function wait_for_pod_up {
-    status_phase=""
-    while [[ $status_phase != "Running" ]]; do
-        new_phase=$(kubectl get pods "$@" | awk 'NR==2{print $3}')
-        if [[ $new_phase != $status_phase ]]; then
-            echo "$(date +%H:%M:%S) - $@ : $new_phase"
-            status_phase=$new_phase
-        fi
-        if [[ $new_phase == "Running" ]]; then
-            echo "Pod $@ is up and running.."
-        fi
-        if [[ $new_phase == "Err"* ]]; then
-            exit 1
-        fi
-    done
-}
-
 function start_nginx_pod {
     kubectl delete deployment -n default nginx --ignore-not-found=true
     kubectl create deployment nginx --image=nginx
@@ -110,7 +94,7 @@ EOF
         sleep 2
         echo "waiting for pod up"
         for pod in $pod_name; do
-            wait_for_pod_up $pod
+            wait_for_pod $pod
         done
         echo "waiting for CPU allocation finished ..."
         rest=$TOTAL
@@ -207,7 +191,7 @@ EOF
         sleep 2
         echo "waiting for pod up"
         for pod in $pod_name; do
-            wait_for_pod_up $pod
+            wait_for_pod $pod
         done
         echo "waiting for CPU allocation finished ..."
         rest=0
diff --git a/kud/tests/kata.sh b/kud/tests/kata.sh
index f55d8cd3..53ca4575 100755
--- a/kud/tests/kata.sh
+++ b/kud/tests/kata.sh
@@ -18,26 +18,10 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-#source _common_test.sh
-#source _common.sh
-#source _functions.sh
+source _functions.sh
 
 kata_pods="kata-qemu kata-clh"
 
-function wait_for_pod {
-    status_phase=""
-    while [[ "$status_phase" != "Running" ]]; do
-        new_phase="$(kubectl get pods -o wide | grep ^$1 | awk '{print $3}')"
-        if [[ "$new_phase" != "$status_phase" ]]; then
-            status_phase="$new_phase"
-        fi
-        if [[ "$new_phase" == "Err"* ]]; then
-            exit 1
-        fi
-        sleep 2
-    done
-}
-
 for pod in ${kata_pods};do
     echo "Deploying ${pod} pod"
     kubectl apply -f ${pod}.yml
diff --git a/kud/tests/nfd.sh b/kud/tests/nfd.sh
index 9937cc65..324d550b 100755
--- a/kud/tests/nfd.sh
+++ b/kud/tests/nfd.sh
@@ -13,6 +13,7 @@ set -o nounset
 set -o pipefail
 
 source _common_test.sh
+source _functions.sh
 
 rm -f $HOME/*.yaml
 pod_name=nfd-pod
@@ -107,21 +108,7 @@ if $(kubectl version &>/dev/null); then
         kubectl create -f $HOME/$pod_name-$podType.yaml --validate=false
 
         for pod in $pod_name; do
-            status_phase=""
-            while [[ $status_phase != "Running" ]]; do
-                new_phase=$(kubectl get pods $pod | awk 'NR==2{print $3}')
-                if [[ $new_phase != $status_phase ]]; then
-                    echo "$(date +%H:%M:%S) - $pod-$podType : $new_phase"
-                    status_phase=$new_phase
-                fi
-
-                if [[ $new_phase == "Running" ]]; then
-                    echo " Test is complete.."
-                fi
-                if [[ $new_phase == "Err"* ]]; then
-                    exit 1
-                fi
-            done
+            wait_for_pod $pod
         done
         kubectl delete pod $pod_name
         while kubectl get pod $pod_name &>/dev/null; do
diff --git a/kud/tests/qat.sh b/kud/tests/qat.sh
index 98c0bb72..54573a97 100755
--- a/kud/tests/qat.sh
+++ b/kud/tests/qat.sh
@@ -10,6 +10,8 @@
 
 set -o pipefail
 
+source _functions.sh
+
 qat_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select((.status.capacity."qat.intel.com/cy2_dc2"!=null) and ((.status.capacity."qat.intel.com/cy2_dc2"|tonumber)>=1)) | .metadata.name')
 if [ -z "$qat_capable_nodes" ]; then
     echo "This test case cannot run. QAT device unavailable."
@@ -57,20 +59,7 @@ spec:
 POD
 kubectl create -f $HOME/$pod_name.yaml --validate=false
     for pod in $pod_name; do
-        status_phase=""
-        while [[ $status_phase != "Running" ]]; do
-            new_phase=$(kubectl get pods $pod | awk 'NR==2{print $3}')
-            if [[ $new_phase != $status_phase ]]; then
-                echo "$(date +%H:%M:%S) - $pod : $new_phase"
-                status_phase=$new_phase
-            fi
-            if [[ $new_phase == "Running" ]]; then
-                echo "Pod is up and running.."
-            fi
-            if [[ $new_phase == "Err"* ]]; then
-                exit 1
-            fi
-        done
+        wait_for_pod $pod_name
     done
 
 allocated_node_resource=$(kubectl describe node | grep "qat.intel.com" | tail -n1 |awk '{print $(NF)}')
diff --git a/kud/tests/sriov-network.sh b/kud/tests/sriov-network.sh
index db1613be..af79f0a7 100755
--- a/kud/tests/sriov-network.sh
+++ b/kud/tests/sriov-network.sh
@@ -10,6 +10,8 @@
 
 set -o pipefail
 
+source _functions.sh
+
 sriov_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select((.status.capacity."intel.com/intel_sriov_nic"!=null) and ((.status.capacity."intel.com/intel_sriov_nic"|tonumber)>=2)) | .metadata.name')
 if [ -z "$sriov_capable_nodes" ]; then
     echo "SRIOV test case cannot run on the cluster."
@@ -78,20 +80,7 @@ for podType in ${POD_TYPE:-single multiple}; do
     kubectl create -f $HOME/$pod_name-$podType.yaml --validate=false
 
         for pod in $pod_name; do
-            status_phase=""
-            while [[ $status_phase != "Running" ]]; do
-                new_phase=$(kubectl get pods $pod | awk 'NR==2{print $3}')
-                if [[ $new_phase != $status_phase ]]; then
-                    echo "$(date +%H:%M:%S) - $pod-$podType : $new_phase"
-                    status_phase=$new_phase
-                fi
-                if [[ $new_phase == "Running" ]]; then
-                    echo "Pod is up and running.."
-                fi
-                if [[ $new_phase == "Err"* ]]; then
-                    exit 1
-                fi
-            done
+            wait_for_pod $pod_name
         done
     allocated_node_resource=$(kubectl describe node | grep "intel.com/intel_sriov_nic" | tail -n1 |awk '{print $(NF)}')
 
diff --git a/kud/tests/sriov.sh b/kud/tests/sriov.sh
index a44aba04..ecc13d87 100755
--- a/kud/tests/sriov.sh
+++ b/kud/tests/sriov.sh
@@ -10,6 +10,8 @@
 
 set -o pipefail
 
+source _functions.sh
+
 sriov_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select((.status.capacity."intel.com/intel_sriov_700"!=null) and ((.status.capacity."intel.com/intel_sriov_700"|tonumber)>=2)) | .metadata.name')
 if [ -z "$sriov_capable_nodes" ]; then
     echo "SRIOV test case cannot run on the cluster."
@@ -78,20 +80,7 @@ for podType in ${POD_TYPE:-single multiple}; do
     kubectl create -f $HOME/$pod_name-$podType.yaml --validate=false
 
         for pod in $pod_name; do
-            status_phase=""
-            while [[ $status_phase != "Running" ]]; do
-                new_phase=$(kubectl get pods $pod | awk 'NR==2{print $3}')
-                if [[ $new_phase != $status_phase ]]; then
-                    echo "$(date +%H:%M:%S) - $pod-$podType : $new_phase"
-                    status_phase=$new_phase
-                fi
-                if [[ $new_phase == "Running" ]]; then
-                    echo "Pod is up and running.."
-                fi
-                if [[ $new_phase == "Err"* ]]; then
-                    exit 1
-                fi
-            done
+            wait_for_pod $pod_name
         done
     allocated_node_resource=$(kubectl describe node | grep "intel.com/intel_sriov_700" | tail -n1 |awk '{print $(NF)}')
 
diff --git a/kud/tests/topology-manager-sriov.sh b/kud/tests/topology-manager-sriov.sh
index 447a7c83..48a1b876 100755
--- a/kud/tests/topology-manager-sriov.sh
+++ b/kud/tests/topology-manager-sriov.sh
@@ -61,20 +61,7 @@ create_pod_yaml ${csar_id}
 kubectl delete pod $pod_name --ignore-not-found=true --now --wait
 kubectl create -f ${CSAR_DIR}/${csar_id}/$pod_name.yaml --validate=false
 
-status_phase=""
-while [[ $status_phase != "Running" ]]; do
-    new_phase=$(kubectl get pods $pod_name | awk 'NR==2{print $3}')
-    if [[ $new_phase != $status_phase ]]; then
-        echo "$(date +%H:%M:%S) - $pod_name : $new_phase"
-        status_phase=$new_phase
-    fi
-    if [[ $new_phase == "Running" ]]; then
-        echo "Pod is up and running.."
-    fi
-    if [[ $new_phase == "Err"* ]]; then
-        exit 1
-    fi
-done
+wait_for_pod $pod_name
 
 uid=$(kubectl get pod pod-topology-manager -o jsonpath='{.metadata.uid}')
 node_name=$(kubectl get pod $pod_name -o jsonpath='{.spec.nodeName}')
diff --git a/kud/tests/topology-manager.sh b/kud/tests/topology-manager.sh
index 772dcfed..a1cd7b73 100755
--- a/kud/tests/topology-manager.sh
+++ b/kud/tests/topology-manager.sh
@@ -61,20 +61,7 @@ create_pod_yaml ${csar_id}
 kubectl delete pod $pod_name --ignore-not-found=true --now --wait
 kubectl create -f ${CSAR_DIR}/${csar_id}/$pod_name.yaml --validate=false
 
-status_phase=""
-while [[ $status_phase != "Running" ]]; do
-    new_phase=$(kubectl get pods $pod_name | awk 'NR==2{print $3}')
-    if [[ $new_phase != $status_phase ]]; then
-        echo "$(date +%H:%M:%S) - $pod_name : $new_phase"
-        status_phase=$new_phase
-    fi
-    if [[ $new_phase == "Running" ]]; then
-        echo "Pod is up and running.."
-    fi
-    if [[ $new_phase == "Err"* ]]; then
-        exit 1
-    fi
-done
+wait_for_pod $pod_name
 
 uid=$(kubectl get pod pod-topology-manager -o jsonpath='{.metadata.uid}')
 node_name=$(kubectl get pod $pod_name -o jsonpath='{.spec.nodeName}')