From 1f92a0ec4ea037089b82e3f80bb030f34fab64f0 Mon Sep 17 00:00:00 2001 From: Todd Malsbary Date: Wed, 19 Jan 2022 13:54:14 -0800 Subject: Use kubectl wait in KuD wait_for_pod test function This handles intermittent Err* Pod status such as ErrImagePull. Additionally, remove the duplication of wait_for_pod among the tests. Issue-ID: MULTICLOUD-1435 Signed-off-by: Todd Malsbary Change-Id: I70c69e0427b80e31b2f97515ebdbc3496a91ced1 --- kud/tests/_functions.sh | 15 ++------------- kud/tests/cmk.sh | 22 +++------------------- kud/tests/kata.sh | 18 +----------------- kud/tests/nfd.sh | 17 ++--------------- kud/tests/qat.sh | 17 +++-------------- kud/tests/sriov-network.sh | 17 +++-------------- kud/tests/sriov.sh | 17 +++-------------- kud/tests/topology-manager-sriov.sh | 15 +-------------- kud/tests/topology-manager.sh | 15 +-------------- 9 files changed, 19 insertions(+), 134 deletions(-) diff --git a/kud/tests/_functions.sh b/kud/tests/_functions.sh index 367888e5..e2d92079 100755 --- a/kud/tests/_functions.sh +++ b/kud/tests/_functions.sh @@ -223,24 +223,13 @@ function wait_deployment { done } -# wait_for_pod() - Wait until first pod matched by kubectl filters is in running status +# wait_for_pod() - Wait until pods matched by kubectl filters is in running status function wait_for_pod { #Example usage: # wait_for_pods example_pod # wait_for_pods --namespace test different_pod # wait_for_pods -n test -l app=plugin_test - - status_phase="" - while [[ "$status_phase" != "Running" ]]; do - new_phase="$(kubectl get pods -o 'go-template={{ index .items 0 "status" "phase" }}' "$@" )" - if [[ "$new_phase" != "$status_phase" ]]; then - echo "$(date +%H:%M:%S) - Filter=[$*] : $new_phase" - status_phase="$new_phase" - fi - if [[ "$new_phase" == "Err"* ]]; then - exit 1 - fi - done + kubectl wait --for=condition=Ready --timeout=600s pod "$@" } # wait_for_deployment() - Wait until the deployment is ready diff --git a/kud/tests/cmk.sh b/kud/tests/cmk.sh index e8e77c6c..3b55fa83 100755 --- a/kud/tests/cmk.sh +++ b/kud/tests/cmk.sh @@ -1,6 +1,7 @@ #!/bin/bash source _common.sh +source _functions.sh ENV=$(kubectl get nodes --all-namespaces | wc -l) if [[ $ENV -gt 2 ]]; then @@ -17,23 +18,6 @@ CORE=2 DIR=/tmp pod_name=cmk-test-pod -function wait_for_pod_up { - status_phase="" - while [[ $status_phase != "Running" ]]; do - new_phase=$(kubectl get pods "$@" | awk 'NR==2{print $3}') - if [[ $new_phase != $status_phase ]]; then - echo "$(date +%H:%M:%S) - $@ : $new_phase" - status_phase=$new_phase - fi - if [[ $new_phase == "Running" ]]; then - echo "Pod $@ is up and running.." - fi - if [[ $new_phase == "Err"* ]]; then - exit 1 - fi - done -} - function start_nginx_pod { kubectl delete deployment -n default nginx --ignore-not-found=true kubectl create deployment nginx --image=nginx @@ -110,7 +94,7 @@ EOF sleep 2 echo "waiting for pod up" for pod in $pod_name; do - wait_for_pod_up $pod + wait_for_pod $pod done echo "waiting for CPU allocation finished ..." rest=$TOTAL @@ -207,7 +191,7 @@ EOF sleep 2 echo "waiting for pod up" for pod in $pod_name; do - wait_for_pod_up $pod + wait_for_pod $pod done echo "waiting for CPU allocation finished ..." rest=0 diff --git a/kud/tests/kata.sh b/kud/tests/kata.sh index f55d8cd3..53ca4575 100755 --- a/kud/tests/kata.sh +++ b/kud/tests/kata.sh @@ -18,26 +18,10 @@ set -o errexit set -o nounset set -o pipefail -#source _common_test.sh -#source _common.sh -#source _functions.sh +source _functions.sh kata_pods="kata-qemu kata-clh" -function wait_for_pod { - status_phase="" - while [[ "$status_phase" != "Running" ]]; do - new_phase="$(kubectl get pods -o wide | grep ^$1 | awk '{print $3}')" - if [[ "$new_phase" != "$status_phase" ]]; then - status_phase="$new_phase" - fi - if [[ "$new_phase" == "Err"* ]]; then - exit 1 - fi - sleep 2 - done -} - for pod in ${kata_pods};do echo "Deploying ${pod} pod" kubectl apply -f ${pod}.yml diff --git a/kud/tests/nfd.sh b/kud/tests/nfd.sh index 9937cc65..324d550b 100755 --- a/kud/tests/nfd.sh +++ b/kud/tests/nfd.sh @@ -13,6 +13,7 @@ set -o nounset set -o pipefail source _common_test.sh +source _functions.sh rm -f $HOME/*.yaml pod_name=nfd-pod @@ -107,21 +108,7 @@ if $(kubectl version &>/dev/null); then kubectl create -f $HOME/$pod_name-$podType.yaml --validate=false for pod in $pod_name; do - status_phase="" - while [[ $status_phase != "Running" ]]; do - new_phase=$(kubectl get pods $pod | awk 'NR==2{print $3}') - if [[ $new_phase != $status_phase ]]; then - echo "$(date +%H:%M:%S) - $pod-$podType : $new_phase" - status_phase=$new_phase - fi - - if [[ $new_phase == "Running" ]]; then - echo " Test is complete.." - fi - if [[ $new_phase == "Err"* ]]; then - exit 1 - fi - done + wait_for_pod $pod done kubectl delete pod $pod_name while kubectl get pod $pod_name &>/dev/null; do diff --git a/kud/tests/qat.sh b/kud/tests/qat.sh index 98c0bb72..54573a97 100755 --- a/kud/tests/qat.sh +++ b/kud/tests/qat.sh @@ -10,6 +10,8 @@ set -o pipefail +source _functions.sh + qat_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select((.status.capacity."qat.intel.com/cy2_dc2"!=null) and ((.status.capacity."qat.intel.com/cy2_dc2"|tonumber)>=1)) | .metadata.name') if [ -z "$qat_capable_nodes" ]; then echo "This test case cannot run. QAT device unavailable." @@ -57,20 +59,7 @@ spec: POD kubectl create -f $HOME/$pod_name.yaml --validate=false for pod in $pod_name; do - status_phase="" - while [[ $status_phase != "Running" ]]; do - new_phase=$(kubectl get pods $pod | awk 'NR==2{print $3}') - if [[ $new_phase != $status_phase ]]; then - echo "$(date +%H:%M:%S) - $pod : $new_phase" - status_phase=$new_phase - fi - if [[ $new_phase == "Running" ]]; then - echo "Pod is up and running.." - fi - if [[ $new_phase == "Err"* ]]; then - exit 1 - fi - done + wait_for_pod $pod_name done allocated_node_resource=$(kubectl describe node | grep "qat.intel.com" | tail -n1 |awk '{print $(NF)}') diff --git a/kud/tests/sriov-network.sh b/kud/tests/sriov-network.sh index db1613be..af79f0a7 100755 --- a/kud/tests/sriov-network.sh +++ b/kud/tests/sriov-network.sh @@ -10,6 +10,8 @@ set -o pipefail +source _functions.sh + sriov_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select((.status.capacity."intel.com/intel_sriov_nic"!=null) and ((.status.capacity."intel.com/intel_sriov_nic"|tonumber)>=2)) | .metadata.name') if [ -z "$sriov_capable_nodes" ]; then echo "SRIOV test case cannot run on the cluster." @@ -78,20 +80,7 @@ for podType in ${POD_TYPE:-single multiple}; do kubectl create -f $HOME/$pod_name-$podType.yaml --validate=false for pod in $pod_name; do - status_phase="" - while [[ $status_phase != "Running" ]]; do - new_phase=$(kubectl get pods $pod | awk 'NR==2{print $3}') - if [[ $new_phase != $status_phase ]]; then - echo "$(date +%H:%M:%S) - $pod-$podType : $new_phase" - status_phase=$new_phase - fi - if [[ $new_phase == "Running" ]]; then - echo "Pod is up and running.." - fi - if [[ $new_phase == "Err"* ]]; then - exit 1 - fi - done + wait_for_pod $pod_name done allocated_node_resource=$(kubectl describe node | grep "intel.com/intel_sriov_nic" | tail -n1 |awk '{print $(NF)}') diff --git a/kud/tests/sriov.sh b/kud/tests/sriov.sh index a44aba04..ecc13d87 100755 --- a/kud/tests/sriov.sh +++ b/kud/tests/sriov.sh @@ -10,6 +10,8 @@ set -o pipefail +source _functions.sh + sriov_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select((.status.capacity."intel.com/intel_sriov_700"!=null) and ((.status.capacity."intel.com/intel_sriov_700"|tonumber)>=2)) | .metadata.name') if [ -z "$sriov_capable_nodes" ]; then echo "SRIOV test case cannot run on the cluster." @@ -78,20 +80,7 @@ for podType in ${POD_TYPE:-single multiple}; do kubectl create -f $HOME/$pod_name-$podType.yaml --validate=false for pod in $pod_name; do - status_phase="" - while [[ $status_phase != "Running" ]]; do - new_phase=$(kubectl get pods $pod | awk 'NR==2{print $3}') - if [[ $new_phase != $status_phase ]]; then - echo "$(date +%H:%M:%S) - $pod-$podType : $new_phase" - status_phase=$new_phase - fi - if [[ $new_phase == "Running" ]]; then - echo "Pod is up and running.." - fi - if [[ $new_phase == "Err"* ]]; then - exit 1 - fi - done + wait_for_pod $pod_name done allocated_node_resource=$(kubectl describe node | grep "intel.com/intel_sriov_700" | tail -n1 |awk '{print $(NF)}') diff --git a/kud/tests/topology-manager-sriov.sh b/kud/tests/topology-manager-sriov.sh index 447a7c83..48a1b876 100755 --- a/kud/tests/topology-manager-sriov.sh +++ b/kud/tests/topology-manager-sriov.sh @@ -61,20 +61,7 @@ create_pod_yaml ${csar_id} kubectl delete pod $pod_name --ignore-not-found=true --now --wait kubectl create -f ${CSAR_DIR}/${csar_id}/$pod_name.yaml --validate=false -status_phase="" -while [[ $status_phase != "Running" ]]; do - new_phase=$(kubectl get pods $pod_name | awk 'NR==2{print $3}') - if [[ $new_phase != $status_phase ]]; then - echo "$(date +%H:%M:%S) - $pod_name : $new_phase" - status_phase=$new_phase - fi - if [[ $new_phase == "Running" ]]; then - echo "Pod is up and running.." - fi - if [[ $new_phase == "Err"* ]]; then - exit 1 - fi -done +wait_for_pod $pod_name uid=$(kubectl get pod pod-topology-manager -o jsonpath='{.metadata.uid}') node_name=$(kubectl get pod $pod_name -o jsonpath='{.spec.nodeName}') diff --git a/kud/tests/topology-manager.sh b/kud/tests/topology-manager.sh index 772dcfed..a1cd7b73 100755 --- a/kud/tests/topology-manager.sh +++ b/kud/tests/topology-manager.sh @@ -61,20 +61,7 @@ create_pod_yaml ${csar_id} kubectl delete pod $pod_name --ignore-not-found=true --now --wait kubectl create -f ${CSAR_DIR}/${csar_id}/$pod_name.yaml --validate=false -status_phase="" -while [[ $status_phase != "Running" ]]; do - new_phase=$(kubectl get pods $pod_name | awk 'NR==2{print $3}') - if [[ $new_phase != $status_phase ]]; then - echo "$(date +%H:%M:%S) - $pod_name : $new_phase" - status_phase=$new_phase - fi - if [[ $new_phase == "Running" ]]; then - echo "Pod is up and running.." - fi - if [[ $new_phase == "Err"* ]]; then - exit 1 - fi -done +wait_for_pod $pod_name uid=$(kubectl get pod pod-topology-manager -o jsonpath='{.metadata.uid}') node_name=$(kubectl get pod $pod_name -o jsonpath='{.spec.nodeName}') -- cgit 1.2.3-korg