From cc10b9aff3fd16df62c0655ec1626624ad2e2fc0 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 30 Oct 2019 23:47:35 -0700 Subject: add cmk in KuD - deploy cmk related pods - untaint compute nodes if necessary - run cmk unit tests: allocate CPUs from exclusive and shared pools - deploy a testing nginx pod along with cmk testing pods - preset 1/2 CPUs for shared/exlusive pools to fit CI server machines users can adjust the parameters to meet their own requirements Test Results: - many rounds of vagrant/5 VMs(controller01/02/03 and compute01/02) based test are all OK - 14 rounds tests on my local server (S2600WFQ (36C/72T) )and PC(HP Z228 (4C/4T)) with all-in-one bare metal deployment are all OK - CI(a 4C/4T machine) results of latest patch set also show that the test of bare metal deployment is OK - NOTE: both my local test and CI use the same testing method of calling aio.sh after applying the latest patch set. Change-Id: I046a4a63b94f92f23347ab76c21a661521e01119 Issue-ID: MULTICLOUD-879 Signed-off-by: Liang Ding --- kud/tests/cmk.sh | 231 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100755 kud/tests/cmk.sh (limited to 'kud/tests') diff --git a/kud/tests/cmk.sh b/kud/tests/cmk.sh new file mode 100755 index 00000000..1a14b5b8 --- /dev/null +++ b/kud/tests/cmk.sh @@ -0,0 +1,231 @@ +#!/bin/bash +ENV=$(kubectl get nodes --all-namespaces | wc -l) +if [[ $ENV -gt 2 ]]; then + COMPUTE_NODE=$(kubectl get nodes --all-namespaces | grep -v master | awk 'NR==2{print $1}') +else + COMPUTE_NODE=$(kubectl get nodes --all-namespaces | grep master | awk 'NR==1{print $1}') +fi +cases=("exclusive ${COMPUTE_NODE} 1" "shared ${COMPUTE_NODE} -1") +case=(null null 0) +num=${#cases[*]} +POOL=0 +NODE=1 +CORE=2 +DIR=/tmp +pod_name=cmk-test-pod + +function wait_for_pod_up { + status_phase="" + while [[ $status_phase != "Running" ]]; do + new_phase=$(kubectl get pods "$@" | awk 'NR==2{print $3}') + if [[ $new_phase != $status_phase ]]; then + echo "$(date +%H:%M:%S) - $@ : $new_phase" + status_phase=$new_phase + fi + if [[ $new_phase == "Running" ]]; then + echo "Pod $@ is up and running.." + fi + if [[ $new_phase == "Err"* ]]; then + exit 1 + fi + done +} + + +function start_nginx_pod { + kubectl delete deployment -n default nginx --ignore-not-found=true + kubectl create deployment nginx --image=nginx + sleep 2 + nginx_pod=$(kubectl get pods --all-namespaces| grep nginx | awk 'NR==1{print $2}') + wait_for_pod_up $nginx_pod + kubectl delete deployment -n default nginx --ignore-not-found=true + pod_status="Running" + until [[ $pod_status == "" ]]; do + pod_status=$(kubectl get pod $nginx_pod --ignore-not-found=true | awk 'NR==2{print $3}') + done +} + +rm -f $DIR/$pod_name.yaml +kubectl delete pod $pod_name --ignore-not-found=true --now --wait +echo +echo "env is $ENV" +echo +for ((i=0;i<$num;i++)); do + inner_case=(${cases[$i]}) + num_inner=${#inner_case[*]} + for ((j=0;j<$num_inner;j++)); do + case[$j]=${inner_case[$j]} + done + echo "##################################" + if [ "${case[$POOL]}" == "exclusive" ]; then + echo "TC: to allocate ${case[$CORE]} CPU(s) from pool of ${case[$POOL]} on node of ${case[$NODE]}" + TOTAL=$(kubectl get cmk-nodereport ${case[$NODE]} -o json | jq .spec.report.description.pools.${case[$POOL]} | jq .cpuLists | awk -F '{' '{print $(NF)}' | awk -F '}' '{print $(NF)}' | awk -F ',' '{print $(NF)}' | grep "\"tasks\": \[" | wc -l) + echo "ready to generate yaml" +cat << EOF > $DIR/$pod_name.yaml + apiVersion: v1 + kind: Pod + metadata: + labels: + app: cmk-test-pod + name: cmk-test-pod + spec: + nodeName: ${case[$NODE]} + containers: + - args: + - "/opt/bin/cmk isolate --conf-dir=/etc/cmk --pool=exclusive sleep -- 3900" + command: + - "sh" + - "-c" + env: + - name: CMK_PROC_FS + value: "/host/proc" + - name: CMK_NUM_CORES + value: "${case[$CORE]}" + image: ubuntu:18.04 + imagePullPolicy: "IfNotPresent" + name: cmk-test + volumeMounts: + - mountPath: "/host/proc" + name: host-proc + - mountPath: "/opt/bin" + name: cmk-install-dir + - mountPath: "/etc/cmk" + name: cmk-conf-dir + restartPolicy: Never + volumes: + - hostPath: + path: "/opt/bin" + name: cmk-install-dir + - hostPath: + path: "/proc" + name: host-proc + - hostPath: + path: "/etc/cmk" + name: cmk-conf-dir +EOF + + echo "ready to create pod" + kubectl create -f $DIR/$pod_name.yaml --validate=false + sleep 2 + echo "waiting for pod up" + for pod in $pod_name; do + wait_for_pod_up $pod + done + echo "waiting for CPU allocation finished ..." + rest=$TOTAL + until [[ $TOTAL -gt $rest ]]; do + rest=$(kubectl get cmk-nodereport ${case[$NODE]} -o json | jq .spec.report.description.pools.exclusive | jq .cpuLists | awk -F '{' '{print $(NF)}' | awk -F '}' '{print $(NF)}' | awk -F ',' '{print $(NF)}' | grep "\"tasks\": \[\]" | wc -l) + done + let allocated=`expr $TOTAL - $rest` + echo "The allocated CPU amount is:" $allocated + echo "deploy a nginx pod" + start_nginx_pod + if [[ $allocated == ${case[$CORE]} ]]; then + echo "CPU was allocated as expected, TC passed !!" + else + echo "failed to allocate CPU, TC failed !!" + fi + rm -f $DIR/$pod_name.yaml + echo "ready to delete pod" + kubectl delete pod $pod_name --ignore-not-found=true --now --wait + echo "Pod was deleted" + echo "##################################" + echo + echo + else + echo "TC: to allocate CPU(s) from pool of ${case[$POOL]} on node of ${case[$NODE]}" + echo "ready to generate yaml" +cat << EOF > $DIR/$pod_name.yaml +apiVersion: v1 +kind: Pod +metadata: + labels: + app: cmk-test-pod + name: cmk-test-pod +spec: + nodeName: ${case[$NODE]} + containers: + - name: share1 + args: + - "/opt/bin/cmk isolate --conf-dir=/etc/cmk --pool=shared sleep -- 3900" + command: + - "sh" + - "-c" + env: + - name: CMK_PROC_FS + value: "/host/proc" + - name: CMK_NUM_CORES + value: "3" + image: ubuntu:18.10 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: "/host/proc" + name: host-proc + - mountPath: "/opt/bin" + name: cmk-install-dir + - mountPath: "/etc/cmk" + name: cmk-conf-dir + - name: share2 + args: + - "/opt/bin/cmk isolate --conf-dir=/etc/cmk --pool=shared sleep -- 3300" + command: + - "sh" + - "-c" + env: + - name: CMK_PROC_FS + value: "/host/proc" + - name: CMK_NUM_CORES + value: "3" + image: ubuntu:18.10 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: "/host/proc" + name: host-proc + - mountPath: "/opt/bin" + name: cmk-install-dir + - mountPath: "/etc/cmk" + name: cmk-conf-dir + volumes: + - hostPath: + path: "/opt/bin" + name: cmk-install-dir + - hostPath: + path: "/proc" + name: host-proc + - hostPath: + path: "/etc/cmk" + name: cmk-conf-dir +EOF + + echo "ready to create pod" + kubectl create -f $DIR/$pod_name.yaml --validate=false + sleep 2 + echo "waiting for pod up" + for pod in $pod_name; do + wait_for_pod_up $pod + done + echo "waiting for CPU allocation finished ..." + rest=0 + timeout=0 + until [ $rest == 2 -o $timeout == 180 ]; do + rest=$(kubectl get cmk-nodereport ${case[$NODE]} -o json | jq .spec.report.description.pools.shared | jq .cpuLists | awk -F '{' '{print $(NF)}' | awk -F '}' '{print $(NF)}' | grep -v "cpus" | grep " "| grep -v "tasks"| grep -v "\]" | wc -l) + sleep -- 1 + let timeout++ + done + echo "The CPU allocated in shared pool for 2 tasks" + echo "deploy a nginx pod" + start_nginx_pod + if [[ $rest == 2 ]]; then + echo "CPU was allocated as expected, TC passed !!" + else + echo "failed to allocate CPU, TC failed !!" + fi + rm -f $DIR/$pod_name.yaml + echo "ready to delete pod" + kubectl delete pod $pod_name --ignore-not-found=true --now --wait + echo "Pod was deleted" + echo "##################################" + echo + echo + fi +done -- cgit 1.2.3-korg