diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/performance/cloud/cloud-based-performance-test.sh | 101 |
1 files changed, 80 insertions, 21 deletions
diff --git a/tools/performance/cloud/cloud-based-performance-test.sh b/tools/performance/cloud/cloud-based-performance-test.sh index b9267a97..283c1301 100755 --- a/tools/performance/cloud/cloud-based-performance-test.sh +++ b/tools/performance/cloud/cloud-based-performance-test.sh @@ -19,6 +19,8 @@ SCRIPT_DIRECTORY="$(pwd "$0")" CONTAINERS_COUNT=1 +LOAD_TEST="false" +TEST_CONFIG_MAP=performance-test-config PROPERTIES_FILE=${SCRIPT_DIRECTORY}/test.properties PRODUCER_APPS_LABEL=hv-collector-producer CONSUMER_APPS_LABEL=hv-collector-kafka-consumer @@ -31,6 +33,7 @@ GRAFANA_DASHBOARD_PROVIDERS=grafana-dashboards-providers ONAP_NAMESPACE=onap MAXIMUM_BACK_OFF_CHECK_ITERATIONS=30 CHECK_NUMBER=0 +COMPLETED_PRODUCERS_SUM=0 NAME_REASON_PATTERN="custom-columns=NAME:.metadata.name,REASON:.status.containerStatuses[].state.waiting.reason" HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector) HVVES_CERT_PATH=/etc/ves-hv/ssl/ @@ -39,7 +42,7 @@ function clean() { echo "Cleaning up environment" echo "Attempting to delete test parameters ConfigMap" - kubectl delete configmap performance-test-config -n ${ONAP_NAMESPACE} + kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE} echo "Attempting to delete prometheus ConfigMap" kubectl delete configmap -l name=${PROMETHEUS_CONF_LABEL} -n ${ONAP_NAMESPACE} @@ -50,13 +53,13 @@ function clean() { echo "Attempting to delete grafana deployment and service" kubectl delete service,deployments -l app=${GRAFANA_APPS_LABEL} -n ${ONAP_NAMESPACE} - echo "Attempting to delete grafana ConfigMap(DASHBOARD)" + echo "Attempting to delete grafana ConfigMap (DASHBOARD)" kubectl delete configmap -l name=${GRAFANA_DASHBOARD} -n ${ONAP_NAMESPACE} - echo "Attempting to delete grafana ConfigMap(GRAFANA_DASHBOARD_PROVIDERS)" + echo "Attempting to delete grafana ConfigMap (GRAFANA_DASHBOARD_PROVIDERS)" kubectl delete configmap -l name=${GRAFANA_DASHBOARD_PROVIDERS} -n ${ONAP_NAMESPACE} - echo "Attempting to delete grafana ConfigMap(GRAFANA_DATASOURCE)" + echo "Attempting to delete grafana ConfigMap (GRAFANA_DATASOURCE)" kubectl delete configmap -l name=${GRAFANA_DATASOURCE} -n ${ONAP_NAMESPACE} echo "Attempting to delete consumer deployments" @@ -85,8 +88,8 @@ function copy_certs_to_hvves() { function create_producers() { echo "Recreating test properties ConfigMap from: $PROPERTIES_FILE" - kubectl delete configmap performance-test-config -n ${ONAP_NAMESPACE} - kubectl create configmap performance-test-config --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE} + kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE} + kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE} set -e for i in $(seq 1 ${CONTAINERS_COUNT}); @@ -104,6 +107,32 @@ function generate_certs() { ./gen-certs.sh } +function handle_backoffs() { + if [[ ${IMAGE_PULL_BACK_OFFS} -gt 0 ]]; then + CHECK_NUMBER=$((CHECK_NUMBER + 1)) + if [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]]; then + echo "Error: Image pull problem" + exit 1 + fi + fi +} + +function handle_key_interrupt() { + trap SIGINT + echo "Script interrupted, attempt to delete producers" + kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} + exit 0 +} + +function print_test_setup_info() { + echo "Starting cloud based performance tests" + echo "________________________________________" + echo "Test configuration:" + echo "Producer containers count: ${CONTAINERS_COUNT}" + echo "Properties file path: ${PROPERTIES_FILE}" + echo "________________________________________" +} + function usage() { echo "" echo "Run cloud based HV-VES performance test" @@ -112,6 +141,7 @@ function usage() { echo " setup : set up ConfigMap and consumers" echo " start : create producers - start the performance test" echo " Optional parameters:" + echo " --load : should test keep defined containers number till script interruption (false)" echo " --containers : number of producer containers to create (1)" echo " --properties-file : path to file with benchmark properties (./test.properties)" echo " clean : remove ConfigMap, HV-VES consumers and producers" @@ -121,6 +151,7 @@ function usage() { echo "./cloud-based-performance-test.sh setup" echo "./cloud-based-performance-test.sh start" echo "./cloud-based-performance-test.sh start --containers 10" + echo "./cloud-based-performance-test.sh start --load true --containers 10" echo "./cloud-based-performance-test.sh start --properties-file ~/other_test.properties" echo "./cloud-based-performance-test.sh clean" exit 1 @@ -128,6 +159,7 @@ function usage() { function setup_environment() { echo "Setting up environment" + echo "Copying certs to hv-ves pod" copy_certs_to_hvves @@ -139,7 +171,7 @@ function setup_environment() { ./configure-consul.sh false echo "Creating test properties ConfigMap from: $PROPERTIES_FILE" - kubectl create configmap performance-test-config --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE} + kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE} echo "Creating consumer deployment" kubectl apply -f consumer-deployment.yaml @@ -175,13 +207,39 @@ function setup_environment() { echo "Setting up environment finished!" } +function start_load_tests() { + print_test_setup_info + + echo "CTRL + C to stop/interrupt this script" + create_producers + + trap "handle_key_interrupt" INT + + echo "Constant producer number keeper started working" + while :; do + COMPLETED_PRODUCERS=$(($(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed")-COMPLETED_PRODUCERS_SUM)) + IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull") + + handle_backoffs + + set -e + for i in $(seq 1 ${COMPLETED_PRODUCERS}); + do + echo "Recreating ${i}/${COMPLETED_PRODUCERS} producer" + kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE} + COMPLETED_PRODUCERS_SUM=$((COMPLETED_PRODUCERS_SUM + 1)) + done + set +e + [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break + sleep 2 + done + + trap SIGINT + exit 0 +} + function start_performance_test() { - echo "Starting cloud based performance tests" - echo "________________________________________" - echo "Test configuration:" - echo "Producer containers count: ${CONTAINERS_COUNT}" - echo "Properties file path: ${PROPERTIES_FILE}" - echo "________________________________________" + print_test_setup_info create_producers @@ -190,13 +248,7 @@ function start_performance_test() { COMPLETED_PRODUCERS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed") IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull") - if [[ ${IMAGE_PULL_BACK_OFFS} -gt 0 ]]; then - CHECK_NUMBER=$((CHECK_NUMBER + 1)) - if [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]]; then - echo "Error: Image pull problem" - exit 1 - fi - fi + handle_backoffs [[ ${COMPLETED_PRODUCERS} -eq ${CONTAINERS_COUNT} || ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break sleep 1 @@ -226,6 +278,9 @@ else shift 1 while [[ $(($#)) -gt 0 ]]; do case "${1}" in + --load) + LOAD_TEST=${2} + ;; --containers) CONTAINERS_COUNT=${2} ;; @@ -239,7 +294,11 @@ else esac shift 2 done - start_performance_test + if [ ${LOAD_TEST} == "true" ] ; then + start_load_tests + else + start_performance_test + fi ;; clean) clean |