diff options
| -rw-r--r-- | INFO.yaml | 15 | ||||
| -rwxr-xr-x | tools/performance/cloud/cloud-based-performance-test.sh | 101 |
2 files changed, 87 insertions, 29 deletions
@@ -28,8 +28,7 @@ meetings: repeats: 'weekly' time: '13:00 UTC (DST), 15:30 UTC (post DST)' repositories: - - 'dcaegen2-collectors-hv-ves' - - 'dcaegen2-services-prh' + - 'dcaegen2/collectors/hv-ves' committers: - <<: *onap_dcaegen2_ptl - name: 'Lusheng Ji' @@ -62,11 +61,6 @@ committers: company: 'ATT' id: 'jflucas' timezone: 'America/New_York' - - name: 'Piotr Jaszczyk' - email: 'piotr.jaszczyk@nokia.com' - company: 'Nokia' - id: 'jaszczur' - timezone: 'Europe/Warsaw' - name: 'Piotr Wielebski' email: 'piotr.wielebski@nokia.com' company: 'Nokia' @@ -74,11 +68,16 @@ committers: timezone: 'Europe/Warsaw' tsc: approval: 'https://lists.onap.org/pipermail/onap-tsc' - changes: + changes: - type: 'Removal' name: 'Przemyslaw Wasala' link: 'https://lists.onap.org/g/onap-tsc/message/4248' - type: 'Addition' name: 'Piotr Jaszczyk' + link: 'https://lists.onap.org/g/onap-tsc/message/4259' + - type: 'Addition' name: 'Piotr Wielebski' link: 'https://lists.onap.org/g/onap-tsc/message/4259' + - type: 'Removal' + name: 'Piotr Jaszczyk' + link: 'https://lists.onap.org/g/onap-tsc/message/5539' diff --git a/tools/performance/cloud/cloud-based-performance-test.sh b/tools/performance/cloud/cloud-based-performance-test.sh index b9267a97..283c1301 100755 --- a/tools/performance/cloud/cloud-based-performance-test.sh +++ b/tools/performance/cloud/cloud-based-performance-test.sh @@ -19,6 +19,8 @@ SCRIPT_DIRECTORY="$(pwd "$0")" CONTAINERS_COUNT=1 +LOAD_TEST="false" +TEST_CONFIG_MAP=performance-test-config PROPERTIES_FILE=${SCRIPT_DIRECTORY}/test.properties PRODUCER_APPS_LABEL=hv-collector-producer CONSUMER_APPS_LABEL=hv-collector-kafka-consumer @@ -31,6 +33,7 @@ GRAFANA_DASHBOARD_PROVIDERS=grafana-dashboards-providers ONAP_NAMESPACE=onap MAXIMUM_BACK_OFF_CHECK_ITERATIONS=30 CHECK_NUMBER=0 +COMPLETED_PRODUCERS_SUM=0 NAME_REASON_PATTERN="custom-columns=NAME:.metadata.name,REASON:.status.containerStatuses[].state.waiting.reason" HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector) HVVES_CERT_PATH=/etc/ves-hv/ssl/ @@ -39,7 +42,7 @@ function clean() { echo "Cleaning up environment" echo "Attempting to delete test parameters ConfigMap" - kubectl delete configmap performance-test-config -n ${ONAP_NAMESPACE} + kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE} echo "Attempting to delete prometheus ConfigMap" kubectl delete configmap -l name=${PROMETHEUS_CONF_LABEL} -n ${ONAP_NAMESPACE} @@ -50,13 +53,13 @@ function clean() { echo "Attempting to delete grafana deployment and service" kubectl delete service,deployments -l app=${GRAFANA_APPS_LABEL} -n ${ONAP_NAMESPACE} - echo "Attempting to delete grafana ConfigMap(DASHBOARD)" + echo "Attempting to delete grafana ConfigMap (DASHBOARD)" kubectl delete configmap -l name=${GRAFANA_DASHBOARD} -n ${ONAP_NAMESPACE} - echo "Attempting to delete grafana ConfigMap(GRAFANA_DASHBOARD_PROVIDERS)" + echo "Attempting to delete grafana ConfigMap (GRAFANA_DASHBOARD_PROVIDERS)" kubectl delete configmap -l name=${GRAFANA_DASHBOARD_PROVIDERS} -n ${ONAP_NAMESPACE} - echo "Attempting to delete grafana ConfigMap(GRAFANA_DATASOURCE)" + echo "Attempting to delete grafana ConfigMap (GRAFANA_DATASOURCE)" kubectl delete configmap -l name=${GRAFANA_DATASOURCE} -n ${ONAP_NAMESPACE} echo "Attempting to delete consumer deployments" @@ -85,8 +88,8 @@ function copy_certs_to_hvves() { function create_producers() { echo "Recreating test properties ConfigMap from: $PROPERTIES_FILE" - kubectl delete configmap performance-test-config -n ${ONAP_NAMESPACE} - kubectl create configmap performance-test-config --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE} + kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE} + kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE} set -e for i in $(seq 1 ${CONTAINERS_COUNT}); @@ -104,6 +107,32 @@ function generate_certs() { ./gen-certs.sh } +function handle_backoffs() { + if [[ ${IMAGE_PULL_BACK_OFFS} -gt 0 ]]; then + CHECK_NUMBER=$((CHECK_NUMBER + 1)) + if [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]]; then + echo "Error: Image pull problem" + exit 1 + fi + fi +} + +function handle_key_interrupt() { + trap SIGINT + echo "Script interrupted, attempt to delete producers" + kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} + exit 0 +} + +function print_test_setup_info() { + echo "Starting cloud based performance tests" + echo "________________________________________" + echo "Test configuration:" + echo "Producer containers count: ${CONTAINERS_COUNT}" + echo "Properties file path: ${PROPERTIES_FILE}" + echo "________________________________________" +} + function usage() { echo "" echo "Run cloud based HV-VES performance test" @@ -112,6 +141,7 @@ function usage() { echo " setup : set up ConfigMap and consumers" echo " start : create producers - start the performance test" echo " Optional parameters:" + echo " --load : should test keep defined containers number till script interruption (false)" echo " --containers : number of producer containers to create (1)" echo " --properties-file : path to file with benchmark properties (./test.properties)" echo " clean : remove ConfigMap, HV-VES consumers and producers" @@ -121,6 +151,7 @@ function usage() { echo "./cloud-based-performance-test.sh setup" echo "./cloud-based-performance-test.sh start" echo "./cloud-based-performance-test.sh start --containers 10" + echo "./cloud-based-performance-test.sh start --load true --containers 10" echo "./cloud-based-performance-test.sh start --properties-file ~/other_test.properties" echo "./cloud-based-performance-test.sh clean" exit 1 @@ -128,6 +159,7 @@ function usage() { function setup_environment() { echo "Setting up environment" + echo "Copying certs to hv-ves pod" copy_certs_to_hvves @@ -139,7 +171,7 @@ function setup_environment() { ./configure-consul.sh false echo "Creating test properties ConfigMap from: $PROPERTIES_FILE" - kubectl create configmap performance-test-config --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE} + kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE} echo "Creating consumer deployment" kubectl apply -f consumer-deployment.yaml @@ -175,13 +207,39 @@ function setup_environment() { echo "Setting up environment finished!" } +function start_load_tests() { + print_test_setup_info + + echo "CTRL + C to stop/interrupt this script" + create_producers + + trap "handle_key_interrupt" INT + + echo "Constant producer number keeper started working" + while :; do + COMPLETED_PRODUCERS=$(($(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed")-COMPLETED_PRODUCERS_SUM)) + IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull") + + handle_backoffs + + set -e + for i in $(seq 1 ${COMPLETED_PRODUCERS}); + do + echo "Recreating ${i}/${COMPLETED_PRODUCERS} producer" + kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE} + COMPLETED_PRODUCERS_SUM=$((COMPLETED_PRODUCERS_SUM + 1)) + done + set +e + [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break + sleep 2 + done + + trap SIGINT + exit 0 +} + function start_performance_test() { - echo "Starting cloud based performance tests" - echo "________________________________________" - echo "Test configuration:" - echo "Producer containers count: ${CONTAINERS_COUNT}" - echo "Properties file path: ${PROPERTIES_FILE}" - echo "________________________________________" + print_test_setup_info create_producers @@ -190,13 +248,7 @@ function start_performance_test() { COMPLETED_PRODUCERS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed") IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull") - if [[ ${IMAGE_PULL_BACK_OFFS} -gt 0 ]]; then - CHECK_NUMBER=$((CHECK_NUMBER + 1)) - if [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]]; then - echo "Error: Image pull problem" - exit 1 - fi - fi + handle_backoffs [[ ${COMPLETED_PRODUCERS} -eq ${CONTAINERS_COUNT} || ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break sleep 1 @@ -226,6 +278,9 @@ else shift 1 while [[ $(($#)) -gt 0 ]]; do case "${1}" in + --load) + LOAD_TEST=${2} + ;; --containers) CONTAINERS_COUNT=${2} ;; @@ -239,7 +294,11 @@ else esac shift 2 done - start_performance_test + if [ ${LOAD_TEST} == "true" ] ; then + start_load_tests + else + start_performance_test + fi ;; clean) clean |