summaryrefslogtreecommitdiffstats
path: root/tools/performance/cloud
diff options
context:
space:
mode:
Diffstat (limited to 'tools/performance/cloud')
-rwxr-xr-xtools/performance/cloud/cloud-based-performance-test.sh302
-rwxr-xr-xtools/performance/cloud/producer-deployment.yaml2
-rwxr-xr-xtools/performance/cloud/producer-proxy.yaml42
-rw-r--r--tools/performance/cloud/producers-config/instant-config.json3
-rw-r--r--tools/performance/cloud/producers-config/interval-config.json12
-rw-r--r--tools/performance/cloud/producers-config/producer-config.json4
-rwxr-xr-xtools/performance/cloud/reboot-test-environment.sh4
-rw-r--r--tools/performance/cloud/test.properties8
8 files changed, 254 insertions, 123 deletions
diff --git a/tools/performance/cloud/cloud-based-performance-test.sh b/tools/performance/cloud/cloud-based-performance-test.sh
index 2365cc4d..a4ec99b5 100755
--- a/tools/performance/cloud/cloud-based-performance-test.sh
+++ b/tools/performance/cloud/cloud-based-performance-test.sh
@@ -18,12 +18,13 @@
# ============LICENSE_END=========================================================
SCRIPT_DIRECTORY="$(pwd "$0")"
-CONTAINERS_COUNT=1
-COMPLETED_PRODUCERS_SUM=0
-LOAD_TEST="false"
+PRODUCERS_COUNT=1
TEST_CONFIG_MAP=performance-test-config
PROPERTIES_FILE=${SCRIPT_DIRECTORY}/test.properties
PRODUCER_APPS_LABEL=hv-collector-producer
+PRODUCER_APPS_DEPLOYMENT=hv-collector-producer-deployment
+PRODUCER_SERVICE=hv-collector-producer
+PRODUCER_PROXY_APPS_LABEL=hv-collector-producer-proxy
CONSUMER_APPS_LABEL=hv-collector-kafka-consumer
PROMETHEUS_CONF_LABEL=prometheus-server-conf
PROMETHEUS_APPS_LABEL=hv-collector-prometheus
@@ -32,9 +33,6 @@ GRAFANA_DATASOURCE=grafana-datasources
GRAFANA_DASHBOARDS=grafana-dashboards
GRAFANA_DASHBOARD_PROVIDERS=grafana-dashboards-providers
ONAP_NAMESPACE=onap
-MAXIMUM_BACK_OFF_CHECK_ITERATIONS=30
-CHECK_NUMBER=0
-PRODUCERS_TO_RECREATE=0
NAME_REASON_PATTERN="custom-columns=NAME:.metadata.name,REASON:.status.containerStatuses[].state.waiting.reason"
HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
HVVES_CERT_PATH=/etc/ves-hv/ssl/server
@@ -44,6 +42,10 @@ CALC_RETENTION_TIME_IN_MS_CMD='expr $KAFKA_RETENTION_TIME_MINUTES \* $MILISECOND
KAFKA_ROUTER_0_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep router-kafka-0)
KAFKA_SET_TOPIC_RETENTION_TIME_CMD='kafka-topics --zookeeper message-router-zookeeper:2181 --alter --topic HV_VES_PERF3GPP --config retention.ms='
HIDE_OUTPUT='grep abc | grep 123'
+CONTENT_TYPE_HEADER='Content-Type: application/json'
+REQUEST_ENDPOINT="/"
+REQUEST_JSON_DATA="{}"
+PRODUCER_INTERNAL_PORT=8080
function clean() {
echo "Cleaning up environment"
@@ -72,9 +74,6 @@ function clean() {
echo "Attempting to delete consumer deployments"
kubectl delete deployments -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE}
- echo "Attempting to delete producer pods"
- kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
-
echo "Attempting to delete client certs secret"
kubectl delete secret cert -n ${ONAP_NAMESPACE}
@@ -84,6 +83,16 @@ function clean() {
echo "Environment clean up finished!"
}
+function clean_all() {
+ clean
+ echo "Attepting to delete producer pods"
+ kubectl delete service,deployments -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
+ echo "Attepting to delete producer-proxy pod"
+ kubectl delete deployments -l app=${PRODUCER_PROXY_APPS_LABEL} -n ${ONAP_NAMESPACE}
+
+ echo "Producers deleted"
+}
+
function copy_certs_to_hvves() {
cd ../../ssl
echo "Attempting to create certs directory in HV-VES"
@@ -100,52 +109,20 @@ function set_kafka_retention_time() {
kubectl exec -it ${KAFKA_ROUTER_0_POD_NAME} -n ${ONAP_NAMESPACE} -- ${KAFKA_SET_TOPIC_RETENTION_TIME_CMD}$(eval $CALC_RETENTION_TIME_IN_MS_CMD) | eval $HIDE_OUTPUT
}
-function create_producers() {
- echo "Recreating test properties ConfigMap from: $PROPERTIES_FILE"
- kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE}
- kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}
-
- set -e
- for i in $(seq 1 ${CONTAINERS_COUNT});
- do
- echo "Creating ${i}/${CONTAINERS_COUNT} producer"
- kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE}
- done
- echo "Producers created"
- set +e
-}
-
function generate_certs() {
echo "Generation of certs"
cd ../../ssl
./gen-certs.sh
}
-function handle_backoffs() {
- IMAGE_PULL_BACK_OFFS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} -o ${NAME_REASON_PATTERN} | grep -c "ImagePullBackOff \| ErrImagePull")
- if [[ ${IMAGE_PULL_BACK_OFFS} -gt 0 ]]; then
- CHECK_NUMBER=$((CHECK_NUMBER + 1))
- if [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]]; then
- echo "Error: Image pull problem"
- exit 1
- fi
- fi
-}
-
-function handle_key_interrupt() {
- trap SIGINT
- echo "Script interrupted, attempt to delete producers"
- echo "Wait with patience"
- COMPLETED_PRODUCERS_SUM=$(($(kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep producer | wc -l) + COMPLETED_PRODUCERS_SUM))
- echo "Total number of completed producers: ${COMPLETED_PRODUCERS_SUM}"
- exit 0
-}
-
function print_test_setup_info() {
+ RUNNING_PRODUCERS=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_APPS_LABEL} | grep -c "Running")
echo "Starting cloud based performance tests"
echo "________________________________________"
echo "Test configuration:"
- echo "Producer containers count: ${CONTAINERS_COUNT}"
+ echo "Running producers: ${RUNNING_PRODUCERS}"
+ echo "Sending scheme:"
+ echo ${REQUEST_JSON_DATA}
echo "Properties file path: ${PROPERTIES_FILE}"
echo "Retention time of kafka messages in minutes: ${KAFKA_RETENTION_TIME_MINUTES}"
echo "________________________________________"
@@ -154,26 +131,36 @@ function print_test_setup_info() {
function usage() {
echo ""
echo "Run cloud based HV-VES performance test"
- echo "Usage $0 gen_certs|setup|start|clean|help"
- echo " gen_certs: generate certs in ../../ssl directory"
- echo " setup : set up ConfigMap and consumers"
- echo " start : create producers - start the performance test"
- echo " Optional parameters:"
- echo " --load : should test keep defined containers number till script interruption (false)"
- echo " --containers : number of producer containers to create (1)"
- echo " --properties-file : path to file with benchmark properties (./test.properties)"
- echo " --retention-time-minutes : messages retention time on kafka in minutes (60)"
- echo " clean : remove ConfigMap, HV-VES consumers and producers"
- echo " help : print usage"
+ echo "Usage $0 gen_certs|setup|send_config|start_interval|start_instant|stop|reset_producers|clean|help"
+ echo " gen_certs : generate certs in ../../ssl directory"
+ echo " setup : set up ConfigMaps and consumers"
+ echo " setup_all : set up ConfigMaps consumers and producers"
+ echo " send_config : send producers configuration (message interval and payload), located in producers-config/producer-config.json to each producer"
+ echo " start_interval : start interval mode, config file is located in producers-config/interval-config.json"
+ echo " Optional parameters:"
+ echo " --producers : number of producers in deployment (10)"
+ echo " --retention-time-minutes : messages retention time on kafka in minutes (60)"
+ echo " start_instant : start instant mode, config file is located in producers-config/instant-config.json"
+ echo " Optional parameters:"
+ echo " --producers : number of producers in deployment (10)"
+ echo " --retention-time-minutes : messages retention time on kafka in minutes (60)"
+ echo " scale_producers : scale producer deployment to number provide in argument"
+ echo " stop : stop all producers"
+ echo " reset_producers : reset all metrics on each producer"
+ echo " clean : remove ConfigMap, HV-VES consumers"
+ echo " clean_all : remove ConfigMap, HV-VES consumers and producers"
+ echo " help : print usage"
echo "Example invocations:"
echo "./cloud-based-performance-test.sh gen_certs"
echo "./cloud-based-performance-test.sh setup"
- echo "./cloud-based-performance-test.sh start"
- echo "./cloud-based-performance-test.sh start --containers 10"
- echo "./cloud-based-performance-test.sh start --load true --containers 10"
- echo "./cloud-based-performance-test.sh start --load true --containers 10 --retention-time-minutes 50"
- echo "./cloud-based-performance-test.sh start --properties-file ~/other_test.properties"
+ echo "./cloud-based-performance-test.sh setup_all"
+ echo "./cloud-based-performance-test.sh send_config"
+ echo "./cloud-based-performance-test.sh start_interval"
+ echo "./cloud-based-performance-test.sh start_interval --producers 8"
+ echo "./cloud-based-performance-test.sh start_instant"
+ echo "./cloud-based-performance-test.sh scale_producers 8"
echo "./cloud-based-performance-test.sh clean"
+ echo "./cloud-based-performance-test.sh clean_all"
exit 1
}
@@ -221,62 +208,112 @@ function setup_environment() {
echo "Setting up environment finished!"
}
-function start_load_tests() {
- print_test_setup_info
+function setup_all() {
+ setup_environment
+ echo "Creating producer deployment"
+ kubectl apply -f producer-deployment.yaml
- set_kafka_retention_time
+ echo "Creating producer-proxy pod"
+ kubectl apply -f producer-proxy.yaml
- echo "CTRL + C to stop/interrupt this script"
- create_producers
-
- trap "handle_key_interrupt" INT
+ echo "Creating producers finished"
+}
- echo "Constant producer number keeper started working"
- while :; do
- PRODUCERS_TO_RECREATE=$((CONTAINERS_COUNT-$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Running")))
- handle_backoffs
+function scale_producer_deployment() {
+ echo "Scaling prodcuer deployment to ${PRODUCERS_COUNT}"
+ kubectl scale --replicas=${PRODUCERS_COUNT} deployment ${PRODUCER_APPS_DEPLOYMENT} -n ${ONAP_NAMESPACE}
+ RUNNING_PRODUCERS=""
+ while [ "${RUNNING_PRODUCERS}" != "${PRODUCERS_COUNT}" ]; do
+ RUNNING_PRODUCERS=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_APPS_LABEL} | grep -c "Running")
+ sleep 1s
+ done
+ echo "Producers are ready"
+}
- set -e
- for i in $(seq 1 ${PRODUCERS_TO_RECREATE});
- do
- echo "Recreating ${i}/${PRODUCERS_TO_RECREATE} producer"
- kubectl create -f producer-pod.yaml -n ${ONAP_NAMESPACE}
- done
- set +e
- COMPLETED_PRODUCERS_SUM=$((COMPLETED_PRODUCERS_SUM + PRODUCERS_TO_RECREATE))
- echo "Attempting to clear completed producers"
- kubectl delete pod --field-selector=status.phase==Succeeded -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
+function set_producers_array_internal_IP() {
+ PRODUCER_IP_ARRAY=$(kubectl -n ${ONAP_NAMESPACE} get endpoints ${PRODUCER_SERVICE} -o jsonpath="{.subsets[*].addresses[*].ip}")
+}
- [[ ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break
- sleep 1
+function send_post_request_to_each_producer() {
+ set_producers_array_internal_IP
+ PROXY_POD=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_PROXY_APPS_LABEL} -o name)
+ echo "Sending POST request to each producer, endpoint: ${REQUEST_ENDPOINT}"
+ REQUEST_ARRAY=""
+ for item in ${PRODUCER_IP_ARRAY[*]}
+ do
+ URL="${item}:${PRODUCER_INTERNAL_PORT}${REQUEST_ENDPOINT}"
+ echo ${URL}
+ REQUEST_ARRAY="curl -H '${CONTENT_TYPE_HEADER}' -X POST -d '${REQUEST_JSON_DATA}' ${URL} ; ${REQUEST_ARRAY}"
done
+ kubectl -n onap exec -it ${PROXY_POD} -- /bin/sh -c "${REQUEST_ARRAY}"
+ echo "Request was send to each producer"
+}
- trap SIGINT
- exit 0
+function send_get_request_to_each_producer() {
+ set_producers_array_internal_IP
+ PROXY_POD=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_PROXY_APPS_LABEL} -o name)
+ echo "Sending GET request to each producer, endpoint: ${REQUEST_ENDPOINT}"
+ REQUEST_ARRAY=""
+ for item in ${PRODUCER_IP_ARRAY[*]}
+ do
+ URL="${item}:${PRODUCER_INTERNAL_PORT}${REQUEST_ENDPOINT}"
+ echo ${URL}
+ REQUEST_ARRAY="curl -X GET ${URL} ; ${REQUEST_ARRAY}"
+ done
+ kubectl -n onap exec -it ${PROXY_POD} -- /bin/sh -c "${REQUEST_ARRAY}"
+ echo "Request was send to each producer"
}
-function start_performance_test() {
- print_test_setup_info
+function send_configuration() {
+ REQUEST_ENDPOINT="/configuration"
+ REQUEST_JSON_DATA=$(cat producers-config/producer-config.json)
+ echo "Sending producer configuration: "
+ echo ${REQUEST_JSON_DATA}
+ send_post_request_to_each_producer
+ echo "Configuration was send to each producer pod"
+ exit 0
+}
- set_kafka_retention_time
+function start_interval_mode() {
+ set_kafka_retention_time
+ REQUEST_ENDPOINT="/interval"
+ REQUEST_JSON_DATA=$(cat producers-config/interval-config.json)
+ print_test_setup_info
+ echo "Sending start interval command to producer pods"
+ send_post_request_to_each_producer
+ echo "Command was send to each producer pod"
+ exit 0
+}
- create_producers
+function start_instant_mode() {
+ set_kafka_retention_time
+ REQUEST_ENDPOINT="/instant"
+ REQUEST_JSON_DATA=$(cat producers-config/instant-config.json)
+ print_test_setup_info
- echo "Waiting for producers completion"
- while :; do
- COMPLETED_PRODUCERS=$(kubectl get pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "Completed")
- handle_backoffs
+ echo "Sending start instant command to producer pods"
+ send_post_request_to_each_producer
+ echo "Command was send to each producer pod"
+ exit 0
+}
+function stop_producer_pods() {
+ REQUEST_ENDPOINT="/cancel"
- [[ ${COMPLETED_PRODUCERS} -eq ${CONTAINERS_COUNT} || ${CHECK_NUMBER} -gt ${MAXIMUM_BACK_OFF_CHECK_ITERATIONS} ]] && break
- sleep 1
- done
+ echo "Sending stop command"
+ send_get_request_to_each_producer
+ echo "Stop command was send to each producer pod"
+ exit 0
+}
+function reset_producer_pods() {
+ REQUEST_ENDPOINT="/monitoring/prometheus/reset"
- echo "Attempting to delete producer pods"
- kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
- echo "Performance test finished"
- exit 0
+ echo "Sending reset command"
+ send_get_request_to_each_producer
+ echo "Reset command was send to each producer pod"
+ exit 0
}
+
cd ${SCRIPT_DIRECTORY}
if [[ $# -eq 0 ]]; then
@@ -291,18 +328,50 @@ else
setup)
setup_environment
;;
+ setup_all)
+ setup_all
+ ;;
start)
+ echo "Option start is deprecated. Last support commit: 3e7de0deb033e485d519c74feaffecc02e7e9dc7"
+ ;;
+ clean)
+ clean
+ ;;
+ clean_all)
+ clean_all
+ ;;
+ send_config)
+ send_configuration
+ ;;
+ start_interval)
shift 1
while [[ $(($#)) -gt 0 ]]; do
case "${1}" in
- --load)
- LOAD_TEST=${2}
+ --producers)
+ PRODUCERS_COUNT=${2}
+ scale_producer_deployment
+ send_configuration
;;
- --containers)
- CONTAINERS_COUNT=${2}
+ --retention-time-minutes)
+ KAFKA_RETENTION_TIME_MINUTES=${2}
+ ;;
+ *)
+ echo "Unknown option: ${1}"
+ usage
;;
- --properties-file)
- PROPERTIES_FILE=${2}
+ esac
+ shift 2
+ done
+ start_interval_mode
+ ;;
+ start_instant)
+ shift 1
+ while [[ $(($#)) -gt 0 ]]; do
+ case "${1}" in
+ --producers)
+ PRODUCERS_COUNT=${2}
+ scale_producer_deployment
+ send_configuration
;;
--retention-time-minutes)
KAFKA_RETENTION_TIME_MINUTES=${2}
@@ -314,14 +383,19 @@ else
esac
shift 2
done
- if [ ${LOAD_TEST} == "true" ] ; then
- start_load_tests
- else
- start_performance_test
- fi
+ start_instant_mode
;;
- clean)
- clean
+ stop)
+ stop_producer_pods
+ ;;
+ reset_producers)
+ reset_producer_pods
+ ;;
+ scale_producers)
+ shift 1
+ PRODUCERS_COUNT=${1}
+ scale_producer_deployment
+ exit 0
;;
help)
usage
diff --git a/tools/performance/cloud/producer-deployment.yaml b/tools/performance/cloud/producer-deployment.yaml
index f13760a6..e81fd7f6 100755
--- a/tools/performance/cloud/producer-deployment.yaml
+++ b/tools/performance/cloud/producer-deployment.yaml
@@ -80,7 +80,7 @@ metadata:
prometheus.io/path: '/monitoring/prometheus'
spec:
selector:
- name: hv-collector-producer
+ app: hv-collector-producer
type: NodePort
ports:
- protocol: TCP
diff --git a/tools/performance/cloud/producer-proxy.yaml b/tools/performance/cloud/producer-proxy.yaml
new file mode 100755
index 00000000..ace46a63
--- /dev/null
+++ b/tools/performance/cloud/producer-proxy.yaml
@@ -0,0 +1,42 @@
+# ============LICENSE_START=======================================================
+# dcaegen2-collectors-veshv
+# ================================================================================
+# Copyright (C) 2020 NOKIA
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: hv-collector-producer-proxy-deployment
+ namespace: onap
+ labels:
+ app: hv-collector-producer-proxy
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: hv-collector-producer-proxy
+ template:
+ metadata:
+ labels:
+ app: hv-collector-producer-proxy
+ name: hv-collector-producer-proxy
+ spec:
+ containers:
+ - name: hv-collector-producer-proxy
+ imagePullPolicy: IfNotPresent
+ image: alpine:latest
+ command: [ "/bin/sh", "-c" ]
+ args: [ "apk add --update curl;rm -rf /var/cache/apk/*; while true; do sleep 30; done;" ]
diff --git a/tools/performance/cloud/producers-config/instant-config.json b/tools/performance/cloud/producers-config/instant-config.json
new file mode 100644
index 00000000..486bb077
--- /dev/null
+++ b/tools/performance/cloud/producers-config/instant-config.json
@@ -0,0 +1,3 @@
+{
+ "connections": 500
+}
diff --git a/tools/performance/cloud/producers-config/interval-config.json b/tools/performance/cloud/producers-config/interval-config.json
new file mode 100644
index 00000000..62deb8c4
--- /dev/null
+++ b/tools/performance/cloud/producers-config/interval-config.json
@@ -0,0 +1,12 @@
+{
+ "intervalConfigs": [
+ {
+ "duration": 1,
+ "interval": 100
+ },
+ {
+ "duration": 2,
+ "interval": 100
+ }
+ ]
+}
diff --git a/tools/performance/cloud/producers-config/producer-config.json b/tools/performance/cloud/producers-config/producer-config.json
new file mode 100644
index 00000000..74972e61
--- /dev/null
+++ b/tools/performance/cloud/producers-config/producer-config.json
@@ -0,0 +1,4 @@
+{
+ "payloadSize": 8192,
+ "messageIntervalMs": 500
+}
diff --git a/tools/performance/cloud/reboot-test-environment.sh b/tools/performance/cloud/reboot-test-environment.sh
index 0fb916f0..3d9a5a1e 100755
--- a/tools/performance/cloud/reboot-test-environment.sh
+++ b/tools/performance/cloud/reboot-test-environment.sh
@@ -63,6 +63,10 @@ function rebootEnvironment(){
./cloud-based-performance-test.sh setup | formatOutput
+ ./cloud-based-performance-test.sh stop | formatOutput
+
+ ./cloud-based-performance-test.sh reset_producers | formatOutput
+
echo "${GREEN}Environment ready!${NO_COLOR}"
}
diff --git a/tools/performance/cloud/test.properties b/tools/performance/cloud/test.properties
index 53a38e23..05a50055 100644
--- a/tools/performance/cloud/test.properties
+++ b/tools/performance/cloud/test.properties
@@ -2,14 +2,6 @@
# HV-VES address
producer.hvVesAddress=dcae-hv-ves-collector.onap:6061
-# Number of clients per pod
-producer.client.count=1
-# Size in bytes of a single message
-producer.message.size=16384
-# Amount of messages to sent by one client in a single pod
-producer.message.count=1000
-# Interval between messages in milliseconds
-producer.message.interval=1
# Path to client cert file
client.cert.path=/ssl/client.p12
# Path to cert pass file