summaryrefslogtreecommitdiffstats
path: root/tools/performance/cloud/cloud-based-performance-test.sh
blob: a4ec99b51eba2a73cd7d1cc6d632ca3921602016 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
#!/usr/bin/env bash
# ============LICENSE_START=======================================================
# dcaegen2-collectors-veshv
# ================================================================================
# Copyright (C) 2019-2020 NOKIA
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============LICENSE_END=========================================================

SCRIPT_DIRECTORY="$(pwd "$0")"
PRODUCERS_COUNT=1
TEST_CONFIG_MAP=performance-test-config
PROPERTIES_FILE=${SCRIPT_DIRECTORY}/test.properties
PRODUCER_APPS_LABEL=hv-collector-producer
PRODUCER_APPS_DEPLOYMENT=hv-collector-producer-deployment
PRODUCER_SERVICE=hv-collector-producer
PRODUCER_PROXY_APPS_LABEL=hv-collector-producer-proxy
CONSUMER_APPS_LABEL=hv-collector-kafka-consumer
PROMETHEUS_CONF_LABEL=prometheus-server-conf
PROMETHEUS_APPS_LABEL=hv-collector-prometheus
GRAFANA_APPS_LABEL=hv-collector-grafana
GRAFANA_DATASOURCE=grafana-datasources
GRAFANA_DASHBOARDS=grafana-dashboards
GRAFANA_DASHBOARD_PROVIDERS=grafana-dashboards-providers
ONAP_NAMESPACE=onap
NAME_REASON_PATTERN="custom-columns=NAME:.metadata.name,REASON:.status.containerStatuses[].state.waiting.reason"
HVVES_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep hv-ves-collector)
HVVES_CERT_PATH=/etc/ves-hv/ssl/server
KAFKA_RETENTION_TIME_MINUTES=60
MILISECONDS_IN_MINUTE=60000
CALC_RETENTION_TIME_IN_MS_CMD='expr $KAFKA_RETENTION_TIME_MINUTES \* $MILISECONDS_IN_MINUTE'
KAFKA_ROUTER_0_POD_NAME=$(kubectl -n ${ONAP_NAMESPACE} get pods --no-headers=true -o custom-columns=:metadata.name | grep router-kafka-0)
KAFKA_SET_TOPIC_RETENTION_TIME_CMD='kafka-topics --zookeeper message-router-zookeeper:2181 --alter --topic HV_VES_PERF3GPP --config retention.ms='
HIDE_OUTPUT='grep abc | grep 123'
CONTENT_TYPE_HEADER='Content-Type: application/json'
REQUEST_ENDPOINT="/"
REQUEST_JSON_DATA="{}"
PRODUCER_INTERNAL_PORT=8080

function clean() {
    echo "Cleaning up environment"

    echo "Attempting to delete test parameters ConfigMap"
    kubectl delete configmap ${TEST_CONFIG_MAP} -n ${ONAP_NAMESPACE}

    echo "Attempting to delete prometheus ConfigMap"
    kubectl delete configmap -l name=${PROMETHEUS_CONF_LABEL} -n ${ONAP_NAMESPACE}

    echo "Attempting to delete prometheus deployment and service"
    kubectl delete service,deployments -l app=${PROMETHEUS_APPS_LABEL} -n ${ONAP_NAMESPACE}

    echo "Attempting to delete grafana deployment and service"
    kubectl delete service,deployments -l app=${GRAFANA_APPS_LABEL} -n ${ONAP_NAMESPACE}

    echo "Attempting to delete grafana ConfigMap (DASHBOARDS)"
    kubectl delete configmap ${GRAFANA_DASHBOARDS} -n ${ONAP_NAMESPACE}

    echo "Attempting to delete grafana ConfigMap (GRAFANA_DASHBOARD_PROVIDERS)"
    kubectl delete configmap -l name=${GRAFANA_DASHBOARD_PROVIDERS} -n ${ONAP_NAMESPACE}

    echo "Attempting to delete grafana ConfigMap (GRAFANA_DATASOURCE)"
    kubectl delete configmap -l name=${GRAFANA_DATASOURCE} -n ${ONAP_NAMESPACE}

    echo "Attempting to delete consumer deployments"
    kubectl delete deployments -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE}

    echo "Attempting to delete client certs secret"
    kubectl delete secret cert -n ${ONAP_NAMESPACE}

    echo "Attempting to turn off SSL"
    ./configure-consul.sh true

    echo "Environment clean up finished!"
}

function clean_all() {
    clean
    echo "Attepting to delete producer pods"
    kubectl delete service,deployments -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
    echo "Attepting to delete producer-proxy pod"
    kubectl delete deployments -l app=${PRODUCER_PROXY_APPS_LABEL} -n ${ONAP_NAMESPACE}

    echo "Producers deleted"
}

function copy_certs_to_hvves() {
    cd ../../ssl
    echo "Attempting to create certs directory in HV-VES"
    kubectl exec -n ${ONAP_NAMESPACE} ${HVVES_POD_NAME} 'mkdir' ${HVVES_CERT_PATH}
    for file in {trust.p12,trust.pass,server.p12,server.pass}
    do
        echo "Copying file: ${file}"
        kubectl cp ${file} ${ONAP_NAMESPACE}/${HVVES_POD_NAME}:${HVVES_CERT_PATH}
    done
}

function set_kafka_retention_time() {
    echo "Setting message retention time"
    kubectl exec -it ${KAFKA_ROUTER_0_POD_NAME} -n ${ONAP_NAMESPACE} -- ${KAFKA_SET_TOPIC_RETENTION_TIME_CMD}$(eval $CALC_RETENTION_TIME_IN_MS_CMD) | eval $HIDE_OUTPUT
}

function generate_certs() {
    echo "Generation of certs"
    cd ../../ssl
    ./gen-certs.sh
}

function print_test_setup_info() {
    RUNNING_PRODUCERS=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_APPS_LABEL} | grep -c "Running")
    echo "Starting cloud based performance tests"
    echo "________________________________________"
    echo "Test configuration:"
    echo "Running producers: ${RUNNING_PRODUCERS}"
    echo "Sending scheme:"
    echo ${REQUEST_JSON_DATA}
    echo "Properties file path: ${PROPERTIES_FILE}"
    echo "Retention time of kafka messages in minutes: ${KAFKA_RETENTION_TIME_MINUTES}"
    echo "________________________________________"
}

function usage() {
    echo ""
    echo "Run cloud based HV-VES performance test"
    echo "Usage $0 gen_certs|setup|send_config|start_interval|start_instant|stop|reset_producers|clean|help"
    echo "  gen_certs :    generate certs in ../../ssl directory"
    echo "  setup     :    set up ConfigMaps and consumers"
    echo "  setup_all     :    set up ConfigMaps consumers and producers"
    echo "  send_config : send producers configuration (message interval and payload), located in producers-config/producer-config.json to each producer"
    echo "  start_interval    : start interval mode, config file is located in producers-config/interval-config.json"
    echo "     Optional parameters:"
    echo "        --producers               : number of producers in deployment (10)"
    echo "        --retention-time-minutes  : messages retention time on kafka in minutes (60)"
    echo "  start_instant     : start instant mode, config file is located in producers-config/instant-config.json"
    echo "     Optional parameters:"
    echo "        --producers               : number of producers in deployment (10)"
    echo "        --retention-time-minutes  : messages retention time on kafka in minutes (60)"
    echo "  scale_producers   : scale producer deployment to number provide in argument"
    echo "  stop      : stop all producers"
    echo "  reset_producers   : reset all metrics on each producer"
    echo "  clean     : remove ConfigMap, HV-VES consumers"
    echo "  clean_all     : remove ConfigMap, HV-VES consumers and producers"
    echo "  help      : print usage"
    echo "Example invocations:"
    echo "./cloud-based-performance-test.sh gen_certs"
    echo "./cloud-based-performance-test.sh setup"
    echo "./cloud-based-performance-test.sh setup_all"
    echo "./cloud-based-performance-test.sh send_config"
    echo "./cloud-based-performance-test.sh start_interval"
    echo "./cloud-based-performance-test.sh start_interval --producers 8"
    echo "./cloud-based-performance-test.sh start_instant"
    echo "./cloud-based-performance-test.sh scale_producers 8"
    echo "./cloud-based-performance-test.sh clean"
    echo "./cloud-based-performance-test.sh clean_all"
    exit 1
}

function setup_environment() {
    echo "Setting up environment"

    echo "Copying certs to hv-ves pod"
    copy_certs_to_hvves

    echo "Creating secrets with clients cert"
    kubectl create secret generic cert --from-file=./client.p12 --from-file=./client.pass -n ${ONAP_NAMESPACE}
    cd ${SCRIPT_DIRECTORY}

    echo "Turning on SSL"
    ./configure-consul.sh false

    echo "Creating test properties ConfigMap from: $PROPERTIES_FILE"
    kubectl create configmap ${TEST_CONFIG_MAP} --from-env-file=${PROPERTIES_FILE} -n ${ONAP_NAMESPACE}

    echo "Creating consumer deployment"
    kubectl apply -f consumer-deployment.yaml

    echo "Creating ConfigMap for prometheus deployment"
    kubectl apply -f prometheus/prometheus-config-map.yaml

    echo "Creating prometheus deployment"
    kubectl apply -f prometheus-deployment.yaml

    echo "Creating ConfigMap for grafana connections dashboard"
    kubectl create configmap ${GRAFANA_DASHBOARDS} -n ${ONAP_NAMESPACE} --from-file grafana/dashboards/

    echo "Creating ConfigMap for grafana datasource"
    kubectl apply -f grafana/datasources/datasource.yaml

    echo "Creating ConfigMap for grafana dashboards-providers"
    kubectl apply -f grafana/dashboards-providers/dashboards.yaml

    echo "Creating grafana deployment"
    kubectl apply -f grafana-deployment.yaml

    echo "Waiting for consumers to be running."
    while [[ $(kubectl get pods -l app=${CONSUMER_APPS_LABEL} -n ${ONAP_NAMESPACE} | grep -c "unhealthy\|starting") -ne 0 ]] ; do
        sleep 1
    done
    echo "Setting up environment finished!"
}

function setup_all() {
    setup_environment
    echo "Creating producer deployment"
    kubectl apply -f producer-deployment.yaml

    echo "Creating producer-proxy pod"
    kubectl apply -f producer-proxy.yaml

    echo "Creating producers finished"
}

function scale_producer_deployment() {
  echo "Scaling prodcuer deployment to ${PRODUCERS_COUNT}"
  kubectl scale --replicas=${PRODUCERS_COUNT} deployment ${PRODUCER_APPS_DEPLOYMENT} -n ${ONAP_NAMESPACE}
  RUNNING_PRODUCERS=""
  while [ "${RUNNING_PRODUCERS}" != "${PRODUCERS_COUNT}" ]; do
      RUNNING_PRODUCERS=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_APPS_LABEL} | grep -c "Running")
      sleep 1s
    done
  echo "Producers are ready"
}

function set_producers_array_internal_IP() {
  PRODUCER_IP_ARRAY=$(kubectl -n ${ONAP_NAMESPACE} get endpoints ${PRODUCER_SERVICE} -o jsonpath="{.subsets[*].addresses[*].ip}")
}

function send_post_request_to_each_producer() {
  set_producers_array_internal_IP
  PROXY_POD=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_PROXY_APPS_LABEL} -o name)
  echo "Sending POST request to each producer, endpoint: ${REQUEST_ENDPOINT}"
  REQUEST_ARRAY=""
  for item in ${PRODUCER_IP_ARRAY[*]}
    do
    URL="${item}:${PRODUCER_INTERNAL_PORT}${REQUEST_ENDPOINT}"
    echo ${URL}
    REQUEST_ARRAY="curl -H '${CONTENT_TYPE_HEADER}' -X POST -d '${REQUEST_JSON_DATA}' ${URL} ; ${REQUEST_ARRAY}"
    done
    kubectl -n onap exec -it ${PROXY_POD} -- /bin/sh -c "${REQUEST_ARRAY}"
  echo "Request was send to each producer"
}

function send_get_request_to_each_producer() {
  set_producers_array_internal_IP
  PROXY_POD=$(kubectl -n ${ONAP_NAMESPACE} get pods -l app=${PRODUCER_PROXY_APPS_LABEL} -o name)
  echo "Sending GET request to each producer, endpoint: ${REQUEST_ENDPOINT}"
  REQUEST_ARRAY=""
  for item in ${PRODUCER_IP_ARRAY[*]}
    do
    URL="${item}:${PRODUCER_INTERNAL_PORT}${REQUEST_ENDPOINT}"
    echo ${URL}
    REQUEST_ARRAY="curl -X GET ${URL} ; ${REQUEST_ARRAY}"
    done
    kubectl -n onap exec -it ${PROXY_POD} -- /bin/sh -c "${REQUEST_ARRAY}"
  echo "Request was send to each producer"
}

function send_configuration() {
  REQUEST_ENDPOINT="/configuration"
  REQUEST_JSON_DATA=$(cat producers-config/producer-config.json)
  echo "Sending producer configuration: "
  echo ${REQUEST_JSON_DATA}
  send_post_request_to_each_producer
  echo "Configuration was send to each producer pod"
  exit 0
}

function start_interval_mode() {
  set_kafka_retention_time
  REQUEST_ENDPOINT="/interval"
  REQUEST_JSON_DATA=$(cat producers-config/interval-config.json)
  print_test_setup_info
  echo "Sending start interval command to producer pods"
  send_post_request_to_each_producer
  echo "Command was send to each producer pod"
  exit 0
}

function start_instant_mode() {
  set_kafka_retention_time
  REQUEST_ENDPOINT="/instant"
  REQUEST_JSON_DATA=$(cat producers-config/instant-config.json)
  print_test_setup_info

  echo "Sending start instant command to producer pods"
  send_post_request_to_each_producer
  echo "Command was send to each producer pod"
  exit 0
}
function stop_producer_pods() {
  REQUEST_ENDPOINT="/cancel"

  echo "Sending stop command"
  send_get_request_to_each_producer
  echo "Stop command was send to each producer pod"
  exit 0
}
function reset_producer_pods() {
  REQUEST_ENDPOINT="/monitoring/prometheus/reset"

  echo "Sending reset command"
  send_get_request_to_each_producer
  echo "Reset command was send to each producer pod"
  exit 0
}


cd ${SCRIPT_DIRECTORY}

if [[ $# -eq 0 ]]; then
    usage
else
    for arg in ${@}
    do
        case ${arg} in
            gen_certs)
            generate_certs
            ;;
            setup)
            setup_environment
            ;;
            setup_all)
            setup_all
            ;;
            start)
            echo "Option start is deprecated. Last support commit: 3e7de0deb033e485d519c74feaffecc02e7e9dc7"
            ;;
            clean)
            clean
            ;;
            clean_all)
            clean_all
            ;;
            send_config)
            send_configuration
            ;;
            start_interval)
            shift 1
            while [[ $(($#)) -gt 0 ]]; do
                case "${1}" in
                    --producers)
                        PRODUCERS_COUNT=${2}
                        scale_producer_deployment
                        send_configuration
                        ;;
                    --retention-time-minutes)
                        KAFKA_RETENTION_TIME_MINUTES=${2}
                        ;;
                    *)
                        echo "Unknown option: ${1}"
                        usage
                        ;;
                esac
                shift 2
            done
            start_interval_mode
            ;;
            start_instant)
            shift 1
            while [[ $(($#)) -gt 0 ]]; do
                case "${1}" in
                    --producers)
                        PRODUCERS_COUNT=${2}
                        scale_producer_deployment
                        send_configuration
                        ;;
                    --retention-time-minutes)
                        KAFKA_RETENTION_TIME_MINUTES=${2}
                        ;;
                    *)
                        echo "Unknown option: ${1}"
                        usage
                        ;;
                esac
                shift 2
            done
            start_instant_mode
            ;;
            stop)
            stop_producer_pods
            ;;
            reset_producers)
            reset_producer_pods
            ;;
            scale_producers)
            shift 1
            PRODUCERS_COUNT=${1}
            scale_producer_deployment
            exit 0
            ;;
            help)
            usage
            ;;
            *)
            echo "Unknown action: ${arg}" >&2
            usage
            ;;
        esac
    done
fi