summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPiotr Jaszczyk <piotr.jaszczyk@nokia.com>2019-07-03 12:39:12 +0000
committerGerrit Code Review <gerrit@onap.org>2019-07-03 12:39:12 +0000
commit2c80da49d88ef2fc81741f99e76e85d31ef2ffa4 (patch)
treecff2c9be11e3c13f1fc520a252dd0cd31cb5a917
parent3e4f108d66fb31e5ea713fef1cf7bc99c0510c21 (diff)
parent83a1d39bca162e8fdabfb3ad9189c5e4f94a44df (diff)
Merge "Create script to run performance test locally"
-rwxr-xr-xtools/development/bin/constants.sh (renamed from development/bin/constants.sh)0
-rwxr-xr-xtools/development/bin/consul.sh (renamed from development/bin/consul.sh)0
-rwxr-xr-xtools/development/bin/dcae-msgs.sh (renamed from development/bin/dcae-msgs.sh)0
-rwxr-xr-xtools/development/bin/dcae-reset.sh (renamed from development/bin/dcae-reset.sh)0
-rwxr-xr-xtools/development/bin/dcae-topic.sh (renamed from development/bin/dcae-topic.sh)0
-rwxr-xr-xtools/development/bin/run-xnf-simulator.sh (renamed from development/bin/run-xnf-simulator.sh)0
-rwxr-xr-xtools/development/bin/start-simulation.sh (renamed from development/bin/start-simulation.sh)0
-rwxr-xr-xtools/development/bin/xnf-simulation.sh (renamed from development/bin/xnf-simulation.sh)0
-rw-r--r--tools/development/configuration/base.json (renamed from development/configuration/base.json)0
-rw-r--r--tools/development/configuration/local.json (renamed from development/configuration/local.json)0
-rw-r--r--tools/development/consul/configuration.hcl (renamed from development/consul/configuration.hcl)0
-rw-r--r--tools/development/docker-compose.yml (renamed from development/docker-compose.yml)2
-rw-r--r--tools/development/grafana/dashboards-providers/dashboard-providers.yaml (renamed from development/grafana/dashboards-providers/dashboard-providers.yaml)0
-rw-r--r--tools/development/grafana/dashboards/connections.json (renamed from development/grafana/dashboards/connections.json)0
-rw-r--r--tools/development/grafana/dashboards/processing.json (renamed from development/grafana/dashboards/processing.json)0
-rw-r--r--tools/development/grafana/datasources/prometheus.yaml (renamed from development/grafana/datasources/prometheus.yaml)0
-rw-r--r--tools/development/logs/.gitignore (renamed from development/logs/.gitignore)0
-rw-r--r--tools/development/prometheus.yml (renamed from development/prometheus.yml)0
-rw-r--r--tools/performance/configuration/base.json11
-rw-r--r--tools/performance/consul/configuration.hcl34
-rw-r--r--tools/performance/docker-compose.yml116
-rwxr-xr-xtools/performance/local-performance-test.sh176
-rw-r--r--tools/performance/logs/.gitignore2
-rw-r--r--tools/performance/prometheus.yml14
-rw-r--r--tools/ssl/.gitignore (renamed from development/ssl/.gitignore)0
-rw-r--r--tools/ssl/Makefile-openssl (renamed from development/ssl/Makefile-openssl)0
-rw-r--r--tools/ssl/README.md (renamed from development/ssl/README.md)0
-rwxr-xr-xtools/ssl/gen-certs.sh (renamed from development/ssl/gen-certs.sh)0
28 files changed, 354 insertions, 1 deletions
diff --git a/development/bin/constants.sh b/tools/development/bin/constants.sh
index f0df9b00..f0df9b00 100755
--- a/development/bin/constants.sh
+++ b/tools/development/bin/constants.sh
diff --git a/development/bin/consul.sh b/tools/development/bin/consul.sh
index 5f9271f2..5f9271f2 100755
--- a/development/bin/consul.sh
+++ b/tools/development/bin/consul.sh
diff --git a/development/bin/dcae-msgs.sh b/tools/development/bin/dcae-msgs.sh
index 84cef972..84cef972 100755
--- a/development/bin/dcae-msgs.sh
+++ b/tools/development/bin/dcae-msgs.sh
diff --git a/development/bin/dcae-reset.sh b/tools/development/bin/dcae-reset.sh
index d2d8ebd0..d2d8ebd0 100755
--- a/development/bin/dcae-reset.sh
+++ b/tools/development/bin/dcae-reset.sh
diff --git a/development/bin/dcae-topic.sh b/tools/development/bin/dcae-topic.sh
index b4c2638d..b4c2638d 100755
--- a/development/bin/dcae-topic.sh
+++ b/tools/development/bin/dcae-topic.sh
diff --git a/development/bin/run-xnf-simulator.sh b/tools/development/bin/run-xnf-simulator.sh
index e4d8d94a..e4d8d94a 100755
--- a/development/bin/run-xnf-simulator.sh
+++ b/tools/development/bin/run-xnf-simulator.sh
diff --git a/development/bin/start-simulation.sh b/tools/development/bin/start-simulation.sh
index 8c63ddbb..8c63ddbb 100755
--- a/development/bin/start-simulation.sh
+++ b/tools/development/bin/start-simulation.sh
diff --git a/development/bin/xnf-simulation.sh b/tools/development/bin/xnf-simulation.sh
index ade0e426..ade0e426 100755
--- a/development/bin/xnf-simulation.sh
+++ b/tools/development/bin/xnf-simulation.sh
diff --git a/development/configuration/base.json b/tools/development/configuration/base.json
index 2a806adb..2a806adb 100644
--- a/development/configuration/base.json
+++ b/tools/development/configuration/base.json
diff --git a/development/configuration/local.json b/tools/development/configuration/local.json
index cfaaaa40..cfaaaa40 100644
--- a/development/configuration/local.json
+++ b/tools/development/configuration/local.json
diff --git a/development/consul/configuration.hcl b/tools/development/consul/configuration.hcl
index f975955e..f975955e 100644
--- a/development/consul/configuration.hcl
+++ b/tools/development/consul/configuration.hcl
diff --git a/development/docker-compose.yml b/tools/development/docker-compose.yml
index 2704722c..fbc38920 100644
--- a/development/docker-compose.yml
+++ b/tools/development/docker-compose.yml
@@ -110,7 +110,7 @@ services:
- config-binding-service
volumes:
- ./configuration/:/etc/ves-hv/configuration/
- - ./ssl/:/etc/ves-hv/ssl/
+ - ../ssl/:/etc/ves-hv/ssl/
- ./logs:/var/log/ONAP/dcae-hv-ves-collector
diff --git a/development/grafana/dashboards-providers/dashboard-providers.yaml b/tools/development/grafana/dashboards-providers/dashboard-providers.yaml
index 78da55c9..78da55c9 100644
--- a/development/grafana/dashboards-providers/dashboard-providers.yaml
+++ b/tools/development/grafana/dashboards-providers/dashboard-providers.yaml
diff --git a/development/grafana/dashboards/connections.json b/tools/development/grafana/dashboards/connections.json
index 2d0182c9..2d0182c9 100644
--- a/development/grafana/dashboards/connections.json
+++ b/tools/development/grafana/dashboards/connections.json
diff --git a/development/grafana/dashboards/processing.json b/tools/development/grafana/dashboards/processing.json
index d74968cb..d74968cb 100644
--- a/development/grafana/dashboards/processing.json
+++ b/tools/development/grafana/dashboards/processing.json
diff --git a/development/grafana/datasources/prometheus.yaml b/tools/development/grafana/datasources/prometheus.yaml
index 80717b08..80717b08 100644
--- a/development/grafana/datasources/prometheus.yaml
+++ b/tools/development/grafana/datasources/prometheus.yaml
diff --git a/development/logs/.gitignore b/tools/development/logs/.gitignore
index 1287e9bd..1287e9bd 100644
--- a/development/logs/.gitignore
+++ b/tools/development/logs/.gitignore
diff --git a/development/prometheus.yml b/tools/development/prometheus.yml
index 201c8f96..201c8f96 100644
--- a/development/prometheus.yml
+++ b/tools/development/prometheus.yml
diff --git a/tools/performance/configuration/base.json b/tools/performance/configuration/base.json
new file mode 100644
index 00000000..2a806adb
--- /dev/null
+++ b/tools/performance/configuration/base.json
@@ -0,0 +1,11 @@
+{
+ "logLevel": "DEBUG",
+ "server.listenPort": 6061,
+ "server.idleTimeoutSec": 60,
+ "cbs.firstRequestDelaySec": 5,
+ "cbs.requestIntervalSec": 5,
+ "security.keys.keyStoreFile": "/etc/ves-hv/ssl/server.p12",
+ "security.keys.keyStorePasswordFile": "/etc/ves-hv/ssl/server.pass",
+ "security.keys.trustStoreFile": "/etc/ves-hv/ssl/trust.p12",
+ "security.keys.trustStorePasswordFile": "/etc/ves-hv/ssl/trust.pass"
+} \ No newline at end of file
diff --git a/tools/performance/consul/configuration.hcl b/tools/performance/consul/configuration.hcl
new file mode 100644
index 00000000..f975955e
--- /dev/null
+++ b/tools/performance/consul/configuration.hcl
@@ -0,0 +1,34 @@
+/*
+ * ============LICENSE_START=======================================================
+ * csit-dcaegen2-collectors-hv-ves
+ * ================================================================================
+ * Copyright (C) 2019 NOKIA
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+server = true
+bootstrap = true
+ui = true
+client_addr = "0.0.0.0"
+
+service {
+ # name under which hv-ves collector should seek cbs
+ # usually set as CONFIG_BINDING_SERVICE environment variable
+ Name = "CBS"
+ # address of CBS as seen by hv-ves collector
+ Address = "config-binding-service"
+ Port = 10000
+}
+
diff --git a/tools/performance/docker-compose.yml b/tools/performance/docker-compose.yml
new file mode 100644
index 00000000..82143235
--- /dev/null
+++ b/tools/performance/docker-compose.yml
@@ -0,0 +1,116 @@
+version: "3.5"
+services:
+
+ message-router-zookeeper:
+ image: nexus3.onap.org:10001/onap/dmaap/zookeeper:4.0.0
+ ports:
+ - "2181:2181"
+
+ message-router-kafka-0:
+ image: nexus3.onap.org:10001/onap/dmaap/kafka111:0.0.6
+ ports:
+ - "9092:9092"
+ - "9093:9093"
+ environment:
+ HOST_IP: 127.0.0.1
+ KAFKA_BROKER_ID: 0
+ ENDPOINT_PORT: 30490
+ KAFKA_ZOOKEEPER_CONNECT: "message-router-zookeeper:2181"
+ KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+ KAFKA_DELETE_TOPIC_ENABLE: "true"
+
+ KAFKA_LISTENERS: "INTERNAL_SASL_PLAINTEXT://0.0.0.0:9092,EXTERNAL_SASL_PLAINTEXT://0.0.0.0:9093"
+ KAFKA_ADVERTISED_LISTENERS: "INTERNAL_SASL_PLAINTEXT://message-router-kafka-0:9092,EXTERNAL_SASL_PLAINTEXT://message-router-kafka-0:9093"
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL_SASL_PLAINTEXT:SASL_PLAINTEXT,EXTERNAL_SASL_PLAINTEXT:SASL_PLAINTEXT"
+ KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL_SASL_PLAINTEXT"
+ KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN"
+ KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: "PLAIN"
+ KAFKA_AUTHORIZER_CLASS_NAME: "org.onap.dmaap.kafkaAuthorize.KafkaCustomAuthorizer"
+
+ aaf_locate_url: https://aaf-locate:8095
+ KAFKA_LOG_DIRS: /opt/kafka/data
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+ KAFKA_DEFAULT_REPLICATION_FACTOR: 1
+ KAFKA_NUM_PARTITIONS: 1
+ volumes:
+ - /var/run/docker.sock:/var/run/docker.sock
+ depends_on:
+ - message-router-zookeeper
+
+ consul-server:
+ image: docker.io/consul:1.0.6
+ ports:
+ - "8500:8500"
+ volumes:
+ - ./consul/:/consul/config
+
+ consul-bootstrap:
+ image: docker.io/consul:1.0.6
+ restart: on-failure
+ command: ["kv", "put", "-http-addr=http://consul-server:8500", "dcae-hv-ves-collector", '{
+ "streams_publishes": {
+ "perf3gpp": {
+ "type": "kafka",
+ "aaf_credentials": {
+ "username": "admin",
+ "password": "admin_secret"
+ },
+ "kafka_info": {
+ "bootstrap_servers": "message-router-kafka-0:9093",
+ "topic_name": "HV_VES_PERF3GPP"
+ }
+ }
+ }
+ }'
+ ]
+ depends_on:
+ - consul-server
+
+ config-binding-service:
+ image: nexus3.onap.org:10001/onap/org.onap.dcaegen2.platform.configbinding.app-app:2.2.4
+ ports:
+ - "10000:10000"
+ environment:
+ CONSUL_HOST: "consul-server"
+ depends_on:
+ - consul-bootstrap
+
+ ves-hv-collector:
+ image: onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-main:latest
+ ports:
+ - "6060:6060"
+ - "6061:6061/tcp"
+ environment:
+ JAVA_OPTS: "-Dio.netty.leakDetection.level=paranoid -Dlogback.configurationFile=/etc/ONAP/dcae-hv-ves-collector/logback.xml"
+ VESHV_CONFIGURATION_FILE: "/etc/ves-hv/configuration/base.json"
+ CONSUL_HOST: "consul-server"
+ CONFIG_BINDING_SERVICE: "CBS"
+ HOSTNAME: "dcae-hv-ves-collector"
+ healthcheck:
+ test: ./healthcheck.sh || exit 1
+ interval: 10s
+ timeout: 3s
+ retries: 3
+ start_period: 15s
+ depends_on:
+ - message-router-kafka-0
+ - config-binding-service
+ volumes:
+ - ./configuration/:/etc/ves-hv/configuration/
+ - ./logs:/var/log/ONAP/dcae-hv-ves-collector
+ - ../ssl/:/etc/ves-hv/ssl/
+
+ kafka-consumer:
+ image: onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-kafka-consumer
+ ports:
+ - "6064:6064/tcp"
+ command: ["--listen-port", "6062"]
+ depends_on:
+ - message-router-kafka-0
+
+ prometheus:
+ image: prom/prometheus
+ ports:
+ - "9090:9090"
+ volumes:
+ - ./prometheus.yml:/etc/prometheus/prometheus.yml
diff --git a/tools/performance/local-performance-test.sh b/tools/performance/local-performance-test.sh
new file mode 100755
index 00000000..cad21ef8
--- /dev/null
+++ b/tools/performance/local-performance-test.sh
@@ -0,0 +1,176 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "$0")"
+
+CERT_FILE=${CERT_FILE:-/ssl/client.p12}
+CERT_PASS_FILE=${CERT_PASS_FILE:-/ssl/client.pass}
+HV_VES_NETWORK=${HV_VES_NETWORK:-performance_default}
+VOLUME_MAPPING=${VOLUME_MAPPING:-$PWD/../ssl/:/ssl}
+PRODUCER_IMAGE_NAME=${PRODUCER_IMAGE_NAME:-the-a-team-registry-local.esisoj70.emea.nsn-net.net/onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-rust-client:latest}
+
+PRODUCER_APP_NAME=hv-ves-producer
+HV_VES_ADDRESS=ves-hv-collector:6061
+CONTAINERS_COUNT=1
+CLIENTS_PER_CONTAINER=1
+MSG_SIZE=16384
+MSG_COUNT=1000
+INTERVAL_MS=0
+
+
+function usage() {
+ echo ""
+ echo "Run HV-VES performance test locally"
+ echo "Usage $0 setup|start|clean|help"
+ echo " setup : generate certs and set up docker components"
+ echo " start : run the performance test"
+ echo " Optional parameters:"
+ echo " --address : HV-VES address in host:port format (ves-hv-collector:6061)"
+ echo " --containers : number of docker containers to create (1)"
+ echo " --clients : number of clients in single container (1)"
+ echo " --msg-size : size in bytes of a single message (16384)"
+ echo " --msg-count : amount of messages to sent by one client in single container (1000)"
+ echo " --interval : interval between messages (0)"
+ echo " clean : remove generated certs, HV-VES components and producers"
+ echo " help : print usage"
+ echo "Example invocations:"
+ echo "./local-performance-test.sh setup"
+ echo "./local-performance-test.sh start --containers 10 --clients 100 --msg-count 10000"
+ echo "./local-performance-test.sh clean"
+ exit 1
+}
+
+function setup_environment(){
+ echo "Setting up"
+ cd ../ssl
+ ./gen-certs.sh
+ cd ../performance
+ docker-compose up -d
+
+ echo "Waiting for components to be healthy.."
+ while [[ $(docker-compose ps | grep -c "unhealthy\|starting") -ne 0 ]] ; do
+ sleep 1
+ done
+
+ echo "All components ready"
+ exit 0
+}
+
+function start_performance_test(){
+
+ TEST_ID=$(date +%s)
+ create_containers ${CONTAINERS_COUNT} ${TEST_ID} &
+
+ while :; do
+ ACTIVE_PRODUCERS=$(docker ps --format "table {{.ID}}\t{{.Status}}" -f "label=id=$TEST_ID")
+ ACTIVE_PRODUCERS_COUNT=$(echo "$ACTIVE_PRODUCERS" | grep -c "Up")
+
+ clear
+ print_test_configuration
+ echo "Active producers ($ACTIVE_PRODUCERS_COUNT/$CONTAINERS_COUNT):"
+ echo "$ACTIVE_PRODUCERS"
+
+ EXITED_CONTAINERS=$(docker ps -aq -f "label=id=$TEST_ID" -f status=exited | wc -l)
+ [[ ${EXITED_CONTAINERS} -eq ${CONTAINERS_COUNT} ]] && break
+
+ sleep 1
+ done
+
+ clear
+ print_test_configuration
+ echo "Test finished"
+ # TODO put test result here
+ exit 0
+}
+
+function print_test_configuration(){
+ echo "PERFORMANCE TEST IN PROGRESS"
+ echo ""
+ echo "Test configuration:"
+ echo "Containers count: $CONTAINERS_COUNT"
+ echo "Clients per container: $CLIENTS_PER_CONTAINER"
+ echo "Message size: $MSG_SIZE"
+ echo "Messages per client: $MSG_COUNT"
+ echo "Interval: $INTERVAL_MS"
+ echo ""
+}
+
+function create_containers(){
+
+ for i in $(seq 1 ${1}); do
+ docker run -d -l id="$2" -l app="$PRODUCER_APP_NAME" -v "$VOLUME_MAPPING" --network="$HV_VES_NETWORK" "$PRODUCER_IMAGE_NAME" \
+ --address "$HV_VES_ADDRESS" \
+ --certfile "$CERT_FILE" \
+ --certpass "$CERT_PASS_FILE" \
+ --containers "$CONTAINERS_COUNT" \
+ --clients "$CLIENTS_PER_CONTAINER" \
+ --msgsize "$MSG_SIZE" \
+ --msgcount "$MSG_COUNT" \
+ --intervalms "$INTERVAL_MS" > /dev/null
+ done
+}
+
+function clean(){
+ echo "Cleaning up"
+
+ echo "Removing active producers"
+ docker rm --force $(docker ps -aqf "label=app=$PRODUCER_APP_NAME")
+
+ echo "Clearing generated certs"
+ cd ../ssl
+ ./gen-certs.sh clean
+ cd ../performance
+
+ echo "Removing HV-VES components"
+ docker-compose down
+ exit 0
+}
+
+if [[ $# -eq 0 ]]; then
+ usage
+else
+ for arg in ${@}
+ do
+ case ${arg} in
+ setup)
+ setup_environment
+ ;;
+ start)
+ shift 1
+ while [[ $(($#)) -gt 0 ]]; do
+ case "${1}" in
+ --address)
+ HV_VES_ADDRESS=${2}
+ ;;
+ --containers)
+ CONTAINERS_COUNT=${2}
+ ;;
+ --clients)
+ CLIENTS_PER_CONTAINER=${2}
+ ;;
+ --msg-size)
+ MSG_SIZE=${2}
+ ;;
+ --msg-count)
+ MSG_COUNT=${2}
+ ;;
+ --interval)
+ INTERVAL_MS=${2}
+ ;;
+ esac
+ shift 2
+ done
+ start_performance_test
+ ;;
+ clean)
+ clean
+ ;;
+ help)
+ usage
+ ;;
+ *)
+ echo "Unknown action: ${arg}" >&2
+ usage
+ ;;
+ esac
+ done
+fi \ No newline at end of file
diff --git a/tools/performance/logs/.gitignore b/tools/performance/logs/.gitignore
new file mode 100644
index 00000000..1287e9bd
--- /dev/null
+++ b/tools/performance/logs/.gitignore
@@ -0,0 +1,2 @@
+**
+!.gitignore
diff --git a/tools/performance/prometheus.yml b/tools/performance/prometheus.yml
new file mode 100644
index 00000000..b9a937c2
--- /dev/null
+++ b/tools/performance/prometheus.yml
@@ -0,0 +1,14 @@
+global:
+ scrape_interval: 5s
+ external_labels:
+ monitor: 'my-monitor'
+
+scrape_configs:
+ - job_name: 'prometheus'
+ static_configs:
+ - targets: ['localhost:9090']
+
+ - job_name: 'kafka-consumer'
+ metrics_path: '/monitoring/prometheus'
+ static_configs:
+ - targets: ['kafka-consumer:6062']
diff --git a/development/ssl/.gitignore b/tools/ssl/.gitignore
index 955c17d1..955c17d1 100644
--- a/development/ssl/.gitignore
+++ b/tools/ssl/.gitignore
diff --git a/development/ssl/Makefile-openssl b/tools/ssl/Makefile-openssl
index 09802ce4..09802ce4 100644
--- a/development/ssl/Makefile-openssl
+++ b/tools/ssl/Makefile-openssl
diff --git a/development/ssl/README.md b/tools/ssl/README.md
index c2819d24..c2819d24 100644
--- a/development/ssl/README.md
+++ b/tools/ssl/README.md
diff --git a/development/ssl/gen-certs.sh b/tools/ssl/gen-certs.sh
index bf28ca02..bf28ca02 100755
--- a/development/ssl/gen-certs.sh
+++ b/tools/ssl/gen-certs.sh