From 2e334500fba9893f1f2d2ab3b9528c638d993a03 Mon Sep 17 00:00:00 2001 From: kjaniak Date: Mon, 7 Oct 2019 11:46:48 +0200 Subject: Add graphs for performance monitoring Performance tests need graphs representing: total, message queue on HV-VES, offset per kafka topic partition, incoming data rate. Added deletion of producers at test end. Gauges for offsets turned to Range vector (distinguishable based on tag). Since Producer is to fast it was reconfigured to send messages with 1 ms interval. Change-Id: I09fc1edbcd824294d0bb89acc2ea2306a6df00ca Issue-ID: DCAEGEN2-1807 Signed-off-by: kjaniak --- .../cloud/cloud-based-performance-test.sh | 2 + .../grafana/dashboards/performance-tests.yaml | 448 ++++++++++++++++----- tools/performance/cloud/test.properties | 4 +- 3 files changed, 361 insertions(+), 93 deletions(-) (limited to 'tools/performance') diff --git a/tools/performance/cloud/cloud-based-performance-test.sh b/tools/performance/cloud/cloud-based-performance-test.sh index cb106a56..937dcf3e 100755 --- a/tools/performance/cloud/cloud-based-performance-test.sh +++ b/tools/performance/cloud/cloud-based-performance-test.sh @@ -164,6 +164,8 @@ function start_performance_test() { sleep 1 done + echo "Attempting to delete producer pods" + kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE} echo "Performance test finished" exit 0 } diff --git a/tools/performance/cloud/grafana/dashboards/performance-tests.yaml b/tools/performance/cloud/grafana/dashboards/performance-tests.yaml index cb8aec41..24626b7a 100644 --- a/tools/performance/cloud/grafana/dashboards/performance-tests.yaml +++ b/tools/performance/cloud/grafana/dashboards/performance-tests.yaml @@ -42,7 +42,7 @@ data: "editable": true, "gnetId": null, "graphTooltip": 0, - "iteration": 1563350058629, + "iteration": 1570533687099, "links": [], "panels": [ { @@ -50,13 +50,361 @@ data: "bars": false, "dashLength": 10, "dashes": false, - "fill": 0, + "datasource": null, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 10, - "w": 10, + "h": 11, + "w": 11, "x": 0, "y": 0 }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(hvves_data_received_bytes_total[30s])", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HV-VES incomming data rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 11, + "x": 11, + "y": 0 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "hvves_messages_received_total - hvves_messages_sent_total", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HV-VES processing message queue", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 11 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "hv_kafka_consumer_offset_partition", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Current offset on partitions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 11, + "x": 11, + "y": 11 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(hv_kafka_consumer_offset_partition)", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total number of messages on topic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 20 + }, "id": 6, "legend": { "alignAsTable": true, @@ -74,7 +422,9 @@ data: "linewidth": 1, "links": [], "nullPointMode": "null", - "options": {}, + "options": { + "dataLinks": [] + }, "percentage": false, "pointradius": 2, "points": false, @@ -154,94 +504,10 @@ data: "align": false, "alignLevel": null } - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorPostfix": false, - "colorPrefix": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 10 - }, - "id": 8, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "hv_kafka_consumer_offset_partition_hv_ves_perf3gpp_0", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Current offset on partition 0", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" } ], "refresh": "5s", - "schemaVersion": 18, + "schemaVersion": 20, "style": "dark", "tags": [], "templating": { @@ -323,5 +589,5 @@ data: "timezone": "", "title": "Performance tests", "uid": "ErPSMaIZk", - "version": 3 + "version": 1 } \ No newline at end of file diff --git a/tools/performance/cloud/test.properties b/tools/performance/cloud/test.properties index 04169e3a..092c51cf 100644 --- a/tools/performance/cloud/test.properties +++ b/tools/performance/cloud/test.properties @@ -8,8 +8,8 @@ producer.client.count=1 producer.message.size=16384 # Amount of messages to sent by one client in a single pod producer.message.count=1000 -# Interval between messages -producer.message.interval=0 +# Interval between messages in milliseconds +producer.message.interval=1 # CONSUMER CONFIGURATION -- cgit 1.2.3-korg