Add graphs for performance monitoring

Performance tests need graphs representing: total, message queue on HV-VES,
offset per kafka topic partition, incoming data rate. Added deletion of producers at test end.
Gauges for offsets turned to Range vector (distinguishable based on tag).
Since Producer is to fast it was reconfigured to send messages with 1 ms interval.

Change-Id: I09fc1edbcd824294d0bb89acc2ea2306a6df00ca
Issue-ID: DCAEGEN2-1807
Signed-off-by: kjaniak <kornel.janiak@nokia.com>

6 files changed, 377 insertions, 104 deletions

diff --git a/sources/hv-collector-kafka-consumer/src/main/kotlin/org/onap/dcae/collectors/veshv/kafkaconsumer/metrics/MicrometerMetrics.kt b/sources/hv-collector-kafka-consumer/src/main/kotlin/org/onap/dcae/collectors/veshv/kafkaconsumer/metrics/MicrometerMetrics.kt
index da6a4676..906fce23 100644
--- a/sources/hv-collector-kafka-consumer/src/main/kotlin/org/onap/dcae/collectors/veshv/kafkaconsumer/metrics/MicrometerMetrics.kt
+++ b/sources/hv-collector-kafka-consumer/src/main/kotlin/org/onap/dcae/collectors/veshv/kafkaconsumer/metrics/MicrometerMetrics.kt
@@ -36,7 +36,7 @@ internal class MicrometerMetrics constructor(
 ) : Metrics {
 
     private val currentOffsetByTopicPartition = { topicPartition: String ->
-        registry.gauge(name(OFFSET, PARTITION, topicPartition.toLowerCase()),
+        registry.gauge(name(OFFSET, PARTITION),
                 listOf(Tag.of(PARTITION, topicPartition)),
                 AtomicLong(0))
     }.memoize<String, AtomicLong>()
diff --git a/sources/hv-collector-kafka-consumer/src/test/kotlin/org/onap/dcae/collectors/veshv/kafkaconsumer/metrics/MicrometerMetricsTest.kt b/sources/hv-collector-kafka-consumer/src/test/kotlin/org/onap/dcae/collectors/veshv/kafkaconsumer/metrics/MicrometerMetricsTest.kt
index cfe67df2..fec7e54e 100644
--- a/sources/hv-collector-kafka-consumer/src/test/kotlin/org/onap/dcae/collectors/veshv/kafkaconsumer/metrics/MicrometerMetricsTest.kt
+++ b/sources/hv-collector-kafka-consumer/src/test/kotlin/org/onap/dcae/collectors/veshv/kafkaconsumer/metrics/MicrometerMetricsTest.kt
@@ -69,33 +69,32 @@ object MicrometerMetricsTest : Spek({
     }
 
     describe("Gauges") {
-        val gaugeName1 = "$PREFIX.offset.partition.sample_topic-0"
-        val gaugeName2 = "$PREFIX.offset.partition.sample_topic-1"
+        val gaugeName = "$PREFIX.offset.partition"
         val offset1 = 966L
         val offset2 = 967L
         val topicPartition1 = TopicPartition("sample_topic", 0)
         val topicPartition2 = TopicPartition("sample_topic", 1)
 
         on("notifyOffsetChanged") {
-            it("should update $gaugeName1") {
+            it("should update $gaugeName") {
                 cut.notifyOffsetChanged(offset1, topicPartition1)
 
-                registry.verifyGauge(gaugeName1) {
+                registry.verifyGauge(name = gaugeName, tagValue = topicPartition1.toString()) {
                     assertThat(it.value()).isCloseTo(offset1.toDouble(), doublePrecision)
                 }
             }
         }
 
         on("two partition update") {
-            it("should update $gaugeName1") {
+            it("should update $gaugeName") {
                 cut.notifyOffsetChanged(offset1, topicPartition1)
                 cut.notifyOffsetChanged(offset2, topicPartition2)
 
-                registry.verifyGauge(gaugeName1) {
+                registry.verifyGauge(name = gaugeName, tagValue = topicPartition1.toString()) {
                     assertThat(it.value()).isCloseTo(offset1.toDouble(), doublePrecision)
                 }
 
-                registry.verifyGauge(gaugeName2) {
+                registry.verifyGauge(name = gaugeName, tagValue = topicPartition2.toString()) {
                     assertThat(it.value()).isCloseTo(offset2.toDouble(), doublePrecision)
                 }
             }
diff --git a/sources/hv-collector-test-utils/src/main/kotlin/org/onap/dcae/collectors/veshv/tests/utils/metrics.kt b/sources/hv-collector-test-utils/src/main/kotlin/org/onap/dcae/collectors/veshv/tests/utils/metrics.kt
index 1aefdb34..52f9ae87 100644
--- a/sources/hv-collector-test-utils/src/main/kotlin/org/onap/dcae/collectors/veshv/tests/utils/metrics.kt
+++ b/sources/hv-collector-test-utils/src/main/kotlin/org/onap/dcae/collectors/veshv/tests/utils/metrics.kt
@@ -32,6 +32,9 @@ import org.assertj.core.api.Assertions
 fun <T> PrometheusMeterRegistry.verifyGauge(name: String, verifier: (Gauge) -> T) =
         verifyMeter(findMeter(name), RequiredSearch::gauge, verifier)
 
+fun <T> PrometheusMeterRegistry.verifyGauge(name: String, tagKey: String = "partition", tagValue: String, verifier: (Gauge) -> T) =
+        verifyMeter(findMeter(name, tagKey, tagValue), RequiredSearch::gauge, verifier)
+
 fun <T> PrometheusMeterRegistry.verifyTimer(name: String, verifier: (Timer) -> T) =
         verifyMeter(findMeter(name), RequiredSearch::timer, verifier)
 
@@ -43,13 +46,16 @@ fun <T> PrometheusMeterRegistry.verifyCounter(name: String, tags: Tags, verifier
 
 private fun PrometheusMeterRegistry.findMeter(meterName: String) = RequiredSearch.`in`(this).name(meterName)
 
+private fun PrometheusMeterRegistry.findMeter(meterName: String, tagKey: String, tagValue: String) =
+        RequiredSearch.`in`(this).tag(tagKey, tagValue).name(meterName)
+
 private fun <T> verifyCounter(search: RequiredSearch, verifier: (Counter) -> T) =
         verifyMeter(search, RequiredSearch::counter, verifier)
 
 private inline fun <M, T> verifyMeter(search: RequiredSearch,
-                               map: (RequiredSearch) -> M,
-                               verifier: (M) -> T) =
+                                      map: (RequiredSearch) -> M,
+                                      verifier: (M) -> T) =
         Try { map(search) }.fold(
                 { ex -> Assertions.assertThat(ex).doesNotThrowAnyException() },
                 verifier
-        )
\ No newline at end of file
+        )
diff --git a/tools/performance/cloud/cloud-based-performance-test.sh b/tools/performance/cloud/cloud-based-performance-test.sh
index cb106a56..937dcf3e 100755
--- a/tools/performance/cloud/cloud-based-performance-test.sh
+++ b/tools/performance/cloud/cloud-based-performance-test.sh
@@ -164,6 +164,8 @@ function start_performance_test() {
         sleep 1
     done
 
+    echo "Attempting to delete producer pods"
+    kubectl delete pods -l app=${PRODUCER_APPS_LABEL} -n ${ONAP_NAMESPACE}
     echo "Performance test finished"
     exit 0
 }
diff --git a/tools/performance/cloud/grafana/dashboards/performance-tests.yaml b/tools/performance/cloud/grafana/dashboards/performance-tests.yaml
index cb8aec41..24626b7a 100644
--- a/tools/performance/cloud/grafana/dashboards/performance-tests.yaml
+++ b/tools/performance/cloud/grafana/dashboards/performance-tests.yaml
@@ -42,7 +42,7 @@ data:
       "editable": true,
       "gnetId": null,
       "graphTooltip": 0,
-      "iteration": 1563350058629,
+      "iteration": 1570533687099,
       "links": [],
       "panels": [
         {
@@ -50,13 +50,361 @@ data:
           "bars": false,
           "dashLength": 10,
           "dashes": false,
-          "fill": 0,
+          "datasource": null,
+          "fill": 1,
+          "fillGradient": 0,
           "gridPos": {
-            "h": 10,
-            "w": 10,
+            "h": 11,
+            "w": 11,
             "x": 0,
             "y": 0
           },
+          "id": 12,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "irate(hvves_data_received_bytes_total[30s])",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "HV-VES incomming data rate",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "Bps",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": null,
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 11,
+            "w": 11,
+            "x": 11,
+            "y": 0
+          },
+          "id": 14,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "hvves_messages_received_total - hvves_messages_sent_total",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "HV-VES processing message queue",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": null,
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 11,
+            "x": 0,
+            "y": 11
+          },
+          "id": 8,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "hv_kafka_consumer_offset_partition",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Current offset on partitions",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "cacheTimeout": null,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": null,
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 11,
+            "x": 11,
+            "y": 11
+          },
+          "id": 10,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(hv_kafka_consumer_offset_partition)",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Total number of messages on topic",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": null,
+          "fill": 0,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 11,
+            "x": 0,
+            "y": 20
+          },
           "id": 6,
           "legend": {
             "alignAsTable": true,
@@ -74,7 +422,9 @@ data:
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
-          "options": {},
+          "options": {
+            "dataLinks": []
+          },
           "percentage": false,
           "pointradius": 2,
           "points": false,
@@ -154,94 +504,10 @@ data:
             "align": false,
             "alignLevel": null
           }
-        },
-        {
-          "cacheTimeout": null,
-          "colorBackground": false,
-          "colorPostfix": false,
-          "colorPrefix": false,
-          "colorValue": false,
-          "colors": [
-            "#299c46",
-            "rgba(237, 129, 40, 0.89)",
-            "#d44a3a"
-          ],
-          "format": "none",
-          "gauge": {
-            "maxValue": 100,
-            "minValue": 0,
-            "show": false,
-            "thresholdLabels": false,
-            "thresholdMarkers": true
-          },
-          "gridPos": {
-            "h": 4,
-            "w": 6,
-            "x": 0,
-            "y": 10
-          },
-          "id": 8,
-          "interval": null,
-          "links": [],
-          "mappingType": 1,
-          "mappingTypes": [
-            {
-              "name": "value to text",
-              "value": 1
-            },
-            {
-              "name": "range to text",
-              "value": 2
-            }
-          ],
-          "maxDataPoints": 100,
-          "nullPointMode": "connected",
-          "nullText": null,
-          "options": {},
-          "postfix": "",
-          "postfixFontSize": "50%",
-          "prefix": "",
-          "prefixFontSize": "50%",
-          "rangeMaps": [
-            {
-              "from": "null",
-              "text": "N/A",
-              "to": "null"
-            }
-          ],
-          "sparkline": {
-            "fillColor": "rgba(31, 118, 189, 0.18)",
-            "full": false,
-            "lineColor": "rgb(31, 120, 193)",
-            "show": true
-          },
-          "tableColumn": "",
-          "targets": [
-            {
-              "expr": "hv_kafka_consumer_offset_partition_hv_ves_perf3gpp_0",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "refId": "A"
-            }
-          ],
-          "thresholds": "",
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Current offset on partition 0",
-          "type": "singlestat",
-          "valueFontSize": "80%",
-          "valueMaps": [
-            {
-              "op": "=",
-              "text": "N/A",
-              "value": "null"
-            }
-          ],
-          "valueName": "current"
         }
       ],
       "refresh": "5s",
-      "schemaVersion": 18,
+      "schemaVersion": 20,
       "style": "dark",
       "tags": [],
       "templating": {
@@ -323,5 +589,5 @@ data:
       "timezone": "",
       "title": "Performance tests",
       "uid": "ErPSMaIZk",
-      "version": 3
+      "version": 1
     }
\ No newline at end of file
diff --git a/tools/performance/cloud/test.properties b/tools/performance/cloud/test.properties
index 04169e3a..092c51cf 100644
--- a/tools/performance/cloud/test.properties
+++ b/tools/performance/cloud/test.properties
@@ -8,8 +8,8 @@ producer.client.count=1
 producer.message.size=16384
 # Amount of messages to sent by one client in a single pod
 producer.message.count=1000
-# Interval between messages
-producer.message.interval=0
+# Interval between messages in milliseconds
+producer.message.interval=1
 
 # CONSUMER CONFIGURATION