Improve grafana dashboards

Issue-ID: DCAEGEN2-1858
Signed-off-by: Remigiusz Janeczek <remigiusz.janeczek@nokia.com>
Change-Id: I6ce040a7e9ad4e70c2f8c1139de66b1145cabe5b

2 files changed, 96 insertions, 155 deletions

diff --git a/tools/performance/cloud/grafana/dashboards/performance-tests.yaml b/tools/performance/cloud/grafana/dashboards/performance-tests.yaml
index b1a5d526..97ca71b1 100644
--- a/tools/performance/cloud/grafana/dashboards/performance-tests.yaml
+++ b/tools/performance/cloud/grafana/dashboards/performance-tests.yaml
@@ -42,8 +42,7 @@ data:
       "editable": true,
       "gnetId": null,
       "graphTooltip": 0,
-      "id": 2,
-      "iteration": 1571311867605,
+      "iteration": 1571408429976,
       "links": [],
       "panels": [
         {
@@ -55,8 +54,8 @@ data:
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
-            "h": 10,
-            "w": 7,
+            "h": 11,
+            "w": 12,
             "x": 0,
             "y": 0
           },
@@ -87,14 +86,15 @@ data:
           "targets": [
             {
               "expr": "irate(hvves_data_received_bytes_total[30s])",
+              "legendFormat": "Incoming data rate",
               "refId": "A"
             }
           ],
           "thresholds": [],
-          "timeFrom": "30m",
+          "timeFrom": "20m",
           "timeRegions": [],
           "timeShift": null,
-          "title": "HV-VES incomming data rate",
+          "title": "HV-VES incoming data rate",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -114,92 +114,7 @@ data:
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": null
-          }
-        },
-        {
-          "aliasColors": {},
-          "bars": false,
-          "dashLength": 10,
-          "dashes": false,
-          "datasource": null,
-          "fill": 1,
-          "fillGradient": 0,
-          "gridPos": {
-            "h": 10,
-            "w": 8,
-            "x": 7,
-            "y": 0
-          },
-          "id": 14,
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 1,
-          "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 2,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "spaceLength": 10,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "expr": "hvves_messages_received_total - hvves_messages_sent_total",
-              "refId": "A"
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": "30m",
-          "timeRegions": [],
-          "timeShift": null,
-          "title": "HV-VES processing message queue",
-          "tooltip": {
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "type": "graph",
-          "xaxis": {
-            "buckets": null,
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -225,23 +140,23 @@ data:
           "fill": 0,
           "fillGradient": 0,
           "gridPos": {
-            "h": 10,
-            "w": 9,
-            "x": 15,
+            "h": 11,
+            "w": 12,
+            "x": 12,
             "y": 0
           },
           "id": 6,
           "legend": {
-            "alignAsTable": true,
-            "avg": true,
-            "current": true,
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
             "hideEmpty": true,
             "hideZero": true,
             "max": false,
             "min": false,
             "show": true,
             "total": false,
-            "values": true
+            "values": false
           },
           "lines": true,
           "linewidth": 1,
@@ -260,40 +175,41 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "histogram_quantile(0.90, sum( rate(hv_kafka_consumer_travel_time_seconds_bucket[$time_range]) )by(le) )",
+              "expr": "rate(hv_kafka_consumer_travel_time_seconds_sum{instance=\"kafka-processing-consumer.onap:6063\"}[1h])/rate(hv_kafka_consumer_travel_time_seconds_count{instance=\"kafka-processing-consumer.onap:6063\"}[1h])",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "90th percentile of avg message travel time",
+              "legendFormat": "average",
               "refId": "A"
             },
             {
-              "expr": "histogram_quantile(0.95, sum( rate(hv_kafka_consumer_travel_time_seconds_bucket[$time_range]) )by(le) )",
+              "expr": "histogram_quantile(0.90, sum( rate(hv_kafka_consumer_travel_time_seconds_bucket[1h]) )by(le) )",
               "format": "time_series",
               "intervalFactor": 1,
-              "legendFormat": "95th percentile of avg message travel time",
+              "legendFormat": "90th percentile",
               "refId": "B"
             },
             {
-              "expr": "histogram_quantile(0.99, sum( rate(hv_kafka_consumer_travel_time_seconds_bucket[$time_range]) )by(le) )",
+              "expr": "histogram_quantile(0.95, sum( rate(hv_kafka_consumer_travel_time_seconds_bucket[1h]) )by(le) )",
               "format": "time_series",
               "intervalFactor": 1,
-              "legendFormat": "99th percentile of avg message travel time",
+              "legendFormat": "95th percentile",
               "refId": "C"
             },
             {
-              "expr": "rate(hv_kafka_consumer_travel_time_seconds_sum[$time_range])/rate(hv_kafka_consumer_travel_time_seconds_count[$time_range])",
+              "expr": "histogram_quantile(0.99, sum( rate(hv_kafka_consumer_travel_time_seconds_bucket[1h]) )by(le) )",
               "format": "time_series",
+              "hide": false,
               "intervalFactor": 1,
-              "legendFormat": "avg message travel time",
+              "legendFormat": "95th percentile",
               "refId": "D"
             }
           ],
           "thresholds": [],
-          "timeFrom": "30m",
+          "timeFrom": "20m",
           "timeRegions": [],
           "timeShift": null,
-          "title": "Avg message travel time",
+          "title": "End to end message travel time",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -309,11 +225,11 @@ data:
           },
           "yaxes": [
             {
-              "format": "ns",
+              "format": "s",
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -333,7 +249,6 @@ data:
         {
           "aliasColors": {},
           "bars": false,
-          "cacheTimeout": null,
           "dashLength": 10,
           "dashes": false,
           "datasource": null,
@@ -343,9 +258,9 @@ data:
             "h": 11,
             "w": 12,
             "x": 0,
-            "y": 10
+            "y": 11
           },
-          "id": 8,
+          "id": 14,
           "legend": {
             "avg": false,
             "current": false,
@@ -357,7 +272,6 @@ data:
           },
           "lines": true,
           "linewidth": 1,
-          "links": [],
           "nullPointMode": "null",
           "options": {
             "dataLinks": []
@@ -372,17 +286,16 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "hv_kafka_consumer_offset_partition",
-              "format": "time_series",
-              "intervalFactor": 1,
+              "expr": "hvves_messages_received_total - hvves_messages_sent_total",
+              "legendFormat": "Processing message queue",
               "refId": "A"
             }
           ],
           "thresholds": [],
-          "timeFrom": "30m",
+          "timeFrom": "20m",
           "timeRegions": [],
           "timeShift": null,
-          "title": "Current offset on partitions",
+          "title": "HV-VES processing message queue",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -402,7 +315,7 @@ data:
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -432,7 +345,7 @@ data:
             "h": 11,
             "w": 12,
             "x": 12,
-            "y": 10
+            "y": 11
           },
           "id": 10,
           "legend": {
@@ -463,11 +376,12 @@ data:
             {
               "expr": "sum(hv_kafka_consumer_offset_partition)",
               "intervalFactor": 1,
+              "legendFormat": "Kafka consumer offset",
               "refId": "A"
             }
           ],
           "thresholds": [],
-          "timeFrom": "30m",
+          "timeFrom": "20m",
           "timeRegions": [],
           "timeShift": null,
           "title": "Total number of messages on topic",
@@ -490,7 +404,7 @@ data:
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -517,9 +431,9 @@ data:
           {
             "allValue": null,
             "current": {
-              "tags": [],
-              "text": "15m",
-              "value": "15m"
+              "selected": false,
+              "text": "1h",
+              "value": "1h"
             },
             "hide": 0,
             "includeAll": false,
@@ -538,7 +452,7 @@ data:
                 "value": "10m"
               },
               {
-                "selected": true,
+                "selected": false,
                 "text": "15m",
                 "value": "15m"
               },
@@ -548,7 +462,7 @@ data:
                 "value": "30m"
               },
               {
-                "selected": false,
+                "selected": true,
                 "text": "1h",
                 "value": "1h"
               }
diff --git a/tools/performance/cloud/grafana/dashboards/processing.yaml b/tools/performance/cloud/grafana/dashboards/processing.yaml
index 959cfb9d..8e429009 100644
--- a/tools/performance/cloud/grafana/dashboards/processing.yaml
+++ b/tools/performance/cloud/grafana/dashboards/processing.yaml
@@ -306,9 +306,9 @@ data:
           },
           "id": 9,
           "legend": {
-            "alignAsTable": true,
-            "avg": true,
-            "current": true,
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
             "hideEmpty": true,
             "hideZero": true,
             "max": false,
@@ -317,7 +317,7 @@ data:
             "show": true,
             "sideWidth": 350,
             "total": false,
-            "values": true
+            "values": false
           },
           "lines": true,
           "linewidth": 1,
@@ -336,30 +336,40 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "histogram_quantile(0.90, sum( rate(hvves_messages_processing_time_seconds_bucket[15m]) )by(le) )",
+              "expr": "rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h])",
               "format": "time_series",
               "hide": false,
               "instant": false,
+              "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "90th percentile of processing time on pods",
-              "refId": "B"
+              "legendFormat": "average",
+              "refId": "A"
             },
             {
-              "expr": "rate(hvves_messages_processing_time_seconds_sum[15m])/rate(hvves_messages_processing_time_seconds_count[15m])",
+              "expr": "histogram_quantile(0.90, sum( rate(hvves_messages_processing_time_seconds_bucket[1h]) )by(le) )",
               "format": "time_series",
               "hide": false,
               "instant": false,
-              "interval": "",
               "intervalFactor": 1,
-              "legendFormat": "{{ kubernetes_pod_name }}",
-              "refId": "A"
+              "legendFormat": "90th percentile",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.95, sum( rate(hvves_messages_processing_time_seconds_bucket[1h]) )by(le) )",
+              "legendFormat": "95th percentile",
+              "refId": "C"
+            },
+            {
+              "expr": "histogram_quantile(0.99, sum( rate(hvves_messages_processing_time_seconds_bucket[1h]) )by(le) )",
+              "legendFormat": "99th percentile",
+              "refId": "D"
             }
           ],
           "thresholds": [],
-          "timeFrom": null,
+          "timeFrom": "20m",
           "timeRegions": [],
           "timeShift": null,
-          "title": "Single message processing time average",
+          "title": "Message processing time in HV-VES",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -411,7 +421,9 @@ data:
             "y": 3
           },
           "id": 8,
+          "interval": "",
           "legend": {
+            "alignAsTable": false,
             "avg": false,
             "current": false,
             "hideEmpty": true,
@@ -444,15 +456,30 @@ data:
               "hide": false,
               "instant": false,
               "intervalFactor": 1,
-              "legendFormat": "{{ kubernetes_pod_name }}",
+              "legendFormat": "average",
               "refId": "A"
+            },
+            {
+              "expr": "histogram_quantile(0.90, sum( rate(hvves_messages_latency_seconds_bucket[1h]) )by(le) )",
+              "legendFormat": "90th percentile",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.90, sum( rate(hvves_messages_latency_seconds_bucket[1h]) )by(le) )",
+              "legendFormat": "95th percentile",
+              "refId": "C"
+            },
+            {
+              "expr": "histogram_quantile(0.99, sum( rate(hvves_messages_latency_seconds_bucket[1h]) )by(le) )",
+              "legendFormat": "99th precentile",
+              "refId": "D"
             }
           ],
           "thresholds": [],
-          "timeFrom": null,
+          "timeFrom": "20m",
           "timeRegions": [],
           "timeShift": null,
-          "title": "Latency [s]",
+          "title": "Message travel time from Producer to HV-VES output",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -469,11 +496,11 @@ data:
           "yaxes": [
             {
               "decimals": 2,
-              "format": "short",
+              "format": "s",
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -619,15 +646,15 @@ data:
               "expr": "sum(process_cpu_usage) by (kubernetes_pod_name)*100",
               "format": "time_series",
               "intervalFactor": 1,
-              "legendFormat": "{{kubernetes_pod_name}}",
+              "legendFormat": "CPU usage",
               "refId": "A"
             }
           ],
           "thresholds": [],
-          "timeFrom": "20 m",
+          "timeFrom": "20m",
           "timeRegions": [],
           "timeShift": null,
-          "title": "System cpu usage",
+          "title": "HV-VES process cpu usage",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -714,13 +741,13 @@ data:
               "hide": false,
               "instant": false,
               "intervalFactor": 1,
-              "legendFormat": "jvm_memory_used_bytes",
+              "legendFormat": "Memory used",
               "refId": "A"
             },
             {
               "expr": "sum(jvm_memory_committed_bytes)/1024/1024/1024",
               "instant": false,
-              "legendFormat": "jvm_memory_committed_bytes",
+              "legendFormat": "Memory commited",
               "refId": "B"
             }
           ],
@@ -728,7 +755,7 @@ data:
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Memory usage per pod [GB]",
+          "title": "Memory usage of HV-VES",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -748,7 +775,7 @@ data:
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -977,7 +1004,7 @@ data:
         "list": []
       },
       "time": {
-        "from": "now-30m",
+        "from": "now-1h",
         "to": "now"
       },
       "timepicker": {