Verify system_cpu_usage metric and fix memory usage per pod in cloud deployment

Issue-ID: DCAEGEN2-1855
Signed-off-by: pawel <pawel.kasperkiewicz@nokia.com>
Change-Id: I99ffdf23515ebe9c070abc5b1993db0cebaedf80

1 files changed, 280 insertions, 220 deletions

diff --git a/tools/performance/cloud/grafana/dashboards/processing.yaml b/tools/performance/cloud/grafana/dashboards/processing.yaml
index 2d68981b..959cfb9d 100644
--- a/tools/performance/cloud/grafana/dashboards/processing.yaml
+++ b/tools/performance/cloud/grafana/dashboards/processing.yaml
@@ -42,7 +42,6 @@ data:
       "editable": true,
       "gnetId": null,
       "graphTooltip": 0,
-      "id": 2,
       "links": [],
       "panels": [
         {
@@ -86,6 +85,7 @@ data:
           "maxDataPoints": 100,
           "nullPointMode": "connected",
           "nullText": null,
+          "options": {},
           "postfix": "",
           "postfixFontSize": "50%",
           "prefix": "",
@@ -135,6 +135,7 @@ data:
             "rgba(237, 129, 40, 0.89)",
             "#d44a3a"
           ],
+          "datasource": null,
           "format": "none",
           "gauge": {
             "maxValue": 100,
@@ -166,6 +167,7 @@ data:
           "maxDataPoints": 100,
           "nullPointMode": "connected",
           "nullText": null,
+          "options": {},
           "postfix": "",
           "postfixFontSize": "50%",
           "prefix": "",
@@ -215,6 +217,7 @@ data:
             "rgba(237, 129, 40, 0.89)",
             "#d44a3a"
           ],
+          "datasource": null,
           "format": "none",
           "gauge": {
             "maxValue": 100,
@@ -246,6 +249,7 @@ data:
           "maxDataPoints": 100,
           "nullPointMode": "connected",
           "nullText": null,
+          "options": {},
           "postfix": "",
           "postfixFontSize": "50%",
           "prefix": "",
@@ -291,7 +295,9 @@ data:
           "bars": false,
           "dashLength": 10,
           "dashes": false,
+          "datasource": null,
           "fill": 0,
+          "fillGradient": 0,
           "gridPos": {
             "h": 12,
             "w": 15,
@@ -317,6 +323,9 @@ data:
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
           "percentage": false,
           "pointradius": 5,
           "points": false,
@@ -370,7 +379,7 @@ data:
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -392,7 +401,9 @@ data:
           "bars": false,
           "dashLength": 10,
           "dashes": false,
+          "datasource": null,
           "fill": 0,
+          "fillGradient": 0,
           "gridPos": {
             "h": 9,
             "w": 9,
@@ -415,6 +426,9 @@ data:
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
           "percentage": false,
           "pointradius": 5,
           "points": false,
@@ -438,7 +452,7 @@ data:
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Latency",
+          "title": "Latency [s]",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -454,6 +468,7 @@ data:
           },
           "yaxes": [
             {
+              "decimals": 2,
               "format": "short",
               "label": null,
               "logBase": 1,
@@ -476,120 +491,85 @@ data:
           }
         },
         {
-          "columns": [
-            {
-              "text": "Current",
-              "value": "current"
-            }
+          "cacheTimeout": null,
+          "colorBackground": false,
+          "colorValue": false,
+          "colors": [
+            "#299c46",
+            "rgba(237, 129, 40, 0.89)",
+            "#d44a3a"
           ],
-          "fontSize": "100%",
+          "datasource": null,
+          "format": "decbytes",
+          "gauge": {
+            "maxValue": 100,
+            "minValue": 0,
+            "show": false,
+            "thresholdLabels": false,
+            "thresholdMarkers": true
+          },
           "gridPos": {
-            "h": 9,
-            "w": 5,
+            "h": 2,
+            "w": 4,
             "x": 0,
             "y": 12
           },
-          "id": 11,
+          "id": 20,
+          "interval": null,
           "links": [],
-          "pageSize": null,
-          "scroll": true,
-          "showHeader": true,
-          "sort": {
-            "col": 0,
-            "desc": true
-          },
-          "styles": [
+          "mappingType": 1,
+          "mappingTypes": [
             {
-              "alias": "Time",
-              "dateFormat": "YYYY-MM-DD HH:mm:ss",
-              "pattern": "Time",
-              "type": "date"
+              "name": "value to text",
+              "value": 1
             },
             {
-              "alias": "",
-              "colorMode": null,
-              "colors": [
-                "rgba(245, 54, 54, 0.9)",
-                "rgba(237, 129, 40, 0.89)",
-                "rgba(50, 172, 45, 0.97)"
-              ],
-              "decimals": 2,
-              "pattern": "/.*/",
-              "thresholds": [],
-              "type": "number",
-              "unit": "short"
-            }
-          ],
-          "targets": [
-            {
-              "expr": "sum(hvves_messages_received_total) by (kubernetes_pod_name, value)",
-              "format": "time_series",
-              "intervalFactor": 1,
-              "legendFormat": "{{kubernetes_pod_name}}",
-              "refId": "A"
+              "name": "range to text",
+              "value": 2
             }
           ],
-          "title": "Messages processed per pod",
-          "transform": "timeseries_aggregations",
-          "type": "table"
-        },
-        {
-          "columns": [
+          "maxDataPoints": 100,
+          "nullPointMode": "connected",
+          "nullText": null,
+          "options": {},
+          "postfix": "",
+          "postfixFontSize": "50%",
+          "prefix": "",
+          "prefixFontSize": "50%",
+          "rangeMaps": [
             {
-              "text": "Current",
-              "value": "current"
+              "from": "null",
+              "text": "N/A",
+              "to": "null"
             }
           ],
-          "fontSize": "100%",
-          "gridPos": {
-            "h": 5,
-            "w": 4,
-            "x": 5,
-            "y": 12
-          },
-          "id": 12,
-          "links": [],
-          "pageSize": null,
-          "scroll": true,
-          "showHeader": true,
-          "sort": {
-            "col": 0,
-            "desc": true
+          "sparkline": {
+            "fillColor": "rgba(31, 118, 189, 0.18)",
+            "full": false,
+            "lineColor": "rgb(31, 120, 193)",
+            "show": false
           },
-          "styles": [
-            {
-              "alias": "Time",
-              "dateFormat": "YYYY-MM-DD HH:mm:ss",
-              "pattern": "Time",
-              "type": "date"
-            },
-            {
-              "alias": "",
-              "colorMode": null,
-              "colors": [
-                "rgba(245, 54, 54, 0.9)",
-                "rgba(237, 129, 40, 0.89)",
-                "rgba(50, 172, 45, 0.97)"
-              ],
-              "decimals": 2,
-              "pattern": "/.*/",
-              "thresholds": [],
-              "type": "number",
-              "unit": "short"
-            }
-          ],
+          "tableColumn": "",
           "targets": [
             {
-              "expr": "sum(hvves_messages_sent_topic_total) by (topic, value)",
+              "expr": "sum(hvves_messages_received_payload_bytes_total)",
               "format": "time_series",
               "intervalFactor": 1,
-              "legendFormat": "{{topic}}",
               "refId": "A"
             }
           ],
-          "title": "Messages sent per topic",
-          "transform": "timeseries_aggregations",
-          "type": "table"
+          "thresholds": "",
+          "title": "Total received payload bytes",
+          "type": "singlestat",
+          "valueFontSize": "80%",
+          "valueMaps": [
+            {
+              "op": "=",
+              "text": "N/A",
+              "value": "null"
+            }
+          ],
+          "valueName": "current"
         },
         {
           "aliasColors": {},
@@ -597,11 +577,12 @@ data:
           "dashLength": 10,
           "dashes": false,
           "datasource": "Prometheus",
-          "fill": 0,
+          "fill": 1,
+          "fillGradient": 3,
           "gridPos": {
-            "h": 9,
-            "w": 9,
-            "x": 9,
+            "h": 10,
+            "w": 10,
+            "x": 4,
             "y": 12
           },
           "id": 16,
@@ -622,6 +603,9 @@ data:
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
+          },
           "percentage": false,
           "pointradius": 5,
           "points": false,
@@ -632,9 +616,9 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(system_cpu_usage) by (kubernetes_pod_name)",
+              "expr": "sum(process_cpu_usage) by (kubernetes_pod_name)*100",
               "format": "time_series",
-              "intervalFactor": 2,
+              "intervalFactor": 1,
               "legendFormat": "{{kubernetes_pod_name}}",
               "refId": "A"
             }
@@ -649,7 +633,6 @@ data:
             "sort": 0,
             "value_type": "individual"
           },
-          "transparent": false,
           "type": "graph",
           "xaxis": {
             "buckets": null,
@@ -660,11 +643,12 @@ data:
           },
           "yaxes": [
             {
-              "format": "short",
+              "decimals": 2,
+              "format": "percent",
               "label": null,
               "logBase": 1,
               "max": null,
-              "min": null,
+              "min": "0",
               "show": true
             },
             {
@@ -682,70 +666,104 @@ data:
           }
         },
         {
-          "columns": [
-            {
-              "text": "Current",
-              "value": "current"
-            },
-            {
-              "text": "Avg",
-              "value": "avg"
-            }
-          ],
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
           "datasource": "Prometheus",
-          "fontSize": "100%",
+          "fill": 0,
+          "fillGradient": 0,
           "gridPos": {
-            "h": 9,
-            "w": 6,
-            "x": 18,
+            "h": 10,
+            "w": 10,
+            "x": 14,
             "y": 12
           },
           "id": 22,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": true,
+            "min": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
           "links": [],
-          "pageSize": null,
-          "scroll": true,
-          "showHeader": true,
-          "sort": {
-            "col": 0,
-            "desc": true
+          "nullPointMode": "null",
+          "options": {
+            "dataLinks": []
           },
-          "styles": [
-            {
-              "alias": "Time",
-              "dateFormat": "YYYY-MM-DD HH:mm:ss",
-              "pattern": "Time",
-              "type": "date"
-            },
-            {
-              "alias": "",
-              "colorMode": null,
-              "colors": [
-                "rgba(245, 54, 54, 0.9)",
-                "rgba(237, 129, 40, 0.89)",
-                "rgba(50, 172, 45, 0.97)"
-              ],
-              "decimals": 2,
-              "pattern": "/.*/",
-              "thresholds": [],
-              "type": "number",
-              "unit": "short"
-            }
-          ],
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
           "targets": [
             {
-              "expr": "(sum(container_memory_usage_bytes{pod_name=~\"dep-dcae-hv-ves-collector.*\"}) by (pod_name)) / 1024 / 1024 / 1024",
+              "expr": "sum(jvm_memory_used_bytes)/1024/1024/1024\n",
               "format": "time_series",
               "hide": false,
-              "instant": true,
+              "instant": false,
               "intervalFactor": 1,
-              "legendFormat": "{{ pod_name }}",
+              "legendFormat": "jvm_memory_used_bytes",
               "refId": "A"
+            },
+            {
+              "expr": "sum(jvm_memory_committed_bytes)/1024/1024/1024",
+              "instant": false,
+              "legendFormat": "jvm_memory_committed_bytes",
+              "refId": "B"
             }
           ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
           "title": "Memory usage per pod [GB]",
-          "transform": "timeseries_aggregations",
-          "transparent": false,
-          "type": "table"
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "gbytes",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         },
         {
           "cacheTimeout": null,
@@ -756,6 +774,7 @@ data:
             "rgba(237, 129, 40, 0.89)",
             "#d44a3a"
           ],
+          "datasource": "Prometheus",
           "format": "decbytes",
           "gauge": {
             "maxValue": 100,
@@ -767,10 +786,10 @@ data:
           "gridPos": {
             "h": 2,
             "w": 4,
-            "x": 5,
-            "y": 17
+            "x": 0,
+            "y": 14
           },
-          "id": 20,
+          "id": 18,
           "interval": null,
           "links": [],
           "mappingType": 1,
@@ -787,6 +806,7 @@ data:
           "maxDataPoints": 100,
           "nullPointMode": "connected",
           "nullText": null,
+          "options": {},
           "postfix": "",
           "postfixFontSize": "50%",
           "prefix": "",
@@ -807,117 +827,157 @@ data:
           "tableColumn": "",
           "targets": [
             {
-              "expr": "sum(hvves_messages_received_payload_bytes_total)",
+              "expr": "sum(hvves_messages_received_payload_bytes_total)/sum(hvves_messages_received_total)",
               "format": "time_series",
+              "hide": false,
+              "instant": false,
               "intervalFactor": 1,
               "refId": "A"
             }
           ],
           "thresholds": "",
-          "title": "Total received payload bytes",
+          "title": "Average received payload bytes",
           "type": "singlestat",
           "valueFontSize": "80%",
           "valueMaps": [
             {
               "op": "=",
-              "text": "N/A",
+              "text": "0",
               "value": "null"
             }
           ],
           "valueName": "current"
         },
         {
-          "cacheTimeout": null,
-          "colorBackground": false,
-          "colorValue": false,
-          "colors": [
-            "#299c46",
-            "rgba(237, 129, 40, 0.89)",
-            "#d44a3a"
+          "columns": [
+            {
+              "text": "Current",
+              "value": "current"
+            }
           ],
-          "datasource": "Prometheus",
-          "format": "decbytes",
-          "gauge": {
-            "maxValue": 100,
-            "minValue": 0,
-            "show": false,
-            "thresholdLabels": false,
-            "thresholdMarkers": true
-          },
+          "datasource": null,
+          "fontSize": "100%",
           "gridPos": {
-            "h": 2,
+            "h": 3,
             "w": 4,
-            "x": 5,
-            "y": 19
+            "x": 0,
+            "y": 16
           },
-          "id": 18,
-          "interval": null,
+          "id": 12,
           "links": [],
-          "mappingType": 1,
-          "mappingTypes": [
+          "options": {},
+          "pageSize": null,
+          "scroll": true,
+          "showHeader": true,
+          "sort": {
+            "col": 0,
+            "desc": true
+          },
+          "styles": [
             {
-              "name": "value to text",
-              "value": 1
+              "alias": "Time",
+              "dateFormat": "YYYY-MM-DD HH:mm:ss",
+              "pattern": "Time",
+              "type": "date"
             },
             {
-              "name": "range to text",
-              "value": 2
+              "alias": "",
+              "colorMode": null,
+              "colors": [
+                "rgba(245, 54, 54, 0.9)",
+                "rgba(237, 129, 40, 0.89)",
+                "rgba(50, 172, 45, 0.97)"
+              ],
+              "decimals": 2,
+              "pattern": "/.*/",
+              "thresholds": [],
+              "type": "number",
+              "unit": "short"
             }
           ],
-          "maxDataPoints": 100,
-          "nullPointMode": "connected",
-          "nullText": null,
-          "postfix": "",
-          "postfixFontSize": "50%",
-          "prefix": "",
-          "prefixFontSize": "50%",
-          "rangeMaps": [
+          "targets": [
             {
-              "from": "null",
-              "text": "N/A",
-              "to": "null"
+              "expr": "sum(hvves_messages_sent_topic_total) by (topic, value)",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{topic}}",
+              "refId": "A"
             }
           ],
-          "sparkline": {
-            "fillColor": "rgba(31, 118, 189, 0.18)",
-            "full": false,
-            "lineColor": "rgb(31, 120, 193)",
-            "show": false
+          "title": "Messages sent per topic",
+          "transform": "timeseries_aggregations",
+          "type": "table"
+        },
+        {
+          "columns": [
+            {
+              "text": "Current",
+              "value": "current"
+            }
+          ],
+          "datasource": null,
+          "fontSize": "100%",
+          "gridPos": {
+            "h": 3,
+            "w": 4,
+            "x": 0,
+            "y": 19
           },
-          "tableColumn": "",
+          "id": 11,
+          "links": [],
+          "options": {},
+          "pageSize": null,
+          "scroll": true,
+          "showHeader": true,
+          "sort": {
+            "col": 0,
+            "desc": true
+          },
+          "styles": [
+            {
+              "alias": "Time",
+              "dateFormat": "YYYY-MM-DD HH:mm:ss",
+              "pattern": "Time",
+              "type": "date"
+            },
+            {
+              "alias": "",
+              "colorMode": null,
+              "colors": [
+                "rgba(245, 54, 54, 0.9)",
+                "rgba(237, 129, 40, 0.89)",
+                "rgba(50, 172, 45, 0.97)"
+              ],
+              "decimals": 2,
+              "pattern": "/.*/",
+              "thresholds": [],
+              "type": "number",
+              "unit": "short"
+            }
+          ],
           "targets": [
             {
-              "expr": "sum(hvves_messages_received_payload_bytes_total)/sum(hvves_messages_received_total)",
+              "expr": "sum(hvves_messages_received_total) by (kubernetes_pod_name, value)",
               "format": "time_series",
-              "hide": false,
-              "instant": false,
               "intervalFactor": 1,
+              "legendFormat": "{{kubernetes_pod_name}}",
               "refId": "A"
             }
           ],
-          "thresholds": "",
-          "title": "Average received payload bytes",
-          "type": "singlestat",
-          "valueFontSize": "80%",
-          "valueMaps": [
-            {
-              "op": "=",
-              "text": "0",
-              "value": "null"
-            }
-          ],
-          "valueName": "current"
+          "title": "Messages processed per pod",
+          "transform": "timeseries_aggregations",
+          "type": "table"
         }
       ],
-      "refresh": "1s",
-      "schemaVersion": 16,
+      "refresh": "5s",
+      "schemaVersion": 20,
       "style": "dark",
       "tags": [],
       "templating": {
         "list": []
       },
       "time": {
-        "from": "now-1h",
+        "from": "now-30m",
         "to": "now"
       },
       "timepicker": {