Add metric for total latency without routing

Add metric for time between Producer and HV-VES output without
sending to Kafka
Refactor metrics test
Add new latencies and individual cores usage to Grafana

Issue-ID: DCAEGEN2-1576
Signed-off-by: Remigiusz Janeczek <remigiusz.janeczek@nokia.com>
Change-Id: I6112db76be1c7108c18336b50f9f12d5ce62c24a

6 files changed, 735 insertions, 551 deletions

diff --git a/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt b/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt
index 12555965..4a5474c6 100644
--- a/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt
+++ b/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt
@@ -42,6 +42,7 @@ class FakeMetrics : Metrics {
     var lastProcessingTimeMicros: Double = -1.0; private set
     var lastProcessingTimeWithoutRoutingMicros: Double = -1.0; private set
     var lastToCollectorTravelTime: Double = -1.0; private set
+    var lastTotalLatencyWithoutRouting: Double = -1.0; private set
     var messagesSentCount: Int = 0; private set
     var clientRejectionCause = mutableMapOf<ClientRejectionCause, Int>(); private set
 
@@ -63,6 +64,7 @@ class FakeMetrics : Metrics {
 
     override fun notifyMessageReadyForRouting(msg: VesMessage) {
         lastProcessingTimeWithoutRoutingMicros = Duration.between(msg.wtpFrame.receivedAt, Instant.now()).toNanos() / 1000.0
+        lastTotalLatencyWithoutRouting = Duration.between(TimeUtils.epochMicroToInstant(msg.header.lastEpochMicrosec), Instant.now()).toNanos() / 1000.0
     }
 
     override fun notifyMessageSent(msg: RoutedMessage) {
diff --git a/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt b/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt
index e0d99fc6..a949803f 100644
--- a/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt
+++ b/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt
@@ -66,6 +66,10 @@ class MicrometerMetrics internal constructor(
             .maximumExpectedValue(MAX_BUCKET_DURATION)
             .publishPercentileHistogram(true)
             .register(registry)
+    private val totalLatencyWithoutRouting = Timer.builder(name(MESSAGES, LATENCY, WITHOUT, ROUTING))
+            .maximumExpectedValue(MAX_BUCKET_DURATION)
+            .publishPercentileHistogram(true)
+            .register(registry)
     private val totalLatency = Timer.builder(name(MESSAGES, LATENCY))
             .maximumExpectedValue(MAX_BUCKET_DURATION)
             .publishPercentileHistogram(true)
@@ -104,7 +108,9 @@ class MicrometerMetrics internal constructor(
     }
 
     override fun notifyMessageReadyForRouting(msg: VesMessage) {
-        processingTimeWithoutRouting.record(Duration.between(msg.wtpFrame.receivedAt, Instant.now()))
+        val now = Instant.now()
+        processingTimeWithoutRouting.record(Duration.between(msg.wtpFrame.receivedAt, now))
+        totalLatencyWithoutRouting.record(Duration.between(epochMicroToInstant(msg.header.lastEpochMicrosec), now))
     }
 
     override fun notifyMessageReceived(msg: WireFrameMessage) {
diff --git a/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt b/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt
index efd353ec..dd206d08 100644
--- a/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt
+++ b/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt
@@ -146,8 +146,66 @@ object MicrometerMetricsTest : Spek({
                     "$PREFIX.messages.received.payload.bytes"
             )
         }
+
+        on("$PREFIX.messages.to.collector.travel.time") {
+            val counterName = "$PREFIX.messages.to.collector.travel.time"
+            val toCollectorTravelTimeMs = 100L
+
+            it("should update timer") {
+                val now = Instant.now()
+                val vesMessage = vesMessageReceivedAt(now, sentAt = now.minusMillis(toCollectorTravelTimeMs))
+                cut.notifyMessageReceived(vesMessage)
+
+                registry.verifyTimer(counterName) { timer ->
+                    assertThat(timer.mean(TimeUnit.MILLISECONDS)).isEqualTo(toCollectorTravelTimeMs.toDouble())
+                }
+
+                verifyCountersAndTimersAreUnchangedBut(counterName)
+            }
+        }
     }
 
+    describe("notifyMessageReadyForRouting"){
+        on("$PREFIX.messages.processing.time.without.routing") {
+            val counterName = "$PREFIX.messages.processing.time.without.routing"
+            val processingTimeMs = 100L
+
+            it("should update timer") {
+
+                cut.notifyMessageReadyForRouting(vesMessageReceivedAt(Instant.now().minusMillis(processingTimeMs)))
+
+                registry.verifyTimer(counterName) { timer ->
+                    assertThat(timer.mean(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(processingTimeMs.toDouble())
+                }
+                verifyCountersAndTimersAreUnchangedBut(
+                        counterName,
+                        "$PREFIX.messages.latency.without.routing"
+                )
+            }
+        }
+
+        on("$PREFIX.messages.latency.without.routing") {
+            val counterName = "$PREFIX.messages.latency.without.routing"
+            val latencyWithoutRoutingMs = 200L
+
+            it("should update timer") {
+
+                val sentAt = Instant.now().minusMillis(latencyWithoutRoutingMs)
+
+                cut.notifyMessageReadyForRouting(vesMessageSentAt(sentAt))
+
+                registry.verifyTimer(counterName) { timer ->
+                    assertThat(timer.mean(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(latencyWithoutRoutingMs.toDouble())
+                }
+                verifyCountersAndTimersAreUnchangedBut(
+                        counterName,
+                        "$PREFIX.messages.processing.time.without.routing"
+                )
+            }
+        }
+    }
+
+
     describe("notifyMessageSent") {
         val topicName1 = "PERF3GPP"
         val topicName2 = "CALLTRACE"
@@ -206,39 +264,6 @@ object MicrometerMetricsTest : Spek({
             }
         }
 
-        on("$PREFIX.messages.to.collector.travel.time") {
-            val counterName = "$PREFIX.messages.to.collector.travel.time"
-            val toCollectorTravelTimeMs = 100L
-
-            it("should update timer") {
-                val now = Instant.now()
-                val vesMessage = vesMessageReceivedAt(now, sentAt = now.minusMillis(toCollectorTravelTimeMs))
-                cut.notifyMessageReceived(vesMessage)
-
-                registry.verifyTimer(counterName) { timer ->
-                    assertThat(timer.mean(TimeUnit.MILLISECONDS)).isEqualTo(toCollectorTravelTimeMs.toDouble())
-                }
-
-                verifyCountersAndTimersAreUnchangedBut(counterName)
-            }
-        }
-
-        on("$PREFIX.messages.processing.time.without.routing") {
-            val counterName = "$PREFIX.messages.processing.time.without.routing"
-            val processingTimeMs = 100L
-
-            it("should update timer") {
-
-                cut.notifyMessageReadyForRouting(vesMessageReceivedAt(Instant.now().minusMillis(processingTimeMs)))
-
-                registry.verifyTimer(counterName) { timer ->
-                    assertThat(timer.mean(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(processingTimeMs.toDouble())
-                }
-
-                verifyCountersAndTimersAreUnchangedBut(counterName)
-            }
-        }
-
         on("$PREFIX.messages.latency") {
             val counterName = "$PREFIX.messages.latency"
             val latencyMs = 1666L
@@ -398,6 +423,13 @@ object MicrometerMetricsTest : Spek({
     }
 })
 
+private fun vesMessageSentAt(sentAt: Instant): VesMessage {
+    val lastEpochMicrosec = sentAt.epochSecond * 1000000 + sentAt.nano / 1000
+    val commonHeader = commonHeader(lastEpochMicrosec = lastEpochMicrosec)
+    return VesMessage(commonHeader,
+            wireProtocolFrame(commonHeader, ByteString.copyFromUtf8("highvolume measurements")))
+}
+
 private fun vesMessageReceivedAt(receivedAt: Instant, sentAt: Instant): VesMessage {
     val lastEpochMicrosec = sentAt.epochSecond * 1000000 + sentAt.nano / 1000
     val commonHeader = commonHeader(lastEpochMicrosec = lastEpochMicrosec)
diff --git a/tools/performance/cloud/grafana/dashboards/k8s-metrics.json b/tools/performance/cloud/grafana/dashboards/k8s-metrics.json
index 676d0a93..7841f77d 100644
--- a/tools/performance/cloud/grafana/dashboards/k8s-metrics.json
+++ b/tools/performance/cloud/grafana/dashboards/k8s-metrics.json
@@ -1,530 +1,619 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 2,
+  "links": [],
+  "panels": [
     {
-      "annotations": {
-        "list": [
-          {
-            "builtIn": 1,
-            "datasource": "-- Grafana --",
-            "enable": true,
-            "hide": true,
-            "iconColor": "rgba(0, 211, 255, 1)",
-            "name": "Annotations & Alerts",
-            "type": "dashboard"
-          }
-        ]
+      "datasource": null,
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 0,
+        "y": 0
       },
-      "editable": true,
-      "gnetId": null,
-      "graphTooltip": 0,
-      "id": 5,
-      "links": [],
-      "panels": [
-        {
-          "datasource": null,
-          "gridPos": {
-            "h": 7,
-            "w": 8,
-            "x": 0,
-            "y": 0
-          },
-          "id": 8,
-          "options": {
-            "fieldOptions": {
-              "calcs": [
-                "mean"
-              ],
-              "defaults": {
-                "mappings": [],
-                "max": 100,
-                "min": 0,
-                "thresholds": {
-                  "mode": "absolute",
-                  "steps": [
-                    {
-                      "color": "green",
-                      "value": null
-                    },
-                    {
-                      "color": "red",
-                      "value": 80
-                    }
-                  ]
+      "id": 6,
+      "options": {
+        "fieldOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "defaults": {
+            "mappings": [],
+            "max": 100,
+            "min": 0,
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
                 },
-                "unit": "percent"
-              },
-              "overrides": [],
-              "values": false
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
             },
-            "orientation": "auto",
-            "showThresholdLabels": false,
-            "showThresholdMarkers": true
+            "unit": "percent"
           },
-          "pluginVersion": "6.7.2",
-          "targets": [
-            {
-              "expr": "sum(container_memory_working_set_bytes{id=\"/\", instance=~\".*worker.*\"}) by(instance) / (sum(machine_memory_bytes{instance=~\".*worker.*\"}) by (instance))*100",
-              "interval": "",
-              "legendFormat": "{{instance}}",
-              "refId": "A"
-            }
-          ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Nodes RAM Usage",
-          "type": "gauge"
+          "overrides": [],
+          "values": false
         },
+        "orientation": "auto",
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "6.7.2",
+      "targets": [
         {
-          "datasource": null,
-          "gridPos": {
-            "h": 7,
-            "w": 8,
-            "x": 8,
-            "y": 0
-          },
-          "id": 6,
-          "options": {
-            "fieldOptions": {
-              "calcs": [
-                "mean"
-              ],
-              "defaults": {
-                "mappings": [],
-                "max": 100,
-                "min": 0,
-                "thresholds": {
-                  "mode": "absolute",
-                  "steps": [
-                    {
-                      "color": "green",
-                      "value": null
-                    },
-                    {
-                      "color": "red",
-                      "value": 80
-                    }
-                  ]
-                },
-                "unit": "percent"
-              },
-              "overrides": [],
-              "values": false
-            },
-            "orientation": "auto",
-            "showThresholdLabels": false,
-            "showThresholdMarkers": true
-          },
-          "pluginVersion": "6.7.2",
-          "targets": [
-            {
-              "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\", instance=~\".*worker.*\"}[1m])) by(instance) / (sum(machine_cpu_cores{instance=~\".*worker.*\"}) by (instance))*100",
-              "interval": "",
-              "legendFormat": "{{instance}}",
-              "refId": "A"
-            }
-          ],
-          "timeFrom": null,
-          "timeShift": null,
-          "title": "Nodes CPU Usage",
-          "type": "gauge"
+          "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\", instance=~\".*worker.*\"}[1m])) by(instance) / (sum(machine_cpu_cores{instance=~\".*worker.*\"}) by (instance))*100",
+          "interval": "",
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Nodes CPU Usage",
+      "type": "gauge"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 0,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 12,
+        "w": 8,
+        "x": 8,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\"}[1m])) by(instance) / (sum(machine_cpu_cores) by (instance))*100",
+          "interval": "",
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": "20m",
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Nodes Total CPU usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "percent",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
         },
         {
-          "aliasColors": {},
-          "bars": false,
-          "dashLength": 10,
-          "dashes": false,
-          "datasource": null,
-          "fill": 0,
-          "fillGradient": 0,
-          "gridPos": {
-            "h": 12,
-            "w": 8,
-            "x": 16,
-            "y": 0
-          },
-          "hiddenSeries": false,
-          "id": 10,
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 1,
-          "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 2,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "spaceLength": 10,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)",
-              "hide": false,
-              "interval": "",
-              "legendFormat": "{{instance}}",
-              "refId": "C"
-            },
-            {
-              "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))",
-              "hide": false,
-              "interval": "",
-              "legendFormat": "Total Receive",
-              "refId": "D"
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": "20m",
-          "timeRegions": [],
-          "timeShift": null,
-          "title": "Network Usage - Receive",
-          "tooltip": {
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "type": "graph",
-          "xaxis": {
-            "buckets": null,
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "format": "Bps",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": null
-          }
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 0,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 12,
+        "w": 8,
+        "x": 16,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "{{instance}}",
+          "refId": "C"
         },
         {
-          "aliasColors": {},
-          "bars": false,
-          "dashLength": 10,
-          "dashes": false,
-          "datasource": null,
-          "fill": 0,
-          "fillGradient": 0,
-          "gridPos": {
-            "h": 16,
-            "w": 8,
-            "x": 0,
-            "y": 7
-          },
-          "hiddenSeries": false,
-          "id": 4,
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 1,
-          "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 2,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "spaceLength": 10,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "expr": "sum(container_memory_working_set_bytes{id=\"/\"}) by (instance) / (sum(machine_memory_bytes) by (instance))*100",
-              "interval": "",
-              "legendFormat": "{{instance}}",
-              "refId": "A"
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": "20m",
-          "timeRegions": [],
-          "timeShift": null,
-          "title": "Nodes Total RAM Usage",
-          "tooltip": {
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "type": "graph",
-          "xaxis": {
-            "buckets": null,
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "format": "percent",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": null
-          }
+          "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Total Receive",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": "20m",
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Network Usage - Receive",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
         },
         {
-          "aliasColors": {},
-          "bars": false,
-          "dashLength": 10,
-          "dashes": false,
-          "datasource": null,
-          "fill": 0,
-          "fillGradient": 0,
-          "gridPos": {
-            "h": 16,
-            "w": 8,
-            "x": 8,
-            "y": 7
-          },
-          "hiddenSeries": false,
-          "id": 2,
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 1,
-          "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 2,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "spaceLength": 10,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\"}[1m])) by(instance) / (sum(machine_cpu_cores) by (instance))*100",
-              "interval": "",
-              "legendFormat": "{{instance}}",
-              "refId": "A"
-            }
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "datasource": null,
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 0,
+        "y": 5
+      },
+      "id": 8,
+      "options": {
+        "fieldOptions": {
+          "calcs": [
+            "mean"
           ],
-          "thresholds": [],
-          "timeFrom": "20m",
-          "timeRegions": [],
-          "timeShift": null,
-          "title": "Nodes Total CPU usage",
-          "tooltip": {
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "type": "graph",
-          "xaxis": {
-            "buckets": null,
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "format": "percent",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
+          "defaults": {
+            "mappings": [],
+            "max": 100,
+            "min": 0,
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
             },
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": null
-          }
+            "unit": "percent"
+          },
+          "overrides": [],
+          "values": false
         },
+        "orientation": "auto",
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "6.7.2",
+      "targets": [
         {
-          "aliasColors": {},
-          "bars": false,
-          "dashLength": 10,
-          "dashes": false,
-          "datasource": null,
-          "fill": 0,
-          "fillGradient": 0,
-          "gridPos": {
-            "h": 11,
-            "w": 8,
-            "x": 16,
-            "y": 12
-          },
-          "hiddenSeries": false,
-          "id": 11,
-          "legend": {
-            "avg": false,
-            "current": false,
-            "max": false,
-            "min": false,
-            "show": true,
-            "total": false,
-            "values": false
-          },
-          "lines": true,
-          "linewidth": 1,
-          "nullPointMode": "null",
-          "options": {
-            "dataLinks": []
-          },
-          "percentage": false,
-          "pointradius": 2,
-          "points": false,
-          "renderer": "flot",
-          "seriesOverrides": [],
-          "spaceLength": 10,
-          "stack": false,
-          "steppedLine": false,
-          "targets": [
-            {
-              "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)",
-              "hide": false,
-              "interval": "",
-              "legendFormat": "{{instance}}",
-              "refId": "A"
-            },
-            {
-              "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))",
-              "hide": false,
-              "interval": "",
-              "legendFormat": "Total Transmit",
-              "refId": "B"
-            }
-          ],
-          "thresholds": [],
-          "timeFrom": "20m",
-          "timeRegions": [],
-          "timeShift": null,
-          "title": "Network Usage - Transmit",
-          "tooltip": {
-            "shared": true,
-            "sort": 0,
-            "value_type": "individual"
-          },
-          "type": "graph",
-          "xaxis": {
-            "buckets": null,
-            "mode": "time",
-            "name": null,
-            "show": true,
-            "values": []
-          },
-          "yaxes": [
-            {
-              "format": "Bps",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            },
-            {
-              "format": "short",
-              "label": null,
-              "logBase": 1,
-              "max": null,
-              "min": null,
-              "show": true
-            }
-          ],
-          "yaxis": {
-            "align": false,
-            "alignLevel": null
-          }
+          "expr": "sum(container_memory_working_set_bytes{id=\"/\", instance=~\".*worker.*\"}) by(instance) / (sum(machine_memory_bytes{instance=~\".*worker.*\"}) by (instance))*100",
+          "interval": "",
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Nodes RAM Usage",
+      "type": "gauge"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 0,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 13,
+        "w": 8,
+        "x": 0,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(container_memory_working_set_bytes{id=\"/\"}) by (instance) / (sum(machine_memory_bytes) by (instance))*100",
+          "interval": "",
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": "20m",
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Nodes Total RAM Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "percent",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 0,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 11,
+        "w": 8,
+        "x": 8,
+        "y": 12
+      },
+      "hiddenSeries": false,
+      "id": 12,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "100- sum(rate(node_cpu_seconds_total{mode=\"idle\"}[1m])) by (node_name, cpu) * 100",
+          "interval": "",
+          "legendFormat": "Instance: {{node_name}} cpu: {{cpu}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": "20m",
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Usage of each core",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "percent",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
         }
       ],
-      "refresh": "5s",
-      "schemaVersion": 22,
-      "style": "dark",
-      "tags": [],
-      "templating": {
-        "list": []
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 0,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 11,
+        "w": 8,
+        "x": 16,
+        "y": 12
+      },
+      "hiddenSeries": false,
+      "id": 11,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
       },
-      "time": {
-        "from": "now-6h",
-        "to": "now"
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
       },
-      "timepicker": {
-        "refresh_intervals": [
-          "5s",
-          "10s",
-          "30s",
-          "1m",
-          "5m",
-          "15m",
-          "30m",
-          "1h",
-          "2h",
-          "1d"
-        ]
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "{{instance}}",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Total Transmit",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": "20m",
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Network Usage - Transmit",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
       },
-      "timezone": "",
-      "title": "K8s metrics",
-      "uid": "RxjnWpjZk",
-      "variables": {
-        "list": []
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
       },
-      "version": 1
+      "yaxes": [
+        {
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
     }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 22,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ]
+  },
+  "timezone": "",
+  "title": "K8s metrics",
+  "uid": "RxjnWpjZk",
+  "variables": {
+    "list": []
+  },
+  "version": 1
+}
diff --git a/tools/performance/cloud/grafana/dashboards/latencies.json b/tools/performance/cloud/grafana/dashboards/latencies.json
index 43ec7f4f..9efe8485 100644
--- a/tools/performance/cloud/grafana/dashboards/latencies.json
+++ b/tools/performance/cloud/grafana/dashboards/latencies.json
@@ -15,7 +15,7 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "id": 6,
+  "id": 5,
   "links": [],
   "panels": [
     {
@@ -33,8 +33,10 @@
         "y": 0
       },
       "hiddenSeries": false,
-      "id": 2,
+      "id": 8,
+      "interval": "",
       "legend": {
+        "alignAsTable": false,
         "avg": false,
         "current": false,
         "hideEmpty": true,
@@ -47,12 +49,13 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null",
       "options": {
         "dataLinks": []
       },
       "percentage": false,
-      "pointradius": 2,
+      "pointradius": 5,
       "points": false,
       "renderer": "flot",
       "seriesOverrides": [],
@@ -61,10 +64,32 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h])",
+          "expr": "rate(hvves_messages_to_collector_travel_time_seconds_sum[1h])/rate(hvves_messages_to_collector_travel_time_seconds_count[1h])",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
           "interval": "",
+          "intervalFactor": 1,
           "legendFormat": "average",
           "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.90, sum( rate(hvves_messages_to_collector_travel_time_seconds_bucket[1h]) )by(le) )",
+          "interval": "",
+          "legendFormat": "90th percentile",
+          "refId": "B"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum( rate(hvves_messages_to_collector_travel_time_seconds_bucket[1h]) )by(le) )",
+          "interval": "",
+          "legendFormat": "95th percentile",
+          "refId": "C"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum( rate(hvves_messages_to_collector_travel_time_seconds_bucket[1h]) )by(le) )",
+          "interval": "",
+          "legendFormat": "99th precentile",
+          "refId": "D"
         }
       ],
       "thresholds": [],
@@ -87,6 +112,7 @@
       },
       "yaxes": [
         {
+          "decimals": 2,
           "format": "s",
           "label": null,
           "logBase": 1,
@@ -123,8 +149,10 @@
         "y": 0
       },
       "hiddenSeries": false,
-      "id": 7,
+      "id": 9,
+      "interval": "",
       "legend": {
+        "alignAsTable": false,
         "avg": false,
         "current": false,
         "hideEmpty": true,
@@ -137,12 +165,13 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null",
       "options": {
         "dataLinks": []
       },
       "percentage": false,
-      "pointradius": 2,
+      "pointradius": 5,
       "points": false,
       "renderer": "flot",
       "seriesOverrides": [],
@@ -151,10 +180,32 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h]) + rate(hvves_messages_processing_time_without_routing_seconds_sum[1h])/rate(hvves_messages_processing_time_without_routing_seconds_count[1h])",
+          "expr": "rate(hvves_messages_latency_without_routing_seconds_sum[1h])/rate(hvves_messages_latency_without_routing_seconds_count[1h])",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
           "interval": "",
+          "intervalFactor": 1,
           "legendFormat": "average",
           "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.90, sum( rate(hvves_messages_latency_without_routing_seconds_bucket[1h]) )by(le) )",
+          "interval": "",
+          "legendFormat": "90th percentile",
+          "refId": "B"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum( rate(hvves_messages_latency_without_routing_seconds_bucket[1h]) )by(le) )",
+          "interval": "",
+          "legendFormat": "95th percentile",
+          "refId": "C"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum( rate(hvves_messages_latency_without_routing_seconds_bucket[1h]) )by(le) )",
+          "interval": "",
+          "legendFormat": "99th precentile",
+          "refId": "D"
         }
       ],
       "thresholds": [],
@@ -177,6 +228,7 @@
       },
       "yaxes": [
         {
+          "decimals": 2,
           "format": "s",
           "label": null,
           "logBase": 1,
@@ -353,19 +405,22 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h])",
+          "expr": "rate(hvves_messages_to_collector_travel_time_seconds_sum[1h])/rate(hvves_messages_to_collector_travel_time_seconds_count[1h])",
+          "hide": false,
           "interval": "",
           "legendFormat": "Producer to HV-VES input",
           "refId": "A"
         },
         {
-          "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h]) + rate(hvves_messages_processing_time_without_routing_seconds_sum[1h])/rate(hvves_messages_processing_time_without_routing_seconds_count[1h])",
+          "expr": "rate(hvves_messages_latency_without_routing_seconds_sum[1h])/rate(hvves_messages_latency_without_routing_seconds_count[1h])",
+          "hide": false,
           "interval": "",
           "legendFormat": "Producer to HV-VES output (without sending to Kafka)",
           "refId": "B"
         },
         {
           "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h])",
+          "hide": false,
           "interval": "",
           "legendFormat": "Producer to HV-VES output (with sending to Kafka)",
           "refId": "C"
@@ -444,5 +499,5 @@
   "variables": {
     "list": []
   },
-  "version": 0
+  "version": 1
 }
diff --git a/tools/performance/cloud/grafana/dashboards/processing.json b/tools/performance/cloud/grafana/dashboards/processing.json
index 7b606e44..f8912578 100644
--- a/tools/performance/cloud/grafana/dashboards/processing.json
+++ b/tools/performance/cloud/grafana/dashboards/processing.json
@@ -660,7 +660,7 @@
         }
       ],
       "thresholds": [],
-      "timeFrom": null,
+      "timeFrom": "20m",
       "timeRegions": [],
       "timeShift": null,
       "title": "Memory usage of HV-VES",