summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRemigiusz Janeczek <remigiusz.janeczek@nokia.com>2020-04-23 08:02:47 +0200
committerRemigiusz Janeczek <remigiusz.janeczek@nokia.com>2020-04-23 13:17:18 +0200
commit66254ca1c5df0d7971764799088c8f20c79f4ca7 (patch)
tree0fe0f7bdedc9b90ebf942b0999c40134e49721b7
parent77f5d0a87ae482c222bf2ef61f0b8ff9b3e5ff68 (diff)
Add metric for total latency without routing
Add metric for time between Producer and HV-VES output without sending to Kafka Refactor metrics test Add new latencies and individual cores usage to Grafana Issue-ID: DCAEGEN2-1576 Signed-off-by: Remigiusz Janeczek <remigiusz.janeczek@nokia.com> Change-Id: I6112db76be1c7108c18336b50f9f12d5ce62c24a
-rw-r--r--sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt2
-rw-r--r--sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt8
-rw-r--r--sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt98
-rw-r--r--tools/performance/cloud/grafana/dashboards/k8s-metrics.json1101
-rw-r--r--tools/performance/cloud/grafana/dashboards/latencies.json75
-rw-r--r--tools/performance/cloud/grafana/dashboards/processing.json2
6 files changed, 735 insertions, 551 deletions
diff --git a/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt b/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt
index 12555965..4a5474c6 100644
--- a/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt
+++ b/sources/hv-collector-ct/src/test/kotlin/org/onap/dcae/collectors/veshv/tests/fakes/metrics.kt
@@ -42,6 +42,7 @@ class FakeMetrics : Metrics {
var lastProcessingTimeMicros: Double = -1.0; private set
var lastProcessingTimeWithoutRoutingMicros: Double = -1.0; private set
var lastToCollectorTravelTime: Double = -1.0; private set
+ var lastTotalLatencyWithoutRouting: Double = -1.0; private set
var messagesSentCount: Int = 0; private set
var clientRejectionCause = mutableMapOf<ClientRejectionCause, Int>(); private set
@@ -63,6 +64,7 @@ class FakeMetrics : Metrics {
override fun notifyMessageReadyForRouting(msg: VesMessage) {
lastProcessingTimeWithoutRoutingMicros = Duration.between(msg.wtpFrame.receivedAt, Instant.now()).toNanos() / 1000.0
+ lastTotalLatencyWithoutRouting = Duration.between(TimeUtils.epochMicroToInstant(msg.header.lastEpochMicrosec), Instant.now()).toNanos() / 1000.0
}
override fun notifyMessageSent(msg: RoutedMessage) {
diff --git a/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt b/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt
index e0d99fc6..a949803f 100644
--- a/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt
+++ b/sources/hv-collector-main/src/main/kotlin/org/onap/dcae/collectors/veshv/main/metrics/MicrometerMetrics.kt
@@ -66,6 +66,10 @@ class MicrometerMetrics internal constructor(
.maximumExpectedValue(MAX_BUCKET_DURATION)
.publishPercentileHistogram(true)
.register(registry)
+ private val totalLatencyWithoutRouting = Timer.builder(name(MESSAGES, LATENCY, WITHOUT, ROUTING))
+ .maximumExpectedValue(MAX_BUCKET_DURATION)
+ .publishPercentileHistogram(true)
+ .register(registry)
private val totalLatency = Timer.builder(name(MESSAGES, LATENCY))
.maximumExpectedValue(MAX_BUCKET_DURATION)
.publishPercentileHistogram(true)
@@ -104,7 +108,9 @@ class MicrometerMetrics internal constructor(
}
override fun notifyMessageReadyForRouting(msg: VesMessage) {
- processingTimeWithoutRouting.record(Duration.between(msg.wtpFrame.receivedAt, Instant.now()))
+ val now = Instant.now()
+ processingTimeWithoutRouting.record(Duration.between(msg.wtpFrame.receivedAt, now))
+ totalLatencyWithoutRouting.record(Duration.between(epochMicroToInstant(msg.header.lastEpochMicrosec), now))
}
override fun notifyMessageReceived(msg: WireFrameMessage) {
diff --git a/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt b/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt
index efd353ec..dd206d08 100644
--- a/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt
+++ b/sources/hv-collector-main/src/test/kotlin/org/onap/dcae/collectors/veshv/main/MicrometerMetricsTest.kt
@@ -146,8 +146,66 @@ object MicrometerMetricsTest : Spek({
"$PREFIX.messages.received.payload.bytes"
)
}
+
+ on("$PREFIX.messages.to.collector.travel.time") {
+ val counterName = "$PREFIX.messages.to.collector.travel.time"
+ val toCollectorTravelTimeMs = 100L
+
+ it("should update timer") {
+ val now = Instant.now()
+ val vesMessage = vesMessageReceivedAt(now, sentAt = now.minusMillis(toCollectorTravelTimeMs))
+ cut.notifyMessageReceived(vesMessage)
+
+ registry.verifyTimer(counterName) { timer ->
+ assertThat(timer.mean(TimeUnit.MILLISECONDS)).isEqualTo(toCollectorTravelTimeMs.toDouble())
+ }
+
+ verifyCountersAndTimersAreUnchangedBut(counterName)
+ }
+ }
}
+ describe("notifyMessageReadyForRouting"){
+ on("$PREFIX.messages.processing.time.without.routing") {
+ val counterName = "$PREFIX.messages.processing.time.without.routing"
+ val processingTimeMs = 100L
+
+ it("should update timer") {
+
+ cut.notifyMessageReadyForRouting(vesMessageReceivedAt(Instant.now().minusMillis(processingTimeMs)))
+
+ registry.verifyTimer(counterName) { timer ->
+ assertThat(timer.mean(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(processingTimeMs.toDouble())
+ }
+ verifyCountersAndTimersAreUnchangedBut(
+ counterName,
+ "$PREFIX.messages.latency.without.routing"
+ )
+ }
+ }
+
+ on("$PREFIX.messages.latency.without.routing") {
+ val counterName = "$PREFIX.messages.latency.without.routing"
+ val latencyWithoutRoutingMs = 200L
+
+ it("should update timer") {
+
+ val sentAt = Instant.now().minusMillis(latencyWithoutRoutingMs)
+
+ cut.notifyMessageReadyForRouting(vesMessageSentAt(sentAt))
+
+ registry.verifyTimer(counterName) { timer ->
+ assertThat(timer.mean(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(latencyWithoutRoutingMs.toDouble())
+ }
+ verifyCountersAndTimersAreUnchangedBut(
+ counterName,
+ "$PREFIX.messages.processing.time.without.routing"
+ )
+ }
+ }
+ }
+
+
describe("notifyMessageSent") {
val topicName1 = "PERF3GPP"
val topicName2 = "CALLTRACE"
@@ -206,39 +264,6 @@ object MicrometerMetricsTest : Spek({
}
}
- on("$PREFIX.messages.to.collector.travel.time") {
- val counterName = "$PREFIX.messages.to.collector.travel.time"
- val toCollectorTravelTimeMs = 100L
-
- it("should update timer") {
- val now = Instant.now()
- val vesMessage = vesMessageReceivedAt(now, sentAt = now.minusMillis(toCollectorTravelTimeMs))
- cut.notifyMessageReceived(vesMessage)
-
- registry.verifyTimer(counterName) { timer ->
- assertThat(timer.mean(TimeUnit.MILLISECONDS)).isEqualTo(toCollectorTravelTimeMs.toDouble())
- }
-
- verifyCountersAndTimersAreUnchangedBut(counterName)
- }
- }
-
- on("$PREFIX.messages.processing.time.without.routing") {
- val counterName = "$PREFIX.messages.processing.time.without.routing"
- val processingTimeMs = 100L
-
- it("should update timer") {
-
- cut.notifyMessageReadyForRouting(vesMessageReceivedAt(Instant.now().minusMillis(processingTimeMs)))
-
- registry.verifyTimer(counterName) { timer ->
- assertThat(timer.mean(TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(processingTimeMs.toDouble())
- }
-
- verifyCountersAndTimersAreUnchangedBut(counterName)
- }
- }
-
on("$PREFIX.messages.latency") {
val counterName = "$PREFIX.messages.latency"
val latencyMs = 1666L
@@ -398,6 +423,13 @@ object MicrometerMetricsTest : Spek({
}
})
+private fun vesMessageSentAt(sentAt: Instant): VesMessage {
+ val lastEpochMicrosec = sentAt.epochSecond * 1000000 + sentAt.nano / 1000
+ val commonHeader = commonHeader(lastEpochMicrosec = lastEpochMicrosec)
+ return VesMessage(commonHeader,
+ wireProtocolFrame(commonHeader, ByteString.copyFromUtf8("highvolume measurements")))
+}
+
private fun vesMessageReceivedAt(receivedAt: Instant, sentAt: Instant): VesMessage {
val lastEpochMicrosec = sentAt.epochSecond * 1000000 + sentAt.nano / 1000
val commonHeader = commonHeader(lastEpochMicrosec = lastEpochMicrosec)
diff --git a/tools/performance/cloud/grafana/dashboards/k8s-metrics.json b/tools/performance/cloud/grafana/dashboards/k8s-metrics.json
index 676d0a93..7841f77d 100644
--- a/tools/performance/cloud/grafana/dashboards/k8s-metrics.json
+++ b/tools/performance/cloud/grafana/dashboards/k8s-metrics.json
@@ -1,530 +1,619 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": 2,
+ "links": [],
+ "panels": [
{
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": "-- Grafana --",
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard"
- }
- ]
+ "datasource": null,
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 0,
+ "y": 0
},
- "editable": true,
- "gnetId": null,
- "graphTooltip": 0,
- "id": 5,
- "links": [],
- "panels": [
- {
- "datasource": null,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 0,
- "y": 0
- },
- "id": 8,
- "options": {
- "fieldOptions": {
- "calcs": [
- "mean"
- ],
- "defaults": {
- "mappings": [],
- "max": 100,
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
+ "id": 6,
+ "options": {
+ "fieldOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "defaults": {
+ "mappings": [],
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
},
- "unit": "percent"
- },
- "overrides": [],
- "values": false
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
},
- "orientation": "auto",
- "showThresholdLabels": false,
- "showThresholdMarkers": true
+ "unit": "percent"
},
- "pluginVersion": "6.7.2",
- "targets": [
- {
- "expr": "sum(container_memory_working_set_bytes{id=\"/\", instance=~\".*worker.*\"}) by(instance) / (sum(machine_memory_bytes{instance=~\".*worker.*\"}) by (instance))*100",
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Nodes RAM Usage",
- "type": "gauge"
+ "overrides": [],
+ "values": false
},
+ "orientation": "auto",
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "6.7.2",
+ "targets": [
{
- "datasource": null,
- "gridPos": {
- "h": 7,
- "w": 8,
- "x": 8,
- "y": 0
- },
- "id": 6,
- "options": {
- "fieldOptions": {
- "calcs": [
- "mean"
- ],
- "defaults": {
- "mappings": [],
- "max": 100,
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "percent"
- },
- "overrides": [],
- "values": false
- },
- "orientation": "auto",
- "showThresholdLabels": false,
- "showThresholdMarkers": true
- },
- "pluginVersion": "6.7.2",
- "targets": [
- {
- "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\", instance=~\".*worker.*\"}[1m])) by(instance) / (sum(machine_cpu_cores{instance=~\".*worker.*\"}) by (instance))*100",
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Nodes CPU Usage",
- "type": "gauge"
+ "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\", instance=~\".*worker.*\"}[1m])) by(instance) / (sum(machine_cpu_cores{instance=~\".*worker.*\"}) by (instance))*100",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Nodes CPU Usage",
+ "type": "gauge"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 12,
+ "w": 8,
+ "x": 8,
+ "y": 0
+ },
+ "hiddenSeries": false,
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\"}[1m])) by(instance) / (sum(machine_cpu_cores) by (instance))*100",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": "20m",
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Nodes Total CPU usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percent",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 12,
- "w": 8,
- "x": 16,
- "y": 0
- },
- "hiddenSeries": false,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
- "percentage": false,
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)",
- "hide": false,
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "C"
- },
- {
- "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))",
- "hide": false,
- "interval": "",
- "legendFormat": "Total Receive",
- "refId": "D"
- }
- ],
- "thresholds": [],
- "timeFrom": "20m",
- "timeRegions": [],
- "timeShift": null,
- "title": "Network Usage - Receive",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 12,
+ "w": 8,
+ "x": 16,
+ "y": 0
+ },
+ "hiddenSeries": false,
+ "id": 10,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "C"
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 16,
- "w": 8,
- "x": 0,
- "y": 7
- },
- "hiddenSeries": false,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
- "percentage": false,
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(container_memory_working_set_bytes{id=\"/\"}) by (instance) / (sum(machine_memory_bytes) by (instance))*100",
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
- ],
- "thresholds": [],
- "timeFrom": "20m",
- "timeRegions": [],
- "timeShift": null,
- "title": "Nodes Total RAM Usage",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percent",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "expr": "sum(irate(container_network_receive_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Total Receive",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": "20m",
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Network Usage - Receive",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "Bps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
},
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 16,
- "w": 8,
- "x": 8,
- "y": 7
- },
- "hiddenSeries": false,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
- "percentage": false,
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(container_cpu_usage_seconds_total{id=\"/\"}[1m])) by(instance) / (sum(machine_cpu_cores) by (instance))*100",
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- }
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "datasource": null,
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 0,
+ "y": 5
+ },
+ "id": 8,
+ "options": {
+ "fieldOptions": {
+ "calcs": [
+ "mean"
],
- "thresholds": [],
- "timeFrom": "20m",
- "timeRegions": [],
- "timeShift": null,
- "title": "Nodes Total CPU usage",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "percent",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
+ "defaults": {
+ "mappings": [],
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
},
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "unit": "percent"
+ },
+ "overrides": [],
+ "values": false
},
+ "orientation": "auto",
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "6.7.2",
+ "targets": [
{
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
- "fill": 0,
- "fillGradient": 0,
- "gridPos": {
- "h": 11,
- "w": 8,
- "x": 16,
- "y": 12
- },
- "hiddenSeries": false,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
- "percentage": false,
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)",
- "hide": false,
- "interval": "",
- "legendFormat": "{{instance}}",
- "refId": "A"
- },
- {
- "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))",
- "hide": false,
- "interval": "",
- "legendFormat": "Total Transmit",
- "refId": "B"
- }
- ],
- "thresholds": [],
- "timeFrom": "20m",
- "timeRegions": [],
- "timeShift": null,
- "title": "Network Usage - Transmit",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
+ "expr": "sum(container_memory_working_set_bytes{id=\"/\", instance=~\".*worker.*\"}) by(instance) / (sum(machine_memory_bytes{instance=~\".*worker.*\"}) by (instance))*100",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Nodes RAM Usage",
+ "type": "gauge"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 13,
+ "w": 8,
+ "x": 0,
+ "y": 10
+ },
+ "hiddenSeries": false,
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(container_memory_working_set_bytes{id=\"/\"}) by (instance) / (sum(machine_memory_bytes) by (instance))*100",
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": "20m",
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Nodes Total RAM Usage",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percent",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 11,
+ "w": 8,
+ "x": 8,
+ "y": 12
+ },
+ "hiddenSeries": false,
+ "id": 12,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": true,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "100- sum(rate(node_cpu_seconds_total{mode=\"idle\"}[1m])) by (node_name, cpu) * 100",
+ "interval": "",
+ "legendFormat": "Instance: {{node_name}} cpu: {{cpu}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": "20m",
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Usage of each core",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percent",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
}
],
- "refresh": "5s",
- "schemaVersion": 22,
- "style": "dark",
- "tags": [],
- "templating": {
- "list": []
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 0,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 11,
+ "w": 8,
+ "x": 16,
+ "y": 12
+ },
+ "hiddenSeries": false,
+ "id": 11,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
},
- "time": {
- "from": "now-6h",
- "to": "now"
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
},
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ]
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m])) by (instance)",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ },
+ {
+ "expr": "sum(irate(container_network_transmit_bytes_total{pod!=\"\", namespace=\"onap\", instance=~\".*worker.*\"}[1m]))",
+ "hide": false,
+ "interval": "",
+ "legendFormat": "Total Transmit",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": "20m",
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Network Usage - Transmit",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
},
- "timezone": "",
- "title": "K8s metrics",
- "uid": "RxjnWpjZk",
- "variables": {
- "list": []
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
},
- "version": 1
+ "yaxes": [
+ {
+ "format": "Bps",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
}
+ ],
+ "refresh": "5s",
+ "schemaVersion": 22,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "",
+ "title": "K8s metrics",
+ "uid": "RxjnWpjZk",
+ "variables": {
+ "list": []
+ },
+ "version": 1
+}
diff --git a/tools/performance/cloud/grafana/dashboards/latencies.json b/tools/performance/cloud/grafana/dashboards/latencies.json
index 43ec7f4f..9efe8485 100644
--- a/tools/performance/cloud/grafana/dashboards/latencies.json
+++ b/tools/performance/cloud/grafana/dashboards/latencies.json
@@ -15,7 +15,7 @@
"editable": true,
"gnetId": null,
"graphTooltip": 0,
- "id": 6,
+ "id": 5,
"links": [],
"panels": [
{
@@ -33,8 +33,10 @@
"y": 0
},
"hiddenSeries": false,
- "id": 2,
+ "id": 8,
+ "interval": "",
"legend": {
+ "alignAsTable": false,
"avg": false,
"current": false,
"hideEmpty": true,
@@ -47,12 +49,13 @@
},
"lines": true,
"linewidth": 1,
+ "links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
- "pointradius": 2,
+ "pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
@@ -61,10 +64,32 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h])",
+ "expr": "rate(hvves_messages_to_collector_travel_time_seconds_sum[1h])/rate(hvves_messages_to_collector_travel_time_seconds_count[1h])",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
"interval": "",
+ "intervalFactor": 1,
"legendFormat": "average",
"refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.90, sum( rate(hvves_messages_to_collector_travel_time_seconds_bucket[1h]) )by(le) )",
+ "interval": "",
+ "legendFormat": "90th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.95, sum( rate(hvves_messages_to_collector_travel_time_seconds_bucket[1h]) )by(le) )",
+ "interval": "",
+ "legendFormat": "95th percentile",
+ "refId": "C"
+ },
+ {
+ "expr": "histogram_quantile(0.99, sum( rate(hvves_messages_to_collector_travel_time_seconds_bucket[1h]) )by(le) )",
+ "interval": "",
+ "legendFormat": "99th precentile",
+ "refId": "D"
}
],
"thresholds": [],
@@ -87,6 +112,7 @@
},
"yaxes": [
{
+ "decimals": 2,
"format": "s",
"label": null,
"logBase": 1,
@@ -123,8 +149,10 @@
"y": 0
},
"hiddenSeries": false,
- "id": 7,
+ "id": 9,
+ "interval": "",
"legend": {
+ "alignAsTable": false,
"avg": false,
"current": false,
"hideEmpty": true,
@@ -137,12 +165,13 @@
},
"lines": true,
"linewidth": 1,
+ "links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
- "pointradius": 2,
+ "pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
@@ -151,10 +180,32 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h]) + rate(hvves_messages_processing_time_without_routing_seconds_sum[1h])/rate(hvves_messages_processing_time_without_routing_seconds_count[1h])",
+ "expr": "rate(hvves_messages_latency_without_routing_seconds_sum[1h])/rate(hvves_messages_latency_without_routing_seconds_count[1h])",
+ "format": "time_series",
+ "hide": false,
+ "instant": false,
"interval": "",
+ "intervalFactor": 1,
"legendFormat": "average",
"refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.90, sum( rate(hvves_messages_latency_without_routing_seconds_bucket[1h]) )by(le) )",
+ "interval": "",
+ "legendFormat": "90th percentile",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.95, sum( rate(hvves_messages_latency_without_routing_seconds_bucket[1h]) )by(le) )",
+ "interval": "",
+ "legendFormat": "95th percentile",
+ "refId": "C"
+ },
+ {
+ "expr": "histogram_quantile(0.99, sum( rate(hvves_messages_latency_without_routing_seconds_bucket[1h]) )by(le) )",
+ "interval": "",
+ "legendFormat": "99th precentile",
+ "refId": "D"
}
],
"thresholds": [],
@@ -177,6 +228,7 @@
},
"yaxes": [
{
+ "decimals": 2,
"format": "s",
"label": null,
"logBase": 1,
@@ -353,19 +405,22 @@
"steppedLine": false,
"targets": [
{
- "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h])",
+ "expr": "rate(hvves_messages_to_collector_travel_time_seconds_sum[1h])/rate(hvves_messages_to_collector_travel_time_seconds_count[1h])",
+ "hide": false,
"interval": "",
"legendFormat": "Producer to HV-VES input",
"refId": "A"
},
{
- "expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h]) - rate(hvves_messages_processing_time_seconds_sum[1h])/rate(hvves_messages_processing_time_seconds_count[1h]) + rate(hvves_messages_processing_time_without_routing_seconds_sum[1h])/rate(hvves_messages_processing_time_without_routing_seconds_count[1h])",
+ "expr": "rate(hvves_messages_latency_without_routing_seconds_sum[1h])/rate(hvves_messages_latency_without_routing_seconds_count[1h])",
+ "hide": false,
"interval": "",
"legendFormat": "Producer to HV-VES output (without sending to Kafka)",
"refId": "B"
},
{
"expr": "rate(hvves_messages_latency_seconds_sum[1h])/rate(hvves_messages_latency_seconds_count[1h])",
+ "hide": false,
"interval": "",
"legendFormat": "Producer to HV-VES output (with sending to Kafka)",
"refId": "C"
@@ -444,5 +499,5 @@
"variables": {
"list": []
},
- "version": 0
+ "version": 1
}
diff --git a/tools/performance/cloud/grafana/dashboards/processing.json b/tools/performance/cloud/grafana/dashboards/processing.json
index 7b606e44..f8912578 100644
--- a/tools/performance/cloud/grafana/dashboards/processing.json
+++ b/tools/performance/cloud/grafana/dashboards/processing.json
@@ -660,7 +660,7 @@
}
],
"thresholds": [],
- "timeFrom": null,
+ "timeFrom": "20m",
"timeRegions": [],
"timeShift": null,
"title": "Memory usage of HV-VES",