From 8838f2044f45451e2c7a86d374c2b3d39714405b Mon Sep 17 00:00:00 2001 From: Remigiusz Janeczek Date: Tue, 21 Apr 2020 15:40:52 +0200 Subject: Add script for loading prometheus snapshots, improve grafana 1. Add scripts for loading prometheus snapshots locally 2. Improve Grafana dashboards for performance tests 3. Improve Grafana dashboards management (now there's one configmap for all dashboards and to add new dashboard only needed change is adding new json file in grafana/dashboards directory) 4. Add prometheus job for scraping node metrics and add deployment yaml for node exporter Issue-ID: DCAEGEN2-1576 Signed-off-by: Remigiusz Janeczek Change-Id: I8870fd3291711b5dc09b76a7bf86ac4d13725237 --- .../grafana/dashboards/kafka-and-producers.yaml | 774 --------------------- 1 file changed, 774 deletions(-) delete mode 100644 tools/performance/cloud/grafana/dashboards/kafka-and-producers.yaml (limited to 'tools/performance/cloud/grafana/dashboards/kafka-and-producers.yaml') diff --git a/tools/performance/cloud/grafana/dashboards/kafka-and-producers.yaml b/tools/performance/cloud/grafana/dashboards/kafka-and-producers.yaml deleted file mode 100644 index 74cad82d..00000000 --- a/tools/performance/cloud/grafana/dashboards/kafka-and-producers.yaml +++ /dev/null @@ -1,774 +0,0 @@ -# ============LICENSE_START======================================================= -# dcaegen2-collectors-veshv -# ================================================================================ -# Copyright (C) 2020 NOKIA -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END========================================================= - -apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-kafka-and-producers - namespace: onap - labels: - name: grafana-dashboards -data: - kafka-and-producers.json: |- - { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 11, - "w": 8, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(container_network_transmit_bytes_total{pod=~\".*producer.*\"}[1m]))", - "interval": "", - "legendFormat": "Total Transmit Producers", - "refId": "A" - }, - { - "expr": "sum(irate(container_network_transmit_bytes_total{pod=~\".*kafka-.\"}[1m]))", - "interval": "", - "legendFormat": "Total Transmit Kafka", - "refId": "B" - }, - { - "expr": "irate(container_network_transmit_bytes_total{pod=~\".*producer.*\"}[1m])", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "C" - }, - { - "expr": "irate(container_network_transmit_bytes_total{pod=~\".*kafka-.\"}[1m])", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": "20m", - "timeRegions": [], - "timeShift": null, - "title": "Network Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 11, - "w": 8, - "x": 8, - "y": 0 - }, - "hiddenSeries": false, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_network_transmit_bytes_total{pod=~\".*kafka-.|.*hv-ves-coll.*|.*producer.*\"}[1m]))", - "hide": false, - "interval": "", - "legendFormat": "Total transmit", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": "20m", - "timeRegions": [], - "timeShift": null, - "title": "Total network usage (producers + hv-ves + kafka)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 11, - "w": 8, - "x": 16, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(container_network_receive_bytes_total{pod=~\"dep-dcae-hv-ves-collector.*\"}[1m])", - "hide": false, - "interval": "", - "legendFormat": "receive", - "refId": "A" - }, - { - "expr": "irate(container_network_transmit_bytes_total{pod=~\"dep-dcae-hv-ves-collector.*\"}[1m])", - "interval": "", - "legendFormat": "transmit", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": "20m", - "timeRegions": [], - "timeShift": null, - "title": "HV-VES Network Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "Bps", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 12, - "w": 8, - "x": 0, - "y": 11 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum (rate (container_cpu_usage_seconds_total{pod=~\".*producer.*\"}[1m])) by (pod)", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - }, - { - "expr": "sum (rate (container_cpu_usage_seconds_total{pod=~\".*router-kafka.*\"}[1m])) by (pod)", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": "20m", - "timeRegions": [], - "timeShift": null, - "title": "CPU usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 12, - "w": 8, - "x": 8, - "y": 11 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_working_set_bytes{image!=\"\", pod=~\".*kafka-.\"}) by (pod)", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - }, - { - "expr": "sum (rate (container_cpu_usage_seconds_total{pod=~\".*router-kafka.*\"}[1m])) by (pod)", - "hide": true, - "interval": "", - "legendFormat": "{{pod}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": "20m", - "timeRegions": [], - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": null, - "gridPos": { - "h": 6, - "w": 4, - "x": 16, - "y": 11 - }, - "id": 15, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ], - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [], - "values": false - }, - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto" - }, - "pluginVersion": "6.7.2", - "targets": [ - { - "expr": "sum(container_network_transmit_packets_dropped_total{pod=~\".*producer.*\"})", - "interval": "", - "legendFormat": "Packets dropped", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Packets Dropped (producers)", - "type": "stat" - }, - { - "datasource": null, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 11 - }, - "id": 4, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ], - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [], - "values": false - }, - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto" - }, - "pluginVersion": "6.7.2", - "targets": [ - { - "expr": "sum(container_network_transmit_packets_dropped_total{pod=~\".*hv-ves-collector.*\"})", - "interval": "", - "legendFormat": "Packets dropped", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Packets Dropped (hv-ves)", - "type": "stat" - }, - { - "datasource": null, - "gridPos": { - "h": 6, - "w": 4, - "x": 16, - "y": 17 - }, - "id": 17, - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "max" - ], - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [], - "values": false - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto" - }, - "pluginVersion": "6.7.2", - "targets": [ - { - "expr": "sum(container_network_transmit_bytes_total{pod=~\".*producer.*\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Sent (producers)", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": null, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 17 - }, - "id": 16, - "links": [], - "options": { - "colorMode": "value", - "fieldOptions": { - "calcs": [ - "max" - ], - "defaults": { - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [], - "values": false - }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal" - }, - "pluginVersion": "6.7.2", - "targets": [ - { - "expr": "container_network_receive_bytes_total{pod=~\".*hv-ves-collector.*\"}", - "interval": "", - "legendFormat": "Packets dropped", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Received (hv-ves)", - "type": "stat" - } - ], - "refresh": "5s", - "schemaVersion": 22, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Kafka and Producers Statistics", - "uid": "-28uHcjZk", - "variables": { - "list": [] - }, - "version": 1 - } -- cgit 1.2.3-korg