From 5f9b80c7b6a121f065624e5785c98fa6e3c03419 Mon Sep 17 00:00:00 2001 From: Alberto Soutullo Date: Tue, 21 Apr 2026 18:23:10 +0200 Subject: [PATCH] Update dashboard --- .../grafana-dashboards/nomos-consumption.yaml | 734 ++++++++++++++++-- 1 file changed, 669 insertions(+), 65 deletions(-) diff --git a/fleet/grafana-dashboards/nomos-consumption.yaml b/fleet/grafana-dashboards/nomos-consumption.yaml index ea3d876..b375f84 100644 --- a/fleet/grafana-dashboards/nomos-consumption.yaml +++ b/fleet/grafana-dashboards/nomos-consumption.yaml @@ -31,6 +31,617 @@ data: "id": 170, "links": [], "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "panels": [], + "title": "Consensus", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "consensus_forks_count", + "legendFormat": "{{service_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Consensus forks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Peers - node-genesis-0", + "Peers - node-genesis-1", + "Peers - node-genesis-10", + "Peers - node-genesis-11", + "Peers - node-genesis-12", + "Peers - node-genesis-13", + "Peers - node-genesis-14", + "Peers - node-genesis-15", + "Peers - node-genesis-2", + "Peers - node-genesis-3", + "Peers - node-genesis-4", + "Peers - node-genesis-5", + "Peers - node-genesis-6", + "Peers - node-genesis-7", + "Peers - node-genesis-8", + "Peers - node-genesis-9", + "Peers - test-0", + "Peers - test-1", + "Peers - test-10", + "Peers - test-11", + "Peers - test-12", + "Peers - test-13", + "Peers - test-14", + "Peers - test-15", + "Peers - test-2", + "Peers - test-3", + "Peers - test-4", + "Peers - test-5", + "Peers - test-6", + "Peers - test-7", + "Peers - test-8", + "Peers - test-9" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "consensus_peers_connected", + "hide": false, + "legendFormat": "Peers - {{service_name}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "editorMode": "code", + "expr": "consensus_connections", + "hide": true, + "instant": false, + "legendFormat": "Connections - {{service_name}}", + "range": true, + "refId": "B" + } + ], + "title": "Peers connected", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "consensus_tip_height", + "legendFormat": "{{service_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Tip Height", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "consensus_finalized_height", + "legendFormat": "{{service_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Finalized Height (LIB)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 11, + "panels": [], + "title": "Blend", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "msgps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "rate(blend_messages_sent_total[5m])", + "legendFormat": "Sent/sec - {{service_name}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "rate(blend_messages_received_total[5m])", + "legendFormat": "Received/sec - {{service_name}}", + "range": true, + "refId": "B" + } + ], + "title": "Message Flow Rate — events/sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "pending_outbound_messages", + "legendFormat": "Pending Outbound - {{service_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pending Outbound Messages — queue depth", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -96,9 +707,9 @@ data: "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 26 }, - "id": 2, + "id": 6, "options": { "legend": { "calcs": [], @@ -116,13 +727,14 @@ data: "targets": [ { "editorMode": "code", - "expr": "sum by (pod) (\n max by (pod, container) (\n rate(\n container_cpu_usage_seconds_total{\n namespace=~\"$namespace\",\n container!=\"\",\n container!=\"POD\"\n }[5m]\n )\n )\n)", - "legendFormat": "__auto", + "expr": "blend_peers_connected", + "hide": false, + "legendFormat": "{{service_name}}", "range": true, "refId": "A" } ], - "title": "CPU Load (Cores)", + "title": "Blend connections", "type": "timeseries" }, { @@ -144,7 +756,7 @@ data: "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, @@ -183,7 +795,7 @@ data: } ] }, - "unit": "decbytes" + "unit": "msgps" }, "overrides": [] }, @@ -191,9 +803,9 @@ data: "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 26 }, - "id": 1, + "id": 8, "options": { "legend": { "calcs": [], @@ -209,21 +821,43 @@ data: }, "pluginVersion": "12.3.0", "targets": [ + { + "editorMode": "code", + "expr": "rate(blend_inbound_messages_failed_total[5m])", + "legendFormat": "Failed Inb/sec - {{service_name}}", + "range": true, + "refId": "A" + }, { "datasource": { "type": "prometheus", "uid": "VictoriaMetrics" }, "editorMode": "code", - "expr": "sum by (pod) (\n max by (pod, container) (\n container_memory_working_set_bytes{\n namespace=~\"$namespace\",\n container!=\"\",\n container!=\"POD\"\n }\n )\n)", - "legendFormat": "__auto", + "expr": "rate(blend_outbound_messages_failed_total[5m])", + "hide": false, + "instant": false, + "legendFormat": "Failed Out/sec - {{service_name}}", "range": true, - "refId": "A" + "refId": "B" } ], - "title": "Memory working set", + "title": "Failed Message Rate — events/sec (missing data)", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 10, + "panels": [], + "title": "Node HW resources", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -289,9 +923,9 @@ data: "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 35 }, - "id": 3, + "id": 2, "options": { "legend": { "calcs": [], @@ -309,13 +943,13 @@ data: "targets": [ { "editorMode": "code", - "expr": "consensus_forks_count", - "legendFormat": "{{service_name}}", + "expr": "sum by (pod) (\n max by (pod, container) (\n rate(\n container_cpu_usage_seconds_total{\n namespace=~\"$namespace\",\n container!=\"\",\n container!=\"POD\"\n }[5m]\n )\n )\n)", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Consensus forks", + "title": "CPU Load (Cores)", "type": "timeseries" }, { @@ -384,9 +1018,9 @@ data: "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 35 }, - "id": 4, + "id": 1, "options": { "legend": { "calcs": [], @@ -403,14 +1037,18 @@ data: "pluginVersion": "12.3.0", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, "editorMode": "code", - "expr": "sum by (pod) (\n max by (pod, container) (\n container_memory_rss{\n namespace=~\"$namespace\",\n container!=\"\",\n container!=\"POD\"\n }\n )\n)", + "expr": "sum by (pod) (\n max by (pod, container) (\n container_memory_working_set_bytes{\n namespace=~\"$namespace\",\n container!=\"\",\n container!=\"POD\"\n }\n )\n)", "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Memory RSS", + "title": "Memory working set", "type": "timeseries" }, { @@ -479,7 +1117,7 @@ data: "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 43 }, "id": 5, "options": { @@ -565,51 +1203,18 @@ data: "value": 80 } ] - } + }, + "unit": "decbytes" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "node-genesis-0", - "node-genesis-1", - "node-genesis-2", - "node-genesis-3", - "node-genesis-4", - "node-genesis-5", - "node-genesis-6", - "node-genesis-7", - "node-genesis-8", - "node-genesis-9" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": true, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 43 }, - "id": 6, + "id": 4, "options": { "legend": { "calcs": [], @@ -627,14 +1232,13 @@ data: "targets": [ { "editorMode": "code", - "expr": "blend_peers_connected", - "hide": false, - "legendFormat": "{{service_name}}", + "expr": "sum by (pod) (\n max by (pod, container) (\n container_memory_rss{\n namespace=~\"$namespace\",\n container!=\"\",\n container!=\"POD\"\n }\n )\n)", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Blend connections", + "title": "Memory RSS", "type": "timeseries" } ],