From bd505f8e2d37172ff35a89f4ac42efec9566a263 Mon Sep 17 00:00:00 2001 From: Joao Marcal Date: Mon, 27 Nov 2023 17:24:23 +0100 Subject: [PATCH] mixins: Adds structured metadata panels (#11087) Co-authored-by: Periklis Tsirakidis --- CHANGELOG.md | 2 + .../dashboards/loki-writes.json | 168 +++++++++++++++- .../dashboards/loki-writes.json | 180 +++++++++++++++++- production/loki-mixin/config.libsonnet | 3 + .../dashboards/loki-writes.libsonnet | 18 ++ 5 files changed, 361 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13136a5b41cb..ae8d9bbb162f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,8 @@ #### Mixins +* [11087](https://github.com/grafana/loki/pull/11087) **JoaoBraveCoding**: Adds structured metadata panels for ingested data + #### Fixes #### FluentD diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-writes.json b/production/loki-mixin-compiled-ssd/dashboards/loki-writes.json index 17d54cf81f78..bcd620e69e4a 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-writes.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-writes.json @@ -210,6 +210,170 @@ "title": "Write Path", "titleSize": "h6" }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval])) / sum(rate(loki_distributor_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Per Total Received Bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (tenant) (rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval])) / ignoring(tenant) group_left sum(rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{tenant}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Per Tenant", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Write Path", + "titleSize": "h6" + }, { "collapse": false, "height": "250px", @@ -229,7 +393,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 3, + "id": 5, "legend": { "avg": false, "current": false, @@ -305,7 +469,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 4, + "id": 6, "legend": { "avg": false, "current": false, diff --git a/production/loki-mixin-compiled/dashboards/loki-writes.json b/production/loki-mixin-compiled/dashboards/loki-writes.json index 3946e7897c49..fdb347f56055 100644 --- a/production/loki-mixin-compiled/dashboards/loki-writes.json +++ b/production/loki-mixin-compiled/dashboards/loki-writes.json @@ -210,6 +210,170 @@ "title": "Distributor", "titleSize": "h6" }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/distributor\",}[$__rate_interval])) / sum(rate(loki_distributor_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/distributor\",}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "bytes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Per Total Received Bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (tenant) (rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/distributor\",}[$__rate_interval])) / ignoring(tenant) group_left sum(rate(loki_distributor_structured_metadata_bytes_received_total{cluster=~\"$cluster\",job=~\"($namespace)/distributor\",}[$__rate_interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{tenant}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Per Tenant", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": 1, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Distributor - Structured Metadata", + "titleSize": "h6" + }, { "collapse": false, "height": "250px", @@ -229,7 +393,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 3, + "id": 5, "legend": { "avg": false, "current": false, @@ -305,7 +469,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 4, + "id": 6, "legend": { "avg": false, "current": false, @@ -417,7 +581,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 5, + "id": 7, "legend": { "avg": false, "current": false, @@ -493,7 +657,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 6, + "id": 8, "legend": { "avg": false, "current": false, @@ -605,7 +769,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 7, + "id": 9, "legend": { "avg": false, "current": false, @@ -681,7 +845,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 8, + "id": 10, "legend": { "avg": false, "current": false, @@ -793,7 +957,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 9, + "id": 11, "legend": { "avg": false, "current": false, @@ -869,7 +1033,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 10, + "id": 12, "legend": { "avg": false, "current": false, diff --git a/production/loki-mixin/config.libsonnet b/production/loki-mixin/config.libsonnet index e0b09677ea45..1fa22f566cc6 100644 --- a/production/loki-mixin/config.libsonnet +++ b/production/loki-mixin/config.libsonnet @@ -20,6 +20,9 @@ enabled: true, }, + // Enable TSDB specific dashboards + tsdb: true, + // SSD related configuration for dashboards. ssd: { // Support Loki SSD mode on dashboards. diff --git a/production/loki-mixin/dashboards/loki-writes.libsonnet b/production/loki-mixin/dashboards/loki-writes.libsonnet index df710bb0700e..a12f4f7cea6e 100644 --- a/production/loki-mixin/dashboards/loki-writes.libsonnet +++ b/production/loki-mixin/dashboards/loki-writes.libsonnet @@ -69,6 +69,24 @@ local utils = import 'mixin-utils/utils.libsonnet'; ) ) ) + .addRowIf( + $._config.tsdb, + $.row(if $._config.ssd.enabled then 'Write Path' else 'Distributor - Structured Metadata') + .addPanel( + $.panel('Per Total Received Bytes') + + $.queryPanel('sum (rate(loki_distributor_structured_metadata_bytes_received_total{%s}[$__rate_interval])) / sum(rate(loki_distributor_bytes_received_total{%s}[$__rate_interval]))' % [dashboards['loki-writes.json'].distributorSelector, dashboards['loki-writes.json'].distributorSelector], 'bytes') + ) + .addPanel( + $.panel('Per Tenant') + + $.queryPanel('sum by (tenant) (rate(loki_distributor_structured_metadata_bytes_received_total{%s}[$__rate_interval])) / ignoring(tenant) group_left sum(rate(loki_distributor_structured_metadata_bytes_received_total{%s}[$__rate_interval]))' % [dashboards['loki-writes.json'].distributorSelector, dashboards['loki-writes.json'].distributorSelector], '{{tenant}}') + { + stack: true, + yaxes: [ + { format: 'short', label: null, logBase: 1, max: 1, min: 0, show: true }, + { format: 'short', label: null, logBase: 1, max: 1, min: null, show: false }, + ], + }, + ) + ) .addRowIf( !$._config.ssd.enabled, $.row('Ingester - Zone Aware')