From 149b69c143772cbd639ada4b92fcd5a3132633cd Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Fri, 18 Oct 2024 14:03:31 +0200 Subject: [PATCH] chore: Run `make loki-mixin` to export to JSON Signed-off-by: Christian Haudum --- ...m-compactor.json => loki-bloom-build.json} | 3735 +++++++++++------ .../dashboards/loki-bloom-gateway.json | 2705 ++++++++---- ...m-compactor.json => loki-bloom-build.json} | 3735 +++++++++++------ .../dashboards/loki-bloom-gateway.json | 2705 ++++++++---- 4 files changed, 8542 insertions(+), 4338 deletions(-) rename production/loki-mixin-compiled-ssd/dashboards/{loki-bloom-compactor.json => loki-bloom-build.json} (71%) rename production/loki-mixin-compiled/dashboards/{loki-bloom-compactor.json => loki-bloom-build.json} (71%) diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-compactor.json b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-build.json similarity index 71% rename from production/loki-mixin-compiled-ssd/dashboards/loki-bloom-compactor.json rename to production/loki-mixin-compiled-ssd/dashboards/loki-bloom-build.json index c365fab0a7e5..bf43f94a94aa 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-compactor.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-build.json @@ -36,29 +36,6 @@ "title": "Overview", "type": "row" }, - { - "gridPos": { - "h": 8, - "w": 14, - "x": 0, - "y": 1 - }, - "id": 35, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "## About the Bloom Compactor\nThe compactor iterates through chunks and creates blooms out of them.\nThe size of the resulting blooms depends on the bloom filter settings, the tokenizer settings, the number of ring tokens per compactor and the total number opf compactors.\n\nCompactors are horizontally scalable and uses a ring to:\n- Shard tenants\n- Shard series fingerprints within a tenant subring.\n\nThe blooms for the series are grouped together in blocks which are flushed to object store.", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "transparent": true, - "type": "text" - }, { "datasource": { "type": "prometheus", @@ -77,6 +54,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "none", @@ -121,13 +99,30 @@ }, "unit": "percentunit" }, - "overrides": [ ] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(Planned|success|failure)/" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] }, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 9 + "y": 1 }, "id": 42, "options": { @@ -144,7 +139,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -152,41 +147,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"})\n/\nsum(count(loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}))", - "hide": false, - "instant": false, - "legendFormat": "avg", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.9, \n sum by (pod) (\n loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p90", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.1, \n sum by (pod) (\n loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n )\n)", + "expr": "sum(loki_bloomplanner_tenant_tasks_completed{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})\n/\nsum(loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})", "hide": false, "instant": false, - "legendFormat": "p10", + "legendFormat": "Progress", "range": true, - "refId": "C" + "refId": "D" } ], - "title": "Progress", + "title": "Overall progress", "type": "timeseries" }, { @@ -194,7 +163,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Uncompressed size of chunks in a series VS the size of the blooms built.", + "description": "Cell-wide compaction progress. Should increase till completion throughout each compaction period.", "fieldConfig": { "defaults": { "color": { @@ -207,8 +176,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 15, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -242,33 +212,29 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] }, - "unit": "bytes" + "unit": "percentunit" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "Ratio" + "id": "byRegexp", + "options": "/(Planned|success|failure)/" }, "properties": [ { "id": "unit", - "value": "percentunit" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Ratio over range" - }, - "properties": [ + "value": "none" + }, { - "id": "unit", - "value": "percentunit" + "id": "custom.fillOpacity", + "value": 0 } ] } @@ -278,9 +244,9 @@ "h": 7, "w": 12, "x": 12, - "y": 9 + "y": 1 }, - "id": 41, + "id": 116, "options": { "legend": { "calcs": [ ], @@ -295,7 +261,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -303,54 +269,158 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by (tenant) (loki_bloomplanner_tenant_tasks_completed{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})\n/\nsum by (tenant) (loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})", "hide": false, "instant": false, - "legendFormat": "Bloom", + "legendFormat": "{{tenant}}", "range": true, - "refId": "A" - }, + "refId": "D" + } + ], + "title": "Progress by tenant", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "description": "Blooms size vs uncompressed chunk size.", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 17, + "x": 0, + "y": 8 + }, + "id": 51, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "type": "loki", + "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "Chunk", - "range": true, + "expr": "{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\"", + "queryType": "range", "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + } + ], + "title": "Errors Planner", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" }, - "editorMode": "code", - "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "Ratio", - "range": true, - "refId": "C" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 17, + "y": 8 + }, + "id": 53, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "type": "loki", + "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "Ratio over range", - "range": true, - "refId": "D" + "expr": "sum(count_over_time({cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\" [$__auto]))", + "legendFormat": "Error rate", + "queryType": "range", + "refId": "A" } ], - "title": "Chunks and Bloom size", + "title": "Errors Rate Planner", "type": "timeseries" }, { @@ -359,13 +429,17 @@ "uid": "${loki_datasource}" }, "description": "Blooms size vs uncompressed chunk size.", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, "gridPos": { "h": 7, "w": 17, "x": 0, - "y": 16 + "y": 15 }, - "id": 51, + "id": 133, "options": { "dedupStrategy": "none", "enableLogDetails": true, @@ -377,7 +451,7 @@ "wrapLogMessage": false }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -385,12 +459,12 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} |= \"level=error\" |= \"component=bloom-compactor\"", + "expr": "{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\"", "queryType": "range", "refId": "B" } ], - "title": "Errors", + "title": "Errors Builder", "type": "logs" }, { @@ -411,6 +485,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "bars", "fillOpacity": 0, "gradientMode": "none", @@ -457,9 +532,9 @@ "h": 7, "w": 7, "x": 17, - "y": 16 + "y": 15 }, - "id": 53, + "id": 134, "options": { "legend": { "calcs": [ ], @@ -474,7 +549,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -482,12 +557,13 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum(count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} |= \"level=error\" |= \"component=bloom-compactor\" [$__auto]))", + "expr": "sum(count_over_time({cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\" [$__auto]))", + "legendFormat": "Error rate", "queryType": "range", "refId": "A" } ], - "title": "Errors Rate", + "title": "Errors Rate Builder", "type": "timeseries" }, { @@ -496,7 +572,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 22 }, "id": 112, "panels": [ @@ -517,9 +593,11 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 15, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -551,7 +629,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -559,47 +638,1093 @@ } ] }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 114, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "none" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (pod) (\n loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n)", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(success|failure)/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 100 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Planned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 15 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Queued" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 125, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Planned", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Completed - {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "loki_bloomplanner_inflight_tasks{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", quantile=\"0.95\"}", + "hide": false, + "instant": false, + "legendFormat": "inflight p95", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_queue_length{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Queued", + "range": true, + "refId": "D" + } + ], + "title": "Tasks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(success|failure)/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 100 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Planned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 15 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Connected builders" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "IDLE Builders" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Builders processing task" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 126, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_connected_builders{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "hide": false, + "instant": false, + "legendFormat": "Connected builders", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(loki_bloombuilder_processing_task{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Builders processing task", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(loki_bloombuilder_processing_task{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} == 0)", + "hide": false, + "instant": false, + "legendFormat": "IDLE Builders", + "range": true, + "refId": "D" + } + ], + "title": "Tasks per builder", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 81, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.75, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p75", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + } + ], + "title": "Series per task (includes series copied from other blocks)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 91, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + } + ], + "title": "Number of bytes from chunks added to blocks during each compaction.", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 117, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Identify the tenant using the **_Progress by tenant_** panel from the overview and set tenant variable", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "Tip", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(success|failure)/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 100 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Planned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 15 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Queued" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 81 + }, + "id": 114, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Planned", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Completed - {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_queue_length{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", user=\"$tenant\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Queued", + "range": true, + "refId": "C" + } + ], + "title": "Tasks per tenant", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 81 + }, + "id": 115, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "editorMode": "code", + "expr": "{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"}\n|= \"level=error\"\n|= \"tenant=$tenant\"", + "queryType": "range", + "refId": "B" + } + ], + "title": "Tenant errors", + "type": "logs" + } + ], + "targets": [ ], + "title": "Tasks", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 95, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many tokens each builder is appending to blooms. Accounts for tokens that are not actually added to the blooms since they are already there. See the panel on the right for a drill down on the collision.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 96, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_tokens_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}))", + "hide": false, + "instant": false, + "legendFormat": "Per core", + "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "C" } ], - "title": "Progress per pod", + "title": "Tokens rate", "type": "timeseries" }, { @@ -607,23 +1732,56 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Collision type may be `false` (no collision), `cache` (found in token cache) or true (found in bloom filter).\n\nType may be either `raw` (the original ngram) or `chunk_prefixed` (the ngram with the chunk prefix)", "fieldConfig": { "defaults": { - "fieldMinMax": false, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "mappings": [ ], "thresholds": { - "mode": "percentage", + "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "#EAB839", - "value": 0 + "color": "green", + "value": null }, { - "color": "green", - "value": 100 + "color": "red", + "value": 80 } ] }, @@ -635,26 +1793,24 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 90 }, - "id": 115, + "id": 97, "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showThresholdLabels": false, - "showThresholdMarkers": false, - "sizing": "auto" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } }, "panels": [ ], - "pluginVersion": "11.0.0-68102", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -662,7 +1818,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (\n loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n)", + "expr": "# tokens/s by type+collision\nsum by (collision) (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -670,55 +1826,15 @@ "refId": "B" } ], - "title": "Current Progress per pod", - "type": "gauge" - } - ], - "targets": [ ], - "title": "Progress per pod", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 56, - "panels": [ - { - "description": "", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 85, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "We use tenant sharding so each compactor will process a subset of the tenants.", - "transparent": true, - "type": "text" + "title": "tokens/s by collision type", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "Shows the expected number of cpu cores we need to provision to build blooms as fast as we ingest data so a compaction iteration doesn't take longer than the compaction interval.\n\nWe may decide to have more to speed up compaction.", + "description": "The sizes of the blooms created by the compactor. We build one bloom per series. The more unique ngrams and chunks the series has, the bigger their blooms will be.", "fieldConfig": { "defaults": { "color": { @@ -731,6 +1847,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -744,7 +1861,8 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "type": "linear" + "log": 2, + "type": "log" }, "showPoints": "auto", "spanNulls": false, @@ -763,19 +1881,24 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [ ] }, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 98 }, - "id": 94, + "id": 98, "options": { "legend": { "calcs": [ ], @@ -790,7 +1913,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -798,12 +1921,12 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n # rep factor\n 3 *\n sum(\n # 1 - dedupe_ratio\n 1 - \n sum(rate(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n /\n sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n )\n)\n/\n(\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n)", + "expr": "histogram_quantile(\n 1.0,\n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, - "legendFormat": "Needed", + "legendFormat": "max", "range": true, - "refId": "B" + "refId": "D" }, { "datasource": { @@ -811,15 +1934,28 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", + "expr": "histogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, - "legendFormat": "Available", + "legendFormat": "p99", "range": true, - "refId": "A" + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(\n 0.50, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "F" } ], - "title": "Required CPUs to not lag behind", + "title": "Bloom size", "type": "timeseries" }, { @@ -827,7 +1963,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "How many chunks are we indexing in the blooms. Either:\n- `copied` from a pre-existing bloom block, or \n- `iterated` through all its entries if processed for the first time.", "fieldConfig": { "defaults": { "color": { @@ -840,8 +1976,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 15, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -850,9 +1987,6 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -882,17 +2016,17 @@ } ] }, - "unit": "Bps" + "unit": "short" }, "overrides": [ ] }, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 98 }, - "id": 72, + "id": 99, "options": { "legend": { "calcs": [ ], @@ -907,7 +2041,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -915,35 +2049,38 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}[$__rate_interval])) by (pod)\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) by (pod)", - "hide": true, - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", + "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])) by (type)", "hide": false, "instant": false, - "legendFormat": "Total", + "legendFormat": "__auto", "range": true, "refId": "B" } ], - "title": "MB/s per core", + "title": "Chunks indexed", "type": "timeseries" - }, + } + ], + "targets": [ ], + "title": "Bloom building", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 56, + "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Shows the expected number of cpu cores we need to provision to build blooms as fast as we ingest data so a build iteration doesn't take longer than the build interval.\n\nWe may decide to have more to speed up building blooms.", "fieldConfig": { "defaults": { "color": { @@ -956,6 +2093,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -998,9 +2136,9 @@ "h": 7, "w": 12, "x": 0, - "y": 33 + "y": 2030 }, - "id": 1, + "id": 94, "options": { "legend": { "calcs": [ ], @@ -1015,7 +2153,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1023,10 +2161,10 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)", + "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n # rep factor\n 3 *\n sum(\n # 1 - dedupe_ratio\n 1 - \n sum(rate(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n /\n sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n )\n)\n/\n(\nsum(rate(loki_bloombuilder_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n)", "hide": false, "instant": false, - "legendFormat": "Request", + "legendFormat": "Needed", "range": true, "refId": "B" }, @@ -1036,22 +2174,10 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)", + "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "Limit", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.99,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)", - "instant": false, - "legendFormat": "p99", + "legendFormat": "Available", "range": true, "refId": "A" }, @@ -1061,28 +2187,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.50,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "avg(\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)\n*\ncount(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, - "legendFormat": "Avg", + "legendFormat": "Provisioned", "range": true, - "refId": "E" + "refId": "C" } ], - "title": "CPU", + "title": "Required CPUs to not lag behind", "type": "timeseries" }, { @@ -1090,6 +2203,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1102,8 +2216,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 15, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1112,6 +2227,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1140,7 +2258,8 @@ "value": 80 } ] - } + }, + "unit": "Bps" }, "overrides": [ ] }, @@ -1148,9 +2267,9 @@ "h": 7, "w": 12, "x": 12, - "y": 33 + "y": 2030 }, - "id": 75, + "id": 72, "options": { "legend": { "calcs": [ ], @@ -1165,7 +2284,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1173,40 +2292,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)", + "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloombuilder_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-builder\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "Request", + "legendFormat": "Total", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)", - "hide": false, - "instant": false, - "legendFormat": "Limit", - "range": true, - "refId": "C" } ], - "title": "CPU per pod", + "title": "MB/s per core", "type": "timeseries" }, { @@ -1226,6 +2320,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1258,14 +2353,9 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - }, - "unit": "bytes" + } }, "overrides": [ ] }, @@ -1273,9 +2363,9 @@ "h": 7, "w": 12, "x": 0, - "y": 40 + "y": 2037 }, - "id": 76, + "id": 1, "options": { "legend": { "calcs": [ ], @@ -1290,7 +2380,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1298,7 +2388,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, "legendFormat": "Request", @@ -1311,7 +2401,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"} > 0)", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, "legendFormat": "Limit", @@ -1324,7 +2414,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile (\n 0.99,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)", + "expr": "quantile(\n 0.99,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", "instant": false, "legendFormat": "p99", "range": true, @@ -1336,7 +2426,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile (\n 0.50,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)", + "expr": "quantile(\n 0.50,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "p50", @@ -1349,15 +2439,28 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)", + "expr": "avg(\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "Avg", "range": true, "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "Max", + "range": true, + "refId": "F" } ], - "title": "Memory (workingset)", + "title": "CPU", "type": "timeseries" }, { @@ -1377,6 +2480,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1415,43 +2519,17 @@ "value": 80 } ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "bloom-compactor-106" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] } - ] + }, + "overrides": [ ] }, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 40 + "y": 2037 }, - "id": 5, + "id": 75, "options": { "legend": { "calcs": [ ], @@ -1466,7 +2544,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1474,7 +2552,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"})", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{pod}}", "range": true, @@ -1486,7 +2564,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, "legendFormat": "Request", @@ -1499,7 +2577,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"} > 0)", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, "legendFormat": "Limit", @@ -1507,7 +2585,7 @@ "refId": "C" } ], - "title": "Memory per pod (workingset)", + "title": "CPU per pod", "type": "timeseries" }, { @@ -1515,7 +2593,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -1528,8 +2605,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 15, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1538,9 +2616,6 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1570,7 +2645,7 @@ } ] }, - "unit": "none" + "unit": "bytes" }, "overrides": [ ] }, @@ -1578,9 +2653,9 @@ "h": 7, "w": 12, "x": 0, - "y": 47 + "y": 2044 }, - "id": 27, + "id": 76, "options": { "legend": { "calcs": [ ], @@ -1595,7 +2670,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1603,140 +2678,86 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[10m]\n )\n) > 0", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"memory\"} > 0)", + "hide": false, "instant": false, - "legendFormat": "Restarts", + "legendFormat": "Request", "range": true, - "refId": "A" - } - ], - "title": "Container restarts", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "unit": "none" + "editorMode": "code", + "expr": "quantile (\n 0.99,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 77, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile (\n 0.50,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "E" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "panels": [ ], - "pluginVersion": "11.1.0-69868", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}\n )\n) > 0", + "expr": "max (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, "instant": false, - "legendFormat": "{{reason}} / {{pod}}", + "legendFormat": "Max", "range": true, - "refId": "A" + "refId": "F" } ], - "title": "Container restarts reason per pod", + "title": "Memory (workingset)", "type": "timeseries" - } - ], - "targets": [ ], - "title": "Resource Usage", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 95, - "panels": [ + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "How many tokens each compactor is appending to blooms. Accounts for tokens that are not actually added to the blooms since they are already there. See the panel on the right for a drill down on the collision.\n", "fieldConfig": { "defaults": { "color": { @@ -1749,6 +2770,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1762,8 +2784,7 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 2, - "type": "log" + "type": "linear" }, "showPoints": "auto", "spanNulls": false, @@ -1788,17 +2809,18 @@ "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [ ] }, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 0, - "y": 55 + "x": 12, + "y": 2044 }, - "id": 96, + "id": 5, "options": { "legend": { "calcs": [ ], @@ -1813,7 +2835,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1821,10 +2843,22 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# tokens checked per pod, millions/s\nsum(rate(loki_bloom_tokens_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}))\n/ 1e6", + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"})", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"memory\"} > 0)", "hide": false, "instant": false, - "legendFormat": "Per core", + "legendFormat": "Request", "range": true, "refId": "B" }, @@ -1834,15 +2868,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) / 1e6", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"} > 0)", "hide": false, "instant": false, - "legendFormat": "Total", + "legendFormat": "Limit", "range": true, "refId": "C" } ], - "title": "Tokens rate (millions)", + "title": "Memory per pod (workingset)", "type": "timeseries" }, { @@ -1850,7 +2884,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Collision type may be `false` (no collision), `cache` (found in token cache) or true (found in bloom filter).\n\nType may be either `raw` (the original ngram) or `chunk_prefixed` (the ngram with the chunk prefix)", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1863,8 +2897,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 15, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1873,6 +2908,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1902,17 +2940,17 @@ } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [ ] }, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 12, - "y": 55 + "x": 0, + "y": 2051 }, - "id": 97, + "id": 27, "options": { "legend": { "calcs": [ ], @@ -1927,7 +2965,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1935,15 +2973,14 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# tokens/s by type+collision\nsum by (collision) (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)", - "hide": false, + "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n) > 0", "instant": false, - "legendFormat": "__auto", + "legendFormat": "Restarts", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "tokens/s by collision type", + "title": "Container restarts", "type": "timeseries" }, { @@ -1951,7 +2988,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "The sizes of the blooms created by the compactor. We build one bloom per series. The more unique ngrams and chunks the series has, the bigger their blooms will be.", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1964,8 +3001,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 15, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1974,6 +3012,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2003,17 +3044,17 @@ } ] }, - "unit": "bytes" + "unit": "none" }, "overrides": [ ] }, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 0, - "y": 63 + "x": 12, + "y": 2051 }, - "id": 98, + "id": 77, "options": { "legend": { "calcs": [ ], @@ -2028,7 +3069,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2036,49 +3077,36 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(\n 0.90, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p90", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(\n 0.50, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, + "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}\n )\n) > 0", "instant": false, - "legendFormat": "p50", + "legendFormat": "{{reason}} / {{pod}}", "range": true, - "refId": "F" + "refId": "A" } ], - "title": "Bloom size", + "title": "Container restarts reason per pod", "type": "timeseries" - }, + } + ], + "targets": [ ], + "title": "Builder Resource Usage", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 118, + "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "How many chunks are we indexing in the blooms. Either:\n- `copied` from a pre-existing bloom block, or \n- `iterated` through all its entries if processed for the first time.", "fieldConfig": { "defaults": { "color": { @@ -2091,6 +3119,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2123,24 +3152,19 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - }, - "unit": "none" + } }, "overrides": [ ] }, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 12, - "y": 63 + "x": 0, + "y": 2302 }, - "id": 99, + "id": 119, "options": { "legend": { "calcs": [ ], @@ -2155,7 +3179,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2163,32 +3187,68 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) by (type)", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, - "legendFormat": "__auto", + "legendFormat": "Request", "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile(\n 0.99,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval])\n)", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile(\n 0.50,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "E" } ], - "title": "Chunks indexed", + "title": "CPU", "type": "timeseries" - } - ], - "targets": [ ], - "title": "Bloom building", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 103, - "panels": [ + }, { "datasource": { "type": "prometheus", @@ -2206,6 +3266,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2238,6 +3299,10 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] } @@ -2247,10 +3312,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 72 + "x": 12, + "y": 2302 }, - "id": 107, + "id": 120, "options": { "legend": { "calcs": [ ], @@ -2258,29 +3323,55 @@ "placement": "bottom", "showLegend": true }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "panels": [ ], - "targets": [ + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloomcompactor_blocks_created_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, - "legendFormat": "Blocks", + "legendFormat": "Limit", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Created Blocks", + "title": "CPU per pod", "type": "timeseries" }, { @@ -2288,7 +3379,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.", "fieldConfig": { "defaults": { "color": { @@ -2301,6 +3391,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2333,19 +3424,24 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [ ] }, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 72 + "x": 0, + "y": 2309 }, - "id": 106, + "id": 121, "options": { "legend": { "calcs": [ ], @@ -2360,6 +3456,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2367,15 +3464,66 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloomcompactor_blocks_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"memory\"} > 0)", "hide": false, "instant": false, - "legendFormat": "Blocks", + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile (\n 0.99,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}\n)", + "instant": false, + "legendFormat": "p99", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile (\n 0.50,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "E" } ], - "title": "Deleted Blocks", + "title": "Memory (workingset)", "type": "timeseries" }, { @@ -2383,7 +3531,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Number of overlapping bloom blocks reused when creating new blocks\n", "fieldConfig": { "defaults": { "color": { @@ -2396,6 +3543,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2428,19 +3576,24 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [ ] }, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 79 + "x": 12, + "y": 2309 }, - "id": 109, + "id": 122, "options": { "legend": { "calcs": [ ], @@ -2455,6 +3608,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2462,37 +3616,48 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloomcompactor_blocks_reused_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", - "hide": false, + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"})", "instant": false, - "legendFormat": "Blocks", + "legendFormat": "{{pod}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"memory\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" } ], - "title": "Blocks reused", + "title": "Memory per pod (workingset)", "type": "timeseries" - } - ], - "targets": [ ], - "title": "Blocks building", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 110, - "panels": [ + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -2505,8 +3670,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 15, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2515,6 +3681,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2537,9 +3706,14 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [ ] }, @@ -2547,9 +3721,9 @@ "h": 7, "w": 12, "x": 0, - "y": 87 + "y": 2316 }, - "id": 108, + "id": 123, "options": { "legend": { "calcs": [ ], @@ -2564,7 +3738,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2572,15 +3746,14 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloomcompactor_metas_created_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", - "hide": false, + "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n) > 0", "instant": false, - "legendFormat": "Metas", + "legendFormat": "Restarts", "range": true, "refId": "A" } ], - "title": "Created Metas", + "title": "Container restarts", "type": "timeseries" }, { @@ -2588,7 +3761,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -2601,8 +3774,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 15, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2611,6 +3785,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2633,9 +3810,14 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [ ] }, @@ -2643,9 +3825,9 @@ "h": 7, "w": 12, "x": 12, - "y": 87 + "y": 2316 }, - "id": 105, + "id": 124, "options": { "legend": { "calcs": [ ], @@ -2660,7 +3842,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2668,20 +3850,19 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloomcompactor_metas_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", - "hide": false, + "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}\n )\n) > 0", "instant": false, - "legendFormat": "Metas", + "legendFormat": "{{reason}} / {{pod}}", "range": true, "refId": "A" } ], - "title": "Deleted Metas", + "title": "Container restarts reason per pod", "type": "timeseries" } ], "targets": [ ], - "title": "Metas building", + "title": "Planner Resource Usage", "type": "row" }, { @@ -2690,35 +3871,10 @@ "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 26 }, - "id": 80, + "id": 110, "panels": [ - { - "description": "", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 95 - }, - "id": 93, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "We use tenant sharding so each compactor will process a subset of the tenants.", - "transparent": true, - "type": "text" - }, { "datasource": { "type": "prometheus", @@ -2736,6 +3892,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2776,11 +3933,11 @@ }, "gridPos": { "h": 7, - "w": 12, + "w": 9, "x": 0, - "y": 96 + "y": 2497 }, - "id": 83, + "id": 108, "options": { "legend": { "calcs": [ ], @@ -2795,6 +3952,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2802,48 +3960,92 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.99,\n increase(\n loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.90,\n increase(\n loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", + "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p90", + "legendFormat": "Metas", "range": true, "refId": "A" + } + ], + "title": "Created Metas", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 9, + "y": 2497 + }, + "id": 140, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.50,\n increase(\n loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[30m]\n )\n)", + "exemplar": false, + "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", + "format": "table", "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "B" + "instant": true, + "legendFormat": "Metas", + "range": false, + "refId": "A" } ], - "title": "Tenants", - "type": "timeseries" + "title": "Created Metas", + "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.", "fieldConfig": { "defaults": { "color": { @@ -2856,6 +4058,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2898,9 +4101,9 @@ "h": 7, "w": 12, "x": 12, - "y": 96 + "y": 2497 }, - "id": 84, + "id": 105, "options": { "legend": { "calcs": [ ], @@ -2915,6 +4118,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2922,42 +4126,32 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (\n increase(\n loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", + "expr": "sum by (phase) (increase(loki_bloomplanner_metas_deleted_total{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "Deleted during {{phase}}", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "Tenants per pod", + "title": "Deleted Metas", "type": "timeseries" - }, - { - "description": "", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 103 - }, - "id": 86, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "Number of tenant tables processed. ", - "transparent": true, - "type": "text" - }, + } + ], + "targets": [ ], + "title": "Metas building", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 103, + "panels": [ { "datasource": { "type": "prometheus", @@ -2975,6 +4169,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3015,11 +4210,11 @@ }, "gridPos": { "h": 7, - "w": 12, + "w": 9, "x": 0, - "y": 104 + "y": 2505 }, - "id": 88, + "id": 107, "options": { "legend": { "calcs": [ ], @@ -3034,6 +4229,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3041,41 +4237,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.99,\n increase(\n loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.90,\n increase(\n loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", + "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p90", + "legendFormat": "Blocks", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.50,\n increase(\n loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "B" } ], - "title": "Tenant Tables", + "title": "Created Blocks", "type": "timeseries" }, { @@ -3086,39 +4256,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [ ], "thresholds": { @@ -3135,25 +4273,30 @@ }, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 104 + "w": 3, + "x": 9, + "y": 2505 }, - "id": 89, + "id": 139, "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3161,48 +4304,25 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (\n increase(\n loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", + "exemplar": false, + "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", + "format": "time_series", "hide": false, - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "C" + "instant": true, + "legendFormat": "Blocks", + "range": false, + "refId": "A" } ], - "title": "Tenant Tables per pod", - "type": "timeseries" - }, - { - "description": "", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 111 - }, - "id": 87, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "Series per compaction (includes series copied from other blocks)", - "transparent": true, - "type": "text" + "title": "Created Blocks", + "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.", "fieldConfig": { "defaults": { "color": { @@ -3215,6 +4335,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3256,10 +4377,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 112 + "x": 12, + "y": 2505 }, - "id": 81, + "id": 106, "options": { "legend": { "calcs": [ ], @@ -3274,6 +4395,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3281,41 +4403,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p90", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", + "expr": "sum by (phase) (increase(loki_bloomplanner_blocks_deleted_total{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p50", + "legendFormat": "Deleted during {{phase}}", "range": true, "refId": "A" } ], - "title": "Series", + "title": "Deleted Blocks", "type": "timeseries" }, { @@ -3323,6 +4419,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Number of overlapping bloom blocks reused when creating new blocks\n", "fieldConfig": { "defaults": { "color": { @@ -3335,6 +4432,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3376,10 +4474,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 112 + "x": 0, + "y": 2512 }, - "id": 82, + "id": 109, "options": { "legend": { "calcs": [ ], @@ -3394,6 +4492,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3401,47 +4500,125 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (\n rate(loki_bloomcompactor_series_per_compaction_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n /\n rate(loki_bloomcompactor_series_per_compaction_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)", + "expr": "sum(increase(loki_bloombuilder_blocks_reused_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "Blocks", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "avg series per compaction by pod", + "title": "Blocks reused", "type": "timeseries" - }, + } + ], + "targets": [ ], + "title": "Blocks building", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 135, + "panels": [ { - "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Is the retention currently running?", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "fieldMinMax": false, + "mappings": [ + { + "options": { + "0": { + "color": "yellow", + "index": 0, + "text": "No" + }, + "1": { + "color": "green", + "index": 1, + "text": "Yes" + } + }, + "type": "value" + } + ], + "max": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, "gridPos": { - "h": 1, - "w": 24, + "h": 7, + "w": 3, "x": 0, - "y": 119 + "y": 2573 }, - "id": 90, + "id": 136, "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "content": "", - "mode": "markdown" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "Number of bytes from chunks added to blocks during each compaction.", - "transparent": true, - "type": "text" + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Running now?", + "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Is the retention currently running?", "fieldConfig": { "defaults": { "color": { @@ -3453,7 +4630,10 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3479,7 +4659,9 @@ "mode": "off" } }, + "fieldMinMax": false, "mappings": [ ], + "max": 2, "thresholds": { "mode": "absolute", "steps": [ @@ -3489,17 +4671,17 @@ } ] }, - "unit": "bytes" + "unit": "bool_yes_no" }, "overrides": [ ] }, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 120 + "w": 9, + "x": 3, + "y": 2573 }, - "id": 91, + "id": 137, "options": { "legend": { "calcs": [ ], @@ -3508,12 +4690,12 @@ "showLegend": true }, "tooltip": { - "maxHeight": 600, "mode": "single", "sort": "none" } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3521,41 +4703,13 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p90", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", + "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Bytes", + "title": "Retention running", "type": "timeseries" }, { @@ -3563,6 +4717,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "How much time applying retention took", "fieldConfig": { "defaults": { "color": { @@ -3575,6 +4730,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3600,6 +4756,7 @@ "mode": "off" } }, + "fieldMinMax": false, "mappings": [ ], "thresholds": { "mode": "absolute", @@ -3610,7 +4767,7 @@ } ] }, - "unit": "bytes" + "unit": "dtdurations" }, "overrides": [ ] }, @@ -3618,9 +4775,9 @@ "h": 7, "w": 12, "x": 12, - "y": 120 + "y": 2573 }, - "id": 92, + "id": 138, "options": { "legend": { "calcs": [ ], @@ -3629,12 +4786,12 @@ "showLegend": true }, "tooltip": { - "maxHeight": 600, "mode": "single", "sort": "none" } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3642,20 +4799,18 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (\n rate(loki_bloomcompactor_bytes_per_compaction_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n /\n rate(loki_bloomcompactor_bytes_per_compaction_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", + "expr": "histogram_quantile(0.9, \n sum by (status, le) (\n rate(loki_bloomplanner_retention_time_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval])\n )\n)", + "legendFormat": "__auto", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "avg bytes per compaction by pod", + "title": "Retention time", "type": "timeseries" } ], "targets": [ ], - "title": "Data processed", + "title": "Retention", "type": "row" }, { @@ -3666,7 +4821,7 @@ "x": 0, "y": 29 }, - "id": 58, + "id": 62, "panels": [ { "description": "", @@ -3675,23 +4830,52 @@ "overrides": [ ] }, "gridPos": { - "h": 3, + "h": 4, "w": 24, "x": 0, - "y": 82 + "y": 2581 + }, + "id": 71, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "During the planning phase, the planner downloads the metas and TSDBs to build the plan.\n\nOnce all blocks and metas are built, the builder flushes them to the object store.\n\nAfter each iteration, the planner deletes the metas and blocks marked for deletion in the tombstones.", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 13, + "w": 2, + "x": 0, + "y": 2585 }, - "id": 47, + "id": 63, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, - "content": "\nCompactors write blocks to the attached PVs before flushing them into the object store.\nIt also download chunks and index files.\n\nAfter compacting a given tenant, all the downloaded index files and chunks, as well as the already flushed blocks are deleted.", + "content": "---\n#### GCS\n", "mode": "markdown" }, "panels": [ ], - "pluginVersion": "11.1.0-69747", + "pluginVersion": "11.4.0-77663", "targets": [ ], "title": "", "transparent": true, @@ -3702,6 +4886,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3714,8 +4899,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3736,35 +4922,30 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "area" + "mode": "off" } }, "mappings": [ ], - "max": 1, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 0.80000000000000004 + "color": "green", + "value": null } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 85 + "h": 7, + "w": 11, + "x": 2, + "y": 2585 }, - "id": 9, + "id": 61, "options": { "legend": { "calcs": [ ], @@ -3779,6 +4960,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3786,40 +4968,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.99,\n max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n / \n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n ) \n and \n count by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n )\n)", - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.90,\n max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n / \n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n ) \n and \n count by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n )\n)", + "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p90", + "legendFormat": "{{operation}} {{status_code}}", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.50,\n max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n / \n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n ) \n and \n count by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "C" } ], - "title": "Disk Utilization", + "title": "QPS Planner", "type": "timeseries" }, { @@ -3827,6 +4984,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3839,8 +4997,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3861,35 +5020,30 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "area" + "mode": "off" } }, "mappings": [ ], - "max": 1, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 0.80000000000000004 + "color": "green", + "value": null } ] }, - "unit": "percentunit" + "unit": "s" }, "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 85 + "h": 7, + "w": 11, + "x": 13, + "y": 2585 }, - "id": 100, + "id": 64, "options": { "legend": { "calcs": [ ], @@ -3904,6 +5058,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3911,14 +5066,41 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"})", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "{{operation}} p99", "range": true, - "refId": "A" + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "F" } ], - "title": "Disk Utilization per pod", + "title": "Latency Planner", "type": "timeseries" }, { @@ -3926,6 +5108,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3938,8 +5121,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3968,25 +5152,22 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": null } ] }, - "unit": "bytes" + "unit": "none" }, "overrides": [ ] }, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 93 + "w": 11, + "x": 2, + "y": 2592 }, - "id": 7, + "id": 127, "options": { "legend": { "calcs": [ ], @@ -4001,6 +5182,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4008,40 +5190,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.99,\n sum by(instance, pod, device) (\n rate(node_disk_written_bytes_total[$__rate_interval])\n ) \n + ignoring(pod) group_right() \n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.90,\n sum by(instance, pod, device) (\n rate(node_disk_written_bytes_total[$__rate_interval])\n ) \n + ignoring(pod) group_right() \n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", + "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p90", + "legendFormat": "{{operation}} {{status_code}}", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.50,\n sum by(instance, pod, device) (\n rate(node_disk_written_bytes_total[$__rate_interval])\n ) \n + ignoring(pod) group_right() \n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "C" } ], - "title": "Disk Writes", + "title": "QPS Builder", "type": "timeseries" }, { @@ -4049,6 +5206,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -4061,8 +5219,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4091,25 +5250,22 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": null } ] }, - "unit": "bytes" + "unit": "s" }, "overrides": [ ] }, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 93 + "w": 11, + "x": 13, + "y": 2592 }, - "id": 101, + "id": 128, "options": { "legend": { "calcs": [ ], @@ -4117,35 +5273,93 @@ "placement": "bottom", "showLegend": true }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "panels": [ ], - "targets": [ + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "E" + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "{{operation}} p50", "range": true, - "refId": "A" + "refId": "F" } ], - "title": "Disk Writes per pod", + "title": "Latency Planner", "type": "timeseries" }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 14, + "w": 2, + "x": 0, + "y": 2598 + }, + "id": 65, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "---\n#### S3\n", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -4158,8 +5372,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4188,25 +5403,22 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": null } ] }, - "unit": "bytes" + "unit": "none" }, "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 100 + "h": 7, + "w": 11, + "x": 2, + "y": 2599 }, - "id": 8, + "id": 67, "options": { "legend": { "calcs": [ ], @@ -4221,6 +5433,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4228,40 +5441,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.99,\n sum by(instance, pod, device) (\n rate(node_disk_read_bytes_total[$__rate_interval])\n ) + ignoring(pod) group_right()\n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.90,\n sum by(instance, pod, device) (\n rate(node_disk_read_bytes_total[$__rate_interval])\n ) + ignoring(pod) group_right()\n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p90", + "legendFormat": "{{operation}} {{status_code}}", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.50,\n sum by(instance, pod, device) (\n rate(node_disk_read_bytes_total[$__rate_interval])\n ) + ignoring(pod) group_right()\n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "C" } ], - "title": "Disk Reads", + "title": "QPS Planner", "type": "timeseries" }, { @@ -4269,6 +5457,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -4281,8 +5470,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4311,25 +5501,22 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": null } ] }, - "unit": "bytes" + "unit": "s" }, "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 100 + "h": 7, + "w": 11, + "x": 13, + "y": 2599 }, - "id": 102, + "id": 69, "options": { "legend": { "calcs": [ ], @@ -4344,6 +5531,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4351,88 +5539,42 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "{{operation}} p90", "range": true, - "refId": "A" + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", job=\"\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "F" } ], - "title": "Disk Reads per pod", + "title": "Latency Planner", "type": "timeseries" - } - ], - "targets": [ ], - "title": "Disk Usage", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 62, - "panels": [ - { - "description": "", - "fieldConfig": { - "defaults": { }, - "overrides": [ ] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 83 - }, - "id": 71, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Once all blocks and metas are built locally, the compactor flushes them to the object store.\n\nAfter each iteration, the compactor deletes the metas and blocks marked for deletion in the tombstones.", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-69747", - "targets": [ ], - "title": "", - "transparent": true, - "type": "text" - }, - { - "description": "", - "fieldConfig": { - "defaults": { }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 2, - "x": 0, - "y": 86 - }, - "id": 63, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "---\n#### GCS\n", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-69747", - "targets": [ ], - "title": "", - "transparent": true, - "type": "text" }, { "datasource": { @@ -4452,6 +5594,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4482,7 +5625,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -4494,9 +5638,9 @@ "h": 7, "w": 11, "x": 2, - "y": 86 + "y": 2606 }, - "id": 61, + "id": 129, "options": { "legend": { "calcs": [ ], @@ -4511,6 +5655,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4518,7 +5663,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -4526,7 +5671,7 @@ "refId": "B" } ], - "title": "QPS", + "title": "QPS Builder", "type": "timeseries" }, { @@ -4547,6 +5692,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4577,7 +5723,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -4589,9 +5736,9 @@ "h": 7, "w": 11, "x": 13, - "y": 86 + "y": 2606 }, - "id": 64, + "id": 130, "options": { "legend": { "calcs": [ ], @@ -4606,6 +5753,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4613,7 +5761,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -4626,7 +5774,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -4639,7 +5787,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -4647,7 +5795,7 @@ "refId": "F" } ], - "title": "Latency", + "title": "Latency Builder", "type": "timeseries" }, { @@ -4660,20 +5808,20 @@ "h": 7, "w": 2, "x": 0, - "y": 93 + "y": 2612 }, - "id": 65, + "id": 66, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, - "content": "---\n#### S3\n", + "content": "---\n#### Azure\nBlob Storage", "mode": "markdown" }, "panels": [ ], - "pluginVersion": "11.1.0-69747", + "pluginVersion": "11.4.0-77663", "targets": [ ], "title": "", "transparent": true, @@ -4697,6 +5845,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4727,7 +5876,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -4739,9 +5889,9 @@ "h": 7, "w": 11, "x": 2, - "y": 93 + "y": 2613 }, - "id": 67, + "id": 68, "options": { "legend": { "calcs": [ ], @@ -4756,6 +5906,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4763,7 +5914,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -4771,7 +5922,7 @@ "refId": "B" } ], - "title": "QPS", + "title": "QPS Planner", "type": "timeseries" }, { @@ -4792,6 +5943,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4822,7 +5974,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -4834,9 +5987,9 @@ "h": 7, "w": 11, "x": 13, - "y": 93 + "y": 2613 }, - "id": 69, + "id": 70, "options": { "legend": { "calcs": [ ], @@ -4851,6 +6004,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4858,7 +6012,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -4871,7 +6025,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -4884,7 +6038,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -4892,38 +6046,9 @@ "refId": "F" } ], - "title": "Latency", + "title": "Latency Planner", "type": "timeseries" }, - { - "description": "", - "fieldConfig": { - "defaults": { }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 2, - "x": 0, - "y": 100 - }, - "id": 66, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "---\n#### Azure\nBlob Storage", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-69747", - "targets": [ ], - "title": "", - "transparent": true, - "type": "text" - }, { "datasource": { "type": "prometheus", @@ -4942,6 +6067,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4972,7 +6098,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -4984,9 +6111,9 @@ "h": 7, "w": 11, "x": 2, - "y": 100 + "y": 2620 }, - "id": 68, + "id": 131, "options": { "legend": { "calcs": [ ], @@ -5001,6 +6128,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -5008,7 +6136,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -5016,7 +6144,7 @@ "refId": "B" } ], - "title": "QPS", + "title": "QPS Builder", "type": "timeseries" }, { @@ -5037,6 +6165,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -5067,7 +6196,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -5079,9 +6209,9 @@ "h": 7, "w": 11, "x": 13, - "y": 100 + "y": 2620 }, - "id": 70, + "id": 132, "options": { "legend": { "calcs": [ ], @@ -5096,6 +6226,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -5103,7 +6234,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -5116,7 +6247,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5129,7 +6260,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5137,7 +6268,7 @@ "refId": "F" } ], - "title": "Latency", + "title": "Latency Builder", "type": "timeseries" } ], @@ -5146,6 +6277,7 @@ "type": "row" } ], + "preload": false, "refresh": "10s", "rows": [ ], "schemaVersion": 14, @@ -5231,7 +6363,6 @@ "from": "now-1h", "to": "now" }, - "timeRangeUpdatedDuringEditOrView": false, "timepicker": { "refresh_intervals": [ "5s", @@ -5258,8 +6389,8 @@ ] }, "timezone": "utc", - "title": "Loki / Bloom Compactor", - "uid": "bloom-compactor", + "title": "Loki / Bloom Build", + "uid": "bloom-build", "version": 0, "weekStart": "" } \ No newline at end of file diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json index 2d5e16a9d7e0..7aa907976c80 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json @@ -21,7 +21,6 @@ "type": "dashboards" } ], - "liveNow": false, "panels": [ { "collapsed": false, @@ -46,8 +45,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds", - "seriesBy": "last" + "mode": "thresholds" }, "custom": { "axisBorderShow": false, @@ -56,9 +54,10 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, - "gradientMode": "none", + "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, @@ -69,7 +68,7 @@ "lineStyle": { "fill": "solid" }, - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -114,7 +113,7 @@ }, "gridPos": { "h": 6, - "w": 6, + "w": 12, "x": 0, "y": 1 }, @@ -133,6 +132,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -140,7 +140,8 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "hide": false, "instant": false, "legendFormat": "Chunks", "range": true, @@ -153,7 +154,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Series", @@ -161,7 +162,7 @@ "refId": "B" } ], - "title": "Filter ratio", + "title": "Filter ratio - Bloom Gateway (server)", "type": "timeseries" }, { @@ -206,7 +207,7 @@ "gridPos": { "h": 6, "w": 6, - "x": 6, + "x": 12, "y": 1 }, "id": 75, @@ -226,7 +227,7 @@ "sizing": "auto" }, "panels": [ ], - "pluginVersion": "11.1.0-70005", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -235,7 +236,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(increase(loki_bloom_gateway_filtered_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", "instant": true, "legendFormat": "Chunks", "range": false, @@ -248,40 +249,12 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(increase(loki_bloom_gateway_filtered_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", "hide": false, "instant": true, "legendFormat": "Series", "range": false, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"})\n/\nsum(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"})", - "hide": true, - "instant": true, - "legendFormat": "Chunks avg", - "range": false, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"})\n/\nsum(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"})", - "hide": true, - "instant": true, - "legendFormat": "Series avg", - "range": false, - "refId": "D" } ], "title": "Filter ratio", @@ -305,6 +278,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -381,7 +355,7 @@ "gridPos": { "h": 6, "w": 6, - "x": 12, + "x": 18, "y": 1 }, "id": 72, @@ -399,6 +373,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -413,19 +388,6 @@ "range": true, "refId": "D" }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(kube_pod_container_status_ready{container=\"bloom-gateway\", cluster=\"$cluster\", namespace=\"$namespace\"})", - "hide": true, - "instant": false, - "legendFormat": "Running", - "range": true, - "refId": "A" - }, { "datasource": { "type": "prometheus", @@ -438,19 +400,6 @@ "legendFormat": "Desired", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "kube_statefulset_status_replicas_available{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", - "hide": true, - "instant": false, - "legendFormat": "Available", - "range": true, - "refId": "C" } ], "title": "Readiness", @@ -461,11 +410,11 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Percentage of chunks that are filtered by using bloom filters", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, "custom": { "axisBorderShow": false, @@ -474,9 +423,10 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 50, - "gradientMode": "none", + "fillOpacity": 0, + "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, @@ -499,34 +449,44 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "off" + "mode": "area" } }, "mappings": [ ], + "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null }, { - "color": "red", - "value": 80 + "color": "orange", + "value": 0.5 + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "green", + "value": 0.90000000000000002 } ] }, - "unit": "none" + "unit": "percentunit" }, "overrides": [ ] }, "gridPos": { "h": 6, - "w": 6, - "x": 18, - "y": 1 + "w": 12, + "x": 0, + "y": 7 }, - "id": 37, + "id": 93, "options": { "legend": { "calcs": [ ], @@ -541,6 +501,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -548,77 +509,135 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n * on (pod) group_left\n sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0", + "expr": "sum(rate(loki_bloom_gateway_querier_chunks_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Chunks", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(loki_bloom_gateway_querier_series_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, - "interval": "", - "legendFormat": "{{pod}} ({{reason}})", + "legendFormat": "Series", "range": true, - "refId": "C" + "refId": "B" } ], - "title": "Container restarts", + "title": "Filter ratio - Index Gateway (client)", "type": "timeseries" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Percentage of chunks that are filtered by using bloom filters", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "green", + "value": 0.90000000000000002 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, "gridPos": { - "h": 9, - "w": 15, - "x": 0, + "h": 6, + "w": 6, + "x": 12, "y": 7 }, - "id": 48, + "id": 94, "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": true + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt", - "queryType": "range", + "exemplar": false, + "expr": "sum(increase(loki_bloom_gateway_querier_chunks_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", + "instant": true, + "legendFormat": "Chunks", + "range": false, "refId": "A" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt", - "hide": true, - "queryType": "range", + "exemplar": false, + "expr": "sum(increase(loki_bloom_gateway_querier_series_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", + "hide": false, + "instant": true, + "legendFormat": "Series", + "range": false, "refId": "B" } ], - "title": "Errors", - "type": "logs" + "title": "Filter ratio", + "type": "gauge" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", - "mode": "fixed" + "mode": "palette-classic" }, "custom": { "axisBorderShow": false, @@ -627,8 +646,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, "gradientMode": "none", "hideFrom": { "legend": false, @@ -637,11 +657,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 2, - "type": "symlog" + "type": "linear" }, "showPoints": "auto", "spanNulls": false, @@ -663,66 +685,21 @@ }, { "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "warn" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "error" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } + "value": 80 } ] }, - { - "matcher": { - "id": "byName", - "options": "panic" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-red", - "mode": "fixed" - } - } - ] - } - ] + "unit": "none" + }, + "overrides": [ ] }, "gridPos": { - "h": 9, - "w": 9, - "x": 15, + "h": 6, + "w": 6, + "x": 18, "y": 7 }, - "id": 52, + "id": 37, "options": { "legend": { "calcs": [ ], @@ -737,44 +714,936 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "editorMode": "code", - "expr": "sum by (level) (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))", - "legendFormat": "{{ level }}", - "queryType": "range", - "refId": "A" - }, - { - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))", + "expr": "(\n max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n * on (pod) group_left\n sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0", "hide": false, - "legendFormat": "panic", - "queryType": "range", - "refId": "B" + "instant": false, + "interval": "", + "legendFormat": "{{pod}} ({{reason}})", + "range": true, + "refId": "C" } ], - "title": "Errors Rate", + "title": "Container restarts", "type": "timeseries" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "id": 56, - "panels": [ + "description": "Percentage of chunks that are filtered by using bloom filters", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "green", + "value": 0.90000000000000002 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 99, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n sum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n -\n sum(rate(loki_index_gateway_postfilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n)\n/\nsum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)", + "instant": false, + "legendFormat": "chunks {{ route}}", + "range": true, + "refId": "A" + } + ], + "title": "Filter ratio - Index Gateway by route", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.10000000000000001 + }, + { + "color": "#EF843C", + "value": 0.25 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 100, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(loki_bloom_gateway_querier_series_skipped_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "series", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(loki_bloom_gateway_querier_chunks_skipped_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "chunks", + "range": true, + "refId": "B" + } + ], + "title": "Data skipped because they don't match any bocks", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 96, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 467 + }, + "id": 97, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77450", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Found/Skipped/Missing chunks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 467 + }, + "id": 98, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77450", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Filtered chunks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 481 + }, + "id": 107, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77450", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Found/Skipped/Missing series", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 481 + }, + "id": 108, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77450", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Filtered series", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Bloom Recorder", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 95, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 9, + "w": 15, + "x": 0, + "y": 598 + }, + "id": 48, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "panels": [ ], + "pluginVersion": "11.4.0-77383", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "editorMode": "code", + "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "editorMode": "code", + "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt", + "hide": true, + "queryType": "range", + "refId": "B" + } + ], + "title": "Errors", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "symlog" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "panic" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 15, + "y": 598 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77383", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "editorMode": "code", + "expr": "sum by (level) (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))", + "legendFormat": "{{ level }}", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "editorMode": "code", + "expr": "sum (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))", + "hide": false, + "legendFormat": "panic", + "queryType": "range", + "refId": "B" + } + ], + "title": "Errors Rate", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Logs", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 56, + "panels": [ { "datasource": { "type": "prometheus", @@ -792,6 +1661,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -822,8 +1692,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -838,7 +1707,7 @@ "h": 14, "w": 12, "x": 0, - "y": 17 + "y": 909 }, "id": 10, "options": { @@ -855,6 +1724,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -915,6 +1785,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -945,8 +1816,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -962,7 +1832,7 @@ "h": 7, "w": 12, "x": 12, - "y": 17 + "y": 909 }, "id": 11, "options": { @@ -979,6 +1849,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1040,6 +1911,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1070,8 +1942,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1087,7 +1958,7 @@ "h": 7, "w": 12, "x": 12, - "y": 24 + "y": 916 }, "id": 81, "options": { @@ -1104,6 +1975,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1179,6 +2051,110 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 923 + }, + "id": 87, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77383", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "GC rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1212,25 +2188,25 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [ ] }, "gridPos": { "h": 7, "w": 8, - "x": 0, - "y": 31 + "x": 8, + "y": 923 }, - "id": 87, + "id": 88, "options": { "legend": { "calcs": [ ], @@ -1245,6 +2221,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1252,15 +2229,14 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum by (pod) (rate(go_gc_duration_seconds_sum{container=\"bloom-gateway\"}[$__rate_interval]))\n/\nsum by (pod) (rate(go_gc_duration_seconds_count{container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, - "instant": false, "legendFormat": "__auto", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "GC rate", + "title": "GC duration", "type": "timeseries" }, { @@ -1281,6 +2257,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1314,8 +2291,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1330,10 +2306,10 @@ "gridPos": { "h": 7, "w": 8, - "x": 8, - "y": 31 + "x": 16, + "y": 923 }, - "id": 88, + "id": 89, "options": { "legend": { "calcs": [ ], @@ -1348,6 +2324,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1355,14 +2332,26 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (rate(go_gc_duration_seconds_sum{container=\"bloom-gateway\"}[$__rate_interval]))\n/\nsum by (pod) (rate(go_gc_duration_seconds_count{container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "__auto", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" } ], - "title": "GC duration", + "title": "GC pauses", "type": "timeseries" }, { @@ -1383,6 +2372,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1393,9 +2383,6 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1416,8 +2403,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1425,17 +2411,17 @@ } ] }, - "unit": "s" + "unit": "binBps" }, "overrides": [ ] }, "gridPos": { "h": 7, - "w": 8, - "x": 16, - "y": 31 + "w": 12, + "x": 0, + "y": 930 }, - "id": 89, + "id": 84, "options": { "legend": { "calcs": [ ], @@ -1450,6 +2436,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1457,30 +2444,21 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "expr": "sum by(instance, pod) (rate(node_disk_read_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)", "hide": false, - "legendFormat": "__auto", + "instant": false, + "interval": "", + "legendFormat": "{{pod}}", "range": true, - "refId": "B" + "refId": "D" } ], - "title": "GC pauses", + "title": "Disk reads", "type": "timeseries" }, { "datasource": { + "default": false, "type": "prometheus", "uid": "${datasource}" }, @@ -1497,6 +2475,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1527,8 +2506,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1543,10 +2521,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 38 + "x": 12, + "y": 930 }, - "id": 84, + "id": 85, "options": { "legend": { "calcs": [ ], @@ -1561,6 +2539,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1568,20 +2547,20 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(instance, pod) (rate(node_disk_read_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)", + "expr": "sum by(instance, pod) (rate(node_disk_written_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)", "hide": false, "instant": false, - "interval": "", "legendFormat": "{{pod}}", "range": true, "refId": "D" } ], - "title": "Disk reads", + "title": "Disk writes", "type": "timeseries" }, { "datasource": { + "default": false, "type": "prometheus", "uid": "${datasource}" }, @@ -1598,6 +2577,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1617,7 +2597,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -1628,8 +2608,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1643,11 +2622,11 @@ }, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 38 + "w": 24, + "x": 0, + "y": 937 }, - "id": 85, + "id": 102, "options": { "legend": { "calcs": [ ], @@ -1662,6 +2641,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1669,15 +2649,29 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(instance, pod) (rate(node_disk_written_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)", + "expr": "sum(sum by (instance) (rate(node_disk_read_bytes_total[$__rate_interval]))\n+ on(instance) group_right() \n(count by (instance) (container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0))", "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "interval": "", + "legendFormat": "Reads", "range": true, "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(sum by(instance) (rate(node_disk_written_bytes_total[$__rate_interval]))\n+ on(instance) group_right() \n(count by(instance) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)) * -1", + "hide": false, + "instant": false, + "legendFormat": "Writes", + "range": true, + "refId": "A" } ], - "title": "Disk writes", + "title": "Disk reads/writes", "type": "timeseries" } ], @@ -1691,7 +2685,7 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 22 }, "id": 2, "panels": [ @@ -1712,6 +2706,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "none", @@ -1742,8 +2737,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1799,7 +2793,7 @@ "h": 7, "w": 12, "x": 0, - "y": 18 + "y": 600 }, "id": 13, "options": { @@ -1816,6 +2810,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1823,7 +2818,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code) (\n rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)", + "expr": "sum by (status_code) (\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\",job=\"$namespace/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -1851,6 +2846,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -1881,8 +2877,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1938,7 +2933,7 @@ "h": 7, "w": 12, "x": 12, - "y": 18 + "y": 600 }, "id": 86, "options": { @@ -1955,6 +2950,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1975,6 +2971,7 @@ }, { "datasource": { + "default": false, "type": "prometheus", "uid": "${datasource}" }, @@ -1991,6 +2988,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -2021,8 +3019,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2034,7 +3031,7 @@ "h": 7, "w": 12, "x": 0, - "y": 25 + "y": 698 }, "id": 14, "options": { @@ -2051,6 +3048,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -2090,6 +3088,19 @@ "legendFormat": "{{ route }} 99th percentile", "range": true, "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}))", + "hide": false, + "instant": false, + "legendFormat": "{{ route }} max", + "range": true, + "refId": "A" } ], "title": "Latency", @@ -2112,6 +3123,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -2142,8 +3154,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2155,7 +3166,7 @@ "h": 7, "w": 12, "x": 12, - "y": 25 + "y": 698 }, "id": 15, "options": { @@ -2172,6 +3183,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -2201,7 +3213,7 @@ "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 23 }, "id": 58, "panels": [ @@ -2223,6 +3235,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -2253,7 +3266,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -2265,7 +3279,7 @@ "h": 7, "w": 8, "x": 0, - "y": 11 + "y": 228 }, "id": 16, "options": { @@ -2282,6 +3296,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2344,6 +3359,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2374,7 +3390,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -2386,7 +3403,7 @@ "h": 7, "w": 8, "x": 8, - "y": 11 + "y": 228 }, "id": 17, "options": { @@ -2403,6 +3420,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2478,6 +3496,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2508,7 +3527,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -2520,7 +3540,7 @@ "h": 7, "w": 8, "x": 16, - "y": 11 + "y": 228 }, "id": 22, "options": { @@ -2537,6 +3557,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2566,7 +3587,7 @@ "h": 1, "w": 24, "x": 0, - "y": 19 + "y": 24 }, "id": 68, "panels": [ @@ -2588,6 +3609,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2618,7 +3640,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2634,7 +3657,7 @@ "h": 8, "w": 8, "x": 0, - "y": 12 + "y": 243 }, "id": 69, "options": { @@ -2651,6 +3674,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2712,6 +3736,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2742,7 +3767,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2758,7 +3784,7 @@ "h": 8, "w": 8, "x": 8, - "y": 12 + "y": 243 }, "id": 70, "options": { @@ -2775,6 +3801,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2836,6 +3863,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2866,7 +3894,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2881,7 +3910,7 @@ "h": 8, "w": 8, "x": 16, - "y": 12 + "y": 243 }, "id": 71, "options": { @@ -2898,6 +3927,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2927,6 +3957,107 @@ ], "title": "Tasks dequeued/processed", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "eps" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 251 + }, + "id": 105, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_gateway_process_duration_seconds_count{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (status)", + "instant": false, + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Iterations per second", + "type": "timeseries" } ], "targets": [ ], @@ -2939,21 +4070,21 @@ "h": 1, "w": 24, "x": 0, - "y": 20 + "y": 25 }, "id": 59, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] }, - "description": "", "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 335 }, "id": 19, "options": { @@ -2966,7 +4097,7 @@ "mode": "markdown" }, "panels": [ ], - "pluginVersion": "11.1.0-70005", + "pluginVersion": "11.4.0-77663", "targets": [ ], "title": "We cache bloom blocks in memory to prevent the gateway from hitting the object store too often", "transparent": true, @@ -2990,6 +4121,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3024,7 +4156,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3036,7 +4169,7 @@ "h": 7, "w": 12, "x": 0, - "y": 14 + "y": 336 }, "id": 20, "options": { @@ -3053,6 +4186,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3102,6 +4236,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3137,7 +4272,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3194,7 +4330,7 @@ "h": 7, "w": 12, "x": 12, - "y": 14 + "y": 336 }, "id": 83, "options": { @@ -3211,6 +4347,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3286,6 +4423,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "none", @@ -3321,7 +4459,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3364,7 +4503,7 @@ "h": 7, "w": 24, "x": 0, - "y": 21 + "y": 343 }, "id": 92, "options": { @@ -3381,6 +4520,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3388,7 +4528,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status) (\n rate(loki_bloom_blocks_cache_fetched_total{container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n rate(loki_bloom_blocks_cache_fetched_total{container=\"bloom-gateway\"}[$__rate_interval])\n)", + "expr": "sum by (status) (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -3417,6 +4557,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3451,7 +4592,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3476,7 +4618,7 @@ "h": 7, "w": 12, "x": 0, - "y": 28 + "y": 350 }, "id": 76, "options": { @@ -3493,6 +4635,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3541,6 +4684,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3575,7 +4719,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3600,254 +4745,9 @@ "h": 7, "w": 12, "x": 12, - "y": 28 - }, - "id": 21, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "metas fetch rate", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "blocks fetch rate", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "hide": false, - "instant": false, - "legendFormat": "p90 blocks size", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "hide": false, - "instant": false, - "legendFormat": "p90 metas size", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "hide": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "hide": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "hide": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "G" - } - ], - "title": "Bloom Store", - "type": "timeseries" - } - ], - "targets": [ ], - "title": "Blocks Cache", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 60, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 61, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "The gateway download bloom meta files and blocks from the object store.", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 2, - "x": 0, - "y": 15 - }, - "id": 24, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "---\n#### GCS\n", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 11, - "x": 2, - "y": 15 + "y": 350 }, - "id": 25, + "id": 21, "options": { "legend": { "calcs": [ ], @@ -3862,6 +4762,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3869,14 +4770,93 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", + "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, "instant": false, - "legendFormat": "{{operation}} {{status_code}}", + "legendFormat": "metas fetch rate", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "blocks fetch rate", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "p90 blocks size", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "p90 metas size", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "G" } ], - "title": "QPS", + "title": "Bloom Store", "type": "timeseries" }, { @@ -3897,6 +4877,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3907,6 +4888,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -3922,26 +4906,41 @@ "mode": "off" } }, + "fieldMinMax": false, "mappings": [ ], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, "unit": "none" }, - "overrides": [ ] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Size (.*)/" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] }, "gridPos": { "h": 7, - "w": 11, - "x": 13, - "y": 15 + "w": 12, + "x": 0, + "y": 357 }, - "id": 29, + "id": 101, "options": { "legend": { "calcs": [ ], @@ -3956,6 +4955,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3963,67 +4963,87 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", - "instant": false, - "legendFormat": "{{operation}} p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "sum by (job)(rate(loki_bloom_store_download_queue_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[1m]))", "hide": false, "instant": false, - "legendFormat": "{{operation}} p90", + "legendFormat": "Size", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "{{operation}} p50", - "range": true, - "refId": "C" } ], - "title": "Latency", + "title": "Block download queue", "type": "timeseries" - }, + } + ], + "targets": [ ], + "title": "Blocks Cache", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 60, + "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 604 + }, + "id": 61, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "", + "mode": "markdown" }, + "panels": [ ], + "pluginVersion": "11.4.0-77383", + "targets": [ ], + "title": "The gateway download bloom meta files and blocks from the object store.", + "transparent": true, + "type": "text" + }, + { "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, "gridPos": { "h": 7, "w": 2, "x": 0, - "y": 22 + "y": 629 }, - "id": 62, + "id": 24, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, - "content": "---\n#### S3\n", + "content": "---\n#### GCS\n", "mode": "markdown" }, "panels": [ ], - "pluginVersion": "11.1.0-70005", + "pluginVersion": "11.4.0-77383", "targets": [ ], + "title": "", "transparent": true, "type": "text" }, @@ -4045,6 +5065,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4087,9 +5108,9 @@ "h": 7, "w": 11, "x": 2, - "y": 22 + "y": 629 }, - "id": 63, + "id": 25, "options": { "legend": { "calcs": [ ], @@ -4104,6 +5125,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -4111,7 +5133,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", "instant": false, "legendFormat": "{{operation}} {{status_code}}", "range": true, @@ -4139,8 +5161,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 25, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4158,7 +5181,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -4181,9 +5204,9 @@ "h": 7, "w": 11, "x": 13, - "y": 22 + "y": 629 }, - "id": 64, + "id": 29, "options": { "legend": { "calcs": [ ], @@ -4198,6 +5221,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -4205,7 +5229,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "instant": false, "legendFormat": "{{operation}} p99", "range": true, @@ -4217,7 +5241,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -4230,7 +5254,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -4242,30 +5266,31 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, "gridPos": { "h": 7, "w": 2, "x": 0, - "y": 29 + "y": 636 }, - "id": 65, + "id": 62, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, - "content": "---\n#### Azure\nBlob Storage\n\n", + "content": "---\n#### S3\n", "mode": "markdown" }, "panels": [ ], - "pluginVersion": "11.1.0-70005", + "pluginVersion": "11.4.0-77383", "targets": [ ], + "title": "", "transparent": true, "type": "text" }, @@ -4287,6 +5312,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4329,9 +5355,9 @@ "h": 7, "w": 11, "x": 2, - "y": 29 + "y": 636 }, - "id": 66, + "id": 63, "options": { "legend": { "calcs": [ ], @@ -4346,6 +5372,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -4353,7 +5380,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", "instant": false, "legendFormat": "{{operation}} {{status_code}}", "range": true, @@ -4381,6 +5408,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4423,9 +5451,9 @@ "h": 7, "w": 11, "x": 13, - "y": 29 + "y": 636 }, - "id": 67, + "id": 64, "options": { "legend": { "calcs": [ ], @@ -4440,6 +5468,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -4447,7 +5476,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "instant": false, "legendFormat": "{{operation}} p99", "range": true, @@ -4459,7 +5488,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -4472,7 +5501,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -4482,26 +5511,40 @@ ], "title": "Latency", "type": "timeseries" - } - ], - "targets": [ ], - "title": "Object Store", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 77, - "panels": [ + }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 0, + "y": 643 + }, + "id": 65, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "---\n#### Azure\nBlob Storage\n\n", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77383", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -4516,8 +5559,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4526,9 +5570,6 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -4538,7 +5579,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -4550,23 +5591,20 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 + "h": 7, + "w": 11, + "x": 2, + "y": 643 }, - "id": 78, + "id": 66, "options": { "legend": { "calcs": [ ], @@ -4581,26 +5619,28 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "topk(3, sum by (tasks) (count_over_time({namespace=\"loki-dev-006\", container=\"bloom-gateway\"} |= \"process tasks with bounds\" | logfmt [5s])))", - "legendFormat": "{{tasks}}", - "queryType": "range", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", + "instant": false, + "legendFormat": "{{operation}} {{status_code}}", + "range": true, "refId": "A" } ], - "title": "Process tasks with bounds", + "title": "QPS", "type": "timeseries" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -4615,8 +5655,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4625,9 +5666,6 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -4637,7 +5675,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -4649,49 +5687,20 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "none" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "max", - "avg" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 + "h": 7, + "w": 11, + "x": 13, + "y": 643 }, - "id": 79, + "id": 67, "options": { "legend": { "calcs": [ ], @@ -4706,91 +5715,81 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(max_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"fetch blocks\" | logfmt | unwrap duration(duration) [$__auto]))", - "hide": false, - "legendFormat": "max", - "queryType": "range", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, "refId": "A" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(avg_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"fetch blocks\" | logfmt | unwrap duration(duration) [$__auto]))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, - "legendFormat": "avg", - "queryType": "range", + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, "refId": "B" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(avg_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"request unavailable blocks in the background\" | logfmt | missing > 0 | unwrap missing [$__auto]))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, - "legendFormat": "avg missing", - "queryType": "range", + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, "refId": "C" } ], - "title": "Download enqueue duration", + "title": "Latency", "type": "timeseries" - }, + } + ], + "targets": [ ], + "title": "Object Store", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 77, + "panels": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "fieldConfig": { "defaults": { - "color": { - "fixedColor": "green", - "mode": "palette-classic" - }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineWidth": 1, "scaleDistribution": { "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 100 - } - ] } }, "overrides": [ ] @@ -4799,145 +5798,127 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 412 }, "id": 80, "options": { - "barRadius": 0, - "barWidth": 0.96999999999999997, - "fullHighlight": false, - "groupWidth": 0.69999999999999996, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1.0000000000000001e-09 + }, "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "show": true + }, + "rowsFrame": { + "layout": "auto" }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", "tooltip": { - "maxHeight": 600, "mode": "single", - "sort": "none" + "showColorScale": false, + "yHistogram": false }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "yAxis": { + "axisPlacement": "left", + "reverse": false + } }, "panels": [ ], - "pluginVersion": "11.0.0-67814", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "sort_desc(topk(10, sum by (tasks) (count_over_time({namespace=\"loki-dev-006\", container=\"bloom-gateway\"} |= \"process tasks with bounds\" | logfmt [$__auto]))))", - "legendFormat": "", - "queryType": "instant", + "exemplar": false, + "expr": "increase(loki_bloom_gateway_dequeue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, "refId": "A" } ], - "title": "Tasks multiplexed", - "type": "barchart" + "title": "Dequeue duration", + "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] } }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Enqueue latency" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - } - ] + "overrides": [ ] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 412 }, - "id": 82, + "id": 106, "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1.0000000000000001e-09 + }, "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "show": true + }, + "rowsFrame": { + "layout": "auto" }, "tooltip": { - "maxHeight": 600, "mode": "single", - "sort": "none" + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4945,27 +5926,17 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_store_download_queue_enqueue_time_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", - "hide": false, - "legendFormat": "Enqueue latency", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_store_download_queue_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "expr": "increase(loki_bloom_gateway_tasks_dequeued_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])", + "format": "heatmap", "hide": false, - "legendFormat": "Size", + "instant": false, + "legendFormat": "__auto", "range": true, "refId": "B" } ], - "title": "Block download queue", - "type": "timeseries" + "title": "Dequeue count", + "type": "heatmap" } ], "targets": [ ], @@ -4973,6 +5944,7 @@ "type": "row" } ], + "preload": false, "refresh": "10s", "rows": [ ], "schemaVersion": 14, @@ -5058,7 +6030,6 @@ "from": "now-1h", "to": "now" }, - "timeRangeUpdatedDuringEditOrView": false, "timepicker": { "refresh_intervals": [ "5s", diff --git a/production/loki-mixin-compiled/dashboards/loki-bloom-compactor.json b/production/loki-mixin-compiled/dashboards/loki-bloom-build.json similarity index 71% rename from production/loki-mixin-compiled/dashboards/loki-bloom-compactor.json rename to production/loki-mixin-compiled/dashboards/loki-bloom-build.json index c365fab0a7e5..bf43f94a94aa 100644 --- a/production/loki-mixin-compiled/dashboards/loki-bloom-compactor.json +++ b/production/loki-mixin-compiled/dashboards/loki-bloom-build.json @@ -36,29 +36,6 @@ "title": "Overview", "type": "row" }, - { - "gridPos": { - "h": 8, - "w": 14, - "x": 0, - "y": 1 - }, - "id": 35, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "## About the Bloom Compactor\nThe compactor iterates through chunks and creates blooms out of them.\nThe size of the resulting blooms depends on the bloom filter settings, the tokenizer settings, the number of ring tokens per compactor and the total number opf compactors.\n\nCompactors are horizontally scalable and uses a ring to:\n- Shard tenants\n- Shard series fingerprints within a tenant subring.\n\nThe blooms for the series are grouped together in blocks which are flushed to object store.", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "transparent": true, - "type": "text" - }, { "datasource": { "type": "prometheus", @@ -77,6 +54,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 15, "gradientMode": "none", @@ -121,13 +99,30 @@ }, "unit": "percentunit" }, - "overrides": [ ] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(Planned|success|failure)/" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] }, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 9 + "y": 1 }, "id": 42, "options": { @@ -144,7 +139,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -152,41 +147,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"})\n/\nsum(count(loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}))", - "hide": false, - "instant": false, - "legendFormat": "avg", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.9, \n sum by (pod) (\n loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p90", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.1, \n sum by (pod) (\n loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n )\n)", + "expr": "sum(loki_bloomplanner_tenant_tasks_completed{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})\n/\nsum(loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})", "hide": false, "instant": false, - "legendFormat": "p10", + "legendFormat": "Progress", "range": true, - "refId": "C" + "refId": "D" } ], - "title": "Progress", + "title": "Overall progress", "type": "timeseries" }, { @@ -194,7 +163,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Uncompressed size of chunks in a series VS the size of the blooms built.", + "description": "Cell-wide compaction progress. Should increase till completion throughout each compaction period.", "fieldConfig": { "defaults": { "color": { @@ -207,8 +176,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 15, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -242,33 +212,29 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] }, - "unit": "bytes" + "unit": "percentunit" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "Ratio" + "id": "byRegexp", + "options": "/(Planned|success|failure)/" }, "properties": [ { "id": "unit", - "value": "percentunit" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Ratio over range" - }, - "properties": [ + "value": "none" + }, { - "id": "unit", - "value": "percentunit" + "id": "custom.fillOpacity", + "value": 0 } ] } @@ -278,9 +244,9 @@ "h": 7, "w": 12, "x": 12, - "y": 9 + "y": 1 }, - "id": 41, + "id": 116, "options": { "legend": { "calcs": [ ], @@ -295,7 +261,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -303,54 +269,158 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by (tenant) (loki_bloomplanner_tenant_tasks_completed{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})\n/\nsum by (tenant) (loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"})", "hide": false, "instant": false, - "legendFormat": "Bloom", + "legendFormat": "{{tenant}}", "range": true, - "refId": "A" - }, + "refId": "D" + } + ], + "title": "Progress by tenant", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "description": "Blooms size vs uncompressed chunk size.", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 17, + "x": 0, + "y": 8 + }, + "id": 51, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "type": "loki", + "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "Chunk", - "range": true, + "expr": "{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\"", + "queryType": "range", "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + } + ], + "title": "Errors Planner", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" }, - "editorMode": "code", - "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "Ratio", - "range": true, - "refId": "C" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 17, + "y": 8 + }, + "id": 53, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ { "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "type": "loki", + "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "Ratio over range", - "range": true, - "refId": "D" + "expr": "sum(count_over_time({cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\" [$__auto]))", + "legendFormat": "Error rate", + "queryType": "range", + "refId": "A" } ], - "title": "Chunks and Bloom size", + "title": "Errors Rate Planner", "type": "timeseries" }, { @@ -359,13 +429,17 @@ "uid": "${loki_datasource}" }, "description": "Blooms size vs uncompressed chunk size.", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, "gridPos": { "h": 7, "w": 17, "x": 0, - "y": 16 + "y": 15 }, - "id": 51, + "id": 133, "options": { "dedupStrategy": "none", "enableLogDetails": true, @@ -377,7 +451,7 @@ "wrapLogMessage": false }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -385,12 +459,12 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} |= \"level=error\" |= \"component=bloom-compactor\"", + "expr": "{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\"", "queryType": "range", "refId": "B" } ], - "title": "Errors", + "title": "Errors Builder", "type": "logs" }, { @@ -411,6 +485,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "bars", "fillOpacity": 0, "gradientMode": "none", @@ -457,9 +532,9 @@ "h": 7, "w": 7, "x": 17, - "y": 16 + "y": 15 }, - "id": 53, + "id": 134, "options": { "legend": { "calcs": [ ], @@ -474,7 +549,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -482,12 +557,13 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum(count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} |= \"level=error\" |= \"component=bloom-compactor\" [$__auto]))", + "expr": "sum(count_over_time({cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\" [$__auto]))", + "legendFormat": "Error rate", "queryType": "range", "refId": "A" } ], - "title": "Errors Rate", + "title": "Errors Rate Builder", "type": "timeseries" }, { @@ -496,7 +572,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 22 }, "id": 112, "panels": [ @@ -517,9 +593,11 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 15, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -551,7 +629,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -559,47 +638,1093 @@ } ] }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 114, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "none" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (pod) (\n loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n)", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(success|failure)/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 100 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Planned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 15 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Queued" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 125, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Planned", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Completed - {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "loki_bloomplanner_inflight_tasks{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", quantile=\"0.95\"}", + "hide": false, + "instant": false, + "legendFormat": "inflight p95", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_queue_length{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Queued", + "range": true, + "refId": "D" + } + ], + "title": "Tasks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(success|failure)/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 100 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Planned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 15 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Connected builders" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "IDLE Builders" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Builders processing task" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 126, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_connected_builders{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "hide": false, + "instant": false, + "legendFormat": "Connected builders", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(loki_bloombuilder_processing_task{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Builders processing task", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(loki_bloombuilder_processing_task{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} == 0)", + "hide": false, + "instant": false, + "legendFormat": "IDLE Builders", + "range": true, + "refId": "D" + } + ], + "title": "Tasks per builder", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 81, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.75, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p75", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + } + ], + "title": "Series per task (includes series copied from other blocks)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 91, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + } + ], + "title": "Number of bytes from chunks added to blocks during each compaction.", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 117, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Identify the tenant using the **_Progress by tenant_** panel from the overview and set tenant variable", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "Tip", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/(success|failure)/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 100 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Planned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 15 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Completed - failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Queued" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 81 + }, + "id": 114, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Planned", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Completed - {{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(loki_bloomplanner_queue_length{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", user=\"$tenant\"}) > 0", + "hide": false, + "instant": false, + "legendFormat": "Queued", + "range": true, + "refId": "C" + } + ], + "title": "Tasks per tenant", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 81 + }, + "id": 115, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${loki_datasource}" + }, + "editorMode": "code", + "expr": "{cluster=~\"$cluster\", job=~\"$namespace/bloom-planner\"}\n|= \"level=error\"\n|= \"tenant=$tenant\"", + "queryType": "range", + "refId": "B" + } + ], + "title": "Tenant errors", + "type": "logs" + } + ], + "targets": [ ], + "title": "Tasks", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 95, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many tokens each builder is appending to blooms. Accounts for tokens that are not actually added to the blooms since they are already there. See the panel on the right for a drill down on the collision.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 96, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_tokens_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}))", + "hide": false, + "instant": false, + "legendFormat": "Per core", + "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "C" } ], - "title": "Progress per pod", + "title": "Tokens rate", "type": "timeseries" }, { @@ -607,23 +1732,56 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Collision type may be `false` (no collision), `cache` (found in token cache) or true (found in bloom filter).\n\nType may be either `raw` (the original ngram) or `chunk_prefixed` (the ngram with the chunk prefix)", "fieldConfig": { "defaults": { - "fieldMinMax": false, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "mappings": [ ], "thresholds": { - "mode": "percentage", + "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "#EAB839", - "value": 0 + "color": "green", + "value": null }, { - "color": "green", - "value": 100 + "color": "red", + "value": 80 } ] }, @@ -635,26 +1793,24 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 90 }, - "id": 115, + "id": 97, "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showThresholdLabels": false, - "showThresholdMarkers": false, - "sizing": "auto" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } }, "panels": [ ], - "pluginVersion": "11.0.0-68102", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -662,7 +1818,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (\n loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n)", + "expr": "# tokens/s by type+collision\nsum by (collision) (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -670,55 +1826,15 @@ "refId": "B" } ], - "title": "Current Progress per pod", - "type": "gauge" - } - ], - "targets": [ ], - "title": "Progress per pod", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 56, - "panels": [ - { - "description": "", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 85, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "We use tenant sharding so each compactor will process a subset of the tenants.", - "transparent": true, - "type": "text" + "title": "tokens/s by collision type", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "Shows the expected number of cpu cores we need to provision to build blooms as fast as we ingest data so a compaction iteration doesn't take longer than the compaction interval.\n\nWe may decide to have more to speed up compaction.", + "description": "The sizes of the blooms created by the compactor. We build one bloom per series. The more unique ngrams and chunks the series has, the bigger their blooms will be.", "fieldConfig": { "defaults": { "color": { @@ -731,6 +1847,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -744,7 +1861,8 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "type": "linear" + "log": 2, + "type": "log" }, "showPoints": "auto", "spanNulls": false, @@ -763,19 +1881,24 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [ ] }, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 98 }, - "id": 94, + "id": 98, "options": { "legend": { "calcs": [ ], @@ -790,7 +1913,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -798,12 +1921,12 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n # rep factor\n 3 *\n sum(\n # 1 - dedupe_ratio\n 1 - \n sum(rate(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n /\n sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n )\n)\n/\n(\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n)", + "expr": "histogram_quantile(\n 1.0,\n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, - "legendFormat": "Needed", + "legendFormat": "max", "range": true, - "refId": "B" + "refId": "D" }, { "datasource": { @@ -811,15 +1934,28 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", + "expr": "histogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, - "legendFormat": "Available", + "legendFormat": "p99", "range": true, - "refId": "A" + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(\n 0.50, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "F" } ], - "title": "Required CPUs to not lag behind", + "title": "Bloom size", "type": "timeseries" }, { @@ -827,7 +1963,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "How many chunks are we indexing in the blooms. Either:\n- `copied` from a pre-existing bloom block, or \n- `iterated` through all its entries if processed for the first time.", "fieldConfig": { "defaults": { "color": { @@ -840,8 +1976,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 15, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -850,9 +1987,6 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -882,17 +2016,17 @@ } ] }, - "unit": "Bps" + "unit": "short" }, "overrides": [ ] }, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 98 }, - "id": 72, + "id": 99, "options": { "legend": { "calcs": [ ], @@ -907,7 +2041,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -915,35 +2049,38 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}[$__rate_interval])) by (pod)\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) by (pod)", - "hide": true, - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", + "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])) by (type)", "hide": false, "instant": false, - "legendFormat": "Total", + "legendFormat": "__auto", "range": true, "refId": "B" } ], - "title": "MB/s per core", + "title": "Chunks indexed", "type": "timeseries" - }, + } + ], + "targets": [ ], + "title": "Bloom building", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 56, + "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Shows the expected number of cpu cores we need to provision to build blooms as fast as we ingest data so a build iteration doesn't take longer than the build interval.\n\nWe may decide to have more to speed up building blooms.", "fieldConfig": { "defaults": { "color": { @@ -956,6 +2093,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -998,9 +2136,9 @@ "h": 7, "w": 12, "x": 0, - "y": 33 + "y": 2030 }, - "id": 1, + "id": 94, "options": { "legend": { "calcs": [ ], @@ -1015,7 +2153,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1023,10 +2161,10 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)", + "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n # rep factor\n 3 *\n sum(\n # 1 - dedupe_ratio\n 1 - \n sum(rate(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n /\n sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n )\n)\n/\n(\nsum(rate(loki_bloombuilder_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n)", "hide": false, "instant": false, - "legendFormat": "Request", + "legendFormat": "Needed", "range": true, "refId": "B" }, @@ -1036,22 +2174,10 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)", + "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "Limit", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.99,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)", - "instant": false, - "legendFormat": "p99", + "legendFormat": "Available", "range": true, "refId": "A" }, @@ -1061,28 +2187,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.50,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "avg(\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)\n*\ncount(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, - "legendFormat": "Avg", + "legendFormat": "Provisioned", "range": true, - "refId": "E" + "refId": "C" } ], - "title": "CPU", + "title": "Required CPUs to not lag behind", "type": "timeseries" }, { @@ -1090,6 +2203,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1102,8 +2216,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 15, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1112,6 +2227,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1140,7 +2258,8 @@ "value": 80 } ] - } + }, + "unit": "Bps" }, "overrides": [ ] }, @@ -1148,9 +2267,9 @@ "h": 7, "w": 12, "x": 12, - "y": 33 + "y": 2030 }, - "id": 75, + "id": 72, "options": { "legend": { "calcs": [ ], @@ -1165,7 +2284,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1173,40 +2292,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)", + "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloombuilder_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-builder\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "Request", + "legendFormat": "Total", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)", - "hide": false, - "instant": false, - "legendFormat": "Limit", - "range": true, - "refId": "C" } ], - "title": "CPU per pod", + "title": "MB/s per core", "type": "timeseries" }, { @@ -1226,6 +2320,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1258,14 +2353,9 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - }, - "unit": "bytes" + } }, "overrides": [ ] }, @@ -1273,9 +2363,9 @@ "h": 7, "w": 12, "x": 0, - "y": 40 + "y": 2037 }, - "id": 76, + "id": 1, "options": { "legend": { "calcs": [ ], @@ -1290,7 +2380,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1298,7 +2388,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, "legendFormat": "Request", @@ -1311,7 +2401,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"} > 0)", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, "legendFormat": "Limit", @@ -1324,7 +2414,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile (\n 0.99,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)", + "expr": "quantile(\n 0.99,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", "instant": false, "legendFormat": "p99", "range": true, @@ -1336,7 +2426,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile (\n 0.50,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)", + "expr": "quantile(\n 0.50,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "p50", @@ -1349,15 +2439,28 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)", + "expr": "avg(\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "Avg", "range": true, "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "Max", + "range": true, + "refId": "F" } ], - "title": "Memory (workingset)", + "title": "CPU", "type": "timeseries" }, { @@ -1377,6 +2480,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1415,43 +2519,17 @@ "value": 80 } ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "bloom-compactor-106" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] } - ] + }, + "overrides": [ ] }, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 40 + "y": 2037 }, - "id": 5, + "id": 75, "options": { "legend": { "calcs": [ ], @@ -1466,7 +2544,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1474,7 +2552,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"})", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{pod}}", "range": true, @@ -1486,7 +2564,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, "legendFormat": "Request", @@ -1499,7 +2577,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"} > 0)", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, "legendFormat": "Limit", @@ -1507,7 +2585,7 @@ "refId": "C" } ], - "title": "Memory per pod (workingset)", + "title": "CPU per pod", "type": "timeseries" }, { @@ -1515,7 +2593,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -1528,8 +2605,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 15, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1538,9 +2616,6 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1570,7 +2645,7 @@ } ] }, - "unit": "none" + "unit": "bytes" }, "overrides": [ ] }, @@ -1578,9 +2653,9 @@ "h": 7, "w": 12, "x": 0, - "y": 47 + "y": 2044 }, - "id": 27, + "id": 76, "options": { "legend": { "calcs": [ ], @@ -1595,7 +2670,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1603,140 +2678,86 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[10m]\n )\n) > 0", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"memory\"} > 0)", + "hide": false, "instant": false, - "legendFormat": "Restarts", + "legendFormat": "Request", "range": true, - "refId": "A" - } - ], - "title": "Container restarts", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "unit": "none" + "editorMode": "code", + "expr": "quantile (\n 0.99,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 77, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile (\n 0.50,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "E" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "panels": [ ], - "pluginVersion": "11.1.0-69868", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}\n )\n) > 0", + "expr": "max (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, "instant": false, - "legendFormat": "{{reason}} / {{pod}}", + "legendFormat": "Max", "range": true, - "refId": "A" + "refId": "F" } ], - "title": "Container restarts reason per pod", + "title": "Memory (workingset)", "type": "timeseries" - } - ], - "targets": [ ], - "title": "Resource Usage", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 95, - "panels": [ + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "How many tokens each compactor is appending to blooms. Accounts for tokens that are not actually added to the blooms since they are already there. See the panel on the right for a drill down on the collision.\n", "fieldConfig": { "defaults": { "color": { @@ -1749,6 +2770,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1762,8 +2784,7 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 2, - "type": "log" + "type": "linear" }, "showPoints": "auto", "spanNulls": false, @@ -1788,17 +2809,18 @@ "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [ ] }, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 0, - "y": 55 + "x": 12, + "y": 2044 }, - "id": 96, + "id": 5, "options": { "legend": { "calcs": [ ], @@ -1813,7 +2835,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1821,10 +2843,22 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# tokens checked per pod, millions/s\nsum(rate(loki_bloom_tokens_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}))\n/ 1e6", + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"})", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\", resource=\"memory\"} > 0)", "hide": false, "instant": false, - "legendFormat": "Per core", + "legendFormat": "Request", "range": true, "refId": "B" }, @@ -1834,15 +2868,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) / 1e6", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"} > 0)", "hide": false, "instant": false, - "legendFormat": "Total", + "legendFormat": "Limit", "range": true, "refId": "C" } ], - "title": "Tokens rate (millions)", + "title": "Memory per pod (workingset)", "type": "timeseries" }, { @@ -1850,7 +2884,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Collision type may be `false` (no collision), `cache` (found in token cache) or true (found in bloom filter).\n\nType may be either `raw` (the original ngram) or `chunk_prefixed` (the ngram with the chunk prefix)", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1863,8 +2897,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 15, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1873,6 +2908,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1902,17 +2940,17 @@ } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [ ] }, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 12, - "y": 55 + "x": 0, + "y": 2051 }, - "id": 97, + "id": 27, "options": { "legend": { "calcs": [ ], @@ -1927,7 +2965,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -1935,15 +2973,14 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# tokens/s by type+collision\nsum by (collision) (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)", - "hide": false, + "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n) > 0", "instant": false, - "legendFormat": "__auto", + "legendFormat": "Restarts", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "tokens/s by collision type", + "title": "Container restarts", "type": "timeseries" }, { @@ -1951,7 +2988,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "The sizes of the blooms created by the compactor. We build one bloom per series. The more unique ngrams and chunks the series has, the bigger their blooms will be.", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1964,8 +3001,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 15, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1974,6 +3012,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2003,17 +3044,17 @@ } ] }, - "unit": "bytes" + "unit": "none" }, "overrides": [ ] }, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 0, - "y": 63 + "x": 12, + "y": 2051 }, - "id": 98, + "id": 77, "options": { "legend": { "calcs": [ ], @@ -2028,7 +3069,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2036,49 +3077,36 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(\n 0.90, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p90", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(\n 0.50, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, + "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}\n )\n) > 0", "instant": false, - "legendFormat": "p50", + "legendFormat": "{{reason}} / {{pod}}", "range": true, - "refId": "F" + "refId": "A" } ], - "title": "Bloom size", + "title": "Container restarts reason per pod", "type": "timeseries" - }, + } + ], + "targets": [ ], + "title": "Builder Resource Usage", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 118, + "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "How many chunks are we indexing in the blooms. Either:\n- `copied` from a pre-existing bloom block, or \n- `iterated` through all its entries if processed for the first time.", "fieldConfig": { "defaults": { "color": { @@ -2091,6 +3119,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2123,24 +3152,19 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - }, - "unit": "none" + } }, "overrides": [ ] }, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 12, - "y": 63 + "x": 0, + "y": 2302 }, - "id": 99, + "id": 119, "options": { "legend": { "calcs": [ ], @@ -2155,7 +3179,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2163,32 +3187,68 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) by (type)", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, - "legendFormat": "__auto", + "legendFormat": "Request", "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile(\n 0.99,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval])\n)", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile(\n 0.50,\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "E" } ], - "title": "Chunks indexed", + "title": "CPU", "type": "timeseries" - } - ], - "targets": [ ], - "title": "Bloom building", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 103, - "panels": [ + }, { "datasource": { "type": "prometheus", @@ -2206,6 +3266,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2238,6 +3299,10 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] } @@ -2247,10 +3312,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 72 + "x": 12, + "y": 2302 }, - "id": 107, + "id": 120, "options": { "legend": { "calcs": [ ], @@ -2258,29 +3323,55 @@ "placement": "bottom", "showLegend": true }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "panels": [ ], - "targets": [ + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloomcompactor_blocks_created_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", + "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"cpu\"} > 0)", "hide": false, "instant": false, - "legendFormat": "Blocks", + "legendFormat": "Limit", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Created Blocks", + "title": "CPU per pod", "type": "timeseries" }, { @@ -2288,7 +3379,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.", "fieldConfig": { "defaults": { "color": { @@ -2301,6 +3391,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2333,19 +3424,24 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [ ] }, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 72 + "x": 0, + "y": 2309 }, - "id": 106, + "id": 121, "options": { "legend": { "calcs": [ ], @@ -2360,6 +3456,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2367,15 +3464,66 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloomcompactor_blocks_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"memory\"} > 0)", "hide": false, "instant": false, - "legendFormat": "Blocks", + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile (\n 0.99,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"}\n)", + "instant": false, + "legendFormat": "p99", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "quantile (\n 0.50,\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-builder\"}\n)", + "hide": false, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "E" } ], - "title": "Deleted Blocks", + "title": "Memory (workingset)", "type": "timeseries" }, { @@ -2383,7 +3531,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Number of overlapping bloom blocks reused when creating new blocks\n", "fieldConfig": { "defaults": { "color": { @@ -2396,6 +3543,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2428,19 +3576,24 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [ ] }, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 79 + "x": 12, + "y": 2309 }, - "id": 109, + "id": 122, "options": { "legend": { "calcs": [ ], @@ -2455,6 +3608,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2462,37 +3616,48 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloomcompactor_blocks_reused_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", - "hide": false, + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"})", "instant": false, - "legendFormat": "Blocks", + "legendFormat": "{{pod}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\", resource=\"memory\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Request", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-planner\"} > 0)", + "hide": false, + "instant": false, + "legendFormat": "Limit", + "range": true, + "refId": "C" } ], - "title": "Blocks reused", + "title": "Memory per pod (workingset)", "type": "timeseries" - } - ], - "targets": [ ], - "title": "Blocks building", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 110, - "panels": [ + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -2505,8 +3670,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 15, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2515,6 +3681,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2537,9 +3706,14 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [ ] }, @@ -2547,9 +3721,9 @@ "h": 7, "w": 12, "x": 0, - "y": 87 + "y": 2316 }, - "id": 108, + "id": 123, "options": { "legend": { "calcs": [ ], @@ -2564,7 +3738,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2572,15 +3746,14 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloomcompactor_metas_created_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", - "hide": false, + "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n) > 0", "instant": false, - "legendFormat": "Metas", + "legendFormat": "Restarts", "range": true, "refId": "A" } ], - "title": "Created Metas", + "title": "Container restarts", "type": "timeseries" }, { @@ -2588,7 +3761,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -2601,8 +3774,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 15, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2611,6 +3785,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2633,9 +3810,14 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [ ] }, @@ -2643,9 +3825,9 @@ "h": 7, "w": 12, "x": 12, - "y": 87 + "y": 2316 }, - "id": 105, + "id": 124, "options": { "legend": { "calcs": [ ], @@ -2660,7 +3842,7 @@ } }, "panels": [ ], - "pluginVersion": "11.1.0-69868", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2668,20 +3850,19 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloomcompactor_metas_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))", - "hide": false, + "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}\n )\n) > 0", "instant": false, - "legendFormat": "Metas", + "legendFormat": "{{reason}} / {{pod}}", "range": true, "refId": "A" } ], - "title": "Deleted Metas", + "title": "Container restarts reason per pod", "type": "timeseries" } ], "targets": [ ], - "title": "Metas building", + "title": "Planner Resource Usage", "type": "row" }, { @@ -2690,35 +3871,10 @@ "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 26 }, - "id": 80, + "id": 110, "panels": [ - { - "description": "", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 95 - }, - "id": 93, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "We use tenant sharding so each compactor will process a subset of the tenants.", - "transparent": true, - "type": "text" - }, { "datasource": { "type": "prometheus", @@ -2736,6 +3892,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2776,11 +3933,11 @@ }, "gridPos": { "h": 7, - "w": 12, + "w": 9, "x": 0, - "y": 96 + "y": 2497 }, - "id": 83, + "id": 108, "options": { "legend": { "calcs": [ ], @@ -2795,6 +3952,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2802,48 +3960,92 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.99,\n increase(\n loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.90,\n increase(\n loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", + "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p90", + "legendFormat": "Metas", "range": true, "refId": "A" + } + ], + "title": "Created Metas", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 9, + "y": 2497 + }, + "id": 140, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.50,\n increase(\n loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[30m]\n )\n)", + "exemplar": false, + "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", + "format": "table", "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "B" + "instant": true, + "legendFormat": "Metas", + "range": false, + "refId": "A" } ], - "title": "Tenants", - "type": "timeseries" + "title": "Created Metas", + "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.", "fieldConfig": { "defaults": { "color": { @@ -2856,6 +4058,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2898,9 +4101,9 @@ "h": 7, "w": 12, "x": 12, - "y": 96 + "y": 2497 }, - "id": 84, + "id": 105, "options": { "legend": { "calcs": [ ], @@ -2915,6 +4118,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2922,42 +4126,32 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (\n increase(\n loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", + "expr": "sum by (phase) (increase(loki_bloomplanner_metas_deleted_total{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "Deleted during {{phase}}", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "Tenants per pod", + "title": "Deleted Metas", "type": "timeseries" - }, - { - "description": "", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 103 - }, - "id": 86, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "Number of tenant tables processed. ", - "transparent": true, - "type": "text" - }, + } + ], + "targets": [ ], + "title": "Metas building", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 103, + "panels": [ { "datasource": { "type": "prometheus", @@ -2975,6 +4169,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3015,11 +4210,11 @@ }, "gridPos": { "h": 7, - "w": 12, + "w": 9, "x": 0, - "y": 104 + "y": 2505 }, - "id": 88, + "id": 107, "options": { "legend": { "calcs": [ ], @@ -3034,6 +4229,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3041,41 +4237,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.99,\n increase(\n loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.90,\n increase(\n loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", + "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p90", + "legendFormat": "Blocks", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.50,\n increase(\n loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "B" } ], - "title": "Tenant Tables", + "title": "Created Blocks", "type": "timeseries" }, { @@ -3086,39 +4256,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [ ], "thresholds": { @@ -3135,25 +4273,30 @@ }, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 104 + "w": 3, + "x": 9, + "y": 2505 }, - "id": 89, + "id": 139, "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3161,48 +4304,25 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (\n increase(\n loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n )\n)", + "exemplar": false, + "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", + "format": "time_series", "hide": false, - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "C" + "instant": true, + "legendFormat": "Blocks", + "range": false, + "refId": "A" } ], - "title": "Tenant Tables per pod", - "type": "timeseries" - }, - { - "description": "", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 111 - }, - "id": 87, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "Series per compaction (includes series copied from other blocks)", - "transparent": true, - "type": "text" + "title": "Created Blocks", + "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.", "fieldConfig": { "defaults": { "color": { @@ -3215,6 +4335,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3256,10 +4377,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 112 + "x": 12, + "y": 2505 }, - "id": 81, + "id": 106, "options": { "legend": { "calcs": [ ], @@ -3274,6 +4395,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3281,41 +4403,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p90", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", + "expr": "sum by (phase) (increase(loki_bloomplanner_blocks_deleted_total{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p50", + "legendFormat": "Deleted during {{phase}}", "range": true, "refId": "A" } ], - "title": "Series", + "title": "Deleted Blocks", "type": "timeseries" }, { @@ -3323,6 +4419,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Number of overlapping bloom blocks reused when creating new blocks\n", "fieldConfig": { "defaults": { "color": { @@ -3335,6 +4432,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3376,10 +4474,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 112 + "x": 0, + "y": 2512 }, - "id": 82, + "id": 109, "options": { "legend": { "calcs": [ ], @@ -3394,6 +4492,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3401,47 +4500,125 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (\n rate(loki_bloomcompactor_series_per_compaction_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n /\n rate(loki_bloomcompactor_series_per_compaction_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)", + "expr": "sum(increase(loki_bloombuilder_blocks_reused_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "Blocks", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "avg series per compaction by pod", + "title": "Blocks reused", "type": "timeseries" - }, + } + ], + "targets": [ ], + "title": "Blocks building", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 135, + "panels": [ { - "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Is the retention currently running?", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "fieldMinMax": false, + "mappings": [ + { + "options": { + "0": { + "color": "yellow", + "index": 0, + "text": "No" + }, + "1": { + "color": "green", + "index": 1, + "text": "Yes" + } + }, + "type": "value" + } + ], + "max": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ ] + }, "gridPos": { - "h": 1, - "w": 24, + "h": 7, + "w": 3, "x": 0, - "y": 119 + "y": 2573 }, - "id": 90, + "id": 136, "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "content": "", - "mode": "markdown" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "Number of bytes from chunks added to blocks during each compaction.", - "transparent": true, - "type": "text" + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Running now?", + "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Is the retention currently running?", "fieldConfig": { "defaults": { "color": { @@ -3453,7 +4630,10 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3479,7 +4659,9 @@ "mode": "off" } }, + "fieldMinMax": false, "mappings": [ ], + "max": 2, "thresholds": { "mode": "absolute", "steps": [ @@ -3489,17 +4671,17 @@ } ] }, - "unit": "bytes" + "unit": "bool_yes_no" }, "overrides": [ ] }, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 120 + "w": 9, + "x": 3, + "y": 2573 }, - "id": 91, + "id": 137, "options": { "legend": { "calcs": [ ], @@ -3508,12 +4690,12 @@ "showLegend": true }, "tooltip": { - "maxHeight": 600, "mode": "single", "sort": "none" } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3521,41 +4703,13 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p90", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", + "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Bytes", + "title": "Retention running", "type": "timeseries" }, { @@ -3563,6 +4717,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "How much time applying retention took", "fieldConfig": { "defaults": { "color": { @@ -3575,6 +4730,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3600,6 +4756,7 @@ "mode": "off" } }, + "fieldMinMax": false, "mappings": [ ], "thresholds": { "mode": "absolute", @@ -3610,7 +4767,7 @@ } ] }, - "unit": "bytes" + "unit": "dtdurations" }, "overrides": [ ] }, @@ -3618,9 +4775,9 @@ "h": 7, "w": 12, "x": 12, - "y": 120 + "y": 2573 }, - "id": 92, + "id": 138, "options": { "legend": { "calcs": [ ], @@ -3629,12 +4786,12 @@ "showLegend": true }, "tooltip": { - "maxHeight": 600, "mode": "single", "sort": "none" } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3642,20 +4799,18 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (\n rate(loki_bloomcompactor_bytes_per_compaction_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n /\n rate(loki_bloomcompactor_bytes_per_compaction_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "p99", + "expr": "histogram_quantile(0.9, \n sum by (status, le) (\n rate(loki_bloomplanner_retention_time_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval])\n )\n)", + "legendFormat": "__auto", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "avg bytes per compaction by pod", + "title": "Retention time", "type": "timeseries" } ], "targets": [ ], - "title": "Data processed", + "title": "Retention", "type": "row" }, { @@ -3666,7 +4821,7 @@ "x": 0, "y": 29 }, - "id": 58, + "id": 62, "panels": [ { "description": "", @@ -3675,23 +4830,52 @@ "overrides": [ ] }, "gridPos": { - "h": 3, + "h": 4, "w": 24, "x": 0, - "y": 82 + "y": 2581 + }, + "id": 71, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "During the planning phase, the planner downloads the metas and TSDBs to build the plan.\n\nOnce all blocks and metas are built, the builder flushes them to the object store.\n\nAfter each iteration, the planner deletes the metas and blocks marked for deletion in the tombstones.", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 13, + "w": 2, + "x": 0, + "y": 2585 }, - "id": 47, + "id": 63, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, - "content": "\nCompactors write blocks to the attached PVs before flushing them into the object store.\nIt also download chunks and index files.\n\nAfter compacting a given tenant, all the downloaded index files and chunks, as well as the already flushed blocks are deleted.", + "content": "---\n#### GCS\n", "mode": "markdown" }, "panels": [ ], - "pluginVersion": "11.1.0-69747", + "pluginVersion": "11.4.0-77663", "targets": [ ], "title": "", "transparent": true, @@ -3702,6 +4886,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3714,8 +4899,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3736,35 +4922,30 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "area" + "mode": "off" } }, "mappings": [ ], - "max": 1, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 0.80000000000000004 + "color": "green", + "value": null } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 85 + "h": 7, + "w": 11, + "x": 2, + "y": 2585 }, - "id": 9, + "id": 61, "options": { "legend": { "calcs": [ ], @@ -3779,6 +4960,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3786,40 +4968,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.99,\n max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n / \n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n ) \n and \n count by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n )\n)", - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.90,\n max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n / \n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n ) \n and \n count by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n )\n)", + "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p90", + "legendFormat": "{{operation}} {{status_code}}", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.50,\n max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n / \n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n ) \n and \n count by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "C" } ], - "title": "Disk Utilization", + "title": "QPS Planner", "type": "timeseries" }, { @@ -3827,6 +4984,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3839,8 +4997,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3861,35 +5020,30 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "area" + "mode": "off" } }, "mappings": [ ], - "max": 1, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 0.80000000000000004 + "color": "green", + "value": null } ] }, - "unit": "percentunit" + "unit": "s" }, "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 85 + "h": 7, + "w": 11, + "x": 13, + "y": 2585 }, - "id": 100, + "id": 64, "options": { "legend": { "calcs": [ ], @@ -3904,6 +5058,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3911,14 +5066,41 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"})", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "{{operation}} p99", "range": true, - "refId": "A" + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "F" } ], - "title": "Disk Utilization per pod", + "title": "Latency Planner", "type": "timeseries" }, { @@ -3926,6 +5108,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3938,8 +5121,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3968,25 +5152,22 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": null } ] }, - "unit": "bytes" + "unit": "none" }, "overrides": [ ] }, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 93 + "w": 11, + "x": 2, + "y": 2592 }, - "id": 7, + "id": 127, "options": { "legend": { "calcs": [ ], @@ -4001,6 +5182,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4008,40 +5190,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.99,\n sum by(instance, pod, device) (\n rate(node_disk_written_bytes_total[$__rate_interval])\n ) \n + ignoring(pod) group_right() \n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.90,\n sum by(instance, pod, device) (\n rate(node_disk_written_bytes_total[$__rate_interval])\n ) \n + ignoring(pod) group_right() \n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", + "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p90", + "legendFormat": "{{operation}} {{status_code}}", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.50,\n sum by(instance, pod, device) (\n rate(node_disk_written_bytes_total[$__rate_interval])\n ) \n + ignoring(pod) group_right() \n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "C" } ], - "title": "Disk Writes", + "title": "QPS Builder", "type": "timeseries" }, { @@ -4049,6 +5206,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -4061,8 +5219,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4091,25 +5250,22 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": null } ] }, - "unit": "bytes" + "unit": "s" }, "overrides": [ ] }, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 93 + "w": 11, + "x": 13, + "y": 2592 }, - "id": 101, + "id": 128, "options": { "legend": { "calcs": [ ], @@ -4117,35 +5273,93 @@ "placement": "bottom", "showLegend": true }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "panels": [ ], - "targets": [ + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, + "refId": "E" + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "{{operation}} p50", "range": true, - "refId": "A" + "refId": "F" } ], - "title": "Disk Writes per pod", + "title": "Latency Planner", "type": "timeseries" }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 14, + "w": 2, + "x": 0, + "y": 2598 + }, + "id": 65, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "---\n#### S3\n", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -4158,8 +5372,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4188,25 +5403,22 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": null } ] }, - "unit": "bytes" + "unit": "none" }, "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 100 + "h": 7, + "w": 11, + "x": 2, + "y": 2599 }, - "id": 8, + "id": 67, "options": { "legend": { "calcs": [ ], @@ -4221,6 +5433,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4228,40 +5441,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "quantile(\n 0.99,\n sum by(instance, pod, device) (\n rate(node_disk_read_bytes_total[$__rate_interval])\n ) + ignoring(pod) group_right()\n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", - "instant": false, - "legendFormat": "p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.90,\n sum by(instance, pod, device) (\n rate(node_disk_read_bytes_total[$__rate_interval])\n ) + ignoring(pod) group_right()\n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "p90", + "legendFormat": "{{operation}} {{status_code}}", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "quantile(\n 0.50,\n sum by(instance, pod, device) (\n rate(node_disk_read_bytes_total[$__rate_interval])\n ) + ignoring(pod) group_right()\n (\n label_replace(\n count by(instance, pod, device) (\n container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n ), \n \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n ) * 0\n )\n)", - "hide": false, - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "C" } ], - "title": "Disk Reads", + "title": "QPS Planner", "type": "timeseries" }, { @@ -4269,6 +5457,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -4281,8 +5470,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4311,25 +5501,22 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": null } ] }, - "unit": "bytes" + "unit": "s" }, "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 100 + "h": 7, + "w": 11, + "x": 13, + "y": 2599 }, - "id": 102, + "id": 69, "options": { "legend": { "calcs": [ ], @@ -4344,6 +5531,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4351,88 +5539,42 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "{{operation}} p90", "range": true, - "refId": "A" + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", job=\"\"} [$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, + "refId": "F" } ], - "title": "Disk Reads per pod", + "title": "Latency Planner", "type": "timeseries" - } - ], - "targets": [ ], - "title": "Disk Usage", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 62, - "panels": [ - { - "description": "", - "fieldConfig": { - "defaults": { }, - "overrides": [ ] - }, - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 83 - }, - "id": 71, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "Once all blocks and metas are built locally, the compactor flushes them to the object store.\n\nAfter each iteration, the compactor deletes the metas and blocks marked for deletion in the tombstones.", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-69747", - "targets": [ ], - "title": "", - "transparent": true, - "type": "text" - }, - { - "description": "", - "fieldConfig": { - "defaults": { }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 2, - "x": 0, - "y": 86 - }, - "id": 63, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "---\n#### GCS\n", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-69747", - "targets": [ ], - "title": "", - "transparent": true, - "type": "text" }, { "datasource": { @@ -4452,6 +5594,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4482,7 +5625,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -4494,9 +5638,9 @@ "h": 7, "w": 11, "x": 2, - "y": 86 + "y": 2606 }, - "id": 61, + "id": 129, "options": { "legend": { "calcs": [ ], @@ -4511,6 +5655,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4518,7 +5663,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -4526,7 +5671,7 @@ "refId": "B" } ], - "title": "QPS", + "title": "QPS Builder", "type": "timeseries" }, { @@ -4547,6 +5692,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4577,7 +5723,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -4589,9 +5736,9 @@ "h": 7, "w": 11, "x": 13, - "y": 86 + "y": 2606 }, - "id": 64, + "id": 130, "options": { "legend": { "calcs": [ ], @@ -4606,6 +5753,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4613,7 +5761,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -4626,7 +5774,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -4639,7 +5787,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -4647,7 +5795,7 @@ "refId": "F" } ], - "title": "Latency", + "title": "Latency Builder", "type": "timeseries" }, { @@ -4660,20 +5808,20 @@ "h": 7, "w": 2, "x": 0, - "y": 93 + "y": 2612 }, - "id": 65, + "id": 66, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, - "content": "---\n#### S3\n", + "content": "---\n#### Azure\nBlob Storage", "mode": "markdown" }, "panels": [ ], - "pluginVersion": "11.1.0-69747", + "pluginVersion": "11.4.0-77663", "targets": [ ], "title": "", "transparent": true, @@ -4697,6 +5845,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4727,7 +5876,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -4739,9 +5889,9 @@ "h": 7, "w": 11, "x": 2, - "y": 93 + "y": 2613 }, - "id": 67, + "id": 68, "options": { "legend": { "calcs": [ ], @@ -4756,6 +5906,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4763,7 +5914,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -4771,7 +5922,7 @@ "refId": "B" } ], - "title": "QPS", + "title": "QPS Planner", "type": "timeseries" }, { @@ -4792,6 +5943,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4822,7 +5974,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -4834,9 +5987,9 @@ "h": 7, "w": 11, "x": 13, - "y": 93 + "y": 2613 }, - "id": 69, + "id": 70, "options": { "legend": { "calcs": [ ], @@ -4851,6 +6004,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4858,7 +6012,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -4871,7 +6025,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -4884,7 +6038,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -4892,38 +6046,9 @@ "refId": "F" } ], - "title": "Latency", + "title": "Latency Planner", "type": "timeseries" }, - { - "description": "", - "fieldConfig": { - "defaults": { }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 2, - "x": 0, - "y": 100 - }, - "id": 66, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "---\n#### Azure\nBlob Storage", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-69747", - "targets": [ ], - "title": "", - "transparent": true, - "type": "text" - }, { "datasource": { "type": "prometheus", @@ -4942,6 +6067,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4972,7 +6098,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -4984,9 +6111,9 @@ "h": 7, "w": 11, "x": 2, - "y": 100 + "y": 2620 }, - "id": 68, + "id": 131, "options": { "legend": { "calcs": [ ], @@ -5001,6 +6128,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -5008,7 +6136,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -5016,7 +6144,7 @@ "refId": "B" } ], - "title": "QPS", + "title": "QPS Builder", "type": "timeseries" }, { @@ -5037,6 +6165,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -5067,7 +6196,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -5079,9 +6209,9 @@ "h": 7, "w": 11, "x": 13, - "y": 100 + "y": 2620 }, - "id": 70, + "id": 132, "options": { "legend": { "calcs": [ ], @@ -5096,6 +6226,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -5103,7 +6234,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -5116,7 +6247,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5129,7 +6260,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5137,7 +6268,7 @@ "refId": "F" } ], - "title": "Latency", + "title": "Latency Builder", "type": "timeseries" } ], @@ -5146,6 +6277,7 @@ "type": "row" } ], + "preload": false, "refresh": "10s", "rows": [ ], "schemaVersion": 14, @@ -5231,7 +6363,6 @@ "from": "now-1h", "to": "now" }, - "timeRangeUpdatedDuringEditOrView": false, "timepicker": { "refresh_intervals": [ "5s", @@ -5258,8 +6389,8 @@ ] }, "timezone": "utc", - "title": "Loki / Bloom Compactor", - "uid": "bloom-compactor", + "title": "Loki / Bloom Build", + "uid": "bloom-build", "version": 0, "weekStart": "" } \ No newline at end of file diff --git a/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json b/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json index 2d5e16a9d7e0..7aa907976c80 100644 --- a/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json +++ b/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json @@ -21,7 +21,6 @@ "type": "dashboards" } ], - "liveNow": false, "panels": [ { "collapsed": false, @@ -46,8 +45,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds", - "seriesBy": "last" + "mode": "thresholds" }, "custom": { "axisBorderShow": false, @@ -56,9 +54,10 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, - "gradientMode": "none", + "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, @@ -69,7 +68,7 @@ "lineStyle": { "fill": "solid" }, - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -114,7 +113,7 @@ }, "gridPos": { "h": 6, - "w": 6, + "w": 12, "x": 0, "y": 1 }, @@ -133,6 +132,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -140,7 +140,8 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "hide": false, "instant": false, "legendFormat": "Chunks", "range": true, @@ -153,7 +154,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Series", @@ -161,7 +162,7 @@ "refId": "B" } ], - "title": "Filter ratio", + "title": "Filter ratio - Bloom Gateway (server)", "type": "timeseries" }, { @@ -206,7 +207,7 @@ "gridPos": { "h": 6, "w": 6, - "x": 6, + "x": 12, "y": 1 }, "id": 75, @@ -226,7 +227,7 @@ "sizing": "auto" }, "panels": [ ], - "pluginVersion": "11.1.0-70005", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -235,7 +236,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(increase(loki_bloom_gateway_filtered_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", "instant": true, "legendFormat": "Chunks", "range": false, @@ -248,40 +249,12 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(increase(loki_bloom_gateway_filtered_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", "hide": false, "instant": true, "legendFormat": "Series", "range": false, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"})\n/\nsum(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"})", - "hide": true, - "instant": true, - "legendFormat": "Chunks avg", - "range": false, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"})\n/\nsum(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"})", - "hide": true, - "instant": true, - "legendFormat": "Series avg", - "range": false, - "refId": "D" } ], "title": "Filter ratio", @@ -305,6 +278,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -381,7 +355,7 @@ "gridPos": { "h": 6, "w": 6, - "x": 12, + "x": 18, "y": 1 }, "id": 72, @@ -399,6 +373,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -413,19 +388,6 @@ "range": true, "refId": "D" }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(kube_pod_container_status_ready{container=\"bloom-gateway\", cluster=\"$cluster\", namespace=\"$namespace\"})", - "hide": true, - "instant": false, - "legendFormat": "Running", - "range": true, - "refId": "A" - }, { "datasource": { "type": "prometheus", @@ -438,19 +400,6 @@ "legendFormat": "Desired", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "kube_statefulset_status_replicas_available{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", - "hide": true, - "instant": false, - "legendFormat": "Available", - "range": true, - "refId": "C" } ], "title": "Readiness", @@ -461,11 +410,11 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Percentage of chunks that are filtered by using bloom filters", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, "custom": { "axisBorderShow": false, @@ -474,9 +423,10 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 50, - "gradientMode": "none", + "fillOpacity": 0, + "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, @@ -499,34 +449,44 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "off" + "mode": "area" } }, "mappings": [ ], + "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null }, { - "color": "red", - "value": 80 + "color": "orange", + "value": 0.5 + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "green", + "value": 0.90000000000000002 } ] }, - "unit": "none" + "unit": "percentunit" }, "overrides": [ ] }, "gridPos": { "h": 6, - "w": 6, - "x": 18, - "y": 1 + "w": 12, + "x": 0, + "y": 7 }, - "id": 37, + "id": 93, "options": { "legend": { "calcs": [ ], @@ -541,6 +501,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -548,77 +509,135 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n * on (pod) group_left\n sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0", + "expr": "sum(rate(loki_bloom_gateway_querier_chunks_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "Chunks", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(loki_bloom_gateway_querier_series_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, - "interval": "", - "legendFormat": "{{pod}} ({{reason}})", + "legendFormat": "Series", "range": true, - "refId": "C" + "refId": "B" } ], - "title": "Container restarts", + "title": "Filter ratio - Index Gateway (client)", "type": "timeseries" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Percentage of chunks that are filtered by using bloom filters", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "green", + "value": 0.90000000000000002 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, "gridPos": { - "h": 9, - "w": 15, - "x": 0, + "h": 6, + "w": 6, + "x": 12, "y": 7 }, - "id": 48, + "id": 94, "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": true + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt", - "queryType": "range", + "exemplar": false, + "expr": "sum(increase(loki_bloom_gateway_querier_chunks_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", + "instant": true, + "legendFormat": "Chunks", + "range": false, "refId": "A" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt", - "hide": true, - "queryType": "range", + "exemplar": false, + "expr": "sum(increase(loki_bloom_gateway_querier_series_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", + "hide": false, + "instant": true, + "legendFormat": "Series", + "range": false, "refId": "B" } ], - "title": "Errors", - "type": "logs" + "title": "Filter ratio", + "type": "gauge" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", - "mode": "fixed" + "mode": "palette-classic" }, "custom": { "axisBorderShow": false, @@ -627,8 +646,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, "gradientMode": "none", "hideFrom": { "legend": false, @@ -637,11 +657,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 2, - "type": "symlog" + "type": "linear" }, "showPoints": "auto", "spanNulls": false, @@ -663,66 +685,21 @@ }, { "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "warn" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "error" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } + "value": 80 } ] }, - { - "matcher": { - "id": "byName", - "options": "panic" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-red", - "mode": "fixed" - } - } - ] - } - ] + "unit": "none" + }, + "overrides": [ ] }, "gridPos": { - "h": 9, - "w": 9, - "x": 15, + "h": 6, + "w": 6, + "x": 18, "y": 7 }, - "id": 52, + "id": 37, "options": { "legend": { "calcs": [ ], @@ -737,44 +714,936 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" - }, - "editorMode": "code", - "expr": "sum by (level) (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))", - "legendFormat": "{{ level }}", - "queryType": "range", - "refId": "A" - }, - { - "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))", + "expr": "(\n max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n * on (pod) group_left\n sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0", "hide": false, - "legendFormat": "panic", - "queryType": "range", - "refId": "B" + "instant": false, + "interval": "", + "legendFormat": "{{pod}} ({{reason}})", + "range": true, + "refId": "C" } ], - "title": "Errors Rate", + "title": "Container restarts", "type": "timeseries" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "id": 56, - "panels": [ + "description": "Percentage of chunks that are filtered by using bloom filters", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "green", + "value": 0.90000000000000002 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 99, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n sum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n -\n sum(rate(loki_index_gateway_postfilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n)\n/\nsum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)", + "instant": false, + "legendFormat": "chunks {{ route}}", + "range": true, + "refId": "A" + } + ], + "title": "Filter ratio - Index Gateway by route", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.10000000000000001 + }, + { + "color": "#EF843C", + "value": 0.25 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 100, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(loki_bloom_gateway_querier_series_skipped_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "series", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(loki_bloom_gateway_querier_chunks_skipped_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "chunks", + "range": true, + "refId": "B" + } + ], + "title": "Data skipped because they don't match any bocks", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 96, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 467 + }, + "id": 97, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77450", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Found/Skipped/Missing chunks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 467 + }, + "id": 98, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77450", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Filtered chunks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 481 + }, + "id": 107, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77450", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Found/Skipped/Missing series", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 481 + }, + "id": 108, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77450", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000134" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Filtered series", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Bloom Recorder", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 95, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 9, + "w": 15, + "x": 0, + "y": 598 + }, + "id": 48, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "panels": [ ], + "pluginVersion": "11.4.0-77383", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "editorMode": "code", + "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "editorMode": "code", + "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt", + "hide": true, + "queryType": "range", + "refId": "B" + } + ], + "title": "Errors", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "symlog" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "panic" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 15, + "y": 598 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77383", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "editorMode": "code", + "expr": "sum by (level) (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))", + "legendFormat": "{{ level }}", + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "${logs}" + }, + "editorMode": "code", + "expr": "sum (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))", + "hide": false, + "legendFormat": "panic", + "queryType": "range", + "refId": "B" + } + ], + "title": "Errors Rate", + "type": "timeseries" + } + ], + "targets": [ ], + "title": "Logs", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 56, + "panels": [ { "datasource": { "type": "prometheus", @@ -792,6 +1661,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -822,8 +1692,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -838,7 +1707,7 @@ "h": 14, "w": 12, "x": 0, - "y": 17 + "y": 909 }, "id": 10, "options": { @@ -855,6 +1724,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -915,6 +1785,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -945,8 +1816,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -962,7 +1832,7 @@ "h": 7, "w": 12, "x": 12, - "y": 17 + "y": 909 }, "id": 11, "options": { @@ -979,6 +1849,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1040,6 +1911,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1070,8 +1942,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1087,7 +1958,7 @@ "h": 7, "w": 12, "x": 12, - "y": 24 + "y": 916 }, "id": 81, "options": { @@ -1104,6 +1975,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1179,6 +2051,110 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 923 + }, + "id": 87, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77383", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "GC rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1212,25 +2188,25 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [ ] }, "gridPos": { "h": 7, "w": 8, - "x": 0, - "y": 31 + "x": 8, + "y": 923 }, - "id": 87, + "id": 88, "options": { "legend": { "calcs": [ ], @@ -1245,6 +2221,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1252,15 +2229,14 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum by (pod) (rate(go_gc_duration_seconds_sum{container=\"bloom-gateway\"}[$__rate_interval]))\n/\nsum by (pod) (rate(go_gc_duration_seconds_count{container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, - "instant": false, "legendFormat": "__auto", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "GC rate", + "title": "GC duration", "type": "timeseries" }, { @@ -1281,6 +2257,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1314,8 +2291,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1330,10 +2306,10 @@ "gridPos": { "h": 7, "w": 8, - "x": 8, - "y": 31 + "x": 16, + "y": 923 }, - "id": 88, + "id": 89, "options": { "legend": { "calcs": [ ], @@ -1348,6 +2324,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1355,14 +2332,26 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (rate(go_gc_duration_seconds_sum{container=\"bloom-gateway\"}[$__rate_interval]))\n/\nsum by (pod) (rate(go_gc_duration_seconds_count{container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "__auto", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" } ], - "title": "GC duration", + "title": "GC pauses", "type": "timeseries" }, { @@ -1383,6 +2372,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1393,9 +2383,6 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1416,8 +2403,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1425,17 +2411,17 @@ } ] }, - "unit": "s" + "unit": "binBps" }, "overrides": [ ] }, "gridPos": { "h": 7, - "w": 8, - "x": 16, - "y": 31 + "w": 12, + "x": 0, + "y": 930 }, - "id": 89, + "id": 84, "options": { "legend": { "calcs": [ ], @@ -1450,6 +2436,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1457,30 +2444,21 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "expr": "sum by(instance, pod) (rate(node_disk_read_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)", "hide": false, - "legendFormat": "__auto", + "instant": false, + "interval": "", + "legendFormat": "{{pod}}", "range": true, - "refId": "B" + "refId": "D" } ], - "title": "GC pauses", + "title": "Disk reads", "type": "timeseries" }, { "datasource": { + "default": false, "type": "prometheus", "uid": "${datasource}" }, @@ -1497,6 +2475,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1527,8 +2506,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1543,10 +2521,10 @@ "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 38 + "x": 12, + "y": 930 }, - "id": 84, + "id": 85, "options": { "legend": { "calcs": [ ], @@ -1561,6 +2539,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1568,20 +2547,20 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(instance, pod) (rate(node_disk_read_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)", + "expr": "sum by(instance, pod) (rate(node_disk_written_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)", "hide": false, "instant": false, - "interval": "", "legendFormat": "{{pod}}", "range": true, "refId": "D" } ], - "title": "Disk reads", + "title": "Disk writes", "type": "timeseries" }, { "datasource": { + "default": false, "type": "prometheus", "uid": "${datasource}" }, @@ -1598,6 +2577,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -1617,7 +2597,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -1628,8 +2608,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1643,11 +2622,11 @@ }, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 38 + "w": 24, + "x": 0, + "y": 937 }, - "id": 85, + "id": 102, "options": { "legend": { "calcs": [ ], @@ -1662,6 +2641,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1669,15 +2649,29 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(instance, pod) (rate(node_disk_written_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)", + "expr": "sum(sum by (instance) (rate(node_disk_read_bytes_total[$__rate_interval]))\n+ on(instance) group_right() \n(count by (instance) (container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0))", "hide": false, "instant": false, - "legendFormat": "{{pod}}", + "interval": "", + "legendFormat": "Reads", "range": true, "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(sum by(instance) (rate(node_disk_written_bytes_total[$__rate_interval]))\n+ on(instance) group_right() \n(count by(instance) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)) * -1", + "hide": false, + "instant": false, + "legendFormat": "Writes", + "range": true, + "refId": "A" } ], - "title": "Disk writes", + "title": "Disk reads/writes", "type": "timeseries" } ], @@ -1691,7 +2685,7 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 22 }, "id": 2, "panels": [ @@ -1712,6 +2706,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "none", @@ -1742,8 +2737,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1799,7 +2793,7 @@ "h": 7, "w": 12, "x": 0, - "y": 18 + "y": 600 }, "id": 13, "options": { @@ -1816,6 +2810,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1823,7 +2818,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code) (\n rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)", + "expr": "sum by (status_code) (\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\",job=\"$namespace/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -1851,6 +2846,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", @@ -1881,8 +2877,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1938,7 +2933,7 @@ "h": 7, "w": 12, "x": 12, - "y": 18 + "y": 600 }, "id": 86, "options": { @@ -1955,6 +2950,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -1975,6 +2971,7 @@ }, { "datasource": { + "default": false, "type": "prometheus", "uid": "${datasource}" }, @@ -1991,6 +2988,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -2021,8 +3019,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2034,7 +3031,7 @@ "h": 7, "w": 12, "x": 0, - "y": 25 + "y": 698 }, "id": 14, "options": { @@ -2051,6 +3048,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -2090,6 +3088,19 @@ "legendFormat": "{{ route }} 99th percentile", "range": true, "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}))", + "hide": false, + "instant": false, + "legendFormat": "{{ route }} max", + "range": true, + "refId": "A" } ], "title": "Latency", @@ -2112,6 +3123,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -2142,8 +3154,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2155,7 +3166,7 @@ "h": 7, "w": 12, "x": 12, - "y": 25 + "y": 698 }, "id": 15, "options": { @@ -2172,6 +3183,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -2201,7 +3213,7 @@ "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 23 }, "id": 58, "panels": [ @@ -2223,6 +3235,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -2253,7 +3266,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -2265,7 +3279,7 @@ "h": 7, "w": 8, "x": 0, - "y": 11 + "y": 228 }, "id": 16, "options": { @@ -2282,6 +3296,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2344,6 +3359,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2374,7 +3390,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -2386,7 +3403,7 @@ "h": 7, "w": 8, "x": 8, - "y": 11 + "y": 228 }, "id": 17, "options": { @@ -2403,6 +3420,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2478,6 +3496,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2508,7 +3527,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -2520,7 +3540,7 @@ "h": 7, "w": 8, "x": 16, - "y": 11 + "y": 228 }, "id": 22, "options": { @@ -2537,6 +3557,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2566,7 +3587,7 @@ "h": 1, "w": 24, "x": 0, - "y": 19 + "y": 24 }, "id": 68, "panels": [ @@ -2588,6 +3609,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2618,7 +3640,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2634,7 +3657,7 @@ "h": 8, "w": 8, "x": 0, - "y": 12 + "y": 243 }, "id": 69, "options": { @@ -2651,6 +3674,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2712,6 +3736,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2742,7 +3767,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2758,7 +3784,7 @@ "h": 8, "w": 8, "x": 8, - "y": 12 + "y": 243 }, "id": 70, "options": { @@ -2775,6 +3801,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2836,6 +3863,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2866,7 +3894,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2881,7 +3910,7 @@ "h": 8, "w": 8, "x": 16, - "y": 12 + "y": 243 }, "id": 71, "options": { @@ -2898,6 +3927,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -2927,6 +3957,107 @@ ], "title": "Tasks dequeued/processed", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "eps" + }, + "overrides": [ ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 251 + }, + "id": 105, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "panels": [ ], + "pluginVersion": "11.4.0-77663", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_gateway_process_duration_seconds_count{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (status)", + "instant": false, + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Iterations per second", + "type": "timeseries" } ], "targets": [ ], @@ -2939,21 +4070,21 @@ "h": 1, "w": 24, "x": 0, - "y": 20 + "y": 25 }, "id": 59, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] }, - "description": "", "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 335 }, "id": 19, "options": { @@ -2966,7 +4097,7 @@ "mode": "markdown" }, "panels": [ ], - "pluginVersion": "11.1.0-70005", + "pluginVersion": "11.4.0-77663", "targets": [ ], "title": "We cache bloom blocks in memory to prevent the gateway from hitting the object store too often", "transparent": true, @@ -2990,6 +4121,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3024,7 +4156,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3036,7 +4169,7 @@ "h": 7, "w": 12, "x": 0, - "y": 14 + "y": 336 }, "id": 20, "options": { @@ -3053,6 +4186,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3102,6 +4236,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3137,7 +4272,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3194,7 +4330,7 @@ "h": 7, "w": 12, "x": 12, - "y": 14 + "y": 336 }, "id": 83, "options": { @@ -3211,6 +4347,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3286,6 +4423,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "none", @@ -3321,7 +4459,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3364,7 +4503,7 @@ "h": 7, "w": 24, "x": 0, - "y": 21 + "y": 343 }, "id": 92, "options": { @@ -3381,6 +4520,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3388,7 +4528,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status) (\n rate(loki_bloom_blocks_cache_fetched_total{container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n rate(loki_bloom_blocks_cache_fetched_total{container=\"bloom-gateway\"}[$__rate_interval])\n)", + "expr": "sum by (status) (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -3417,6 +4557,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3451,7 +4592,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3476,7 +4618,7 @@ "h": 7, "w": 12, "x": 0, - "y": 28 + "y": 350 }, "id": 76, "options": { @@ -3493,6 +4635,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3541,6 +4684,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3575,7 +4719,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -3600,254 +4745,9 @@ "h": 7, "w": 12, "x": 12, - "y": 28 - }, - "id": 21, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "panels": [ ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "metas fetch rate", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "blocks fetch rate", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "hide": false, - "instant": false, - "legendFormat": "p90 blocks size", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "hide": false, - "instant": false, - "legendFormat": "p90 metas size", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "hide": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "hide": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "hide": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "G" - } - ], - "title": "Bloom Store", - "type": "timeseries" - } - ], - "targets": [ ], - "title": "Blocks Cache", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 60, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 61, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "title": "The gateway download bloom meta files and blocks from the object store.", - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "gridPos": { - "h": 7, - "w": 2, - "x": 0, - "y": 15 - }, - "id": 24, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "---\n#### GCS\n", - "mode": "markdown" - }, - "panels": [ ], - "pluginVersion": "11.1.0-70005", - "targets": [ ], - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 7, - "w": 11, - "x": 2, - "y": 15 + "y": 350 }, - "id": 25, + "id": 21, "options": { "legend": { "calcs": [ ], @@ -3862,6 +4762,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3869,14 +4770,93 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", + "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, "instant": false, - "legendFormat": "{{operation}} {{status_code}}", + "legendFormat": "metas fetch rate", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "blocks fetch rate", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "p90 blocks size", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "p90 metas size", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "G" } ], - "title": "QPS", + "title": "Bloom Store", "type": "timeseries" }, { @@ -3897,6 +4877,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3907,6 +4888,9 @@ }, "insertNulls": false, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -3922,26 +4906,41 @@ "mode": "off" } }, + "fieldMinMax": false, "mappings": [ ], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, "unit": "none" }, - "overrides": [ ] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Size (.*)/" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] }, "gridPos": { "h": 7, - "w": 11, - "x": 13, - "y": 15 + "w": 12, + "x": 0, + "y": 357 }, - "id": 29, + "id": 101, "options": { "legend": { "calcs": [ ], @@ -3956,6 +4955,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -3963,67 +4963,87 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", - "instant": false, - "legendFormat": "{{operation}} p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "sum by (job)(rate(loki_bloom_store_download_queue_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[1m]))", "hide": false, "instant": false, - "legendFormat": "{{operation}} p90", + "legendFormat": "Size", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "{{operation}} p50", - "range": true, - "refId": "C" } ], - "title": "Latency", + "title": "Block download queue", "type": "timeseries" - }, + } + ], + "targets": [ ], + "title": "Blocks Cache", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 60, + "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 604 + }, + "id": 61, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "", + "mode": "markdown" }, + "panels": [ ], + "pluginVersion": "11.4.0-77383", + "targets": [ ], + "title": "The gateway download bloom meta files and blocks from the object store.", + "transparent": true, + "type": "text" + }, + { "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, "gridPos": { "h": 7, "w": 2, "x": 0, - "y": 22 + "y": 629 }, - "id": 62, + "id": 24, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, - "content": "---\n#### S3\n", + "content": "---\n#### GCS\n", "mode": "markdown" }, "panels": [ ], - "pluginVersion": "11.1.0-70005", + "pluginVersion": "11.4.0-77383", "targets": [ ], + "title": "", "transparent": true, "type": "text" }, @@ -4045,6 +5065,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4087,9 +5108,9 @@ "h": 7, "w": 11, "x": 2, - "y": 22 + "y": 629 }, - "id": 63, + "id": 25, "options": { "legend": { "calcs": [ ], @@ -4104,6 +5125,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -4111,7 +5133,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", "instant": false, "legendFormat": "{{operation}} {{status_code}}", "range": true, @@ -4139,8 +5161,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 25, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4158,7 +5181,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -4181,9 +5204,9 @@ "h": 7, "w": 11, "x": 13, - "y": 22 + "y": 629 }, - "id": 64, + "id": 29, "options": { "legend": { "calcs": [ ], @@ -4198,6 +5221,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -4205,7 +5229,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "instant": false, "legendFormat": "{{operation}} p99", "range": true, @@ -4217,7 +5241,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -4230,7 +5254,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -4242,30 +5266,31 @@ "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, "gridPos": { "h": 7, "w": 2, "x": 0, - "y": 29 + "y": 636 }, - "id": 65, + "id": 62, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, - "content": "---\n#### Azure\nBlob Storage\n\n", + "content": "---\n#### S3\n", "mode": "markdown" }, "panels": [ ], - "pluginVersion": "11.1.0-70005", + "pluginVersion": "11.4.0-77383", "targets": [ ], + "title": "", "transparent": true, "type": "text" }, @@ -4287,6 +5312,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4329,9 +5355,9 @@ "h": 7, "w": 11, "x": 2, - "y": 29 + "y": 636 }, - "id": 66, + "id": 63, "options": { "legend": { "calcs": [ ], @@ -4346,6 +5372,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -4353,7 +5380,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", "instant": false, "legendFormat": "{{operation}} {{status_code}}", "range": true, @@ -4381,6 +5408,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -4423,9 +5451,9 @@ "h": 7, "w": 11, "x": 13, - "y": 29 + "y": 636 }, - "id": 67, + "id": 64, "options": { "legend": { "calcs": [ ], @@ -4440,6 +5468,7 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { @@ -4447,7 +5476,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "instant": false, "legendFormat": "{{operation}} p99", "range": true, @@ -4459,7 +5488,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -4472,7 +5501,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -4482,26 +5511,40 @@ ], "title": "Latency", "type": "timeseries" - } - ], - "targets": [ ], - "title": "Object Store", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 77, - "panels": [ + }, + { + "description": "", + "fieldConfig": { + "defaults": { }, + "overrides": [ ] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 0, + "y": 643 + }, + "id": 65, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "---\n#### Azure\nBlob Storage\n\n", + "mode": "markdown" + }, + "panels": [ ], + "pluginVersion": "11.4.0-77383", + "targets": [ ], + "title": "", + "transparent": true, + "type": "text" + }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -4516,8 +5559,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4526,9 +5570,6 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -4538,7 +5579,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -4550,23 +5591,20 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 + "h": 7, + "w": 11, + "x": 2, + "y": 643 }, - "id": 78, + "id": 66, "options": { "legend": { "calcs": [ ], @@ -4581,26 +5619,28 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "topk(3, sum by (tasks) (count_over_time({namespace=\"loki-dev-006\", container=\"bloom-gateway\"} |= \"process tasks with bounds\" | logfmt [5s])))", - "legendFormat": "{{tasks}}", - "queryType": "range", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))", + "instant": false, + "legendFormat": "{{operation}} {{status_code}}", + "range": true, "refId": "A" } ], - "title": "Process tasks with bounds", + "title": "QPS", "type": "timeseries" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "description": "", "fieldConfig": { @@ -4615,8 +5655,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.59999999999999998, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4625,9 +5666,6 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -4637,7 +5675,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -4649,49 +5687,20 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "none" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "max", - "avg" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [ ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 + "h": 7, + "w": 11, + "x": 13, + "y": 643 }, - "id": 79, + "id": 67, "options": { "legend": { "calcs": [ ], @@ -4706,91 +5715,81 @@ } }, "panels": [ ], + "pluginVersion": "11.4.0-77383", "targets": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(max_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"fetch blocks\" | logfmt | unwrap duration(duration) [$__auto]))", - "hide": false, - "legendFormat": "max", - "queryType": "range", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", + "instant": false, + "legendFormat": "{{operation}} p99", + "range": true, "refId": "A" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(avg_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"fetch blocks\" | logfmt | unwrap duration(duration) [$__auto]))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, - "legendFormat": "avg", - "queryType": "range", + "instant": false, + "legendFormat": "{{operation}} p90", + "range": true, "refId": "B" }, { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(avg_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"request unavailable blocks in the background\" | logfmt | missing > 0 | unwrap missing [$__auto]))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))", "hide": false, - "legendFormat": "avg missing", - "queryType": "range", + "instant": false, + "legendFormat": "{{operation}} p50", + "range": true, "refId": "C" } ], - "title": "Download enqueue duration", + "title": "Latency", "type": "timeseries" - }, + } + ], + "targets": [ ], + "title": "Object Store", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 77, + "panels": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "fieldConfig": { "defaults": { - "color": { - "fixedColor": "green", - "mode": "palette-classic" - }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineWidth": 1, "scaleDistribution": { "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 100 - } - ] } }, "overrides": [ ] @@ -4799,145 +5798,127 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 412 }, "id": 80, "options": { - "barRadius": 0, - "barWidth": 0.96999999999999997, - "fullHighlight": false, - "groupWidth": 0.69999999999999996, + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1.0000000000000001e-09 + }, "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "show": true + }, + "rowsFrame": { + "layout": "auto" }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", "tooltip": { - "maxHeight": 600, "mode": "single", - "sort": "none" + "showColorScale": false, + "yHistogram": false }, - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "yAxis": { + "axisPlacement": "left", + "reverse": false + } }, "panels": [ ], - "pluginVersion": "11.0.0-67814", + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { - "type": "loki", - "uid": "${loki_datasource}" + "type": "prometheus", + "uid": "${datasource}" }, "editorMode": "code", - "expr": "sort_desc(topk(10, sum by (tasks) (count_over_time({namespace=\"loki-dev-006\", container=\"bloom-gateway\"} |= \"process tasks with bounds\" | logfmt [$__auto]))))", - "legendFormat": "", - "queryType": "instant", + "exemplar": false, + "expr": "increase(loki_bloom_gateway_dequeue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, "refId": "A" } ], - "title": "Tasks multiplexed", - "type": "barchart" + "title": "Dequeue duration", + "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] } }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Enqueue latency" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - } - ] + "overrides": [ ] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 412 }, - "id": 82, + "id": 106, "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1.0000000000000001e-09 + }, "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "show": true + }, + "rowsFrame": { + "layout": "auto" }, "tooltip": { - "maxHeight": 600, "mode": "single", - "sort": "none" + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false } }, "panels": [ ], + "pluginVersion": "11.4.0-77663", "targets": [ { "datasource": { @@ -4945,27 +5926,17 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_store_download_queue_enqueue_time_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", - "hide": false, - "legendFormat": "Enqueue latency", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_store_download_queue_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "expr": "increase(loki_bloom_gateway_tasks_dequeued_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])", + "format": "heatmap", "hide": false, - "legendFormat": "Size", + "instant": false, + "legendFormat": "__auto", "range": true, "refId": "B" } ], - "title": "Block download queue", - "type": "timeseries" + "title": "Dequeue count", + "type": "heatmap" } ], "targets": [ ], @@ -4973,6 +5944,7 @@ "type": "row" } ], + "preload": false, "refresh": "10s", "rows": [ ], "schemaVersion": 14, @@ -5058,7 +6030,6 @@ "from": "now-1h", "to": "now" }, - "timeRangeUpdatedDuringEditOrView": false, "timepicker": { "refresh_intervals": [ "5s",