Add capability to disable specific alert rules Loki (grafana#11241)

**What this PR does / why we need it**: Currently using the Loki Chart we can only either enable\disable ALL alert rules. For specific environments and use-cases sometimes not all alert Rules are useful to have enabled. With this PR change, we can cleanly and through the Chart values disable specific Alerts.
mraboosk · Jan 17, 2024 · dcf03ce · dcf03ce
1 parent e08f08c
commit dcf03ce
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 2 deletions.
diff --git a/production/helm/loki/CHANGELOG.md b/production/helm/loki/CHANGELOG.md
@@ -13,6 +13,11 @@ Entries should include a reference to the pull request that introduced the chang
 
 [//]: # (<AUTOMATED_UPDATES_LOCATOR> : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.)
 
+
+## 5.41.7
+
+- [FEATURE] Add support to disable specific alert rules
+
 ## 5.41.6
 
 - [BUGFIX] Added missing namespace to query-scheduler-discovery service when deploying loki in a specific namespace.

diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml
@@ -3,7 +3,7 @@ name: loki
 description: Helm chart for Grafana Loki in simple, scalable mode
 type: application
 appVersion: 2.9.3
-version: 5.41.6
+version: 5.41.7
 home: https://grafana.github.io/helm-charts
 sources:
   - https://github.com/grafana/loki

diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md
@@ -1,6 +1,6 @@
 # loki
 
-![Version: 5.41.6](https://img.shields.io/badge/Version-5.41.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.9.3](https://img.shields.io/badge/AppVersion-2.9.3-informational?style=flat-square)
+![Version: 5.41.7](https://img.shields.io/badge/Version-5.41.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.9.3](https://img.shields.io/badge/AppVersion-2.9.3-informational?style=flat-square)
 
 Helm chart for Grafana Loki in simple, scalable mode
 

diff --git a/production/helm/loki/src/alerts.yaml.tpl b/production/helm/loki/src/alerts.yaml.tpl
@@ -2,6 +2,7 @@
 groups:
   - name: "loki_alerts"
     rules:
+{{- if not (.Values.monitoring.rules.disabled.LokiRequestErrors | default false) }}
       - alert: "LokiRequestErrors"
         annotations:
           message: |
@@ -17,6 +18,8 @@ groups:
 {{- if .Values.monitoring.rules.additionalRuleLabels }}
 {{ toYaml .Values.monitoring.rules.additionalRuleLabels | indent 10 }}
 {{- end }}
+{{- end }}
+{{- if not (.Values.monitoring.rules.disabled.LokiRequestPanics | default false) }}
       - alert: "LokiRequestPanics"
         annotations:
           message: |
@@ -28,6 +31,8 @@ groups:
 {{- if .Values.monitoring.rules.additionalRuleLabels }}
 {{ toYaml .Values.monitoring.rules.additionalRuleLabels | indent 10 }}
 {{- end }}
+{{- end }}
+{{- if not (.Values.monitoring.rules.disabled.LokiRequestLatency | default false) }}
       - alert: "LokiRequestLatency"
         annotations:
           message: |
@@ -40,6 +45,8 @@ groups:
 {{- if .Values.monitoring.rules.additionalRuleLabels }}
 {{ toYaml .Values.monitoring.rules.additionalRuleLabels | indent 10 }}
 {{- end }}
+{{- end }}
+{{- if not (.Values.monitoring.rules.disabled.LokiTooManyCompactorsRunning | default false) }}
       - alert: "LokiTooManyCompactorsRunning"
         annotations:
           message: |
@@ -52,6 +59,8 @@ groups:
 {{- if .Values.monitoring.rules.additionalRuleLabels }}
 {{ toYaml .Values.monitoring.rules.additionalRuleLabels | indent 10 }}
 {{- end }}
+{{- end }}
+{{- if not (.Values.monitoring.rules.disabled.LokiCanaryLatency | default false) }}
   - name: "loki_canaries_alerts"
     rules:
       - alert: "LokiCanaryLatency"
@@ -66,3 +75,4 @@ groups:
 {{- if .Values.monitoring.rules.additionalRuleLabels }}
 {{ toYaml .Values.monitoring.rules.additionalRuleLabels | indent 10 }}
 {{- end }}
+{{- end }}
diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml
@@ -583,6 +583,12 @@ monitoring:
     enabled: true
     # -- Include alerting rules
     alerting: true
+    # -- Specify which individual alerts should be disabled
+    # -- Instead of turning off each alert one by one, set the .monitoring.rules.alerting value to false instead.
+    # -- If you disable all the alerts and keep .monitoring.rules.alerting set to true, the chart will fail to render.
+    disabled: {}
+    #  LokiRequestErrors: true
+    #  LokiRequestPanics: true
     # -- Alternative namespace to create PrometheusRule resources in
     namespace: null
     # -- Additional annotations for the rules PrometheusRule resource