Merge pull request #714 from mlibrary/add-prep-storage-alerts

Add a ticket level alert for prep volume disk pressure
mlibrary · Sep 16, 2024 · 1c32413 · 1c32413
2 parents 8b47e1b + 26bc2fc
commit 1c32413
Showing 1 changed file with 18 additions and 0 deletions.
diff --git a/templates/profile/prometheus/rules.yml.erb b/templates/profile/prometheus/rules.yml.erb
@@ -137,6 +137,24 @@ groups:
     for: 5m
     labels:
       severity: page
+  - alert: PrepDiskPressure
+    annotations:
+      summary: 'Prep filesystem {{$labels.hostname}}:{{$labels.mountpoint}} is more than 98% full.'
+    expr: >
+      (
+        (
+          avg_over_time(
+            node_filesystem_size_bytes{mountpoint=~"prep"}[1m]
+          ) - avg_over_time(
+            node_filesystem_avail_bytes[1m]
+          )
+        ) / avg_over_time(
+          node_filesystem_size_bytes[1m]
+        )
+      ) > 0.98
+    for: 30m
+    labels:
+      severity: ticket
   - alert: HTDataDenIsFull
     annotations:
       summary: 'Filesystem {{$labels.hostname}}:{{$labels.mountpoint}} is full.'