feat: implement some default alerts

src/alert_rules/prometheus/high_cpu_usage.rule

@@ -0,0 +1,12 @@
+alert: SlurmHighCPUUsage
+expr: (slurm_cpu_load{%%juju_topology%%} / slurm_cpus_total{%%juju_topology%%}) * 100 > 90
+for: 5m
+labels:
+  severity: warning
+annotations:
+  summary: CPU usage for the cluster managed by the Slurm controller {{ $labels.juju_model }}/{{ $labels.juju_unit }} reached 90%
+  description: >
+    The total CPU usage for all nodes in the cluster managed by the Slurm controller
+    {{ $labels.juju_model }}/{{ $labels.juju_unit }} reached 90%. This could indicate that the cluster
+    is reaching its maximum computing capacity.
+    LABELS = {{ $labels }}

src/alert_rules/prometheus/unreachable_slurmdbd.rule

@@ -0,0 +1,12 @@
+alert: SlurmTooManyFailedDbdMessages
+expr: max_over_time(slurm_dbd_agent_queue_size{%%juju_topology%%}[10s]) > 50
+for: 1s
+labels:
+  severity: critical
+annotations:
+  summary: Slurm controller {{ $labels.juju_model }}/{{ $labels.juju_unit }} cannot reach SlurmDBD
+  description: >
+    The maximum amount of pending messages from the Slurm controller {{ $labels.juju_model }}/{{ $labels.juju_unit }}
+    to SlurmDBD exceeded 5000 in the past minute. This can indicate a problem to reach SlurmDBD
+    or its backing database.
+    LABELS = {{ $labels }}

src/templates/prometheus-slurm-exporter.service

@@ -3,7 +3,7 @@ Wants=network.target
 After=network.target slurmctld.service
 
 [Service]
-ExecStart=/usr/bin/prometheus-slurm-exporter
+ExecStart=/usr/bin/prometheus-slurm-exporter -slurm.cli-fallback -slurm.collect-diags
 SyslogIdentifier=prometheus-slurm-exporter
 Restart=always
 RestartSec=15