Install prometheus-postfix-exporter

As a data-obsessed administrator I want to have more data such that I
can widen my sense of power.

This also installs rsyslog, because prometheus-postfix-exporter doesn't
work with journald's binary log format.

ansible/host_vars/lovelace/prometheus.yml

@@ -55,6 +55,13 @@ prometheus_configuration: |
           - {{ hostvars[host]['ansible_wg0']['ipv4']['address'] }}:9187
           {%- endfor %}
 
+    - job_name: postfix
+      static_configs:
+        - targets:
+          {%- for host in groups['mail'] %}
+          - {{ hostvars[host]['ansible_wg0']['ipv4']['address'] }}:9154
+          {%- endfor %}
+
     - job_name: blackbox-ssh
       metrics_path: /probe
       params:

ansible/playbook.yml

@@ -26,6 +26,7 @@
     - dovecot
     - spamassassin
     - postfix
+    - prometheus-postfix-exporter
 
 - name: Deploy our monitoring stack
   hosts: monitoring

ansible/roles/prometheus-postfix-exporter/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - postfix

ansible/roles/prometheus-postfix-exporter/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+- name: Install prometheus-postfix-exporter
+  package:
+    name: prometheus-postfix-exporter
+    state: present
+  tags:
+    - role::prometheus-postfix-exporter

kubernetes/namespaces/monitoring/alerts/alerts.d/mail.yaml

@@ -0,0 +1,22 @@
+groups:
+  - name: postfix
+    rules:
+      - alert: postfix/down
+        expr: postfix_up != 1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: Postfix is down (instance {{ $labels.instance }})
+      - alert: postfix/smtp-temporary-errors
+        expr: rate(postfix_smtpd_messages_rejected_total{code=~"^4.*"}[15m]) > 0
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: Postfix is rejecting messages due to errors (instance {{ $labels.instance }})
+          description: Postfix has seen code {{ $labels.code }} errors recently
+            and temporarily rejected emails.
+            https://en.wikipedia.org/wiki/List_of_SMTP_server_return_codes and
+            `sudo journalctl -xeu postfix@-` may provide more information on
+            the current issue.