kube-prometheus: add dead man's switch

assets/alertmanager/alertmanager.yaml

@@ -5,8 +5,10 @@ route:
   group_wait: 30s
   group_interval: 5m
   repeat_interval: 12h
-  receiver: 'webhook'
+  receiver: 'null'
+  routes:
+  - match:
+      alertname: DeadMansSwitch
+    receiver: 'null'
 receivers:
-- name: 'webhook'
+- name: 'null'
-  webhook_configs:
-  - url: 'http://alertmanagerwh:30500/'

assets/prometheus/rules/general.rules

@@ -11,6 +11,18 @@ Alert TargetDown
     description = "A target of type {{ $labels.job }} is down."
   }
 
+### Dead man's switch ###
+
+ALERT DeadMansSwitch
+  IF vector(1)
+  LABELS {
+    severity = "none",
+  }
+  ANNOTATIONS {
+    summary = "Alerting DeadMansSwitch",
+    description = "This is a DeadMansSwitch meant to ensure that the entire Alerting pipeline is functional.",
+  }
+
 ### File descriptor alerts ###
 
 ALERT TooManyOpenFiles

manifests/alertmanager/alertmanager-config.yaml

@@ -3,4 +3,4 @@ kind: Secret
 metadata:
   name: alertmanager-main
 data:
-  alertmanager.yaml: Z2xvYmFsOgogIHJlc29sdmVfdGltZW91dDogNW0Kcm91dGU6CiAgZ3JvdXBfYnk6IFsnam9iJ10KICBncm91cF93YWl0OiAzMHMKICBncm91cF9pbnRlcnZhbDogNW0KICByZXBlYXRfaW50ZXJ2YWw6IDEyaAogIHJlY2VpdmVyOiAnd2ViaG9vaycKcmVjZWl2ZXJzOgotIG5hbWU6ICd3ZWJob29rJwogIHdlYmhvb2tfY29uZmlnczoKICAtIHVybDogJ2h0dHA6Ly9hbGVydG1hbmFnZXJ3aDozMDUwMC8nCg==
+  alertmanager.yaml: Z2xvYmFsOgogIHJlc29sdmVfdGltZW91dDogNW0Kcm91dGU6CiAgZ3JvdXBfYnk6IFsnam9iJ10KICBncm91cF93YWl0OiAzMHMKICBncm91cF9pbnRlcnZhbDogNW0KICByZXBlYXRfaW50ZXJ2YWw6IDEyaAogIHJlY2VpdmVyOiAnbnVsbCcKICByb3V0ZXM6CiAgLSBtYXRjaDoKICAgICAgYWxlcnRuYW1lOiBEZWFkTWFuc1N3aXRjaAogICAgcmVjZWl2ZXI6ICdudWxsJwpyZWNlaXZlcnM6Ci0gbmFtZTogJ251bGwnCg==

manifests/prometheus/prometheus-k8s-rules.yaml

@@ -235,6 +235,18 @@ data:
         description = "A target of type {{ $labels.job }} is down."
       }
     
+    ### Dead man's switch ###
+    
+    ALERT DeadMansSwitch
+      IF vector(1)
+      LABELS {
+        severity = "none",
+      }
+      ANNOTATIONS {
+        summary = "Alerting DeadMansSwitch",
+        description = "This is a DeadMansSwitch meant to ensure that the entire Alerting pipeline is functional.",
+      }
+    
     ### File descriptor alerts ###
     
     ALERT TooManyOpenFiles