Inhibit info alerts unless other alerts fire (#1507)

* Inhibit info alerts unless other alerts fire

* Fix nits

jsonnet/kube-prometheus/components/alertmanager.libsonnet

@@ -35,6 +35,10 @@ local defaults = {
       source_matchers: ['severity = warning'],
       target_matchers: ['severity = info'],
       equal: ['namespace', 'alertname'],
+    }, {
+      source_matchers: ['alertname = InfoInhibitor'],
+      target_matchers: ['severity = info'],
+      equal: ['namespace'],
     }],
     route: {
       group_by: ['namespace'],
@@ -44,6 +48,7 @@ local defaults = {
       receiver: 'Default',
       routes: [
         { receiver: 'Watchdog', matchers: ['alertname = Watchdog'] },
+        { receiver: 'null', matchers: ['alertname = InfoInhibitor'] },
         { receiver: 'Critical', matchers: ['severity = critical'] },
       ],
     },
@@ -51,6 +56,7 @@ local defaults = {
       { name: 'Default' },
       { name: 'Watchdog' },
       { name: 'Critical' },
+      { name: 'null' },
     ],
   },
   replicas: 3,

jsonnet/kube-prometheus/components/mixin/alerts/general.libsonnet

@@ -33,6 +33,24 @@
               severity: 'none',
             },
           },
+          {
+            alert: 'InfoInhibitor',
+            annotations: {
+              summary: 'Info-level alert inhibition.',
+              description: |||
+                This is an alert that is used to inhibit info alerts.
+                By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with
+                other alerts.
+                This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a
+                severity of 'warning' or 'critical' starts firing on the same namespace.
+                This alert should be routed to a null receiver and configured to inhibit alerts with severity="info".
+              |||,
+            },
+            expr: 'ALERTS{severity = "info"} == 1 unless on(namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1',
+            labels: {
+              severity: 'none',
+            },
+          },
         ],
       },
     ],

manifests/alertmanager-secret.yaml

@@ -28,10 +28,17 @@ stringData:
       - "severity = warning"
       "target_matchers":
       - "severity = info"
+    - "equal":
+      - "namespace"
+      "source_matchers":
+      - "alertname = InfoInhibitor"
+      "target_matchers":
+      - "severity = info"
     "receivers":
     - "name": "Default"
     - "name": "Watchdog"
     - "name": "Critical"
+    - "name": "null"
     "route":
       "group_by":
       - "namespace"
@@ -43,6 +50,9 @@ stringData:
       - "matchers":
         - "alertname = Watchdog"
         "receiver": "Watchdog"
+      - "matchers":
+        - "alertname = InfoInhibitor"
+        "receiver": "null"
       - "matchers":
         - "severity = critical"
         "receiver": "Critical"

manifests/kubePrometheus-prometheusRule.yaml

@@ -38,6 +38,21 @@ spec:
       expr: vector(1)
       labels:
         severity: none
+    - alert: InfoInhibitor
+      annotations:
+        description: |
+          This is an alert that is used to inhibit info alerts.
+          By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with
+          other alerts.
+          This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a
+          severity of 'warning' or 'critical' starts firing on the same namespace.
+          This alert should be routed to a null receiver and configured to inhibit alerts with severity="info".
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/infoinhibitor
+        summary: Info-level alert inhibition.
+      expr: ALERTS{severity = "info"} == 1 unless on(namespace) ALERTS{alertname !=
+        "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1
+      labels:
+        severity: none
   - name: node-network
     rules:
     - alert: NodeNetworkInterfaceFlapping