Inhibit info alerts unless other alerts fire (#1507)

* Inhibit info alerts unless other alerts fire

* Fix nits

manifests/alertmanager-secret.yaml

@@ -28,10 +28,17 @@ stringData:
       - "severity = warning"
       "target_matchers":
       - "severity = info"
+    - "equal":
+      - "namespace"
+      "source_matchers":
+      - "alertname = InfoInhibitor"
+      "target_matchers":
+      - "severity = info"
     "receivers":
     - "name": "Default"
     - "name": "Watchdog"
     - "name": "Critical"
+    - "name": "null"
     "route":
       "group_by":
       - "namespace"
@@ -43,6 +50,9 @@ stringData:
       - "matchers":
         - "alertname = Watchdog"
         "receiver": "Watchdog"
+      - "matchers":
+        - "alertname = InfoInhibitor"
+        "receiver": "null"
       - "matchers":
         - "severity = critical"
         "receiver": "Critical"

manifests/kubePrometheus-prometheusRule.yaml

@@ -38,6 +38,21 @@ spec:
       expr: vector(1)
       labels:
         severity: none
+    - alert: InfoInhibitor
+      annotations:
+        description: |
+          This is an alert that is used to inhibit info alerts.
+          By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with
+          other alerts.
+          This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a
+          severity of 'warning' or 'critical' starts firing on the same namespace.
+          This alert should be routed to a null receiver and configured to inhibit alerts with severity="info".
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/infoinhibitor
+        summary: Info-level alert inhibition.
+      expr: ALERTS{severity = "info"} == 1 unless on(namespace) ALERTS{alertname !=
+        "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1
+      labels:
+        severity: none
   - name: node-network
     rules:
     - alert: NodeNetworkInterfaceFlapping