Merge pull request #150 from brancz/prom-mixin

Use upstream Prometheus monitoring mixin

Makefile

@@ -16,6 +16,11 @@ generate-in-docker:
 	@echo ">> Compiling assets and generating Kubernetes manifests"
 	$(CONTAINER_CMD) make $(MFLAGS) generate
 
+.PHONY: clean
+clean:
+	# Remove all files and directories ignored by git.
+	git clean -Xfd .
+
 generate: manifests **.md
 
 **.md: $(EMBEDMD_BINARY) $(shell find examples) build.sh example.jsonnet

jsonnet/kube-prometheus/alerts/alerts.libsonnet

@@ -1,5 +1,4 @@
 (import 'alertmanager.libsonnet') +
 (import 'general.libsonnet') +
 (import 'node.libsonnet') +
-(import 'prometheus.libsonnet') +
 (import 'prometheus-operator.libsonnet')

jsonnet/kube-prometheus/alerts/prometheus.libsonnet

@@ -1,151 +0,0 @@
-{
-  prometheusAlerts+:: {
-    groups+: [
-      {
-        name: 'prometheus.rules',
-        rules: [
-          {
-            alert: 'PrometheusConfigReloadFailed',
-            annotations: {
-              description: "Reloading Prometheus' configuration has failed for {{$labels.namespace}}/{{$labels.pod}}",
-              summary: "Reloading Prometheus' configuration failed",
-            },
-            expr: |||
-              prometheus_config_last_reload_successful{%(prometheusSelector)s} == 0
-            ||| % $._config,
-            'for': '10m',
-            labels: {
-              severity: 'warning',
-            },
-          },
-          {
-            alert: 'PrometheusNotificationQueueRunningFull',
-            annotations: {
-              description: "Prometheus' alert notification queue is running full for {{$labels.namespace}}/{{ $labels.pod}}",
-              summary: "Prometheus' alert notification queue is running full",
-            },
-            expr: |||
-              predict_linear(prometheus_notifications_queue_length{%(prometheusSelector)s}[5m], 60 * 30) > prometheus_notifications_queue_capacity{%(prometheusSelector)s}
-            ||| % $._config,
-            'for': '10m',
-            labels: {
-              severity: 'warning',
-            },
-          },
-          {
-            alert: 'PrometheusErrorSendingAlerts',
-            annotations: {
-              description: 'Errors while sending alerts from Prometheus {{$labels.namespace}}/{{ $labels.pod}} to Alertmanager {{$labels.Alertmanager}}',
-              summary: 'Errors while sending alert from Prometheus',
-            },
-            expr: |||
-              rate(prometheus_notifications_errors_total{%(prometheusSelector)s}[5m]) / rate(prometheus_notifications_sent_total{%(prometheusSelector)s}[5m]) > 0.01
-            ||| % $._config,
-            'for': '10m',
-            labels: {
-              severity: 'warning',
-            },
-          },
-          {
-            alert: 'PrometheusErrorSendingAlerts',
-            annotations: {
-              description: 'Errors while sending alerts from Prometheus {{$labels.namespace}}/{{ $labels.pod}} to Alertmanager {{$labels.Alertmanager}}',
-              summary: 'Errors while sending alerts from Prometheus',
-            },
-            expr: |||
-              rate(prometheus_notifications_errors_total{%(prometheusSelector)s}[5m]) / rate(prometheus_notifications_sent_total{%(prometheusSelector)s}[5m]) > 0.03
-            ||| % $._config,
-            'for': '10m',
-            labels: {
-              severity: 'critical',
-            },
-          },
-          {
-            alert: 'PrometheusNotConnectedToAlertmanagers',
-            annotations: {
-              description: 'Prometheus {{ $labels.namespace }}/{{ $labels.pod}} is not connected to any Alertmanagers',
-              summary: 'Prometheus is not connected to any Alertmanagers',
-            },
-            expr: |||
-              prometheus_notifications_alertmanagers_discovered{%(prometheusSelector)s} < 1
-            ||| % $._config,
-            'for': '10m',
-            labels: {
-              severity: 'warning',
-            },
-          },
-          {
-            alert: 'PrometheusTSDBReloadsFailing',
-            annotations: {
-              description: '{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}} reload failures over the last four hours.',
-              summary: 'Prometheus has issues reloading data blocks from disk',
-            },
-            expr: |||
-              increase(prometheus_tsdb_reloads_failures_total{%(prometheusSelector)s}[2h]) > 0
-            ||| % $._config,
-            'for': '12h',
-            labels: {
-              severity: 'warning',
-            },
-          },
-          {
-            alert: 'PrometheusTSDBCompactionsFailing',
-            annotations: {
-              description: '{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}} compaction failures over the last four hours.',
-              summary: 'Prometheus has issues compacting sample blocks',
-            },
-            expr: |||
-              increase(prometheus_tsdb_compactions_failed_total{%(prometheusSelector)s}[2h]) > 0
-            ||| % $._config,
-            'for': '12h',
-            labels: {
-              severity: 'warning',
-            },
-          },
-          {
-            alert: 'PrometheusTSDBWALCorruptions',
-            annotations: {
-              description: '{{$labels.job}} at {{$labels.instance}} has a corrupted write-ahead log (WAL).',
-              summary: 'Prometheus write-ahead log is corrupted',
-            },
-            expr: |||
-              prometheus_tsdb_wal_corruptions_total{%(prometheusSelector)s} > 0
-            ||| % $._config,
-            'for': '4h',
-            labels: {
-              severity: 'warning',
-            },
-          },
-          {
-            alert: 'PrometheusNotIngestingSamples',
-            annotations: {
-              description: "Prometheus {{ $labels.namespace }}/{{ $labels.pod}} isn't ingesting samples.",
-              summary: "Prometheus isn't ingesting samples",
-            },
-            expr: |||
-              rate(prometheus_tsdb_head_samples_appended_total{%(prometheusSelector)s}[5m]) <= 0
-            ||| % $._config,
-            'for': '10m',
-            labels: {
-              severity: 'warning',
-            },
-          },
-          {
-            alert: 'PrometheusTargetScrapesDuplicate',
-            annotations: {
-              description: '{{$labels.namespace}}/{{$labels.pod}} has many samples rejected due to duplicate timestamps but different values',
-              summary: 'Prometheus has many samples rejected',
-            },
-            expr: |||
-              increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{%(prometheusSelector)s}[5m]) > 0
-            ||| % $._config,
-            'for': '10m',
-            labels: {
-              severity: 'warning',
-            },
-          },
-        ],
-      },
-    ],
-  },
-}

jsonnet/kube-prometheus/jsonnetfile.json

@@ -49,6 +49,16 @@
                 }
             },
             "version": "master"
+        },
+        {
+            "name": "prometheus",
+            "source": {
+                "git": {
+                    "remote": "https://github.com/prometheus/prometheus",
+                    "subdir": "documentation/prometheus-mixin"
+                }
+            },
+            "version": "master"
         }
     ]
 }

jsonnet/kube-prometheus/kube-prometheus.libsonnet

@@ -10,6 +10,7 @@ local configMapList = k3.core.v1.configMapList;
 (import 'prometheus/prometheus.libsonnet') +
 (import 'prometheus-adapter/prometheus-adapter.libsonnet') +
 (import 'kubernetes-mixin/mixin.libsonnet') +
+(import 'prometheus/mixin.libsonnet') +
 (import 'alerts/alerts.libsonnet') +
 (import 'rules/rules.libsonnet') + {
   kubePrometheus+:: {
@@ -89,6 +90,7 @@ local configMapList = k3.core.v1.configMapList;
 
     alertmanagerSelector: 'job="alertmanager-main",namespace="' + $._config.namespace + '"',
     prometheusSelector: 'job="prometheus-' + $._config.prometheus.name + '",namespace="' + $._config.namespace + '"',
+    prometheusName: '{{$labels.namespace}}/{{$labels.pod}}',
     prometheusOperatorSelector: 'job="prometheus-operator",namespace="' + $._config.namespace + '"',
 
     jobs: {
@@ -111,5 +113,6 @@ local configMapList = k3.core.v1.configMapList;
     grafana+:: {
       dashboards: $.grafanaDashboards,
     },
+
   },
 }

jsonnetfile.lock.json

@@ -8,7 +8,7 @@
                     "subdir": "jsonnet/kube-prometheus"
                 }
             },
-            "version": "daf2e76bc6615ca264d424203ca1584da7cad36e"
+            "version": "c8c850ef2b1b6f985075ee06fb86b347f9119b16"
         },
         {
             "name": "ksonnet",
@@ -48,7 +48,7 @@
                     "subdir": "grafana-builder"
                 }
             },
-            "version": "6daf3fa9b87e45009c377247f64005f6949c886f"
+            "version": "3daf42722ee2008cae15267ee7380a58988d60cd"
         },
         {
             "name": "grafana",
@@ -78,7 +78,17 @@
                     "subdir": "Documentation/etcd-mixin"
                 }
             },
-            "version": "eb7dd97135a5c60d9d9259886100823114e45815"
+            "version": "b2274efee09b49d6cdea7d6d5e6a9c090fc9d8ca"
+        },
+        {
+            "name": "prometheus",
+            "source": {
+                "git": {
+                    "remote": "https://github.com/prometheus/prometheus",
+                    "subdir": "documentation/prometheus-mixin"
+                }
+            },
+            "version": "fb6c709a5e493f003a6dc66bc4c36c2af882de6a"
         }
     ]
 }

manifests/grafana-deployment.yaml

@@ -81,6 +81,12 @@ spec:
         - mountPath: /grafana-dashboard-definitions/0/pods
           name: grafana-dashboard-pods
           readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/prometheus-remote-write
+          name: grafana-dashboard-prometheus-remote-write
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/prometheus
+          name: grafana-dashboard-prometheus
+          readOnly: false
         - mountPath: /grafana-dashboard-definitions/0/proxy
           name: grafana-dashboard-proxy
           readOnly: false
@@ -144,6 +150,12 @@ spec:
       - configMap:
           name: grafana-dashboard-pods
         name: grafana-dashboard-pods
+      - configMap:
+          name: grafana-dashboard-prometheus-remote-write
+        name: grafana-dashboard-prometheus-remote-write
+      - configMap:
+          name: grafana-dashboard-prometheus
+        name: grafana-dashboard-prometheus
       - configMap:
           name: grafana-dashboard-proxy
         name: grafana-dashboard-proxy

manifests/prometheus-rules.yaml

@@ -860,6 +860,200 @@ spec:
         apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400
       labels:
         severity: critical
+  - name: prometheus
+    rules:
+    - alert: PrometheusBadConfig
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to
+          reload its configuration.
+        summary: Failed Prometheus configuration reload.
+      expr: |
+        # Without max_over_time, failed scrapes could create false negatives, see
+        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
+        max_over_time(prometheus_config_last_reload_successful{job="prometheus-k8s",namespace="monitoring"}[5m]) == 0
+      for: 10m
+      labels:
+        severity: critical
+    - alert: PrometheusNotificationQueueRunningFull
+      annotations:
+        description: Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}}
+          is running full.
+        summary: Prometheus alert notification queue predicted to run full in less
+          than 30m.
+      expr: |
+        # Without min_over_time, failed scrapes could create false negatives, see
+        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
+        (
+          predict_linear(prometheus_notifications_queue_length{job="prometheus-k8s",namespace="monitoring"}[5m], 60 * 30)
+        >
+          min_over_time(prometheus_notifications_queue_capacity{job="prometheus-k8s",namespace="monitoring"}[5m])
+        )
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers
+      annotations:
+        description: '{{ printf "%.1f" $value }}% errors while sending alerts from
+          Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}}.'
+        summary: Prometheus has encountered more than 1% errors sending alerts to
+          a specific Alertmanager.
+      expr: |
+        (
+          rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m])
+        /
+          rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring"}[5m])
+        )
+        * 100
+        > 1
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusErrorSendingAlertsToAnyAlertmanager
+      annotations:
+        description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts
+          from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager.'
+        summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager.
+      expr: |
+        min without(alertmanager) (
+          rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m])
+        /
+          rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring"}[5m])
+        )
+        * 100
+        > 3
+      for: 15m
+      labels:
+        severity: critical
+    - alert: PrometheusNotConnectedToAlertmanagers
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected
+          to any Alertmanagers.
+        summary: Prometheus is not connected to any Alertmanagers.
+      expr: |
+        # Without max_over_time, failed scrapes could create false negatives, see
+        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
+        max_over_time(prometheus_notifications_alertmanagers_discovered{job="prometheus-k8s",namespace="monitoring"}[5m]) < 1
+      for: 10m
+      labels:
+        severity: warning
+    - alert: PrometheusTSDBReloadsFailing
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected
+          {{$value | humanize}} reload failures over the last 3h.
+        summary: Prometheus has issues reloading blocks from disk.
+      expr: |
+        increase(prometheus_tsdb_reloads_failures_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0
+      for: 4h
+      labels:
+        severity: warning
+    - alert: PrometheusTSDBCompactionsFailing
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected
+          {{$value | humanize}} compaction failures over the last 3h.
+        summary: Prometheus has issues compacting blocks.
+      expr: |
+        increase(prometheus_tsdb_compactions_failed_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0
+      for: 4h
+      labels:
+        severity: warning
+    - alert: PrometheusTSDBWALCorruptions
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected
+          {{$value | humanize}} corruptions of the write-ahead log (WAL) over the
+          last 3h.
+        summary: Prometheus is detecting WAL corruptions.
+      expr: |
+        increase(tsdb_wal_corruptions_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0
+      for: 4h
+      labels:
+        severity: warning
+    - alert: PrometheusNotIngestingSamples
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting
+          samples.
+        summary: Prometheus is not ingesting samples.
+      expr: |
+        rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-k8s",namespace="monitoring"}[5m]) <= 0
+      for: 10m
+      labels:
+        severity: warning
+    - alert: PrometheusDuplicateTimestamps
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping
+          {{$value | humanize}} samples/s with different values but duplicated timestamp.
+        summary: Prometheus is dropping samples with duplicate timestamps.
+      expr: |
+        rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 10m
+      labels:
+        severity: warning
+    - alert: PrometheusOutOfOrderTimestamps
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping
+          {{$value | humanize}} samples/s with timestamps arriving out of order.
+        summary: Prometheus drops samples with out-of-order timestamps.
+      expr: |
+        rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 10m
+      labels:
+        severity: warning
+    - alert: PrometheusRemoteStorageFailures
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send
+          {{ printf "%.1f" $value }}% of the samples to queue {{$labels.queue}}.
+        summary: Prometheus fails to send samples to remote storage.
+      expr: |
+        (
+          rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m])
+        /
+          (
+            rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m])
+          +
+            rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m])
+          )
+        )
+        * 100
+        > 1
+      for: 15m
+      labels:
+        severity: critical
+    - alert: PrometheusRemoteWriteBehind
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write
+          is {{ printf "%.1f" $value }}s behind for queue {{$labels.queue}}.
+        summary: Prometheus remote write is behind.
+      expr: |
+        # Without max_over_time, failed scrapes could create false negatives, see
+        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
+        (
+          max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="prometheus-k8s",namespace="monitoring"}[5m])
+        - on(job, instance) group_right
+          max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="prometheus-k8s",namespace="monitoring"}[5m])
+        )
+        > 120
+      for: 15m
+      labels:
+        severity: critical
+    - alert: PrometheusRuleFailures
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to
+          evaluate {{ printf "%.0f" $value }} rules in the last 5m.
+        summary: Prometheus is failing rule evaluations.
+      expr: |
+        increase(prometheus_rule_evaluation_failures_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: critical
+    - alert: PrometheusMissingRuleEvaluations
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{
+          printf "%.0f" $value }} rule group evaluations in the last 5m.
+        summary: Prometheus is missing rule evaluations due to slow rule group evaluation.
+      expr: |
+        increase(prometheus_rule_group_iterations_missed_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: warning
   - name: alertmanager.rules
     rules:
     - alert: AlertmanagerConfigInconsistent
@@ -970,107 +1164,6 @@ spec:
       for: 2m
       labels:
         severity: warning
-  - name: prometheus.rules
-    rules:
-    - alert: PrometheusConfigReloadFailed
-      annotations:
-        description: Reloading Prometheus' configuration has failed for {{$labels.namespace}}/{{$labels.pod}}
-        summary: Reloading Prometheus' configuration failed
-      expr: |
-        prometheus_config_last_reload_successful{job="prometheus-k8s",namespace="monitoring"} == 0
-      for: 10m
-      labels:
-        severity: warning
-    - alert: PrometheusNotificationQueueRunningFull
-      annotations:
-        description: Prometheus' alert notification queue is running full for {{$labels.namespace}}/{{
-          $labels.pod}}
-        summary: Prometheus' alert notification queue is running full
-      expr: |
-        predict_linear(prometheus_notifications_queue_length{job="prometheus-k8s",namespace="monitoring"}[5m], 60 * 30) > prometheus_notifications_queue_capacity{job="prometheus-k8s",namespace="monitoring"}
-      for: 10m
-      labels:
-        severity: warning
-    - alert: PrometheusErrorSendingAlerts
-      annotations:
-        description: Errors while sending alerts from Prometheus {{$labels.namespace}}/{{
-          $labels.pod}} to Alertmanager {{$labels.Alertmanager}}
-        summary: Errors while sending alert from Prometheus
-      expr: |
-        rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m]) / rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0.01
-      for: 10m
-      labels:
-        severity: warning
-    - alert: PrometheusErrorSendingAlerts
-      annotations:
-        description: Errors while sending alerts from Prometheus {{$labels.namespace}}/{{
-          $labels.pod}} to Alertmanager {{$labels.Alertmanager}}
-        summary: Errors while sending alerts from Prometheus
-      expr: |
-        rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m]) / rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0.03
-      for: 10m
-      labels:
-        severity: critical
-    - alert: PrometheusNotConnectedToAlertmanagers
-      annotations:
-        description: Prometheus {{ $labels.namespace }}/{{ $labels.pod}} is not connected
-          to any Alertmanagers
-        summary: Prometheus is not connected to any Alertmanagers
-      expr: |
-        prometheus_notifications_alertmanagers_discovered{job="prometheus-k8s",namespace="monitoring"} < 1
-      for: 10m
-      labels:
-        severity: warning
-    - alert: PrometheusTSDBReloadsFailing
-      annotations:
-        description: '{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}}
-          reload failures over the last four hours.'
-        summary: Prometheus has issues reloading data blocks from disk
-      expr: |
-        increase(prometheus_tsdb_reloads_failures_total{job="prometheus-k8s",namespace="monitoring"}[2h]) > 0
-      for: 12h
-      labels:
-        severity: warning
-    - alert: PrometheusTSDBCompactionsFailing
-      annotations:
-        description: '{{$labels.job}} at {{$labels.instance}} had {{$value | humanize}}
-          compaction failures over the last four hours.'
-        summary: Prometheus has issues compacting sample blocks
-      expr: |
-        increase(prometheus_tsdb_compactions_failed_total{job="prometheus-k8s",namespace="monitoring"}[2h]) > 0
-      for: 12h
-      labels:
-        severity: warning
-    - alert: PrometheusTSDBWALCorruptions
-      annotations:
-        description: '{{$labels.job}} at {{$labels.instance}} has a corrupted write-ahead
-          log (WAL).'
-        summary: Prometheus write-ahead log is corrupted
-      expr: |
-        prometheus_tsdb_wal_corruptions_total{job="prometheus-k8s",namespace="monitoring"} > 0
-      for: 4h
-      labels:
-        severity: warning
-    - alert: PrometheusNotIngestingSamples
-      annotations:
-        description: Prometheus {{ $labels.namespace }}/{{ $labels.pod}} isn't ingesting
-          samples.
-        summary: Prometheus isn't ingesting samples
-      expr: |
-        rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-k8s",namespace="monitoring"}[5m]) <= 0
-      for: 10m
-      labels:
-        severity: warning
-    - alert: PrometheusTargetScrapesDuplicate
-      annotations:
-        description: '{{$labels.namespace}}/{{$labels.pod}} has many samples rejected
-          due to duplicate timestamps but different values'
-        summary: Prometheus has many samples rejected
-      expr: |
-        increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
-      for: 10m
-      labels:
-        severity: warning
   - name: prometheus-operator
     rules:
     - alert: PrometheusOperatorReconcileErrors