Merge pull request #1494 from PhilipGough/revert-1406-dropped-cadvisor-metrics-6

[bot] [release-0.6] Automated version update

.github/workflows/ci.yaml

@@ -0,0 +1,54 @@
+name: ci
+on:
+  - push
+  - pull_request
+env:
+  golang-version: '1.15'
+  kind-version: 'v0.11.1'
+jobs:
+  generate:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - macos-latest
+          - ubuntu-latest
+    name: Generate
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ${{ env.golang-version }}
+    - run: make --always-make generate && git diff --exit-code
+  unit-tests:
+    runs-on: ubuntu-latest
+    name: Unit tests
+    steps:
+    - uses: actions/checkout@v2
+    - run: make --always-make test
+  e2e-tests:
+    name: E2E tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        kind-image:
+          - 'kindest/node:v1.19.11'
+    steps:
+    - uses: actions/checkout@v2
+    - name: Start KinD
+      uses: engineerd/setup-kind@v0.5.0
+      with:
+        version: ${{ env.kind-version }}
+        image: ${{ matrix.kind-image }}
+        wait: 300s
+    - name: Wait for cluster to finish bootstraping
+      run: kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s
+    - name: Create kube-prometheus stack
+      run: |
+        kubectl create -f manifests/setup
+        until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
+        kubectl create -f manifests/
+    - name: Run tests
+      run: |
+        export KUBECONFIG="${HOME}/.kube/config"
+        make test-e2e

.travis.yml

@@ -1,21 +0,0 @@
-sudo: required
-dist: xenial
-language: go
-
-go:
-- "1.13.x"
-go_import_path: github.com/coreos/kube-prometheus
-
-cache:
-  directories:
-  - $GOCACHE
-  - $GOPATH/pkg/mod
-
-jobs:
-  include:
-  - name: Check generated files
-    script: make --always-make generate && git diff --exit-code
-  - name: Run tests
-    script: make --always-make test
-  - name: Run e2e tests
-    script: ./tests/e2e/travis-e2e.sh

Makefile

@@ -33,6 +33,10 @@ vendor: $(JB_BIN) jsonnetfile.json jsonnetfile.lock.json
 	rm -rf vendor
 	$(JB_BIN) install
 
+.PHONY: update
+update: $(JB_BIN)
+	$(JB_BIN) update
+
 .PHONY: fmt
 fmt: $(JSONNETFMT_BIN)
 	find . -name 'vendor' -prune -o -name '*.libsonnet' -o -name '*.jsonnet' -print | \

README.md

@@ -96,7 +96,7 @@ The following versions are supported and work as we test against these versions
 | `release-0.3`         | ✔               | ✔               | ✔               | ✔               | ✗               | ✗               |
 | `release-0.4`         | ✗               | ✗               | ✔ (v1.16.5+)    | ✔               | ✗               | ✗               |
 | `release-0.5`         | ✗               | ✗               | ✗               | ✗               | ✔               | ✗               |
-| `release-0.6`         | ✗               | ✗               | ✗               | ✗               | ✔               | ✗               |
+| `release-0.6`         | ✗               | ✗               | ✗               | ✗               | ✔               | ✔               |
 | `HEAD`                | ✗               | ✗               | ✗               | ✗               | ✔               | ✗               |
 
 Note: Due to [two](https://github.com/kubernetes/kubernetes/issues/83778) [bugs](https://github.com/kubernetes/kubernetes/issues/86359) in Kubernetes v1.16.1, and prior to Kubernetes v1.16.5 the kube-prometheus release-0.4 branch only supports v1.16.5 and higher.  The `extension-apiserver-authentication-reader` role in the kube-system namespace can be manually edited to include list and watch permissions in order to workaround the second issue with Kubernetes v1.16.2 through v1.16.4.

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/google/go-jsonnet v0.16.1-0.20200703153429-aaf50f5b655f
 	github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
-	github.com/jsonnet-bundler/jsonnet-bundler v0.3.1
+	github.com/jsonnet-bundler/jsonnet-bundler v0.4.0
 	github.com/kr/pretty v0.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.7 // indirect
 	github.com/pkg/errors v0.8.1

go.sum

@@ -39,8 +39,6 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-jsonnet v0.16.0 h1:Nb4EEOp+rdeGGyB1rQ5eisgSAqrTnhf9ip+X6lzZbY0=
-github.com/google/go-jsonnet v0.16.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw=
 github.com/google/go-jsonnet v0.16.1-0.20200703153429-aaf50f5b655f h1:mw4KoMG5/DXLPhpKXQRYTEIZFkFo0a1HU2R1HbeYpek=
 github.com/google/go-jsonnet v0.16.1-0.20200703153429-aaf50f5b655f/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
@@ -52,8 +50,8 @@ github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jsonnet-bundler/jsonnet-bundler v0.3.1 h1:KmNzitX12fFoyqjhU8cRifEB5D8x1NT1UAcK7FQ0zpY=
-github.com/jsonnet-bundler/jsonnet-bundler v0.3.1/go.mod h1:/by7P/OoohkI3q4CgSFqcoFsVY+IaNbzOVDknEsKDeU=
+github.com/jsonnet-bundler/jsonnet-bundler v0.4.0 h1:4BKZ6LDqPc2wJDmaKnmYD/vDjUptJtnUpai802MibFc=
+github.com/jsonnet-bundler/jsonnet-bundler v0.4.0/go.mod h1:/by7P/OoohkI3q4CgSFqcoFsVY+IaNbzOVDknEsKDeU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=

jsonnet/kube-prometheus/alerts/prometheus-operator.libsonnet

@@ -7,7 +7,7 @@
           {
             alert: 'PrometheusOperatorListErrors',
             expr: |||
-              (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.4
+              (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{%(prometheusOperatorSelector)s}[10m])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{%(prometheusOperatorSelector)s}[10m]))) > 0.4
             ||| % $._config,
             labels: {
               severity: 'warning',
@@ -20,7 +20,7 @@
           {
             alert: 'PrometheusOperatorWatchErrors',
             expr: |||
-              (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[1h]))) > 0.4
+              (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{%(prometheusOperatorSelector)s}[10m])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{%(prometheusOperatorSelector)s}[10m]))) > 0.4
             ||| % $._config,
             labels: {
               severity: 'warning',

jsonnet/kube-prometheus/jsonnetfile.json

@@ -17,7 +17,7 @@
           "subdir": "Documentation/etcd-mixin"
         }
       },
-      "version": "master"
+      "version": "e8ba375032e8e48d009759dfb285f7812e7bcb8c"
     },
     {
       "source": {
@@ -26,7 +26,7 @@
           "subdir": "jsonnet/prometheus-operator"
         }
       },
-      "version": "release-0.40"
+      "version": "release-0.42"
     },
     {
       "source": {
@@ -35,7 +35,7 @@
           "subdir": ""
         }
       },
-      "version": "master",
+      "version": "0d2f82676817bbf9e4acf6495b2090205f323b9f",
       "name": "ksonnet"
     },
     {
@@ -45,7 +45,7 @@
           "subdir": ""
         }
       },
-      "version": "master"
+      "version": "release-0.5"
     },
     {
       "source": {
@@ -72,7 +72,7 @@
           "subdir": "docs/node-mixin"
         }
       },
-      "version": "master"
+      "version": "ff2ff3410f4ea8195e51f5fb8d84151684f91b3f"
     },
     {
       "source": {

jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet

@@ -37,6 +37,23 @@
                   regex: 'container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)',
                   action: 'drop',
                 },
+                // Drop cAdvisor metrics with no (pod, namespace) labels while preserving ability to monitor system services resource usage (cardinality estimation)
+                {
+                  sourceLabels: ['__name__', 'pod', 'namespace'],
+                  action: 'drop',
+                  regex: '(' + std.join('|',
+                                        [
+                                          'container_fs_.*',  // add filesystem read/write data (nodes*disks*services*4)
+                                          'container_spec_.*',  // everything related to cgroup specification and thus static data (nodes*services*5)
+                                          'container_blkio_device_usage_total',  // useful for containers, but not for system services (nodes*disks*services*operations*2)
+                                          'container_file_descriptors',  // file descriptors limits and global numbers are exposed via (nodes*services)
+                                          'container_sockets',  // used sockets in cgroup. Usually not important for system services (nodes*services)
+                                          'container_threads_max',  // max number of threads in cgroup. Usually for system services it is not limited (nodes*services)
+                                          'container_threads',  // used threads in cgroup. Usually not important for system services (nodes*services)
+                                          'container_start_time_seconds',  // container start. Possibly not needed for system services (nodes*services)
+                                          'container_last_seen',  // not needed as system services are always running (nodes*services)
+                                        ]) + ');;',
+                },
               ],
             },
           ],

jsonnet/kube-prometheus/node-exporter/node-exporter.libsonnet

@@ -137,6 +137,7 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
       daemonset.mixin.metadata.withNamespace($._config.namespace) +
       daemonset.mixin.metadata.withLabels(podLabels) +
       daemonset.mixin.spec.selector.withMatchLabels(selectorLabels) +
+      daemonset.mixin.spec.updateStrategy.rollingUpdate.withMaxUnavailable('10%') +
       daemonset.mixin.spec.template.metadata.withLabels(podLabels) +
       daemonset.mixin.spec.template.spec.withTolerations([existsToleration]) +
       daemonset.mixin.spec.template.spec.withNodeSelector({ 'kubernetes.io/os': 'linux' }) +

jsonnet/kube-prometheus/prometheus/prometheus.libsonnet

@@ -322,6 +322,23 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
                   regex: 'container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)',
                   action: 'drop',
                 },
+              // Drop cAdvisor metrics with no (pod, namespace) labels while preserving ability to monitor system services resource usage (cardinality estimation)
+              {
+                sourceLabels: ['__name__', 'pod', 'namespace'],
+                action: 'drop',
+                regex: '(' + std.join('|',
+                                      [
+                                        'container_fs_.*',  // add filesystem read/write data (nodes*disks*services*4)
+                                        'container_spec_.*',  // everything related to cgroup specification and thus static data (nodes*services*5)
+                                        'container_blkio_device_usage_total',  // useful for containers, but not for system services (nodes*disks*services*operations*2)
+                                        'container_file_descriptors',  // file descriptors limits and global numbers are exposed via (nodes*services)
+                                        'container_sockets',  // used sockets in cgroup. Usually not important for system services (nodes*services)
+                                        'container_threads_max',  // max number of threads in cgroup. Usually for system services it is not limited (nodes*services)
+                                        'container_threads',  // used threads in cgroup. Usually not important for system services (nodes*services)
+                                        'container_start_time_seconds',  // container start. Possibly not needed for system services (nodes*services)
+                                        'container_last_seen',  // not needed as system services are always running (nodes*services)
+                                      ]) + ');;',
+              },
               ],
             },
             {

jsonnetfile.lock.json

@@ -4,7 +4,7 @@
     {
       "source": {
         "git": {
-          "remote": "https://github.com/brancz/kubernetes-grafana",
+          "remote": "https://github.com/brancz/kubernetes-grafana.git",
           "subdir": "grafana"
         }
       },
@@ -14,47 +14,47 @@
     {
       "source": {
         "git": {
-          "remote": "https://github.com/coreos/etcd",
+          "remote": "https://github.com/coreos/etcd.git",
           "subdir": "Documentation/etcd-mixin"
         }
       },
-      "version": "cc564110bddeadc1d806ef0698745be702cb17ea",
-      "sum": "Uv8ysXlEACF7BafoCkHnrBmJ2AHh/VldI5mm3BuMiy0="
+      "version": "e8ba375032e8e48d009759dfb285f7812e7bcb8c",
+      "sum": "EgKKzxcW3ttt7gjPMX//DNTqNcn/0o2VAIaWJ/HSLEc="
     },
     {
       "source": {
         "git": {
-          "remote": "https://github.com/coreos/prometheus-operator",
+          "remote": "https://github.com/coreos/prometheus-operator.git",
           "subdir": "jsonnet/prometheus-operator"
         }
       },
-      "version": "0dca0f21ffff72a063db8855b5d515e15ab0dccb",
-      "sum": "WggWVWZ+CBEUThQCztSaRELbtqdXf9s3OFzf06HbYNA="
+      "version": "cd331ce9bb58bb926e391c6ae807621cb12cc29e",
+      "sum": "nM1eDP5vftqAeQSmVYzSBAh+lG0SN6zu46QiocQiVhk="
     },
     {
       "source": {
         "git": {
-          "remote": "https://github.com/grafana/grafonnet-lib",
+          "remote": "https://github.com/grafana/grafonnet-lib.git",
           "subdir": "grafonnet"
         }
       },
-      "version": "3336c69715f8f7a4d637582504c9fabd9d9ca081",
-      "sum": "w6zS28Rjs9EzRN/WoLLIdi028BvumxDTyLefYVoql2k="
+      "version": "3626fc4dc2326931c530861ac5bebe39444f6cbf",
+      "sum": "gF8foHByYcB25jcUOBqP6jxk0OPifQMjPvKY0HaCk6w="
     },
     {
       "source": {
         "git": {
-          "remote": "https://github.com/grafana/jsonnet-libs",
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
           "subdir": "grafana-builder"
         }
       },
-      "version": "08a81efce8314b3954665d3616d3ad9e545fa6ff",
-      "sum": "N65Fv0M2JvFE3GN8ZxP5xh1U5a314ey8geLAioJLzF8="
+      "version": "2ed138b205717af721af57b572bc7cd63bda62fd",
+      "sum": "U34Nd1ViO2LZ3D8IzygPPRfUcy6zOgCnTMVHZ+9O/QE="
     },
     {
       "source": {
         "git": {
-          "remote": "https://github.com/ksonnet/ksonnet-lib",
+          "remote": "https://github.com/ksonnet/ksonnet-lib.git",
           "subdir": ""
         }
       },
@@ -65,61 +65,61 @@
     {
       "source": {
         "git": {
-          "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin",
+          "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin.git",
           "subdir": ""
         }
       },
-      "version": "dc563cbb03da396d23bc49f33d4f7ae28db514a4",
-      "sum": "ZBRziwnNo3LPC4XhIjpWahz+gT+w3i2+klIcHx2r7d0="
+      "version": "7acc2fa2cad8d0038646e23656986dfb179cfa78",
+      "sum": "Of/1Y2kgQZSI/wutrkLtsq6GOMzbYXOilcTEMqaUXCQ="
     },
     {
       "source": {
         "git": {
-          "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin",
+          "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin.git",
           "subdir": "lib/promgrafonnet"
         }
       },
-      "version": "dc563cbb03da396d23bc49f33d4f7ae28db514a4",
-      "sum": "VhgBM39yv0f4bKv8VfGg4FXkg573evGDRalip9ypKbc="
+      "version": "06d00e40b43e4e618afbebe8e453b5650c659015",
+      "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps="
     },
     {
       "source": {
         "git": {
-          "remote": "https://github.com/kubernetes/kube-state-metrics",
+          "remote": "https://github.com/kubernetes/kube-state-metrics.git",
           "subdir": "jsonnet/kube-state-metrics"
         }
       },
-      "version": "eef2b125b5f09d0cc5245c0db897a27c9b74ca9b",
+      "version": "e72315512a38653b19dcfe4429f93eadedc0ea96",
       "sum": "zD/pbQLnQq+5hegEelaheHS8mn1h09GTktFO74iwlBI="
     },
     {
       "source": {
         "git": {
-          "remote": "https://github.com/kubernetes/kube-state-metrics",
+          "remote": "https://github.com/kubernetes/kube-state-metrics.git",
           "subdir": "jsonnet/kube-state-metrics-mixin"
         }
       },
-      "version": "eef2b125b5f09d0cc5245c0db897a27c9b74ca9b",
+      "version": "e72315512a38653b19dcfe4429f93eadedc0ea96",
       "sum": "E1GGavnf9PCWBm4WVrxWnc0FIj72UcbcweqGioWrOdU="
     },
     {
       "source": {
         "git": {
-          "remote": "https://github.com/prometheus/node_exporter",
+          "remote": "https://github.com/prometheus/node_exporter.git",
           "subdir": "docs/node-mixin"
         }
       },
-      "version": "503e4fc8486c0082d6bd8c53fad646bcfafeedf6",
-      "sum": "3jFV2qsc/GZe2GADswTYqxxP2zGOiANTj73W/VNFGqc="
+      "version": "ff2ff3410f4ea8195e51f5fb8d84151684f91b3f",
+      "sum": "znDrZiHvvascm7Xuj3lTASIOfwX4Vmx7PELmKKw4YiI="
     },
     {
       "source": {
         "git": {
-          "remote": "https://github.com/prometheus/prometheus",
+          "remote": "https://github.com/prometheus/prometheus.git",
           "subdir": "documentation/prometheus-mixin"
         }
       },
-      "version": "e5a06b483527d4fe0704b8fa3a2b475b661c526f",
+      "version": "983ebb4a513302315a8117932ab832815f85e3d2",
       "sum": "TBq4SL7YsPInARbJqwz25JaBvvAegcnRCsuz3K9niWc=",
       "name": "prometheus"
     },

kustomization.yaml

@@ -57,6 +57,7 @@ resources:
 - ./manifests/setup/0namespace-namespace.yaml
 - ./manifests/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
 - ./manifests/setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml
+- ./manifests/setup/prometheus-operator-0probeCustomResourceDefinition.yaml
 - ./manifests/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml
 - ./manifests/setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
 - ./manifests/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml

manifests/node-exporter-daemonset.yaml

@@ -88,3 +88,6 @@ spec:
       - hostPath:
           path: /
         name: root
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 10%

manifests/prometheus-operator-serviceMonitor.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: controller
     app.kubernetes.io/name: prometheus-operator
-    app.kubernetes.io/version: v0.40.0
+    app.kubernetes.io/version: v0.42.1
   name: prometheus-operator
   namespace: monitoring
 spec:
@@ -19,4 +19,4 @@ spec:
     matchLabels:
       app.kubernetes.io/component: controller
       app.kubernetes.io/name: prometheus-operator
-      app.kubernetes.io/version: v0.40.0
+      app.kubernetes.io/version: v0.42.1

manifests/prometheus-rules.yaml

@@ -1020,6 +1020,28 @@ spec:
       for: 10m
       labels:
         severity: warning
+    - alert: NodeRAIDDegraded
+      annotations:
+        description: RAID array '{{ $labels.device }}' on {{ $labels.instance }} is
+          in degraded state due to one or more disks failures. Number of spare drives
+          is insufficient to fix issue automatically.
+        runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-noderaiddegraded
+        summary: RAID Array is degraded
+      expr: |
+        node_md_disks_required - ignoring (state) (node_md_disks{state="active"}) > 0
+      for: 15m
+      labels:
+        severity: critical
+    - alert: NodeRAIDDiskFailure
+      annotations:
+        description: At least one device in RAID array on {{ $labels.instance }} failed.
+          Array '{{ $labels.device }}' needs attention and possibly a disk swap.
+        runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-noderaiddiskfailure
+        summary: Failed device in RAID array
+      expr: |
+        node_md_disks{state="fail"} > 0
+      labels:
+        severity: warning
   - name: kubernetes-apps
     rules:
     - alert: KubePodCrashLooping
@@ -1301,6 +1323,20 @@ spec:
       for: 5m
       labels:
         severity: warning
+    - alert: KubeQuotaAlmostFull
+      annotations:
+        description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
+          }} of its {{ $labels.resource }} quota.
+        runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaalmostfull
+        summary: Namespace quota is going to be full.
+      expr: |
+        kube_resourcequota{job="kube-state-metrics", type="used"}
+          / ignoring(instance, job, type)
+        (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
+          > 0.9 < 1
+      for: 15m
+      labels:
+        severity: info
     - alert: KubeQuotaFullyUsed
       annotations:
         message: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
@@ -1310,10 +1346,24 @@ spec:
         kube_resourcequota{job="kube-state-metrics", type="used"}
           / ignoring(instance, job, type)
         (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
-          >= 1
+          == 1
       for: 15m
       labels:
         severity: info
+    - alert: KubeQuotaExceeded
+      annotations:
+        description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage
+          }} of its {{ $labels.resource }} quota.
+        runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded
+        summary: Namespace quota has exceeded the limits.
+      expr: |
+        kube_resourcequota{job="kube-state-metrics", type="used"}
+          / ignoring(instance, job, type)
+        (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
+          > 1
+      for: 15m
+      labels:
+        severity: warning
     - alert: CPUThrottlingHigh
       annotations:
         message: '{{ $value | humanizePercentage }} throttling of CPU in namespace
@@ -1875,7 +1925,7 @@ spec:
         message: Errors while performing List operations in controller {{$labels.controller}}
           in {{$labels.namespace}} namespace.
       expr: |
-        (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.4
+        (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[10m])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{job="prometheus-operator",namespace="monitoring"}[10m]))) > 0.4
       for: 15m
       labels:
         severity: warning
@@ -1884,7 +1934,7 @@ spec:
         message: Errors while performing Watch operations in controller {{$labels.controller}}
           in {{$labels.namespace}} namespace.
       expr: |
-        (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[1h]))) > 0.4
+        (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[10m])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[10m]))) > 0.4
       for: 15m
       labels:
         severity: warning

manifests/prometheus-serviceMonitorKubelet.yaml

@@ -59,6 +59,12 @@ spec:
       regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)
       sourceLabels:
       - __name__
+    - action: drop
+      regex: (container_fs_.*|container_spec_.*|container_blkio_device_usage_total|container_file_descriptors|container_sockets|container_threads_max|container_threads|container_start_time_seconds|container_last_seen);;
+      sourceLabels:
+      - __name__
+      - pod
+      - namespace
     path: /metrics/cadvisor
     port: https-metrics
     relabelings:

manifests/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml

@@ -645,7 +645,8 @@ spec:
                     type: object
                 type: object
               baseImage:
-                description: Base image that is used to deploy pods, without tag.
+                description: 'Base image that is used to deploy pods, without tag.
+                  Deprecated: use ''image'' instead'
                 type: string
               clusterAdvertiseAddress:
                 description: 'ClusterAdvertiseAddress is the explicit address to advertise
@@ -1716,6 +1717,12 @@ spec:
                   under. This is necessary to generate correct URLs. This is necessary
                   if Alertmanager is not served from root of a DNS name.
                 type: string
+              forceEnableClusterMode:
+                description: ForceEnableClusterMode ensures Alertmanager does not
+                  deactivate the cluster mode when running with a single replica.
+                  Use case is e.g. spanning an Alertmanager cluster across Kubernetes
+                  clusters with a single replica in each.
+                type: boolean
               image:
                 description: Image if specified has precedence over baseImage, tag
                   and sha combinations. Specifying the version is still necessary
@@ -3014,10 +3021,11 @@ spec:
                   to use to run the Prometheus Pods.
                 type: string
               sha:
-                description: SHA of Alertmanager container image to be deployed. Defaults
-                  to the value of `version`. Similar to a tag, but the SHA explicitly
-                  deploys an immutable container image. Version and Tag are ignored
-                  if SHA is set.
+                description: 'SHA of Alertmanager container image to be deployed.
+                  Defaults to the value of `version`. Similar to a tag, but the SHA
+                  explicitly deploys an immutable container image. Version and Tag
+                  are ignored if SHA is set. Deprecated: use ''image'' instead.  The
+                  image digest can be specified as part of the image URL.'
                 type: string
               storage:
                 description: Storage is the definition of how storage will be used
@@ -3279,8 +3287,10 @@ spec:
                     type: object
                 type: object
               tag:
-                description: Tag of Alertmanager container image to be deployed. Defaults
-                  to the value of `version`. Version is ignored if Tag is set.
+                description: 'Tag of Alertmanager container image to be deployed.
+                  Defaults to the value of `version`. Version is ignored if Tag is
+                  set. Deprecated: use ''image'' instead.  The image tag can be specified
+                  as part of the image URL.'
                 type: string
               tolerations:
                 description: If specified, the pod's tolerations.

manifests/setup/prometheus-operator-0probeCustomResourceDefinition.yaml

@@ -0,0 +1,212 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.4
+  creationTimestamp: null
+  name: probes.monitoring.coreos.com
+spec:
+  group: monitoring.coreos.com
+  names:
+    kind: Probe
+    listKind: ProbeList
+    plural: probes
+    singular: probe
+  scope: Namespaced
+  versions:
+  - name: v1
+    schema:
+      openAPIV3Schema:
+        description: Probe defines monitoring for a set of static targets or ingresses.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Specification of desired Ingress selection for target discovery
+              by Prometheus.
+            properties:
+              interval:
+                description: Interval at which targets are probed using the configured
+                  prober. If not specified Prometheus' global scrape interval is used.
+                type: string
+              jobName:
+                description: The job name assigned to scraped metrics by default.
+                type: string
+              module:
+                description: 'The module to use for probing specifying how to probe
+                  the target. Example module configuring in the blackbox exporter:
+                  https://github.com/prometheus/blackbox_exporter/blob/master/example.yml'
+                type: string
+              prober:
+                description: Specification for the prober to use for probing targets.
+                  The prober.URL parameter is required. Targets cannot be probed if
+                  left empty.
+                properties:
+                  path:
+                    description: Path to collect metrics from. Defaults to `/probe`.
+                    type: string
+                  scheme:
+                    description: HTTP scheme to use for scraping. Defaults to `http`.
+                    type: string
+                  url:
+                    description: Mandatory URL of the prober.
+                    type: string
+                required:
+                - url
+                type: object
+              scrapeTimeout:
+                description: Timeout for scraping metrics from the Prometheus exporter.
+                type: string
+              targets:
+                description: Targets defines a set of static and/or dynamically discovered
+                  targets to be probed using the prober.
+                properties:
+                  ingress:
+                    description: Ingress defines the set of dynamically discovered
+                      ingress objects which hosts are considered for probing.
+                    properties:
+                      namespaceSelector:
+                        description: Select Ingress objects by namespace.
+                        properties:
+                          any:
+                            description: Boolean describing whether all namespaces
+                              are selected in contrast to a list restricting them.
+                            type: boolean
+                          matchNames:
+                            description: List of namespace names.
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      relabelingConfigs:
+                        description: 'RelabelConfigs to apply to samples before ingestion.
+                          More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
+                        items:
+                          description: 'RelabelConfig allows dynamic rewriting of
+                            the label set, being applied to samples before ingestion.
+                            It defines `<metric_relabel_configs>`-section of Prometheus
+                            configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
+                          properties:
+                            action:
+                              description: Action to perform based on regex matching.
+                                Default is 'replace'
+                              type: string
+                            modulus:
+                              description: Modulus to take of the hash of the source
+                                label values.
+                              format: int64
+                              type: integer
+                            regex:
+                              description: Regular expression against which the extracted
+                                value is matched. Default is '(.*)'
+                              type: string
+                            replacement:
+                              description: Replacement value against which a regex
+                                replace is performed if the regular expression matches.
+                                Regex capture groups are available. Default is '$1'
+                              type: string
+                            separator:
+                              description: Separator placed between concatenated source
+                                label values. default is ';'.
+                              type: string
+                            sourceLabels:
+                              description: The source labels select values from existing
+                                labels. Their content is concatenated using the configured
+                                separator and matched against the configured regular
+                                expression for the replace, keep, and drop actions.
+                              items:
+                                type: string
+                              type: array
+                            targetLabel:
+                              description: Label to which the resulting value is written
+                                in a replace action. It is mandatory for replace actions.
+                                Regex capture groups are available.
+                              type: string
+                          type: object
+                        type: array
+                      selector:
+                        description: Select Ingress objects by labels.
+                        properties:
+                          matchExpressions:
+                            description: matchExpressions is a list of label selector
+                              requirements. The requirements are ANDed.
+                            items:
+                              description: A label selector requirement is a selector
+                                that contains values, a key, and an operator that
+                                relates the key and values.
+                              properties:
+                                key:
+                                  description: key is the label key that the selector
+                                    applies to.
+                                  type: string
+                                operator:
+                                  description: operator represents a key's relationship
+                                    to a set of values. Valid operators are In, NotIn,
+                                    Exists and DoesNotExist.
+                                  type: string
+                                values:
+                                  description: values is an array of string values.
+                                    If the operator is In or NotIn, the values array
+                                    must be non-empty. If the operator is Exists or
+                                    DoesNotExist, the values array must be empty.
+                                    This array is replaced during a strategic merge
+                                    patch.
+                                  items:
+                                    type: string
+                                  type: array
+                              required:
+                              - key
+                              - operator
+                              type: object
+                            type: array
+                          matchLabels:
+                            additionalProperties:
+                              type: string
+                            description: matchLabels is a map of {key,value} pairs.
+                              A single {key,value} in the matchLabels map is equivalent
+                              to an element of matchExpressions, whose key field is
+                              "key", the operator is "In", and the values array contains
+                              only "value". The requirements are ANDed.
+                            type: object
+                        type: object
+                    type: object
+                  staticConfig:
+                    description: 'StaticConfig defines static targets which are considers
+                      for probing. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config.'
+                    properties:
+                      labels:
+                        additionalProperties:
+                          type: string
+                        description: Labels assigned to all metrics scraped from the
+                          targets.
+                        type: object
+                      static:
+                        description: Targets is a list of URLs to probe using the
+                          configured prober.
+                        items:
+                          type: string
+                        type: array
+                    type: object
+                type: object
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

manifests/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml

@@ -757,6 +757,10 @@ spec:
                         scheme:
                           description: Scheme to use when firing alerts.
                           type: string
+                        timeout:
+                          description: Timeout is a per-target Alertmanager timeout
+                            when pushing alerts.
+                          type: string
                         tlsConfig:
                           description: TLS Config to use for alertmanager connection.
                           properties:
@@ -900,6 +904,11 @@ spec:
                 required:
                 - alertmanagers
                 type: object
+              allowOverlappingBlocks:
+                description: AllowOverlappingBlocks enables vertical compaction and
+                  vertical query merge in Prometheus. This is still experimental in
+                  Prometheus so it may change in any upcoming release.
+                type: boolean
               apiserverConfig:
                 description: APIServerConfig allows specifying a host and auth methods
                   to access apiserver. If left empty, Prometheus is assumed to run
@@ -1097,7 +1106,8 @@ spec:
                     type: boolean
                 type: object
               baseImage:
-                description: Base image to use for a Prometheus deployment.
+                description: 'Base image to use for a Prometheus deployment. Deprecated:
+                  use ''image'' instead'
                 type: string
               configMaps:
                 description: ConfigMaps is a list of ConfigMaps in the same namespace
@@ -3432,6 +3442,95 @@ spec:
               priorityClassName:
                 description: Priority class assigned to the Pods
                 type: string
+              probeNamespaceSelector:
+                description: '*Experimental* Namespaces to be selected for Probe discovery.
+                  If nil, only check own namespace.'
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: A label selector requirement is a selector that
+                        contains values, a key, and an operator that relates the key
+                        and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: operator represents a key's relationship to
+                            a set of values. Valid operators are In, NotIn, Exists
+                            and DoesNotExist.
+                          type: string
+                        values:
+                          description: values is an array of string values. If the
+                            operator is In or NotIn, the values array must be non-empty.
+                            If the operator is Exists or DoesNotExist, the values
+                            array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: matchLabels is a map of {key,value} pairs. A single
+                      {key,value} in the matchLabels map is equivalent to an element
+                      of matchExpressions, whose key field is "key", the operator
+                      is "In", and the values array contains only "value". The requirements
+                      are ANDed.
+                    type: object
+                type: object
+              probeSelector:
+                description: '*Experimental* Probes to be selected for target discovery.'
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: A label selector requirement is a selector that
+                        contains values, a key, and an operator that relates the key
+                        and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: operator represents a key's relationship to
+                            a set of values. Valid operators are In, NotIn, Exists
+                            and DoesNotExist.
+                          type: string
+                        values:
+                          description: values is an array of string values. If the
+                            operator is In or NotIn, the values array must be non-empty.
+                            If the operator is Exists or DoesNotExist, the values
+                            array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: matchLabels is a map of {key,value} pairs. A single
+                      {key,value} in the matchLabels map is equivalent to an element
+                      of matchExpressions, whose key field is "key", the operator
+                      is "In", and the values array contains only "value". The requirements
+                      are ANDed.
+                    type: object
+                type: object
               prometheusExternalLabelName:
                 description: Name of Prometheus external label used to denote Prometheus
                   instance name. Defaults to the value of `prometheus`. External label
@@ -4374,10 +4473,11 @@ spec:
                     type: object
                 type: object
               sha:
-                description: SHA of Prometheus container image to be deployed. Defaults
+                description: 'SHA of Prometheus container image to be deployed. Defaults
                   to the value of `version`. Similar to a tag, but the SHA explicitly
                   deploys an immutable container image. Version and Tag are ignored
-                  if SHA is set.
+                  if SHA is set. Deprecated: use ''image'' instead.  The image digest
+                  can be specified as part of the image URL.'
                 type: string
               storage:
                 description: Storage spec to specify how storage shall be used.
@@ -4638,8 +4738,10 @@ spec:
                     type: object
                 type: object
               tag:
-                description: Tag of Prometheus container image to be deployed. Defaults
-                  to the value of `version`. Version is ignored if Tag is set.
+                description: 'Tag of Prometheus container image to be deployed. Defaults
+                  to the value of `version`. Version is ignored if Tag is set. Deprecated:
+                  use ''image'' instead.  The image tag can be specified as part of
+                  the image URL.'
                 type: string
               thanos:
                 description: "Thanos configuration allows configuring various aspects
@@ -4649,7 +4751,8 @@ spec:
                   without backward compatibility in any release."
                 properties:
                   baseImage:
-                    description: Thanos base image if other than default.
+                    description: 'Thanos base image if other than default. Deprecated:
+                      use ''image'' instead'
                     type: string
                   grpcServerTlsConfig:
                     description: 'GRPCServerTLSConfig configures the gRPC server from
@@ -4842,15 +4945,17 @@ spec:
                         type: object
                     type: object
                   sha:
-                    description: SHA of Thanos container image to be deployed. Defaults
+                    description: 'SHA of Thanos container image to be deployed. Defaults
                       to the value of `version`. Similar to a tag, but the SHA explicitly
                       deploys an immutable container image. Version and Tag are ignored
-                      if SHA is set.
+                      if SHA is set. Deprecated: use ''image'' instead.  The image
+                      digest can be specified as part of the image URL.'
                     type: string
                   tag:
-                    description: Tag of Thanos sidecar container image to be deployed.
+                    description: 'Tag of Thanos sidecar container image to be deployed.
                       Defaults to the value of `version`. Version is ignored if Tag
-                      is set.
+                      is set. Deprecated: use ''image'' instead.  The image tag can
+                      be specified as part of the image URL.'
                     type: string
                   tracingConfig:
                     description: TracingConfig configures tracing in Thanos. This

manifests/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml

@@ -238,8 +238,9 @@ spec:
                       anyOf:
                       - type: integer
                       - type: string
-                      description: Name or number of the pod port this endpoint refers
-                        to. Mutually exclusive with port.
+                      description: Name or number of the target port of the Pod behind
+                        the Service, the port must be specified with container port
+                        property. Mutually exclusive with port.
                       x-kubernetes-int-or-string: true
                     tlsConfig:
                       description: TLS configuration to use when scraping the endpoint

manifests/setup/prometheus-operator-clusterRole.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: controller
     app.kubernetes.io/name: prometheus-operator
-    app.kubernetes.io/version: v0.40.0
+    app.kubernetes.io/version: v0.42.1
   name: prometheus-operator
 rules:
 - apiGroups:
@@ -18,6 +18,7 @@ rules:
   - thanosrulers/finalizers
   - servicemonitors
   - podmonitors
+  - probes
   - prometheusrules
   verbs:
   - '*'

manifests/setup/prometheus-operator-clusterRoleBinding.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: controller
     app.kubernetes.io/name: prometheus-operator
-    app.kubernetes.io/version: v0.40.0
+    app.kubernetes.io/version: v0.42.1
   name: prometheus-operator
 roleRef:
   apiGroup: rbac.authorization.k8s.io

manifests/setup/prometheus-operator-deployment.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: controller
     app.kubernetes.io/name: prometheus-operator
-    app.kubernetes.io/version: v0.40.0
+    app.kubernetes.io/version: v0.42.1
   name: prometheus-operator
   namespace: monitoring
 spec:
@@ -18,15 +18,15 @@ spec:
       labels:
         app.kubernetes.io/component: controller
         app.kubernetes.io/name: prometheus-operator
-        app.kubernetes.io/version: v0.40.0
+        app.kubernetes.io/version: v0.42.1
     spec:
       containers:
       - args:
         - --kubelet-service=kube-system/kubelet
         - --logtostderr=true
-        - --config-reloader-image=jimmidyson/configmap-reload:v0.3.0
-        - --prometheus-config-reloader=quay.io/coreos/prometheus-config-reloader:v0.40.0
-        image: quay.io/coreos/prometheus-operator:v0.40.0
+        - --config-reloader-image=jimmidyson/configmap-reload:v0.4.0
+        - --prometheus-config-reloader=quay.io/prometheus-operator/prometheus-config-reloader:v0.42.1
+        image: quay.io/prometheus-operator/prometheus-operator:v0.42.1
         name: prometheus-operator
         ports:
         - containerPort: 8080

manifests/setup/prometheus-operator-service.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: controller
     app.kubernetes.io/name: prometheus-operator
-    app.kubernetes.io/version: v0.40.0
+    app.kubernetes.io/version: v0.42.1
   name: prometheus-operator
   namespace: monitoring
 spec:

manifests/setup/prometheus-operator-serviceAccount.yaml

@@ -4,6 +4,6 @@ metadata:
   labels:
     app.kubernetes.io/component: controller
     app.kubernetes.io/name: prometheus-operator
-    app.kubernetes.io/version: v0.40.0
+    app.kubernetes.io/version: v0.42.1
   name: prometheus-operator
   namespace: monitoring

tests/e2e/main_test.go

@@ -87,7 +87,7 @@ func TestQueryPrometheus(t *testing.T) {
 	}
 
 	// Wait for pod to respond at queries at all. Then start verifying their results.
-	err := wait.Poll(5*time.Second, 1*time.Minute, func() (bool, error) {
+	err := wait.Poll(5*time.Second, 5*time.Minute, func() (bool, error) {
 		_, err := promClient.query("up")
 		return err == nil, nil
 	})

tests/e2e/travis-e2e.sh

@@ -13,17 +13,30 @@ chmod +x kubectl
 curl -Lo kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-linux-amd64
 chmod +x kind
 
-./kind create cluster --image=kindest/node:v1.18.0
-# the default kube config location used by kind
-export KUBECONFIG="${HOME}/.kube/config"
+run_e2e_tests() {
+    cluster_version=$1
 
-# create namespace, permissions, and CRDs
-./kubectl create -f manifests/setup
+    ./kind create cluster --image=kindest/node:$cluster_version
+    # the default kube config location used by kind
+    export KUBECONFIG="${HOME}/.kube/config"
+    
+    # create namespace, permissions, and CRDs
+    ./kubectl create -f manifests/setup
 
-# wait for CRD creation to complete
-until ./kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
+    # wait for CRD creation to complete
+    until ./kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
 
-# create monitoring components
-./kubectl create -f manifests/
+    # create monitoring components
+    ./kubectl create -f manifests/
 
-make test-e2e
+    make test-e2e
+
+    ./kind delete cluster
+}
+
+cluster_compatible_versions=("v1.18.8" "v1.19.0")
+
+for cluster_version in "${cluster_compatible_versions[@]}"
+do
+    run_e2e_tests $cluster_version
+done