easylinux/kube-prometheus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1244 from flurreN/prom-rules-hpa

Browse Source
This commit is contained in:
Paweł Krupa
2021-07-07 10:30:18 +02:00
committed by GitHub
parent acd7cdcde0 3e6865d776
commit 60b4b3023d
3 changed files with 46 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
jsonnetfile.lock.json
View File

@@ -18,8 +18,8 @@
"subdir": "contrib/mixin" "subdir": "contrib/mixin"
} }
}, },
"version": "556447683112b6ccc00b27a1bd0d0a4edc361692", "version": "f4fad92e0d0240d1495cc6bf9d2f4d1fbeab4e4b",
"sum": "W/Azptf1PoqjyMwJON96UY69MFugDA4IAYiKURscryc=" "sum": "RqOj1ITnyH6jm/6qBtEE+Ob5KTk/nKX1zlhj/SWqARE="
}, },
{ {
"source": { "source": {
@@ -38,7 +38,7 @@
"subdir": "grafana-builder" "subdir": "grafana-builder"
} }
}, },
"version": "054a2cecd449b88582386cd12a5865daca10439e", "version": "d8d3c4b74089f6f6a83b6c99d7425e2d646a1b30",
"sum": "GRf2GvwEU4jhXV+JOonXSZ4wdDv8mnHBPCQ6TUVd+g8=" "sum": "GRf2GvwEU4jhXV+JOonXSZ4wdDv8mnHBPCQ6TUVd+g8="
}, },
{ {
@@ -48,8 +48,8 @@
"subdir": "" "subdir": ""
} }
}, },
"version": "e0dc3563dcbf2e54e0ffe8e83f3f51b237ef33be", "version": "6a4b21fb685af4a26983a4e51103f62b3ca244f7",
"sum": "egi2xHFco6VkCxettVvAju/yrsGnB3AFoPpCGKfWhtU=" "sum": "Z/G2a7I4uXmDSgcwJqZ1f4haO+CInIJgYiLaCLCpm6U="
}, },
{ {
"source": { "source": {
@@ -58,7 +58,7 @@
"subdir": "lib/promgrafonnet" "subdir": "lib/promgrafonnet"
} }
}, },
"version": "e0dc3563dcbf2e54e0ffe8e83f3f51b237ef33be", "version": "6a4b21fb685af4a26983a4e51103f62b3ca244f7",
"sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps=" "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps="
}, },
{ {
@@ -68,7 +68,7 @@
"subdir": "jsonnet/kube-state-metrics" "subdir": "jsonnet/kube-state-metrics"
} }
}, },
"version": "d94da5292d7e213c5b2f3f508d162e6044a68ca9", "version": "7594278d264753220921eef1a59af4927b375573",
"sum": "S5qI+PJUdNeYOv76jH5nxwYS9N6U7CRxvyuB1wI4cTE=" "sum": "S5qI+PJUdNeYOv76jH5nxwYS9N6U7CRxvyuB1wI4cTE="
}, },
{ {
@@ -78,7 +78,7 @@
"subdir": "jsonnet/kube-state-metrics-mixin" "subdir": "jsonnet/kube-state-metrics-mixin"
} }
}, },
"version": "d94da5292d7e213c5b2f3f508d162e6044a68ca9", "version": "7594278d264753220921eef1a59af4927b375573",
"sum": "u8gaydJoxEjzizQ8jY8xSjYgWooPmxw+wIWdDxifMAk=" "sum": "u8gaydJoxEjzizQ8jY8xSjYgWooPmxw+wIWdDxifMAk="
}, },
{ {
@@ -88,7 +88,7 @@
"subdir": "jsonnet/mixin" "subdir": "jsonnet/mixin"
} }
}, },
"version": "1c35faa561f2557fa156438c13c5763d3b6ac6c4", "version": "2388bfa557c9836d5ff01e620a129d33852670ff",
"sum": "6reUygVmQrLEWQzTKcH8ceDbvM+2ztK3z2VBR2K2l+U=", "sum": "6reUygVmQrLEWQzTKcH8ceDbvM+2ztK3z2VBR2K2l+U=",
"name": "prometheus-operator-mixin" "name": "prometheus-operator-mixin"
}, },
@@ -99,7 +99,7 @@
"subdir": "jsonnet/prometheus-operator" "subdir": "jsonnet/prometheus-operator"
} }
}, },
"version": "1c35faa561f2557fa156438c13c5763d3b6ac6c4", "version": "2388bfa557c9836d5ff01e620a129d33852670ff",
"sum": "eHJp7oFWvBEsSmwoRML356DLK80n7rRt8XKRZ+YawvQ=" "sum": "eHJp7oFWvBEsSmwoRML356DLK80n7rRt8XKRZ+YawvQ="
}, },
{ {
@@ -109,7 +109,7 @@
"subdir": "doc/alertmanager-mixin" "subdir": "doc/alertmanager-mixin"
} }
}, },
"version": "fd0929ba9fc58737a9c91f24771862692fa72d17", "version": "8491f816296b26bd4efe3aa4164fe72a7cd329de",
"sum": "pep+dHzfIjh2SU5pEkwilMCAT/NoL6YYflV4x8cr7vU=", "sum": "pep+dHzfIjh2SU5pEkwilMCAT/NoL6YYflV4x8cr7vU=",
"name": "alertmanager" "name": "alertmanager"
}, },
@@ -120,7 +120,7 @@
"subdir": "docs/node-mixin" "subdir": "docs/node-mixin"
} }
}, },
"version": "90d469805ac04cf1be31fdb50fcd895146332e92", "version": "35a2de21d4ae686f4d3e9e0a54059a6c7ef93885",
"sum": "os3VfjBdFdDaTYzI+A/RahIhQcgQ7KoaLL68s1kiCbA=" "sum": "os3VfjBdFdDaTYzI+A/RahIhQcgQ7KoaLL68s1kiCbA="
}, },
{ {
@@ -130,7 +130,7 @@
"subdir": "documentation/prometheus-mixin" "subdir": "documentation/prometheus-mixin"
} }
}, },
"version": "d8219f29f4fec899edde3d3055acf3013dc3f77a", "version": "62598878dd080d0488457e2238fb6e5a847a8247",
"sum": "G3mFWvwIrrhG6hlPz/hQdE6ZNSim88DlbSDJN7enkhY=", "sum": "G3mFWvwIrrhG6hlPz/hQdE6ZNSim88DlbSDJN7enkhY=",
"name": "prometheus" "name": "prometheus"
}, },
@@ -141,7 +141,7 @@
"subdir": "mixin" "subdir": "mixin"
} }
}, },
"version": "8f5021166d21005f9a63c3c44fd876113fd0c05d", "version": "e7d3ec77ee6ad2580e50b371c00465ad277340a1",
"sum": "cajthvLKDjYgYHCKQU2g/pTMRkxcbuJEvTnCyJOihl8=", "sum": "cajthvLKDjYgYHCKQU2g/pTMRkxcbuJEvTnCyJOihl8=",
"name": "thanos-mixin" "name": "thanos-mixin"
}, },

40
manifests/grafana-dashboardDefinitions.yaml
View File

@@ -7504,7 +7504,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -7590,7 +7590,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -7688,7 +7688,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -7774,7 +7774,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -10210,7 +10210,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -10296,7 +10296,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -10394,7 +10394,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -10480,7 +10480,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -13362,7 +13362,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -13449,7 +13449,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -13548,7 +13548,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -13635,7 +13635,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -16004,7 +16004,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -16090,7 +16090,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -16188,7 +16188,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -16274,7 +16274,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -18166,7 +18166,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -18252,7 +18252,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -18350,7 +18350,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,
@@ -18436,7 +18436,7 @@ items:
}, },
"yaxes": [ "yaxes": [
{ {
"format": "Bps", "format": "pps",
"label": null, "label": null,
"logBase": 1, "logBase": 1,
"max": null, "max": null,

13
manifests/kubernetes-prometheusRule.yaml
View File

@@ -60,7 +60,7 @@ spec:
expr: | expr: |
( (
kube_deployment_spec_replicas{job="kube-state-metrics"} kube_deployment_spec_replicas{job="kube-state-metrics"}
!= >
kube_deployment_status_replicas_available{job="kube-state-metrics"} kube_deployment_status_replicas_available{job="kube-state-metrics"}
) and ( ) and (
changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m]) changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m])
@@ -212,7 +212,7 @@ spec:
severity: warning severity: warning
- alert: KubeHpaReplicasMismatch - alert: KubeHpaReplicasMismatch
annotations: annotations:
description: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the desired number of replicas for longer than 15 minutes. description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has not matched the desired number of replicas for longer than 15 minutes.
runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubehpareplicasmismatch runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubehpareplicasmismatch
summary: HPA has not matched descired number of replicas. summary: HPA has not matched descired number of replicas.
expr: | expr: |
@@ -234,7 +234,7 @@ spec:
severity: warning severity: warning
- alert: KubeHpaMaxedOut - alert: KubeHpaMaxedOut
annotations: annotations:
description: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has been running at max replicas for longer than 15 minutes. description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes.
runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubehpamaxedout runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubehpamaxedout
summary: HPA is running at max replicas summary: HPA is running at max replicas
expr: | expr: |
@@ -362,10 +362,13 @@ spec:
runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubepersistentvolumefillingup runbook_url: https://github.com/prometheus-operator/kube-prometheus/wiki/kubepersistentvolumefillingup
summary: PersistentVolume is filling up. summary: PersistentVolume is filling up.
expr: | expr: |
(
kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"} kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}
/ /
kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"} kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}
< 0.03 ) < 0.03
and
kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0
for: 1m for: 1m
labels: labels:
severity: critical severity: critical
@@ -381,6 +384,8 @@ spec:
kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"} kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}
) < 0.15 ) < 0.15
and and
kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0
and
predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
for: 1h for: 1h
labels: labels: