From 03471fd86ff358d1e25694323862fa14daefdce8 Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Wed, 1 Sep 2021 13:29:29 +0530 Subject: [PATCH 1/3] Adjust threshold for SpaceFillingUp warning alert Reduce threshold of NodeFilesystemSpaceFillingUp warning alert to 20% space available, instead of 40% (default). This will align the threshold according to default kubelet GC values below[1], "imageMinimumGCAge": "2m0s", "imageGCHighThresholdPercent": 85, "imageGCLowThresholdPercent": 80, [1] https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/ Signed-off-by: Arunprasad Rajkumar --- jsonnet/kube-prometheus/components/node-exporter.libsonnet | 1 + 1 file changed, 1 insertion(+) diff --git a/jsonnet/kube-prometheus/components/node-exporter.libsonnet b/jsonnet/kube-prometheus/components/node-exporter.libsonnet index d59d22e7..2498cc81 100644 --- a/jsonnet/kube-prometheus/components/node-exporter.libsonnet +++ b/jsonnet/kube-prometheus/components/node-exporter.libsonnet @@ -28,6 +28,7 @@ local defaults = { ruleLabels: {}, _config: { nodeExporterSelector: 'job="' + defaults.name + '"', + fsSpaceFillingUpWarningThreshold: 20, fsSpaceFillingUpCriticalThreshold: 15, diskDeviceSelector: 'device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"', runbookURLPattern: 'https://runbooks.prometheus-operator.dev/runbooks/node/%s', From 16ef18ec54be99186eede4b02a57e9609eba8616 Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Wed, 1 Sep 2021 13:33:41 +0530 Subject: [PATCH 2/3] assets: regenerate Signed-off-by: Arunprasad Rajkumar --- manifests/node-exporter-prometheusRule.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/node-exporter-prometheusRule.yaml b/manifests/node-exporter-prometheusRule.yaml index e24216fa..7f36e602 100644 --- a/manifests/node-exporter-prometheusRule.yaml +++ b/manifests/node-exporter-prometheusRule.yaml @@ -21,7 +21,7 @@ spec: summary: Filesystem is predicted to run out of space within the next 24 hours. expr: | ( - node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 40 + node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 20 and predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0 and From 4de44139ec15fa2533803231290f004cf57c945c Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Thu, 2 Sep 2021 17:38:02 +0530 Subject: [PATCH 3/3] add comments to reason fsSpaceFilling threshold adjustment Signed-off-by: Arunprasad Rajkumar --- jsonnet/kube-prometheus/components/node-exporter.libsonnet | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jsonnet/kube-prometheus/components/node-exporter.libsonnet b/jsonnet/kube-prometheus/components/node-exporter.libsonnet index 2498cc81..1452174a 100644 --- a/jsonnet/kube-prometheus/components/node-exporter.libsonnet +++ b/jsonnet/kube-prometheus/components/node-exporter.libsonnet @@ -28,6 +28,11 @@ local defaults = { ruleLabels: {}, _config: { nodeExporterSelector: 'job="' + defaults.name + '"', + // Adjust NodeFilesystemSpaceFillingUp warning and critical thresholds according to the following default kubelet + // GC values, + // imageGCLowThresholdPercent: 80 + // imageGCHighThresholdPercent: 85 + // See https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/ for more details. fsSpaceFillingUpWarningThreshold: 20, fsSpaceFillingUpCriticalThreshold: 15, diskDeviceSelector: 'device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"',