easylinux/kube-prometheus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kube-prometheus: Improve disk alert

Browse Source 
Only trigger disk running full alerts when the disk capacity is over 85%
and use recording rule of the kubernetes-mixin to only consider real
physical filesystems.
This commit is contained in:
Frederic Branczyk
2018-09-05 22:42:28 +02:00
parent b338940208
commit 6536496716
1 changed files with 4 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
jsonnet/kube-prometheus/alerts/node.libsonnet
View File

@@ -7,11 +7,10 @@
{ {
alert: 'NodeDiskRunningFull', alert: 'NodeDiskRunningFull',
annotations: { annotations: {
description: 'device {{$labels.device}} on node {{$labels.instance}} is running full within the next 24 hours (mounted at {{$labels.mountpoint}})', message: 'Device {{ $labels.device }} of node-exporter {{ $labels.namespace }}/{{ $labels.pod }} is running full within the next 24 hours.',
summary: 'Node disk is running full within 24 hours',
}, },
expr: ||| expr: |||
predict_linear(node_filesystem_free{%(nodeExporterSelector)s,mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[6h], 3600 * 24) < 0 and on(instance) up{%(nodeExporterSelector)s} (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0)
||| % $._config, ||| % $._config,
'for': '30m', 'for': '30m',
labels: { labels: {
@@ -21,11 +20,10 @@
{ {
alert: 'NodeDiskRunningFull', alert: 'NodeDiskRunningFull',
annotations: { annotations: {
description: 'device {{$labels.device}} on node {{$labels.instance}} is running full within the next 2 hours (mounted at {{$labels.mountpoint}})', message: 'Device {{ $labels.device }} of node-exporter {{ $labels.namespace }}/{{ $labels.pod }} is running full within the next 2 hours.',
summary: 'Node disk is running full within 2 hours',
}, },
expr: ||| expr: |||
predict_linear(node_filesystem_free{%(nodeExporterSelector)s,mountpoint!~"^/etc/(?:resolv.conf|hosts|hostname)$"}[30m], 3600 * 2) < 0 and on(instance) up{%(nodeExporterSelector)s} (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0)
||| % $._config, ||| % $._config,
'for': '10m', 'for': '10m',
labels: { labels: {