98 lines
2.8 KiB
Plaintext
98 lines
2.8 KiB
Plaintext
### Up Alerting ###
|
|
|
|
Alert TargetDown
|
|
IF up == 0
|
|
FOR 10m
|
|
LABELS {
|
|
severity = "warning"
|
|
}
|
|
ANNOTATIONS {
|
|
summary = "target is down",
|
|
description = "A target of type {{ $labels.job }} is down."
|
|
}
|
|
|
|
### File descriptor alerts ###
|
|
|
|
ALERT TooManyOpenFiles
|
|
IF 100*process_open_fds / process_max_fds > 50
|
|
FOR 10m
|
|
LABELS {
|
|
severity = "warning"
|
|
}
|
|
ANNOTATIONS {
|
|
summary = "too many open file descriptors",
|
|
description = "{{ $labels.job }}: {{ $labels.namespace }}/{{ $labels.pod }} is using {{ $value }}% of the available file/socket descriptors.",
|
|
}
|
|
|
|
ALERT K8STooManyOpenFiles
|
|
IF 100*process_open_fds / process_max_fds > 80
|
|
FOR 10m
|
|
LABELS {
|
|
severity = "critical"
|
|
}
|
|
ANNOTATIONS {
|
|
summary = "too many open file descriptors",
|
|
description = "{{ $labels.job }}: {{ $labels.namespace }}/{{ $labels.pod }} is using {{ $value }}% of the available file/socket descriptors.",
|
|
}
|
|
|
|
instance:fd_utilization = process_open_fds / process_max_fds
|
|
|
|
# alert if file descriptors are likely to exhaust within the next 4 hours
|
|
ALERT FdExhaustionClose
|
|
IF predict_linear(instance:fd_utilization[1h], 3600 * 4) > 1
|
|
FOR 10m
|
|
LABELS {
|
|
severity = "warning"
|
|
}
|
|
ANNOTATIONS {
|
|
summary = "file descriptors soon exhausted",
|
|
description = "{{ $labels.job }}: {{ $labels.namespace }}/{{ $labels.pod }} instance {{ $labels.namespace }}/{{ $labels.pod }} will exhaust in file descriptors soon",
|
|
}
|
|
|
|
# alert if file descriptors are likely to exhaust within the next hour
|
|
ALERT FdExhaustionClose
|
|
IF predict_linear(instance:fd_utilization[10m], 3600) > 1
|
|
FOR 10m
|
|
LABELS {
|
|
severity = "critical"
|
|
}
|
|
ANNOTATIONS {
|
|
summary = "file descriptors soon exhausted",
|
|
description = "{{ $labels.job }}: {{ $labels.namespace }}/{{ $labels.pod }} instance {{ $labels.namespace }}/{{ $labels.pod }} will exhaust in file descriptors soon",
|
|
}
|
|
|
|
### Contrack alerts ###
|
|
|
|
# To catch the conntrack sysctl de-tuning when it happens
|
|
ALERT ConntrackTuningMissing
|
|
IF node_nf_conntrack_udp_timeout > 10
|
|
FOR 10m
|
|
LABELS {
|
|
severity = "warning",
|
|
}
|
|
ANNOTATIONS {
|
|
summary = "Node does not have the correct conntrack tunings",
|
|
description = "Nodes keep un-setting the correct tunings, investigate when it happens.",
|
|
}
|
|
|
|
ALERT ConntrackTableFull
|
|
IF 100*node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 50
|
|
FOR 10m
|
|
LABELS {
|
|
severity = "warning"
|
|
}
|
|
ANNOTATIONS {
|
|
summary = "Number of tracked connections is near the limit",
|
|
description = "The nf_conntrack table is {{ $value }}% full.",
|
|
}
|
|
|
|
ALERT ConntrackTableFull
|
|
IF 100*node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 90
|
|
LABELS {
|
|
severity = "critical"
|
|
}
|
|
ANNOTATIONS {
|
|
summary = "Number of tracked connections is near the limit",
|
|
description = "The nf_conntrack table is {{ $value }}% full.",
|
|
}
|