598d6779cd
* If any of the rules is active > 10m * If all daemonsets are not ready * If all daemonsets are not scheduled * If some are miss scheduled
77 lines
2.7 KiB
YAML
77 lines
2.7 KiB
YAML
groups:
|
|
- name: ./kubelet.rules
|
|
rules:
|
|
- alert: K8SNodeNotReady
|
|
expr: kube_node_status_condition{condition="Ready",status="true"} == 0
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: The Kubelet on {{ $labels.node }} has not checked in with the API,
|
|
or has set itself to NotReady, for more than an hour
|
|
summary: Node status is NotReady
|
|
- alert: K8SManyNodesNotReady
|
|
expr: count(kube_node_status_condition{condition="Ready",status="true"} == 0)
|
|
> 1 and (count(kube_node_status_condition{condition="Ready",status="true"} ==
|
|
0) / count(kube_node_status_condition{condition="Ready",status="true"})) > 0.2
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
description: '{{ $value }} Kubernetes nodes (more than 10% are in the NotReady
|
|
state).'
|
|
summary: Many Kubernetes nodes are Not Ready
|
|
- alert: K8SKubeletDown
|
|
expr: count(up{job="kubelet"} == 0) / count(up{job="kubelet"}) > 0.03
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: Prometheus failed to scrape {{ $value }}% of kubelets.
|
|
summary: Many Kubelets cannot be scraped
|
|
- alert: K8SKubeletDown
|
|
expr: absent(up{job="kubelet"} == 1) or count(up{job="kubelet"} == 0) / count(up{job="kubelet"})
|
|
> 0.1
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
description: Prometheus failed to scrape {{ $value }}% of kubelets, or all Kubelets
|
|
have disappeared from service discovery.
|
|
summary: Many Kubelets cannot be scraped
|
|
- alert: K8SKubeletTooManyPods
|
|
expr: kubelet_running_pod_count > 100
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: Kubelet {{$labels.instance}} is running {{$value}} pods, close
|
|
to the limit of 110
|
|
summary: Kubelet is close to pod limit
|
|
- alert: K8SDaemonSetsNotScheduled
|
|
expr: kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled
|
|
> 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: A number of daemonsets are not scheduled.
|
|
summary: Daemonsets are not scheduled correctly
|
|
- alert: K8SDaemonSetsNotRunning
|
|
expr: kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_number_ready
|
|
> 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: A number of daemonsets are not ready.
|
|
summary: Daemonsets are not ready
|
|
- alert: K8SDaemonSetsMissScheduled
|
|
expr: kube_daemonset_status_number_misscheduled > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
description: A number of daemonsets are running where they are not supposed
|
|
to run.
|
|
summary: Daemonsets are not scheduled correctly
|