easylinux/kube-prometheus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #162 from LiliC/relabel-nodename

Browse Source 
Include node name in NodeDiskRunningFull alert
This commit is contained in:
Frederic Branczyk
2019-07-24 12:11:25 -07:00
committed by GitHub
parent 16a49f00d6 05f2a18974
commit 5157534678
5 changed files with 27 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
jsonnet/kube-prometheus/alerts/node.libsonnet
View File

@@ -7,7 +7,7 @@
{ {
alert: 'NodeDiskRunningFull', alert: 'NodeDiskRunningFull',
annotations: { annotations: {
message: 'Device {{ $labels.device }} of node-exporter {{ $labels.namespace }}/{{ $labels.pod }} will be full within the next 24 hours.', message: 'Device {{ $labels.device }} on node {{ $labels.instance }} will be full within the next 24 hours.',
}, },
expr: ||| expr: |||
(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0) (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0)
@@ -20,7 +20,7 @@
{ {
alert: 'NodeDiskRunningFull', alert: 'NodeDiskRunningFull',
annotations: { annotations: {
message: 'Device {{ $labels.device }} of node-exporter {{ $labels.namespace }}/{{ $labels.pod }} will be full within the next 2 hours.', message: 'Device {{ $labels.device }} on node {{ $labels.instance }} will be full within the next 2 hours.',
}, },
expr: ||| expr: |||
(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0) (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0)

9
jsonnet/kube-prometheus/node-exporter/node-exporter.libsonnet
View File

@@ -170,6 +170,15 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
scheme: 'https', scheme: 'https',
interval: '30s', interval: '30s',
bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
relabelings: [
{
action: 'replace',
regex: '(.*)',
replacment: '$1',
sourceLabels: ['__meta_kubernetes_pod_node_name'],
targetLabel: 'instance',
},
],
tlsConfig: { tlsConfig: {
insecureSkipVerify: true, insecureSkipVerify: true,
}, },

10
jsonnetfile.lock.json
View File

@@ -8,7 +8,7 @@
"subdir": "jsonnet/kube-prometheus" "subdir": "jsonnet/kube-prometheus"
} }
}, },
"version": "176a187117e56a5ea2ee0f9bbaeee45ddb6f6972" "version": "aa7df507e98312f638c6a2aa7429906bc63949a6"
}, },
{ {
"name": "ksonnet", "name": "ksonnet",
@@ -38,7 +38,7 @@
"subdir": "grafonnet" "subdir": "grafonnet"
} }
}, },
"version": "3264a8ab6efa23d55da45ea3a3d3b39e86696c76" "version": "69bc267211790a1c3f4ea6e6211f3e8ffe22f987"
}, },
{ {
"name": "grafana-builder", "name": "grafana-builder",
@@ -48,7 +48,7 @@
"subdir": "grafana-builder" "subdir": "grafana-builder"
} }
}, },
"version": "3daf42722ee2008cae15267ee7380a58988d60cd" "version": "dbc94ab71afa538b2cf467f06751b1836920dce9"
}, },
{ {
"name": "grafana", "name": "grafana",
@@ -78,7 +78,7 @@
"subdir": "Documentation/etcd-mixin" "subdir": "Documentation/etcd-mixin"
} }
}, },
"version": "b2274efee09b49d6cdea7d6d5e6a9c090fc9d8ca" "version": "d137fa9d4ad7aa242f6fed04186a700ce082fdda"
}, },
{ {
"name": "prometheus", "name": "prometheus",
@@ -88,7 +88,7 @@
"subdir": "documentation/prometheus-mixin" "subdir": "documentation/prometheus-mixin"
} }
}, },
"version": "fb6c709a5e493f003a6dc66bc4c36c2af882de6a" "version": "87a0fe0c75a42d66fcfc82a0ad89bd2549fcfadf"
} }
] ]
} }

7
manifests/node-exporter-serviceMonitor.yaml
View File

@@ -10,6 +10,13 @@ spec:
- bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
interval: 30s interval: 30s
port: https port: https
relabelings:
- action: replace
regex: (.*)
replacment: $1
sourceLabels:
- __meta_kubernetes_pod_node_name
targetLabel: instance
scheme: https scheme: https
tlsConfig: tlsConfig:
insecureSkipVerify: true insecureSkipVerify: true

8
manifests/prometheus-rules.yaml
View File

@@ -1108,8 +1108,8 @@ spec:
rules: rules:
- alert: NodeDiskRunningFull - alert: NodeDiskRunningFull
annotations: annotations:
message: Device {{ $labels.device }} of node-exporter {{ $labels.namespace message: Device {{ $labels.device }} on node {{ $labels.instance }} will be
}}/{{ $labels.pod }} will be full within the next 24 hours. full within the next 24 hours.
expr: | expr: |
(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0) (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0)
for: 30m for: 30m
@@ -1117,8 +1117,8 @@ spec:
severity: warning severity: warning
- alert: NodeDiskRunningFull - alert: NodeDiskRunningFull
annotations: annotations:
message: Device {{ $labels.device }} of node-exporter {{ $labels.namespace message: Device {{ $labels.device }} on node {{ $labels.instance }} will be
}}/{{ $labels.pod }} will be full within the next 2 hours. full within the next 2 hours.
expr: | expr: |
(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0) (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0)
for: 10m for: 10m