Merge pull request #162 from LiliC/relabel-nodename

Include node name in NodeDiskRunningFull alert

jsonnet/kube-prometheus/alerts/node.libsonnet

@@ -7,7 +7,7 @@
           {
             alert: 'NodeDiskRunningFull',
             annotations: {
-              message: 'Device {{ $labels.device }} of node-exporter {{ $labels.namespace }}/{{ $labels.pod }} will be full within the next 24 hours.',
+              message: 'Device {{ $labels.device }} on node {{ $labels.instance }} will be full within the next 24 hours.',
             },
             expr: |||
               (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0)
@@ -20,7 +20,7 @@
           {
             alert: 'NodeDiskRunningFull',
             annotations: {
-              message: 'Device {{ $labels.device }} of node-exporter {{ $labels.namespace }}/{{ $labels.pod }} will be full within the next 2 hours.',
+              message: 'Device {{ $labels.device }} on node {{ $labels.instance }} will be full within the next 2 hours.',
             },
             expr: |||
               (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0)

jsonnet/kube-prometheus/node-exporter/node-exporter.libsonnet

@@ -170,6 +170,15 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
               scheme: 'https',
               interval: '30s',
               bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+              relabelings: [
+                {
+                  action: 'replace',
+                  regex: '(.*)',
+                  replacment: '$1',
+                  sourceLabels: ['__meta_kubernetes_pod_node_name'],
+                  targetLabel: 'instance',
+                },
+              ],
               tlsConfig: {
                 insecureSkipVerify: true,
               },

jsonnetfile.lock.json

@@ -8,7 +8,7 @@
                     "subdir": "jsonnet/kube-prometheus"
                 }
             },
-            "version": "176a187117e56a5ea2ee0f9bbaeee45ddb6f6972"
+            "version": "aa7df507e98312f638c6a2aa7429906bc63949a6"
         },
         {
             "name": "ksonnet",
@@ -38,7 +38,7 @@
                     "subdir": "grafonnet"
                 }
             },
-            "version": "3264a8ab6efa23d55da45ea3a3d3b39e86696c76"
+            "version": "69bc267211790a1c3f4ea6e6211f3e8ffe22f987"
         },
         {
             "name": "grafana-builder",
@@ -48,7 +48,7 @@
                     "subdir": "grafana-builder"
                 }
             },
-            "version": "3daf42722ee2008cae15267ee7380a58988d60cd"
+            "version": "dbc94ab71afa538b2cf467f06751b1836920dce9"
         },
         {
             "name": "grafana",
@@ -78,7 +78,7 @@
                     "subdir": "Documentation/etcd-mixin"
                 }
             },
-            "version": "b2274efee09b49d6cdea7d6d5e6a9c090fc9d8ca"
+            "version": "d137fa9d4ad7aa242f6fed04186a700ce082fdda"
         },
         {
             "name": "prometheus",
@@ -88,7 +88,7 @@
                     "subdir": "documentation/prometheus-mixin"
                 }
             },
-            "version": "fb6c709a5e493f003a6dc66bc4c36c2af882de6a"
+            "version": "87a0fe0c75a42d66fcfc82a0ad89bd2549fcfadf"
         }
     ]
 }

manifests/node-exporter-serviceMonitor.yaml

@@ -10,6 +10,13 @@ spec:
   - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
     interval: 30s
     port: https
+    relabelings:
+    - action: replace
+      regex: (.*)
+      replacment: $1
+      sourceLabels:
+      - __meta_kubernetes_pod_node_name
+      targetLabel: instance
     scheme: https
     tlsConfig:
       insecureSkipVerify: true

manifests/prometheus-rules.yaml

@@ -1108,8 +1108,8 @@ spec:
     rules:
     - alert: NodeDiskRunningFull
       annotations:
-        message: Device {{ $labels.device }} of node-exporter {{ $labels.namespace
-          }}/{{ $labels.pod }} will be full within the next 24 hours.
+        message: Device {{ $labels.device }} on node {{ $labels.instance }} will be
+          full within the next 24 hours.
       expr: |
         (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0)
       for: 30m
@@ -1117,8 +1117,8 @@ spec:
         severity: warning
     - alert: NodeDiskRunningFull
       annotations:
-        message: Device {{ $labels.device }} of node-exporter {{ $labels.namespace
-          }}/{{ $labels.pod }} will be full within the next 2 hours.
+        message: Device {{ $labels.device }} on node {{ $labels.instance }} will be
+          full within the next 2 hours.
       expr: |
         (node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0)
       for: 10m