This change drops pod-centric metrics without a non-empty 'container' label.

Previously we dropped pod-centric metrics without a (pod, namespace) label set
however these can be critical for debugging.

Keep 'container_fs_.*' metrics from cAdvisor

jsonnet/kube-prometheus/kube-prometheus-insecure-kubelet.libsonnet

@@ -43,9 +43,7 @@
                   action: 'drop',
                   regex: '(' + std.join('|',
                                         [
-                                          'container_fs_.*',  // add filesystem read/write data (nodes*disks*services*4)
                                           'container_spec_.*',  // everything related to cgroup specification and thus static data (nodes*services*5)
-                                          'container_blkio_device_usage_total',  // useful for containers, but not for system services (nodes*disks*services*operations*2)
                                           'container_file_descriptors',  // file descriptors limits and global numbers are exposed via (nodes*services)
                                           'container_sockets',  // used sockets in cgroup. Usually not important for system services (nodes*services)
                                           'container_threads_max',  // max number of threads in cgroup. Usually for system services it is not limited (nodes*services)
@@ -54,6 +52,14 @@
                                           'container_last_seen',  // not needed as system services are always running (nodes*services)
                                         ]) + ');;',
                 },
+                {
+                  sourceLabels: ['__name__', 'container'],
+                  action: 'drop',
+                  regex: '(' + std.join('|',
+                                        [
+                                          'container_blkio_device_usage_total',
+                                        ]) + ');.+',
+                },
               ],
             },
           ],

jsonnet/kube-prometheus/prometheus/prometheus.libsonnet

@@ -328,9 +328,7 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
                 action: 'drop',
                 regex: '(' + std.join('|',
                                       [
-                                        'container_fs_.*',  // add filesystem read/write data (nodes*disks*services*4)
                                         'container_spec_.*',  // everything related to cgroup specification and thus static data (nodes*services*5)
-                                        'container_blkio_device_usage_total',  // useful for containers, but not for system services (nodes*disks*services*operations*2)
                                         'container_file_descriptors',  // file descriptors limits and global numbers are exposed via (nodes*services)
                                         'container_sockets',  // used sockets in cgroup. Usually not important for system services (nodes*services)
                                         'container_threads_max',  // max number of threads in cgroup. Usually for system services it is not limited (nodes*services)
@@ -339,6 +337,14 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
                                         'container_last_seen',  // not needed as system services are always running (nodes*services)
                                       ]) + ');;',
               },
+              {
+                sourceLabels: ['__name__', 'container'],
+                action: 'drop',
+                regex: '(' + std.join('|',
+                                      [
+                                        'container_blkio_device_usage_total',
+                                      ]) + ');.+',
+              },
               ],
             },
             {

manifests/prometheus-serviceMonitorKubelet.yaml

@@ -60,11 +60,16 @@ spec:
       sourceLabels:
       - __name__
     - action: drop
-      regex: (container_fs_.*|container_spec_.*|container_blkio_device_usage_total|container_file_descriptors|container_sockets|container_threads_max|container_threads|container_start_time_seconds|container_last_seen);;
+      regex: (container_spec_.*|container_file_descriptors|container_sockets|container_threads_max|container_threads|container_start_time_seconds|container_last_seen);;
       sourceLabels:
       - __name__
       - pod
       - namespace
+    - action: drop
+      regex: (container_blkio_device_usage_total);.+
+      sourceLabels:
+      - __name__
+      - container
     path: /metrics/cadvisor
     port: https-metrics
     relabelings: