contrib/kube-prometheus: Create prometheus-adapter component

contrib/kube-prometheus: Add section on Prometheus Adapter requirements to README.md

README.md

@@ -45,13 +45,16 @@ This stack is meant for cluster monitoring, so it is pre-configured to collect m
 
 ## Prerequisites
 
-You will need a Kubernetes cluster, that's it! By default it is assumed, that the kubelet uses token authN and authZ, as otherwise Prometheus needs a client certificate, which gives it full access to the kubelet, rather than just the metrics. Token authN and authZ allows more fine grained and easier access control.
+You will need a Kubernetes cluster, that's it! By default it is assumed, that the kubelet uses token authentication and authorization, as otherwise Prometheus needs a client certificate, which gives it full access to the kubelet, rather than just the metrics. Token authentication and authorization allows more fine grained and easier access control.
 
 This means the kubelet configuration must contain these flags:
 
 * `--authentication-token-webhook=true` This flag enables, that a `ServiceAccount` token can be used to authenticate against the kubelet(s).
 * `--authorization-mode=Webhook` This flag enables, that the kubelet will perform an RBAC request with the API to determine, whether the requesting entity (Prometheus in this case) is allow to access a resource, in specific for this project the `/metrics` endpoint.
 
+This stack provides [resource metrics](https://github.com/kubernetes/metrics#resource-metrics-api) by deploying the [Prometheus Adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter/).
+This adapter is an Extension API Server and Kubernetes needs to be have this feature enabled, otherwise the adapter has no effect, but is still deployed.
+
 ### minikube
 
 In order to just try out this stack, start minikube with the following command:
@@ -155,6 +158,7 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + {
 { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
 { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
 { ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
+{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
 { ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }
 ```
 

docs/developing-prometheus-rules-and-grafana-dashboards.md

@@ -22,6 +22,7 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + {
 { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
 { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
 { ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
+{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
 { ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }
 ```
 

example.jsonnet

@@ -10,4 +10,5 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + {
 { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
 { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
 { ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
+{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
 { ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }

jsonnet/kube-prometheus/kube-prometheus.libsonnet

@@ -7,6 +7,7 @@ local configMapList = k.core.v1.configMapList;
 (import 'alertmanager/alertmanager.libsonnet') +
 (import 'prometheus-operator/prometheus-operator.libsonnet') +
 (import 'prometheus/prometheus.libsonnet') +
+(import 'prometheus-adapter/prometheus-adapter.libsonnet') +
 (import 'kubernetes-mixin/mixin.libsonnet') +
 (import 'alerts/alerts.libsonnet') +
 (import 'rules/rules.libsonnet') + {

jsonnet/kube-prometheus/prometheus-adapter/prometheus-adapter.libsonnet

@@ -0,0 +1,198 @@
+local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet';
+
+{
+  _config+:: {
+    namespace: 'default',
+
+    versions+:: {
+      prometheusAdapter: 'v0.3.0',
+    },
+
+    imageRepos+:: {
+      prometheusAdapter: 'quay.io/coreos/k8s-prometheus-adapter-amd64',
+    },
+
+    prometheusAdapter+:: {
+      name: 'prometheus-adapter',
+      labels: { name: $._config.prometheusAdapter.name },
+      config: |||
+        resourceRules:
+          cpu:
+            containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
+            nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[1m])) by (<<.GroupBy>>)
+            resources:
+              overrides:
+                node:
+                  resource: node
+                namespace:
+                  resource: namespace
+                pod_name:
+                  resource: pod
+            containerLabel: container_name
+          memory:
+            containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+            nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
+            resources:
+              overrides:
+                node:
+                  resource: node
+                namespace:
+                  resource: namespace
+                pod_name:
+                  resource: pod
+            containerLabel: container_name
+          window: 1m
+      |||,
+    },
+  },
+
+  prometheusAdapter+:: {
+    apiService:
+      {
+        apiVersion: 'apiregistration.k8s.io/v1beta1',
+        kind: 'APIService',
+        metadata: {
+          name: 'v1beta1.metrics.k8s.io',
+        },
+        spec: {
+          service: {
+            name: $.prometheusAdapter.service.metadata.name,
+            namespace: $._config.namespace,
+          },
+          group: 'metrics.k8s.io',
+          version: 'v1beta1',
+          insecureSkipTLSVerify: true,
+          groupPriorityMinimum: 100,
+          versionPriority: 100,
+        },
+      },
+
+    configMap:
+      local configmap = k.core.v1.configMap;
+
+      configmap.new('adapter-config', { 'config.yaml': $._config.prometheusAdapter.config }) +
+      configmap.mixin.metadata.withNamespace($._config.namespace),
+
+    service:
+      local service = k.core.v1.service;
+      local servicePort = k.core.v1.service.mixin.spec.portsType;
+
+      service.new(
+        $._config.prometheusAdapter.name,
+        $._config.prometheusAdapter.labels,
+        servicePort.newNamed('https', 443, 6443),
+      ) +
+      service.mixin.metadata.withNamespace($._config.namespace) +
+      service.mixin.metadata.withLabels($._config.prometheusAdapter.labels),
+
+    deployment:
+      local deployment = k.apps.v1beta2.deployment;
+      local volume = deployment.mixin.spec.template.spec.volumesType;
+      local container = deployment.mixin.spec.template.spec.containersType;
+      local containerVolumeMount = container.volumeMountsType;
+
+      local c =
+        container.new($._config.prometheusAdapter.name, $._config.imageRepos.prometheusAdapter + ':' + $._config.versions.prometheusAdapter) +
+        container.withArgs([
+          '--cert-dir=/var/run/serving-cert',
+          '--config=/etc/adapter/config.yaml',
+          '--logtostderr=true',
+          '--metrics-relist-interval=1m',
+          '--prometheus-url=http://prometheus-' + $._config.prometheus.name + '.' + $._config.namespace + '.svc:9090/',
+          '--secure-port=6443',
+        ]) +
+        container.withPorts([{ containerPort: 6443 }]) +
+        container.withVolumeMounts([
+          containerVolumeMount.new('volume-serving-cert', '/var/run/serving-cert'),
+          containerVolumeMount.new('config', '/etc/adapter'),
+        ],);
+
+      deployment.new($._config.prometheusAdapter.name, 1, c, $._config.prometheusAdapter.labels) +
+      deployment.mixin.metadata.withNamespace($._config.namespace) +
+      deployment.mixin.spec.selector.withMatchLabels($._config.prometheusAdapter.labels) +
+      deployment.mixin.spec.template.spec.withServiceAccountName($.prometheusAdapter.serviceAccount.metadata.name) +
+      deployment.mixin.spec.template.spec.withVolumes([
+        // volume.fromSecret('volume-serving-cert', 'cm-adapter-serving-certs'),
+        volume.fromEmptyDir(name='volume-serving-cert'),
+        { name: 'config', configMap: { name: 'adapter-config' } },
+      ]),
+
+    serviceAccount:
+      local serviceAccount = k.core.v1.serviceAccount;
+
+      serviceAccount.new($._config.prometheusAdapter.name) +
+      serviceAccount.mixin.metadata.withNamespace($._config.namespace),
+
+    clusterRole:
+      local clusterRole = k.rbac.v1.clusterRole;
+      local policyRule = clusterRole.rulesType;
+
+      local rules =
+        policyRule.new() +
+        policyRule.withApiGroups(['']) +
+        policyRule.withResources(['nodes', 'namespaces', 'pods', 'services']) +
+        policyRule.withVerbs(['get', 'list', 'watch']);
+
+      clusterRole.new() +
+      clusterRole.mixin.metadata.withName($._config.prometheusAdapter.name) +
+      clusterRole.withRules(rules),
+
+    clusterRoleBinding:
+      local clusterRoleBinding = k.rbac.v1.clusterRoleBinding;
+
+      clusterRoleBinding.new() +
+      clusterRoleBinding.mixin.metadata.withName($._config.prometheusAdapter.name) +
+      clusterRoleBinding.mixin.metadata.withNamespace($._config.namespace) +
+      clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') +
+      clusterRoleBinding.mixin.roleRef.withName($.prometheusAdapter.clusterRole.metadata.name) +
+      clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) +
+      clusterRoleBinding.withSubjects([{
+        kind: 'ServiceAccount',
+        name: $.prometheusAdapter.serviceAccount.metadata.name,
+        namespace: $._config.namespace,
+      }]),
+
+    clusterRoleBindingDelegator:
+      local clusterRoleBinding = k.rbac.v1.clusterRoleBinding;
+
+      clusterRoleBinding.new() +
+      clusterRoleBinding.mixin.metadata.withName('resource-metrics:system:auth-delegator') +
+      clusterRoleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') +
+      clusterRoleBinding.mixin.roleRef.withName('system:auth-delegator') +
+      clusterRoleBinding.mixin.roleRef.mixinInstance({ kind: 'ClusterRole' }) +
+      clusterRoleBinding.withSubjects([{
+        kind: 'ServiceAccount',
+        name: $.prometheusAdapter.serviceAccount.metadata.name,
+        namespace: $._config.namespace,
+      }]),
+
+    clusterRoleServerResources:
+      local clusterRole = k.rbac.v1.clusterRole;
+      local policyRule = clusterRole.rulesType;
+
+      local rules =
+        policyRule.new() +
+        policyRule.withApiGroups(['metrics.k8s.io']) +
+        policyRule.withResources(['*']) +
+        policyRule.withVerbs(['*']);
+
+      clusterRole.new() +
+      clusterRole.mixin.metadata.withName('resource-metrics-server-resources') +
+      clusterRole.withRules(rules),
+
+    roleBindingAuthReader:
+      local roleBinding = k.rbac.v1.roleBinding;
+
+      roleBinding.new() +
+      roleBinding.mixin.metadata.withName('resource-metrics-auth-reader') +
+      roleBinding.mixin.metadata.withNamespace('kube-system') +
+      roleBinding.mixin.roleRef.withApiGroup('rbac.authorization.k8s.io') +
+      roleBinding.mixin.roleRef.withName('extension-apiserver-authentication-reader') +
+      roleBinding.mixin.roleRef.mixinInstance({ kind: 'Role' }) +
+      roleBinding.withSubjects([{
+        kind: 'ServiceAccount',
+        name: $.prometheusAdapter.serviceAccount.metadata.name,
+        namespace: $._config.namespace,
+      }]),
+  },
+}

jsonnetfile.lock.json

@@ -8,7 +8,7 @@
                     "subdir": "contrib/kube-prometheus/jsonnet/kube-prometheus"
                 }
             },
-            "version": "fa0a0ae33a16a23845da8ab9973dd4eed50a20df"
+            "version": "d00e8996976492005174b6412a9194421548b247"
         },
         {
             "name": "ksonnet",

manifests/prometheus-adapter-apiService.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+spec:
+  group: metrics.k8s.io
+  groupPriorityMinimum: 100
+  insecureSkipTLSVerify: true
+  service:
+    name: prometheus-adapter
+    namespace: monitoring
+  version: v1beta1
+  versionPriority: 100

manifests/prometheus-adapter-clusterRole.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-adapter
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  - namespaces
+  - pods
+  - services
+  verbs:
+  - get
+  - list
+  - watch

manifests/prometheus-adapter-clusterRoleBinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-adapter
+  namespace: monitoring
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-adapter
+subjects:
+- kind: ServiceAccount
+  name: prometheus-adapter
+  namespace: monitoring

manifests/prometheus-adapter-clusterRoleBindingDelegator.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: resource-metrics:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: prometheus-adapter
+  namespace: monitoring

manifests/prometheus-adapter-clusterRoleServerResources.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: resource-metrics-server-resources
+rules:
+- apiGroups:
+  - metrics.k8s.io
+  resources:
+  - '*'
+  verbs:
+  - '*'

manifests/prometheus-adapter-configMap.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+data:
+  config.yaml: |
+    resourceRules:
+      cpu:
+        containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
+        nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[1m])) by (<<.GroupBy>>)
+        resources:
+          overrides:
+            node:
+              resource: node
+            namespace:
+              resource: namespace
+            pod_name:
+              resource: pod
+        containerLabel: container_name
+      memory:
+        containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+        nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
+        resources:
+          overrides:
+            node:
+              resource: node
+            namespace:
+              resource: namespace
+            pod_name:
+              resource: pod
+        containerLabel: container_name
+      window: 1m
+kind: ConfigMap
+metadata:
+  name: adapter-config
+  namespace: monitoring

manifests/prometheus-adapter-deployment.yaml

@@ -0,0 +1,41 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: prometheus-adapter
+  namespace: monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: prometheus-adapter
+  template:
+    metadata:
+      labels:
+        name: prometheus-adapter
+    spec:
+      containers:
+      - args:
+        - --cert-dir=/var/run/serving-cert
+        - --config=/etc/adapter/config.yaml
+        - --logtostderr=true
+        - --metrics-relist-interval=1m
+        - --prometheus-url=http://prometheus-k8s.monitoring.svc:9090/
+        - --secure-port=6443
+        image: directxman12/k8s-prometheus-adapter-amd64:latest
+        name: prometheus-adapter
+        ports:
+        - containerPort: 6443
+        volumeMounts:
+        - mountPath: /var/run/serving-cert
+          name: volume-serving-cert
+          readOnly: false
+        - mountPath: /etc/adapter
+          name: config
+          readOnly: false
+      serviceAccountName: prometheus-adapter
+      volumes:
+      - emptyDir: {}
+        name: volume-serving-cert
+      - configMap:
+          name: adapter-config
+        name: config

manifests/prometheus-adapter-roleBindingAuthReader.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: resource-metrics-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: prometheus-adapter
+  namespace: monitoring

manifests/prometheus-adapter-service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    name: prometheus-adapter
+  name: prometheus-adapter
+  namespace: monitoring
+spec:
+  ports:
+  - name: https
+    port: 443
+    targetPort: 6443
+  selector:
+    name: prometheus-adapter

manifests/prometheus-adapter-serviceAccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus-adapter
+  namespace: monitoring