Merge pull request #1552 from brancz/static-etcd

kube-prometheus: Add kube-prometheus mixin to configure static etcd

README.md

@@ -285,6 +285,37 @@ In the above example the configuration has been inlined, but can just as well be
    },
  }).alertmanager.secret
 ```
+### Static etcd configuration
+
+In order to configure a static etcd cluster to scrape there is a simple mixin prepared, so only the IPs and certificate information need to be configured. Simply append the `kube-prometheus/kube-prometheus-static-etcd.libsonnet` mixin to the rest of the configuration, and configure the `ips` to be the IPs to scrape, and the `clientCA`, `clientKey` and `clientCert` to values that are valid to scrape etcd metrics with.
+
+Most likely these certificates are generated somewhere in an infrastructure repository, so using the jsonnet `importstr` function can be useful here. All the sensitive information on the certificates will end up in a Kubernetes Secret.
+
+[embedmd]:# (examples/etcd.jsonnet)
+```jsonnet
+local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') +
+           (import 'kube-prometheus/kube-prometheus-static-etcd.libsonnet') + {
+  _config+:: {
+    namespace: 'monitoring',
+
+    etcd+:: {
+      ips: ['127.0.0.1'],
+      clientCA: importstr 'etcd-client-ca.crt',
+      clientKey: importstr 'etcd-client.key',
+      clientCert: importstr 'etcd-client.crt',
+      serverName: 'etcd.my-cluster.local',
+    },
+  },
+};
+
+{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
+{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
+{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
+{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
+{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
+{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
+{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }
+```
 
 ### Customizing Prometheus alerting/recording rules and Grafana dashboards
 

examples/etcd.jsonnet

@@ -0,0 +1,22 @@
+local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') +
+           (import 'kube-prometheus/kube-prometheus-static-etcd.libsonnet') + {
+  _config+:: {
+    namespace: 'monitoring',
+
+    etcd+:: {
+      ips: ['127.0.0.1'],
+      clientCA: importstr 'etcd-client-ca.crt',
+      clientKey: importstr 'etcd-client.key',
+      clientCert: importstr 'etcd-client.crt',
+      serverName: 'etcd.my-cluster.local',
+    },
+  },
+};
+
+{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
+{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
+{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
+{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
+{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
+{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
+{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }

jsonnet/kube-prometheus/jsonnetfile.json

@@ -39,6 +39,16 @@
                 }
             },
             "version": "master"
+        },
+        {
+            "name": "etcd-mixin",
+            "source": {
+                "git": {
+                    "remote": "https://github.com/coreos/etcd",
+                    "subdir": "Documentation/etcd-mixin"
+                }
+            },
+            "version": "master"
         }
     ]
 }

jsonnet/kube-prometheus/kube-prometheus-static-etcd.libsonnet

@@ -0,0 +1,95 @@
+local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet';
+
+(import 'etcd-mixin/mixin.libsonnet') + {
+  _config+:: {
+    etcd: {
+      ips: [],
+      clientCA: null,
+      clientKey: null,
+      clientCert: null,
+      serverName: null,
+    },
+  },
+  prometheus+:: {
+    serviceEtcd:
+      local service = k.core.v1.service;
+      local servicePort = k.core.v1.service.mixin.spec.portsType;
+
+      local etcdServicePort = servicePort.newNamed('metrics', 2379, 2379);
+
+      service.new('etcd', null, etcdServicePort) +
+      service.mixin.metadata.withNamespace('kube-system') +
+      service.mixin.metadata.withLabels({ 'k8s-app': 'etcd' }) +
+      service.mixin.spec.withClusterIp('None'),
+    endpointsEtcd:
+      local endpoints = k.core.v1.endpoints;
+      local endpointSubset = endpoints.subsetsType;
+      local endpointPort = endpointSubset.portsType;
+
+      local etcdPort = endpointPort.new() +
+                       endpointPort.withName('metrics') +
+                       endpointPort.withPort(2379) +
+                       endpointPort.withProtocol('TCP');
+
+      local subset = endpointSubset.new() +
+                     endpointSubset.withAddresses([
+                       { ip: etcdIP }
+                       for etcdIP in $._config.etcd.ips
+                     ]) +
+                     endpointSubset.withPorts(etcdPort);
+
+      endpoints.new() +
+      endpoints.mixin.metadata.withName('etcd') +
+      endpoints.mixin.metadata.withNamespace('kube-system') +
+      endpoints.mixin.metadata.withLabels({ 'k8s-app': 'etcd' }) +
+      endpoints.withSubsets(subset),
+    serviceMonitorEtcd:
+      {
+        apiVersion: 'monitoring.coreos.com/v1',
+        kind: 'ServiceMonitor',
+        metadata: {
+          name: 'etcd',
+          namespace: 'kube-system',
+          labels: {
+            'k8s-app': 'etcd',
+          },
+        },
+        spec: {
+          jobLabel: 'k8s-app',
+          endpoints: [
+            {
+              port: 'metrics',
+              interval: '30s',
+              scheme: 'https',
+              tlsConfig: {
+                caFile: '/etc/prometheus/secrets/kube-etcd-client-certs/etcd-client-ca.crt',
+                keyFile: '/etc/prometheus/secrets/kube-etcd-client-certs/etcd-client.key',
+                certFile: '/etc/prometheus/secrets/kube-etcd-client-certs/etcd-client.crt',
+                serverName: $._config.etcd.serverName,
+              },
+            },
+          ],
+          selector: {
+            matchLabels: {
+              'k8s-app': 'etcd',
+            },
+          },
+        },
+      },
+    secretEtcdCerts:
+      local secret = k.core.v1.secret;
+
+      secret.new('kube-etcd-client-certs', {
+        'etcd-client-ca.crt': std.base64($._config.etcd.clientCA),
+        'etcd-client.key': std.base64($._config.etcd.clientKey),
+        'etcd-client.crt': std.base64($._config.etcd.clientCert),
+      }) +
+      secret.mixin.metadata.withNamespace($._config.namespace),
+    prometheus+:
+      {
+        spec+: {
+          secrets+: [$.prometheus.secretEtcdCerts.metadata.name],
+        },
+      },
+  },
+}