From 4410a80e4ec95a5623723d4b9e66a81bfe61c655 Mon Sep 17 00:00:00 2001 From: tafkam Date: Sat, 25 Jul 2020 18:27:17 +0200 Subject: [PATCH 1/4] secure scheduler/controller metrics ports, kubeadm discovery services --- .../kube-prometheus/kube-prometheus-kubeadm.libsonnet | 4 ++-- .../kube-prometheus/prometheus/prometheus.libsonnet | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/jsonnet/kube-prometheus/kube-prometheus-kubeadm.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-kubeadm.libsonnet index 9e497cd6..1ef808a8 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-kubeadm.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-kubeadm.libsonnet @@ -5,12 +5,12 @@ local servicePort = k.core.v1.service.mixin.spec.portsType; { prometheus+: { kubeControllerManagerPrometheusDiscoveryService: - service.new('kube-controller-manager-prometheus-discovery', { component: 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10252, 10252)) + + service.new('kube-controller-manager-prometheus-discovery', { component: 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10257, 10257)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + service.mixin.spec.withClusterIp('None'), kubeSchedulerPrometheusDiscoveryService: - service.new('kube-scheduler-prometheus-discovery', { component: 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10251, 10251)) + + service.new('kube-scheduler-prometheus-discovery', { component: 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10259, 10259)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + service.mixin.spec.withClusterIp('None'), diff --git a/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet b/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet index e4673f50..924ce636 100644 --- a/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet +++ b/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet @@ -248,6 +248,11 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; { port: 'http-metrics', interval: '30s', + scheme: "https", + bearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token", + tlsConfig: { + insecureSkipVerify: true + } }, ], selector: { @@ -349,6 +354,11 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; { port: 'http-metrics', interval: '30s', + scheme: "https", + bearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token", + tlsConfig: { + insecureSkipVerify: true + }, metricRelabelings: (import 'kube-prometheus/dropping-deprecated-metrics-relabelings.libsonnet') + [ { sourceLabels: ['__name__'], From c1304caa28a681d40656f04ab410531d1a197b52 Mon Sep 17 00:00:00 2001 From: tafkam Date: Sat, 25 Jul 2020 18:30:07 +0200 Subject: [PATCH 2/4] update secure ports for other cluster --- jsonnet/kube-prometheus/kube-prometheus-bootkube.libsonnet | 4 ++-- jsonnet/kube-prometheus/kube-prometheus-kops.libsonnet | 4 ++-- jsonnet/kube-prometheus/kube-prometheus-kube-aws.libsonnet | 4 ++-- jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/jsonnet/kube-prometheus/kube-prometheus-bootkube.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-bootkube.libsonnet index 3e2094a5..2ac441b2 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-bootkube.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-bootkube.libsonnet @@ -5,12 +5,12 @@ local servicePort = k.core.v1.service.mixin.spec.portsType; { prometheus+:: { kubeControllerManagerPrometheusDiscoveryService: - service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10252, 10252)) + + service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10257, 10257)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + service.mixin.spec.withClusterIp('None'), kubeSchedulerPrometheusDiscoveryService: - service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10251, 10251)) + + service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10259, 10259)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + service.mixin.spec.withClusterIp('None'), diff --git a/jsonnet/kube-prometheus/kube-prometheus-kops.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-kops.libsonnet index 01b6e2e7..1223ab28 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-kops.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-kops.libsonnet @@ -5,12 +5,12 @@ local servicePort = k.core.v1.service.mixin.spec.portsType; { prometheus+:: { kubeControllerManagerPrometheusDiscoveryService: - service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10252, 10252)) + + service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10257, 10257)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + service.mixin.spec.withClusterIp('None'), kubeSchedulerPrometheusDiscoveryService: - service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10251, 10251)) + + service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10259, 10259)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + service.mixin.spec.withClusterIp('None'), diff --git a/jsonnet/kube-prometheus/kube-prometheus-kube-aws.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-kube-aws.libsonnet index 23b33c9f..fd8dcdd3 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-kube-aws.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-kube-aws.libsonnet @@ -5,12 +5,12 @@ local servicePort = k.core.v1.service.mixin.spec.portsType; { prometheus+: { kubeControllerManagerPrometheusDiscoveryService: - service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10252, 10252)) + + service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10257, 10257)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + service.mixin.spec.withClusterIp('None'), kubeSchedulerPrometheusDiscoveryService: - service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10251, 10251)) + + service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10259, 10259)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + service.mixin.spec.withClusterIp('None'), diff --git a/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet index 6ef1a34e..17b1c6fa 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet @@ -6,12 +6,12 @@ local servicePort = k.core.v1.service.mixin.spec.portsType; prometheus+: { kubeControllerManagerPrometheusDiscoveryService: - service.new('kube-controller-manager-prometheus-discovery', { 'component': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10252, 10252)) + + service.new('kube-controller-manager-prometheus-discovery', { 'component': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10257, 10257)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + service.mixin.spec.withClusterIp('None'), kubeSchedulerPrometheusDiscoveryService: - service.new('kube-scheduler-prometheus-discovery', { 'component': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10251, 10251)) + + service.new('kube-scheduler-prometheus-discovery', { 'component': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10259, 10259)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + service.mixin.spec.withClusterIp('None'), From 6dfbcf35f28bf1872054db4c7afa93429560c294 Mon Sep 17 00:00:00 2001 From: tafkam Date: Mon, 27 Jul 2020 10:27:14 +0200 Subject: [PATCH 3/4] port https-metrics --- jsonnet/kube-prometheus/kube-prometheus-bootkube.libsonnet | 4 ++-- jsonnet/kube-prometheus/kube-prometheus-kops.libsonnet | 4 ++-- jsonnet/kube-prometheus/kube-prometheus-kube-aws.libsonnet | 4 ++-- jsonnet/kube-prometheus/kube-prometheus-kubeadm.libsonnet | 4 ++-- jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet | 4 ++-- jsonnet/kube-prometheus/prometheus/prometheus.libsonnet | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/jsonnet/kube-prometheus/kube-prometheus-bootkube.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-bootkube.libsonnet index 2ac441b2..6281c664 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-bootkube.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-bootkube.libsonnet @@ -5,12 +5,12 @@ local servicePort = k.core.v1.service.mixin.spec.portsType; { prometheus+:: { kubeControllerManagerPrometheusDiscoveryService: - service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10257, 10257)) + + service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('https-metrics', 10257, 10257)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + service.mixin.spec.withClusterIp('None'), kubeSchedulerPrometheusDiscoveryService: - service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10259, 10259)) + + service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('https-metrics', 10259, 10259)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + service.mixin.spec.withClusterIp('None'), diff --git a/jsonnet/kube-prometheus/kube-prometheus-kops.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-kops.libsonnet index 1223ab28..b5cb1f84 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-kops.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-kops.libsonnet @@ -5,12 +5,12 @@ local servicePort = k.core.v1.service.mixin.spec.portsType; { prometheus+:: { kubeControllerManagerPrometheusDiscoveryService: - service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10257, 10257)) + + service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('https-metrics', 10257, 10257)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + service.mixin.spec.withClusterIp('None'), kubeSchedulerPrometheusDiscoveryService: - service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10259, 10259)) + + service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('https-metrics', 10259, 10259)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + service.mixin.spec.withClusterIp('None'), diff --git a/jsonnet/kube-prometheus/kube-prometheus-kube-aws.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-kube-aws.libsonnet index fd8dcdd3..7a3326c2 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-kube-aws.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-kube-aws.libsonnet @@ -5,12 +5,12 @@ local servicePort = k.core.v1.service.mixin.spec.portsType; { prometheus+: { kubeControllerManagerPrometheusDiscoveryService: - service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10257, 10257)) + + service.new('kube-controller-manager-prometheus-discovery', { 'k8s-app': 'kube-controller-manager' }, servicePort.newNamed('https-metrics', 10257, 10257)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + service.mixin.spec.withClusterIp('None'), kubeSchedulerPrometheusDiscoveryService: - service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10259, 10259)) + + service.new('kube-scheduler-prometheus-discovery', { 'k8s-app': 'kube-scheduler' }, servicePort.newNamed('https-metrics', 10259, 10259)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + service.mixin.spec.withClusterIp('None'), diff --git a/jsonnet/kube-prometheus/kube-prometheus-kubeadm.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-kubeadm.libsonnet index 1ef808a8..e981a128 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-kubeadm.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-kubeadm.libsonnet @@ -5,12 +5,12 @@ local servicePort = k.core.v1.service.mixin.spec.portsType; { prometheus+: { kubeControllerManagerPrometheusDiscoveryService: - service.new('kube-controller-manager-prometheus-discovery', { component: 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10257, 10257)) + + service.new('kube-controller-manager-prometheus-discovery', { component: 'kube-controller-manager' }, servicePort.newNamed('https-metrics', 10257, 10257)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + service.mixin.spec.withClusterIp('None'), kubeSchedulerPrometheusDiscoveryService: - service.new('kube-scheduler-prometheus-discovery', { component: 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10259, 10259)) + + service.new('kube-scheduler-prometheus-discovery', { component: 'kube-scheduler' }, servicePort.newNamed('https-metrics', 10259, 10259)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + service.mixin.spec.withClusterIp('None'), diff --git a/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet index 17b1c6fa..9889963a 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-kubespray.libsonnet @@ -6,12 +6,12 @@ local servicePort = k.core.v1.service.mixin.spec.portsType; prometheus+: { kubeControllerManagerPrometheusDiscoveryService: - service.new('kube-controller-manager-prometheus-discovery', { 'component': 'kube-controller-manager' }, servicePort.newNamed('http-metrics', 10257, 10257)) + + service.new('kube-controller-manager-prometheus-discovery', { 'component': 'kube-controller-manager' }, servicePort.newNamed('https-metrics', 10257, 10257)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controller-manager' }) + service.mixin.spec.withClusterIp('None'), kubeSchedulerPrometheusDiscoveryService: - service.new('kube-scheduler-prometheus-discovery', { 'component': 'kube-scheduler' }, servicePort.newNamed('http-metrics', 10259, 10259)) + + service.new('kube-scheduler-prometheus-discovery', { 'component': 'kube-scheduler' }, servicePort.newNamed('https-metrics', 10259, 10259)) + service.mixin.metadata.withNamespace('kube-system') + service.mixin.metadata.withLabels({ 'k8s-app': 'kube-scheduler' }) + service.mixin.spec.withClusterIp('None'), diff --git a/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet b/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet index 924ce636..ec7dccc2 100644 --- a/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet +++ b/jsonnet/kube-prometheus/prometheus/prometheus.libsonnet @@ -246,7 +246,7 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; jobLabel: 'k8s-app', endpoints: [ { - port: 'http-metrics', + port: 'https-metrics', interval: '30s', scheme: "https", bearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token", @@ -352,7 +352,7 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; jobLabel: 'k8s-app', endpoints: [ { - port: 'http-metrics', + port: 'https-metrics', interval: '30s', scheme: "https", bearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token", From 3a6a0d0837bcbde51110e5c2b6a05b47c890987f Mon Sep 17 00:00:00 2001 From: root Date: Mon, 27 Jul 2020 10:29:31 +0200 Subject: [PATCH 4/4] make generate --- .../prometheus-serviceMonitorKubeControllerManager.yaml | 8 ++++++-- manifests/prometheus-serviceMonitorKubeScheduler.yaml | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/manifests/prometheus-serviceMonitorKubeControllerManager.yaml b/manifests/prometheus-serviceMonitorKubeControllerManager.yaml index 7f20fce3..0f23d84d 100644 --- a/manifests/prometheus-serviceMonitorKubeControllerManager.yaml +++ b/manifests/prometheus-serviceMonitorKubeControllerManager.yaml @@ -7,7 +7,8 @@ metadata: namespace: monitoring spec: endpoints: - - interval: 30s + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + interval: 30s metricRelabelings: - action: drop regex: kubelet_(pod_worker_latency_microseconds|pod_start_latency_microseconds|cgroup_manager_latency_microseconds|pod_worker_start_latency_microseconds|pleg_relist_latency_microseconds|pleg_relist_interval_microseconds|runtime_operations|runtime_operations_latency_microseconds|runtime_operations_errors|eviction_stats_age_microseconds|device_plugin_registration_count|device_plugin_alloc_latency_microseconds|network_plugin_operations_latency_microseconds) @@ -45,7 +46,10 @@ spec: regex: etcd_(debugging|disk|request|server).* sourceLabels: - __name__ - port: http-metrics + port: https-metrics + scheme: https + tlsConfig: + insecureSkipVerify: true jobLabel: k8s-app namespaceSelector: matchNames: diff --git a/manifests/prometheus-serviceMonitorKubeScheduler.yaml b/manifests/prometheus-serviceMonitorKubeScheduler.yaml index f00db0e4..8073eaca 100644 --- a/manifests/prometheus-serviceMonitorKubeScheduler.yaml +++ b/manifests/prometheus-serviceMonitorKubeScheduler.yaml @@ -7,8 +7,12 @@ metadata: namespace: monitoring spec: endpoints: - - interval: 30s - port: http-metrics + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + interval: 30s + port: https-metrics + scheme: https + tlsConfig: + insecureSkipVerify: true jobLabel: k8s-app namespaceSelector: matchNames: