Merge pull request #453 from paulfantom/secure-metrics

Secure metrics endpoint
2020-03-24 11:06:00 +01:00
parent 75c532df17 1dd5bbeb58
commit 68505af1f9
12 changed files with 281 additions and 148 deletions
--- a/2
+++ b/2
@@ -63,4 +63,4 @@ test-in-docker:
 .PHONY: generate generate-in-docker test test-in-docker fmt

 $(JB_BINARY):
-	go get github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.2.0
+	go get github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.3.1
--- a/go.mod
+++ b/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be // indirect
-	github.com/jsonnet-bundler/jsonnet-bundler v0.2.0
+	github.com/jsonnet-bundler/jsonnet-bundler v0.3.1
 	github.com/kr/pretty v0.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.4 // indirect
 	github.com/mattn/go-isatty v0.0.10 // indirect
--- a/go.sum
+++ b/go.sum
@@ -28,6 +28,8 @@ github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be h1:AHimNtVIpiBjPU
 github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jsonnet-bundler/jsonnet-bundler v0.2.0 h1:qL1v+2mjdEOmvNJp+ab+wQH81TQY71w1A666CRUg+1U=
 github.com/jsonnet-bundler/jsonnet-bundler v0.2.0/go.mod h1:/by7P/OoohkI3q4CgSFqcoFsVY+IaNbzOVDknEsKDeU=
+github.com/jsonnet-bundler/jsonnet-bundler v0.3.1 h1:KmNzitX12fFoyqjhU8cRifEB5D8x1NT1UAcK7FQ0zpY=
+github.com/jsonnet-bundler/jsonnet-bundler v0.3.1/go.mod h1:/by7P/OoohkI3q4CgSFqcoFsVY+IaNbzOVDknEsKDeU=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
--- a/jsonnet/kube-prometheus/kube-prometheus.libsonnet
+++ b/jsonnet/kube-prometheus/kube-prometheus.libsonnet
@@ -50,7 +50,62 @@ local configMapList = k3.core.v1.configMapList;
        preserveUnknownFields: null,
      }),
    },
+    service+: {
+      spec+: {
+        ports: [
+          {
+            name: 'https',
+            port: 8443,
+            targetPort: 'https',
          },
+        ],
+      },
+    },
+    serviceMonitor+: {
+      spec+: {
+        endpoints: [
+          {
+            port: 'https',
+            scheme: 'https',
+            honorLabels: true,
+            bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+            tlsConfig: {
+              insecureSkipVerify: true,
+            },
+          },
+        ]
+      },
+    },
+    clusterRole+: {
+      rules+: [
+        {
+          apiGroups: ['authentication.k8s.io'],
+          resources: ['tokenreviews'],
+          verbs: ['create'],
+        },
+        {
+          apiGroups: ['authorization.k8s.io'],
+          resources: ['subjectaccessreviews'],
+          verbs: ['create'],
+        },
+      ],
+    },
+  } +
+  ((import 'kube-prometheus/kube-rbac-proxy/container.libsonnet') {
+    config+:: {
+      kubeRbacProxy: {
+        local cfg = self,
+        image: $._config.imageRepos.kubeRbacProxy + ':' + $._config.versions.kubeRbacProxy,
+        name: 'kube-rbac-proxy',
+        securePortName: 'https',
+        securePort: 8443,
+        secureListenAddress: ':%d' % self.securePort,
+        upstream: 'http://127.0.0.1:8080/',
+        tlsCipherSuites: $._config.tlsCipherSuites,
+      },
+    },
+  }).deploymentMixin,
+
  grafana+:: {
    dashboardDefinitions: configMapList.new(super.dashboardDefinitions),
    serviceMonitor: {
--- a/jsonnetfile.json
+++ b/jsonnetfile.json
@@ -1,7 +1,7 @@
 {
+  "version": 1,
  "dependencies": [
    {
-      "name": "kube-prometheus",
      "source": {
        "local": {
          "directory": "jsonnet/kube-prometheus"
@@ -9,5 +9,6 @@
      },
      "version": ""
    }
-  ]
+  ],
+  "legacyImports": true
 }
--- a/jsonnetfile.lock.json
+++ b/jsonnetfile.lock.json
@@ -1,18 +1,7 @@
 {
+  "version": 1,
  "dependencies": [
    {
-      "name": "etcd-mixin",
-      "source": {
-        "git": {
-          "remote": "https://github.com/coreos/etcd",
-          "subdir": "Documentation/etcd-mixin"
-        }
-      },
-      "version": "e5c90ebf90cb3692c26240d19406de47414a2b38",
-      "sum": "Ko3qhNfC2vN/houLh6C0Ryacjv70gl0DVPGU/PQ4OD0="
-    },
-    {
-      "name": "grafana",
      "source": {
        "git": {
          "remote": "https://github.com/brancz/kubernetes-grafana",
@@ -23,104 +12,16 @@
      "sum": "b8faWX1qqLGyN67sA36oRqYZ5HX+tHBRMPtrWRqIysE="
    },
    {
-      "name": "grafana-builder",
      "source": {
        "git": {
-          "remote": "https://github.com/grafana/jsonnet-libs",
-          "subdir": "grafana-builder"
+          "remote": "https://github.com/coreos/etcd",
+          "subdir": "Documentation/etcd-mixin"
        }
      },
-      "version": "7ac7da1a0fe165b68cdb718b2521b560d51bd1f4",
-      "sum": "slxrtftVDiTlQK22ertdfrg4Epnq97gdrLI63ftUfaE="
+      "version": "e5c90ebf90cb3692c26240d19406de47414a2b38",
+      "sum": "Ko3qhNfC2vN/houLh6C0Ryacjv70gl0DVPGU/PQ4OD0="
    },
    {
-      "name": "grafonnet",
-      "source": {
-        "git": {
-          "remote": "https://github.com/grafana/grafonnet-lib",
-          "subdir": "grafonnet"
-        }
-      },
-      "version": "c459106d2d2b583dd3a83f6c75eb52abee3af764",
-      "sum": "CeM3LRgUCUJTolTdMnerfMPGYmhClx7gX5ajrQVEY2Y="
-    },
-    {
-      "name": "ksonnet",
-      "source": {
-        "git": {
-          "remote": "https://github.com/ksonnet/ksonnet-lib",
-          "subdir": ""
-        }
-      },
-      "version": "0d2f82676817bbf9e4acf6495b2090205f323b9f",
-      "sum": "h28BXZ7+vczxYJ2sCt8JuR9+yznRtU/iA6DCpQUrtEg="
-    },
-    {
-      "name": "kube-prometheus",
-      "source": {
-        "local": {
-          "directory": "jsonnet/kube-prometheus"
-        }
-      },
-      "version": ""
-    },
-    {
-      "name": "kube-state-metrics",
-      "source": {
-        "git": {
-          "remote": "https://github.com/kubernetes/kube-state-metrics",
-          "subdir": "jsonnet/kube-state-metrics"
-        }
-      },
-      "version": "89ede10b19d7ef0145777717351cabe14b113c01",
-      "sum": "cJjGZaLBjcIGrLHZLjRPU9c3KL+ep9rZTb9dbALSKqA="
-    },
-    {
-      "name": "kube-state-metrics-mixin",
-      "source": {
-        "git": {
-          "remote": "https://github.com/kubernetes/kube-state-metrics",
-          "subdir": "jsonnet/kube-state-metrics-mixin"
-        }
-      },
-      "version": "89ede10b19d7ef0145777717351cabe14b113c01",
-      "sum": "E1GGavnf9PCWBm4WVrxWnc0FIj72UcbcweqGioWrOdU="
-    },
-    {
-      "name": "kubernetes-mixin",
-      "source": {
-        "git": {
-          "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin",
-          "subdir": ""
-        }
-      },
-      "version": "b2d7f762bd22be3ba5e7d54a1fcecfe1092f214b",
-      "sum": "NqrJQnQnRDzkCbrHg7L1zX8XPAzfoE4DS2XBEj6WC8g="
-    },
-    {
-      "name": "node-mixin",
-      "source": {
-        "git": {
-          "remote": "https://github.com/prometheus/node_exporter",
-          "subdir": "docs/node-mixin"
-        }
-      },
-      "version": "0107bc794204f50d887898da60032da890637471",
-      "sum": "VKdF0zPMSCiuIuXWblSz2VOeBaXzQ7fp40vz9sxj+Bo="
-    },
-    {
-      "name": "prometheus",
-      "source": {
-        "git": {
-          "remote": "https://github.com/prometheus/prometheus",
-          "subdir": "documentation/prometheus-mixin"
-        }
-      },
-      "version": "1c321ed047ac57e34688e40a55349c9dfe2b72c8",
-      "sum": "u1YS9CVuBTcw2vks0PZbLb1gtlI/7bVGDVBZsjWFLTw="
-    },
-    {
-      "name": "prometheus-operator",
      "source": {
        "git": {
          "remote": "https://github.com/coreos/prometheus-operator",
@@ -131,7 +32,47 @@
      "sum": "vegTm8VSDazwYflBQGLkjs3ystWahwUv0fUyuMbpNRg="
    },
    {
-      "name": "promgrafonnet",
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafonnet-lib",
+          "subdir": "grafonnet"
+        }
+      },
+      "version": "c459106d2d2b583dd3a83f6c75eb52abee3af764",
+      "sum": "CeM3LRgUCUJTolTdMnerfMPGYmhClx7gX5ajrQVEY2Y="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs",
+          "subdir": "grafana-builder"
+        }
+      },
+      "version": "7ac7da1a0fe165b68cdb718b2521b560d51bd1f4",
+      "sum": "slxrtftVDiTlQK22ertdfrg4Epnq97gdrLI63ftUfaE="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/ksonnet/ksonnet-lib",
+          "subdir": ""
+        }
+      },
+      "version": "0d2f82676817bbf9e4acf6495b2090205f323b9f",
+      "sum": "h28BXZ7+vczxYJ2sCt8JuR9+yznRtU/iA6DCpQUrtEg=",
+      "name": "ksonnet"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin",
+          "subdir": ""
+        }
+      },
+      "version": "b2d7f762bd22be3ba5e7d54a1fcecfe1092f214b",
+      "sum": "NqrJQnQnRDzkCbrHg7L1zX8XPAzfoE4DS2XBEj6WC8g="
+    },
+    {
      "source": {
        "git": {
          "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin",
@@ -142,7 +83,26 @@
      "sum": "VhgBM39yv0f4bKv8VfGg4FXkg573evGDRalip9ypKbc="
    },
    {
-      "name": "slo-libsonnet",
+      "source": {
+        "git": {
+          "remote": "https://github.com/kubernetes/kube-state-metrics",
+          "subdir": "jsonnet/kube-state-metrics"
+        }
+      },
+      "version": "89ede10b19d7ef0145777717351cabe14b113c01",
+      "sum": "cJjGZaLBjcIGrLHZLjRPU9c3KL+ep9rZTb9dbALSKqA="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/kubernetes/kube-state-metrics",
+          "subdir": "jsonnet/kube-state-metrics-mixin"
+        }
+      },
+      "version": "89ede10b19d7ef0145777717351cabe14b113c01",
+      "sum": "E1GGavnf9PCWBm4WVrxWnc0FIj72UcbcweqGioWrOdU="
+    },
+    {
      "source": {
        "git": {
          "remote": "https://github.com/metalmatze/slo-libsonnet",
@@ -151,6 +111,36 @@
      },
      "version": "437c402c5f3ad86c3c16db8471f1649284fef0ee",
      "sum": "2Zcyku1f558VrUpMaJnI78fahDksPLcS1idmxxwcQ7Q="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/node_exporter",
+          "subdir": "docs/node-mixin"
        }
-  ]
+      },
+      "version": "0107bc794204f50d887898da60032da890637471",
+      "sum": "VKdF0zPMSCiuIuXWblSz2VOeBaXzQ7fp40vz9sxj+Bo="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/prometheus/prometheus",
+          "subdir": "documentation/prometheus-mixin"
+        }
+      },
+      "version": "1c321ed047ac57e34688e40a55349c9dfe2b72c8",
+      "sum": "u1YS9CVuBTcw2vks0PZbLb1gtlI/7bVGDVBZsjWFLTw=",
+      "name": "prometheus"
+    },
+    {
+      "source": {
+        "local": {
+          "directory": "jsonnet/kube-prometheus"
+        }
+      },
+      "version": ""
+    }
+  ],
+  "legacyImports": false
 }
--- a/manifests/prometheus-operator-serviceMonitor.yaml
+++ b/manifests/prometheus-operator-serviceMonitor.yaml
@@ -9,8 +9,12 @@ metadata:
  namespace: monitoring
 spec:
  endpoints:
-  - honorLabels: true
-    port: http
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    port: https
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
  selector:
    matchLabels:
      app.kubernetes.io/component: controller
--- a/manifests/setup/prometheus-operator-clusterRole.yaml
+++ b/manifests/setup/prometheus-operator-clusterRole.yaml
@@ -87,3 +87,15 @@ rules:
  - get
  - list
  - watch
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
--- a/manifests/setup/prometheus-operator-deployment.yaml
+++ b/manifests/setup/prometheus-operator-deployment.yaml
@@ -40,6 +40,18 @@ spec:
            memory: 100Mi
        securityContext:
          allowPrivilegeEscalation: false
+      - args:
+        - --logtostderr
+        - --secure-listen-address=:8443
+        - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+        - --upstream=http://127.0.0.1:8080/
+        image: quay.io/coreos/kube-rbac-proxy:v0.4.1
+        name: kube-rbac-proxy
+        ports:
+        - containerPort: 8443
+          name: https
+        securityContext:
+          runAsUser: 65534
      nodeSelector:
        beta.kubernetes.io/os: linux
      securityContext:
--- a/manifests/setup/prometheus-operator-service.yaml
+++ b/manifests/setup/prometheus-operator-service.yaml
@@ -10,9 +10,9 @@ metadata:
 spec:
  clusterIP: None
  ports:
-  - name: http
-    port: 8080
-    targetPort: http
+  - name: https
+    port: 8443
+    targetPort: https
  selector:
    app.kubernetes.io/component: controller
    app.kubernetes.io/name: prometheus-operator
--- a/tests/e2e/main_test.go
+++ b/tests/e2e/main_test.go
@@ -123,7 +123,7 @@ func TestDroppedMetrics(t *testing.T) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	for _, k := range md.Data {
+	for _, k := range md {
 		// check if the metric' help text contains Deprecated
 		if strings.Contains(k.Help, "Deprecated") {
 			// query prometheus for the Deprecated metric
@@ -138,3 +138,27 @@ func TestDroppedMetrics(t *testing.T) {

 	}
 }
+
+func TestTargetsScheme(t *testing.T) {
+	// query targets for all endpoints
+	tgs, err := promClient.targets()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// exclude jobs from checking for http endpoints
+	// TODO(paulfantom): This should be reduced as we secure connections for those components
+	exclude := map[string]bool{
+		"alertmanager-main": true,
+		"prometheus-k8s":    true,
+		"kube-dns":          true,
+		"grafana":           true,
+	}
+
+	for _, k := range tgs.Active {
+		job := k.Labels["job"]
+		if k.DiscoveredLabels["__scheme__"] == "http" && !exclude[string(job)] {
+			log.Fatalf("target exposing metrics over HTTP instead of HTTPS: %+v", k)
+		}
+	}
+}
--- a/tests/e2e/prometheus_client.go
+++ b/tests/e2e/prometheus_client.go
@@ -22,6 +22,7 @@ import (
 	"k8s.io/client-go/kubernetes"

 	"github.com/Jeffail/gabs"
+	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 )

 type prometheusClient struct {
@@ -32,6 +33,42 @@ func newPrometheusClient(kubeClient kubernetes.Interface) *prometheusClient {
 	return &prometheusClient{kubeClient}
 }

+// Response hold API response in a form similar to apiResponse struct from prometheus/client_golang
+// https://github.com/prometheus/client_golang/blob/master/api/prometheus/v1/api.go
+type Response struct {
+	Status string          `json:"status"`
+	Data   json.RawMessage `json:"data"`
+}
+
+// apiRequest makes a request against specified Prometheus API endpoint
+func (c *prometheusClient) apiRequest(endpoint string, selector string, query string) (Response, error) {
+	req := c.kubeClient.CoreV1().RESTClient().Get().
+		Namespace("monitoring").
+		Resource("pods").
+		SubResource("proxy").
+		Name("prometheus-k8s-0:9090").
+		Suffix(endpoint).Param(selector, query)
+
+	var data Response
+	b, err := req.DoRaw()
+	if err != nil {
+		return data, err
+	}
+
+	r := bytes.NewReader(b)
+	decoder := json.NewDecoder(r)
+	err = decoder.Decode(&data)
+	if err != nil {
+		return data, err
+	}
+
+	if data.Status != "success" {
+		return data, fmt.Errorf("status of returned response was not successful; status: %s", data.Status)
+	}
+
+	return data, err
+}
+
 // Query makes a request against the Prometheus /api/v1/query endpoint.
 func (c *prometheusClient) query(query string) (int, error) {
 	req := c.kubeClient.CoreV1().RESTClient().Get().
@@ -55,40 +92,36 @@ func (c *prometheusClient) query(query string) (int, error) {
 	return n, err
 }

-type Metadata struct {
-	Status string `json:"status,omitempty"`
-	Data   []Data `json:"data,omitempty"`
-}
-
-type Data struct {
-	Metric string `json:"metric,omitempty"`
-	Help   string `json:"help,omitempty"`
-}
-
 // metadata makes a request against the Prometheus /api/v1/targets/metadata endpoint.
 // It returns all the metrics and its metadata.
-func (c *prometheusClient) metadata(query string) (Metadata, error) {
-	req := c.kubeClient.CoreV1().RESTClient().Get().
-		Namespace("monitoring").
-		Resource("pods").
-		SubResource("proxy").
-		Name("prometheus-k8s-0:9090").
-		Suffix("/api/v1/targets/metadata").Param("match_target", query)
+func (c *prometheusClient) metadata(query string) ([]promv1.MetricMetadata, error) {
+	var metadata []promv1.MetricMetadata
+	rsp, err := c.apiRequest("/api/v1/targets/metadata", "match_target", query)

-	var data Metadata
-	b, err := req.DoRaw()
-	if err != nil {
-		return data, err
-	}
-
-	r := bytes.NewReader(b)
+	r := bytes.NewReader(rsp.Data)
 	decoder := json.NewDecoder(r)
-	err = decoder.Decode(&data)
+	err = decoder.Decode(&metadata)
 	if err != nil {
-		return data, err
+		return metadata, err
 	}
-	if data.Status != "success" {
-		return data, fmt.Errorf("status of returned response was not successful; status: %s", data.Status)
-	}
-	return data, err
+	return metadata, err
+}
+
+// targets makes a request against the Prometheus /api/v1/targets endpoint.
+// It returns all targets registered in prometheus.
+func (c *prometheusClient) targets() (promv1.TargetsResult, error) {
+	var targets promv1.TargetsResult
+	rsp, err := c.apiRequest("/api/v1/targets", "state", "any")
+	if err != nil {
+		return targets, err
+	}
+
+	r := bytes.NewReader(rsp.Data)
+	decoder := json.NewDecoder(r)
+	err = decoder.Decode(&targets)
+	if err != nil {
+		return targets, err
+	}
+
+	return targets, err
 }