diff --git a/Makefile b/Makefile index ea6889e3..0c4cc079 100644 --- a/Makefile +++ b/Makefile @@ -63,4 +63,4 @@ test-in-docker: .PHONY: generate generate-in-docker test test-in-docker fmt $(JB_BINARY): - go get github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.2.0 + go get github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.3.1 diff --git a/go.mod b/go.mod index 77b15efc..bf7efcac 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d // indirect github.com/imdario/mergo v0.3.7 // indirect github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be // indirect - github.com/jsonnet-bundler/jsonnet-bundler v0.2.0 + github.com/jsonnet-bundler/jsonnet-bundler v0.3.1 github.com/kr/pretty v0.2.0 // indirect github.com/mattn/go-colorable v0.1.4 // indirect github.com/mattn/go-isatty v0.0.10 // indirect diff --git a/go.sum b/go.sum index 6496f5b6..47a57104 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be h1:AHimNtVIpiBjPU github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jsonnet-bundler/jsonnet-bundler v0.2.0 h1:qL1v+2mjdEOmvNJp+ab+wQH81TQY71w1A666CRUg+1U= github.com/jsonnet-bundler/jsonnet-bundler v0.2.0/go.mod h1:/by7P/OoohkI3q4CgSFqcoFsVY+IaNbzOVDknEsKDeU= +github.com/jsonnet-bundler/jsonnet-bundler v0.3.1 h1:KmNzitX12fFoyqjhU8cRifEB5D8x1NT1UAcK7FQ0zpY= +github.com/jsonnet-bundler/jsonnet-bundler v0.3.1/go.mod h1:/by7P/OoohkI3q4CgSFqcoFsVY+IaNbzOVDknEsKDeU= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/jsonnet/kube-prometheus/kube-prometheus.libsonnet b/jsonnet/kube-prometheus/kube-prometheus.libsonnet index 71715250..92262460 100644 --- a/jsonnet/kube-prometheus/kube-prometheus.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus.libsonnet @@ -50,7 +50,62 @@ local configMapList = k3.core.v1.configMapList; preserveUnknownFields: null, }), }, - }, + service+: { + spec+: { + ports: [ + { + name: 'https', + port: 8443, + targetPort: 'https', + }, + ], + }, + }, + serviceMonitor+: { + spec+: { + endpoints: [ + { + port: 'https', + scheme: 'https', + honorLabels: true, + bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', + tlsConfig: { + insecureSkipVerify: true, + }, + }, + ] + }, + }, + clusterRole+: { + rules+: [ + { + apiGroups: ['authentication.k8s.io'], + resources: ['tokenreviews'], + verbs: ['create'], + }, + { + apiGroups: ['authorization.k8s.io'], + resources: ['subjectaccessreviews'], + verbs: ['create'], + }, + ], + }, + } + + ((import 'kube-prometheus/kube-rbac-proxy/container.libsonnet') { + config+:: { + kubeRbacProxy: { + local cfg = self, + image: $._config.imageRepos.kubeRbacProxy + ':' + $._config.versions.kubeRbacProxy, + name: 'kube-rbac-proxy', + securePortName: 'https', + securePort: 8443, + secureListenAddress: ':%d' % self.securePort, + upstream: 'http://127.0.0.1:8080/', + tlsCipherSuites: $._config.tlsCipherSuites, + }, + }, + }).deploymentMixin, + grafana+:: { dashboardDefinitions: configMapList.new(super.dashboardDefinitions), serviceMonitor: { diff --git a/jsonnetfile.json b/jsonnetfile.json index ad5fe5bc..73f15687 100644 --- a/jsonnetfile.json +++ b/jsonnetfile.json @@ -1,7 +1,7 @@ { + "version": 1, "dependencies": [ { - "name": "kube-prometheus", "source": { "local": { "directory": "jsonnet/kube-prometheus" @@ -9,5 +9,6 @@ }, "version": "" } - ] + ], + "legacyImports": true } diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json index c887ca49..49392738 100644 --- a/jsonnetfile.lock.json +++ b/jsonnetfile.lock.json @@ -1,18 +1,7 @@ { + "version": 1, "dependencies": [ { - "name": "etcd-mixin", - "source": { - "git": { - "remote": "https://github.com/coreos/etcd", - "subdir": "Documentation/etcd-mixin" - } - }, - "version": "e5c90ebf90cb3692c26240d19406de47414a2b38", - "sum": "Ko3qhNfC2vN/houLh6C0Ryacjv70gl0DVPGU/PQ4OD0=" - }, - { - "name": "grafana", "source": { "git": { "remote": "https://github.com/brancz/kubernetes-grafana", @@ -23,104 +12,16 @@ "sum": "b8faWX1qqLGyN67sA36oRqYZ5HX+tHBRMPtrWRqIysE=" }, { - "name": "grafana-builder", "source": { "git": { - "remote": "https://github.com/grafana/jsonnet-libs", - "subdir": "grafana-builder" + "remote": "https://github.com/coreos/etcd", + "subdir": "Documentation/etcd-mixin" } }, - "version": "7ac7da1a0fe165b68cdb718b2521b560d51bd1f4", - "sum": "slxrtftVDiTlQK22ertdfrg4Epnq97gdrLI63ftUfaE=" + "version": "e5c90ebf90cb3692c26240d19406de47414a2b38", + "sum": "Ko3qhNfC2vN/houLh6C0Ryacjv70gl0DVPGU/PQ4OD0=" }, { - "name": "grafonnet", - "source": { - "git": { - "remote": "https://github.com/grafana/grafonnet-lib", - "subdir": "grafonnet" - } - }, - "version": "c459106d2d2b583dd3a83f6c75eb52abee3af764", - "sum": "CeM3LRgUCUJTolTdMnerfMPGYmhClx7gX5ajrQVEY2Y=" - }, - { - "name": "ksonnet", - "source": { - "git": { - "remote": "https://github.com/ksonnet/ksonnet-lib", - "subdir": "" - } - }, - "version": "0d2f82676817bbf9e4acf6495b2090205f323b9f", - "sum": "h28BXZ7+vczxYJ2sCt8JuR9+yznRtU/iA6DCpQUrtEg=" - }, - { - "name": "kube-prometheus", - "source": { - "local": { - "directory": "jsonnet/kube-prometheus" - } - }, - "version": "" - }, - { - "name": "kube-state-metrics", - "source": { - "git": { - "remote": "https://github.com/kubernetes/kube-state-metrics", - "subdir": "jsonnet/kube-state-metrics" - } - }, - "version": "89ede10b19d7ef0145777717351cabe14b113c01", - "sum": "cJjGZaLBjcIGrLHZLjRPU9c3KL+ep9rZTb9dbALSKqA=" - }, - { - "name": "kube-state-metrics-mixin", - "source": { - "git": { - "remote": "https://github.com/kubernetes/kube-state-metrics", - "subdir": "jsonnet/kube-state-metrics-mixin" - } - }, - "version": "89ede10b19d7ef0145777717351cabe14b113c01", - "sum": "E1GGavnf9PCWBm4WVrxWnc0FIj72UcbcweqGioWrOdU=" - }, - { - "name": "kubernetes-mixin", - "source": { - "git": { - "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin", - "subdir": "" - } - }, - "version": "b2d7f762bd22be3ba5e7d54a1fcecfe1092f214b", - "sum": "NqrJQnQnRDzkCbrHg7L1zX8XPAzfoE4DS2XBEj6WC8g=" - }, - { - "name": "node-mixin", - "source": { - "git": { - "remote": "https://github.com/prometheus/node_exporter", - "subdir": "docs/node-mixin" - } - }, - "version": "0107bc794204f50d887898da60032da890637471", - "sum": "VKdF0zPMSCiuIuXWblSz2VOeBaXzQ7fp40vz9sxj+Bo=" - }, - { - "name": "prometheus", - "source": { - "git": { - "remote": "https://github.com/prometheus/prometheus", - "subdir": "documentation/prometheus-mixin" - } - }, - "version": "1c321ed047ac57e34688e40a55349c9dfe2b72c8", - "sum": "u1YS9CVuBTcw2vks0PZbLb1gtlI/7bVGDVBZsjWFLTw=" - }, - { - "name": "prometheus-operator", "source": { "git": { "remote": "https://github.com/coreos/prometheus-operator", @@ -131,7 +32,47 @@ "sum": "vegTm8VSDazwYflBQGLkjs3ystWahwUv0fUyuMbpNRg=" }, { - "name": "promgrafonnet", + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet-lib", + "subdir": "grafonnet" + } + }, + "version": "c459106d2d2b583dd3a83f6c75eb52abee3af764", + "sum": "CeM3LRgUCUJTolTdMnerfMPGYmhClx7gX5ajrQVEY2Y=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs", + "subdir": "grafana-builder" + } + }, + "version": "7ac7da1a0fe165b68cdb718b2521b560d51bd1f4", + "sum": "slxrtftVDiTlQK22ertdfrg4Epnq97gdrLI63ftUfaE=" + }, + { + "source": { + "git": { + "remote": "https://github.com/ksonnet/ksonnet-lib", + "subdir": "" + } + }, + "version": "0d2f82676817bbf9e4acf6495b2090205f323b9f", + "sum": "h28BXZ7+vczxYJ2sCt8JuR9+yznRtU/iA6DCpQUrtEg=", + "name": "ksonnet" + }, + { + "source": { + "git": { + "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin", + "subdir": "" + } + }, + "version": "b2d7f762bd22be3ba5e7d54a1fcecfe1092f214b", + "sum": "NqrJQnQnRDzkCbrHg7L1zX8XPAzfoE4DS2XBEj6WC8g=" + }, + { "source": { "git": { "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin", @@ -142,7 +83,26 @@ "sum": "VhgBM39yv0f4bKv8VfGg4FXkg573evGDRalip9ypKbc=" }, { - "name": "slo-libsonnet", + "source": { + "git": { + "remote": "https://github.com/kubernetes/kube-state-metrics", + "subdir": "jsonnet/kube-state-metrics" + } + }, + "version": "89ede10b19d7ef0145777717351cabe14b113c01", + "sum": "cJjGZaLBjcIGrLHZLjRPU9c3KL+ep9rZTb9dbALSKqA=" + }, + { + "source": { + "git": { + "remote": "https://github.com/kubernetes/kube-state-metrics", + "subdir": "jsonnet/kube-state-metrics-mixin" + } + }, + "version": "89ede10b19d7ef0145777717351cabe14b113c01", + "sum": "E1GGavnf9PCWBm4WVrxWnc0FIj72UcbcweqGioWrOdU=" + }, + { "source": { "git": { "remote": "https://github.com/metalmatze/slo-libsonnet", @@ -151,6 +111,36 @@ }, "version": "437c402c5f3ad86c3c16db8471f1649284fef0ee", "sum": "2Zcyku1f558VrUpMaJnI78fahDksPLcS1idmxxwcQ7Q=" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus/node_exporter", + "subdir": "docs/node-mixin" + } + }, + "version": "0107bc794204f50d887898da60032da890637471", + "sum": "VKdF0zPMSCiuIuXWblSz2VOeBaXzQ7fp40vz9sxj+Bo=" + }, + { + "source": { + "git": { + "remote": "https://github.com/prometheus/prometheus", + "subdir": "documentation/prometheus-mixin" + } + }, + "version": "1c321ed047ac57e34688e40a55349c9dfe2b72c8", + "sum": "u1YS9CVuBTcw2vks0PZbLb1gtlI/7bVGDVBZsjWFLTw=", + "name": "prometheus" + }, + { + "source": { + "local": { + "directory": "jsonnet/kube-prometheus" + } + }, + "version": "" } - ] + ], + "legacyImports": false } diff --git a/manifests/prometheus-operator-serviceMonitor.yaml b/manifests/prometheus-operator-serviceMonitor.yaml index be604c37..571f5e25 100644 --- a/manifests/prometheus-operator-serviceMonitor.yaml +++ b/manifests/prometheus-operator-serviceMonitor.yaml @@ -9,8 +9,12 @@ metadata: namespace: monitoring spec: endpoints: - - honorLabels: true - port: http + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: true + port: https + scheme: https + tlsConfig: + insecureSkipVerify: true selector: matchLabels: app.kubernetes.io/component: controller diff --git a/manifests/setup/prometheus-operator-clusterRole.yaml b/manifests/setup/prometheus-operator-clusterRole.yaml index 4bcb185d..054414f2 100644 --- a/manifests/setup/prometheus-operator-clusterRole.yaml +++ b/manifests/setup/prometheus-operator-clusterRole.yaml @@ -87,3 +87,15 @@ rules: - get - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/manifests/setup/prometheus-operator-deployment.yaml b/manifests/setup/prometheus-operator-deployment.yaml index c595b0ae..2aeec682 100644 --- a/manifests/setup/prometheus-operator-deployment.yaml +++ b/manifests/setup/prometheus-operator-deployment.yaml @@ -40,6 +40,18 @@ spec: memory: 100Mi securityContext: allowPrivilegeEscalation: false + - args: + - --logtostderr + - --secure-listen-address=:8443 + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 + - --upstream=http://127.0.0.1:8080/ + image: quay.io/coreos/kube-rbac-proxy:v0.4.1 + name: kube-rbac-proxy + ports: + - containerPort: 8443 + name: https + securityContext: + runAsUser: 65534 nodeSelector: beta.kubernetes.io/os: linux securityContext: diff --git a/manifests/setup/prometheus-operator-service.yaml b/manifests/setup/prometheus-operator-service.yaml index 51f7a06c..fbb448d9 100644 --- a/manifests/setup/prometheus-operator-service.yaml +++ b/manifests/setup/prometheus-operator-service.yaml @@ -10,9 +10,9 @@ metadata: spec: clusterIP: None ports: - - name: http - port: 8080 - targetPort: http + - name: https + port: 8443 + targetPort: https selector: app.kubernetes.io/component: controller app.kubernetes.io/name: prometheus-operator diff --git a/tests/e2e/main_test.go b/tests/e2e/main_test.go index eb73108c..36b6a14b 100644 --- a/tests/e2e/main_test.go +++ b/tests/e2e/main_test.go @@ -123,7 +123,7 @@ func TestDroppedMetrics(t *testing.T) { if err != nil { log.Fatal(err) } - for _, k := range md.Data { + for _, k := range md { // check if the metric' help text contains Deprecated if strings.Contains(k.Help, "Deprecated") { // query prometheus for the Deprecated metric @@ -138,3 +138,27 @@ func TestDroppedMetrics(t *testing.T) { } } + +func TestTargetsScheme(t *testing.T) { + // query targets for all endpoints + tgs, err := promClient.targets() + if err != nil { + log.Fatal(err) + } + + // exclude jobs from checking for http endpoints + // TODO(paulfantom): This should be reduced as we secure connections for those components + exclude := map[string]bool{ + "alertmanager-main": true, + "prometheus-k8s": true, + "kube-dns": true, + "grafana": true, + } + + for _, k := range tgs.Active { + job := k.Labels["job"] + if k.DiscoveredLabels["__scheme__"] == "http" && !exclude[string(job)] { + log.Fatalf("target exposing metrics over HTTP instead of HTTPS: %+v", k) + } + } +} diff --git a/tests/e2e/prometheus_client.go b/tests/e2e/prometheus_client.go index 89e25c0a..a932c0b5 100644 --- a/tests/e2e/prometheus_client.go +++ b/tests/e2e/prometheus_client.go @@ -22,6 +22,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/Jeffail/gabs" + promv1 "github.com/prometheus/client_golang/api/prometheus/v1" ) type prometheusClient struct { @@ -32,6 +33,42 @@ func newPrometheusClient(kubeClient kubernetes.Interface) *prometheusClient { return &prometheusClient{kubeClient} } +// Response hold API response in a form similar to apiResponse struct from prometheus/client_golang +// https://github.com/prometheus/client_golang/blob/master/api/prometheus/v1/api.go +type Response struct { + Status string `json:"status"` + Data json.RawMessage `json:"data"` +} + +// apiRequest makes a request against specified Prometheus API endpoint +func (c *prometheusClient) apiRequest(endpoint string, selector string, query string) (Response, error) { + req := c.kubeClient.CoreV1().RESTClient().Get(). + Namespace("monitoring"). + Resource("pods"). + SubResource("proxy"). + Name("prometheus-k8s-0:9090"). + Suffix(endpoint).Param(selector, query) + + var data Response + b, err := req.DoRaw() + if err != nil { + return data, err + } + + r := bytes.NewReader(b) + decoder := json.NewDecoder(r) + err = decoder.Decode(&data) + if err != nil { + return data, err + } + + if data.Status != "success" { + return data, fmt.Errorf("status of returned response was not successful; status: %s", data.Status) + } + + return data, err +} + // Query makes a request against the Prometheus /api/v1/query endpoint. func (c *prometheusClient) query(query string) (int, error) { req := c.kubeClient.CoreV1().RESTClient().Get(). @@ -55,40 +92,36 @@ func (c *prometheusClient) query(query string) (int, error) { return n, err } -type Metadata struct { - Status string `json:"status,omitempty"` - Data []Data `json:"data,omitempty"` -} - -type Data struct { - Metric string `json:"metric,omitempty"` - Help string `json:"help,omitempty"` -} - // metadata makes a request against the Prometheus /api/v1/targets/metadata endpoint. // It returns all the metrics and its metadata. -func (c *prometheusClient) metadata(query string) (Metadata, error) { - req := c.kubeClient.CoreV1().RESTClient().Get(). - Namespace("monitoring"). - Resource("pods"). - SubResource("proxy"). - Name("prometheus-k8s-0:9090"). - Suffix("/api/v1/targets/metadata").Param("match_target", query) +func (c *prometheusClient) metadata(query string) ([]promv1.MetricMetadata, error) { + var metadata []promv1.MetricMetadata + rsp, err := c.apiRequest("/api/v1/targets/metadata", "match_target", query) - var data Metadata - b, err := req.DoRaw() - if err != nil { - return data, err - } - - r := bytes.NewReader(b) + r := bytes.NewReader(rsp.Data) decoder := json.NewDecoder(r) - err = decoder.Decode(&data) + err = decoder.Decode(&metadata) if err != nil { - return data, err + return metadata, err } - if data.Status != "success" { - return data, fmt.Errorf("status of returned response was not successful; status: %s", data.Status) - } - return data, err + return metadata, err +} + +// targets makes a request against the Prometheus /api/v1/targets endpoint. +// It returns all targets registered in prometheus. +func (c *prometheusClient) targets() (promv1.TargetsResult, error) { + var targets promv1.TargetsResult + rsp, err := c.apiRequest("/api/v1/targets", "state", "any") + if err != nil { + return targets, err + } + + r := bytes.NewReader(rsp.Data) + decoder := json.NewDecoder(r) + err = decoder.Decode(&targets) + if err != nil { + return targets, err + } + + return targets, err }