Merge pull request #750 from brancz/metrics-servers

Metrics servers

manifests/custom-metrics-api/.gitignore

@@ -0,0 +1,7 @@
+apiserver-key.pem
+apiserver.csr
+apiserver.pem
+metrics-ca-config.json
+metrics-ca.crt
+metrics-ca.key
+cm-adapter-serving-certs.yaml

manifests/custom-metrics-api/README.md

@@ -0,0 +1,11 @@
+# Custom Metrics API
+
+The custom metrics API allows the HPA v2 to scale on arbirary metrics.
+
+This directory contains an example deployment of the custom metrics API adapter using Prometheus as the backing monitoring system.
+
+In order to deploy the custom metrics adapter for Prometheus you need to generate TLS certficates used to serve the API. An example of how these could be generated can be found in `./gencerts.sh`, note that this is _not_ recommended to be used in production. You need to employ a secure PKI strategy, this is merely an example to get started and try it out quickly.
+
+Once the generated `Secret` with the certificates is in place, you can deploy everything in the `monitoring` namespace using `./deploy.sh`.
+
+When you're done, you can teardown using the `./teardown.sh` script.

manifests/custom-metrics-api/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: custom-metrics:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-apiserver
+  namespace: monitoring

manifests/custom-metrics-api/custom-metrics-apiserver-auth-reader-role-binding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: custom-metrics-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-apiserver
+  namespace: monitoring

manifests/custom-metrics-api/custom-metrics-apiserver-deployment.yaml

@@ -0,0 +1,41 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    app: custom-metrics-apiserver
+  name: custom-metrics-apiserver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: custom-metrics-apiserver
+  template:
+    metadata:
+      labels:
+        app: custom-metrics-apiserver
+      name: custom-metrics-apiserver
+    spec:
+      serviceAccountName: custom-metrics-apiserver
+      containers:
+      - name: custom-metrics-apiserver
+        image: quay.io/coreos/k8s-prometheus-adapter-amd64:v0.2.0
+        args:
+        - /adapter
+        - --secure-port=6443
+        - --tls-cert-file=/var/run/serving-cert/serving.crt
+        - --tls-private-key-file=/var/run/serving-cert/serving.key
+        - --logtostderr=true
+        - --prometheus-url=http://prometheus-k8s.monitoring.svc:9090/
+        - --metrics-relist-interval=30s
+        - --rate-interval=5m
+        - --v=10
+        ports:
+        - containerPort: 6443
+        volumeMounts:
+        - mountPath: /var/run/serving-cert
+          name: volume-serving-cert
+          readOnly: true
+      volumes:
+      - name: volume-serving-cert
+        secret:
+          secretName: cm-adapter-serving-certs

manifests/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: custom-metrics-resource-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: custom-metrics-resource-reader
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-apiserver
+  namespace: monitoring

manifests/custom-metrics-api/custom-metrics-apiserver-service-account.yaml

@@ -0,0 +1,4 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: custom-metrics-apiserver

manifests/custom-metrics-api/custom-metrics-apiserver-service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: custom-metrics-apiserver
+spec:
+  ports:
+  - port: 443
+    targetPort: 6443
+  selector:
+    app: custom-metrics-apiserver

manifests/custom-metrics-api/custom-metrics-apiservice.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1beta1.custom.metrics.k8s.io
+spec:
+  service:
+    name: custom-metrics-apiserver
+    namespace: monitoring
+  group: custom.metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100

manifests/custom-metrics-api/custom-metrics-cluster-role.yaml

@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: custom-metrics-server-resources
+rules:
+- apiGroups:
+  - custom.metrics.k8s.io
+  resources: ["*"]
+  verbs: ["*"]

manifests/custom-metrics-api/custom-metrics-resource-reader-cluster-role.yaml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: custom-metrics-resource-reader
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - pods
+  - services
+  verbs:
+  - get
+  - list

manifests/custom-metrics-api/deploy.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+kubectl create -f custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
+kubectl create -f custom-metrics-apiserver-auth-reader-role-binding.yaml
+kubectl -n monitoring create -f cm-adapter-serving-certs.yaml
+kubectl -n monitoring create -f custom-metrics-apiserver-deployment.yaml
+kubectl create -f custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
+kubectl -n monitoring create -f custom-metrics-apiserver-service-account.yaml
+kubectl -n monitoring create -f custom-metrics-apiserver-service.yaml
+kubectl create -f custom-metrics-apiservice.yaml
+kubectl create -f custom-metrics-cluster-role.yaml
+kubectl create -f custom-metrics-resource-reader-cluster-role.yaml
+kubectl create -f hpa-custom-metrics-cluster-role-binding.yaml

manifests/custom-metrics-api/gencerts.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+go get -v -u github.com/cloudflare/cfssl/cmd/...
+
+export PURPOSE=metrics
+openssl req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout ${PURPOSE}-ca.key -out ${PURPOSE}-ca.crt -subj "/CN=ca"
+echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","'${PURPOSE}'"]}}}' > "${PURPOSE}-ca-config.json"
+
+export SERVICE_NAME=custom-metrics-apiserver
+export ALT_NAMES='"custom-metrics-apiserver.monitoring","custom-metrics-apiserver.monitoring.svc"'
+echo '{"CN":"'${SERVICE_NAME}'","hosts":['${ALT_NAMES}'],"key":{"algo":"rsa","size":2048}}' | cfssl gencert -ca=metrics-ca.crt -ca-key=metrics-ca.key -config=metrics-ca-config.json - | cfssljson -bare apiserver
+
+cat <<-EOF > cm-adapter-serving-certs.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cm-adapter-serving-certs
+data:
+  serving.crt: $(cat apiserver.pem | base64 --wrap=0)
+  serving.key: $(cat apiserver-key.pem | base64 --wrap=0)
+EOF

manifests/custom-metrics-api/hpa-custom-metrics-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: hpa-controller-custom-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: custom-metrics-server-resources
+subjects:
+- kind: ServiceAccount
+  name: horizontal-pod-autoscaler
+  namespace: kube-system

manifests/custom-metrics-api/teardown.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+kubectl delete -f custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
+kubectl delete -f custom-metrics-apiserver-auth-reader-role-binding.yaml
+kubectl -n monitoring delete -f cm-adapter-serving-certs.yaml
+kubectl -n monitoring delete -f custom-metrics-apiserver-deployment.yaml
+kubectl delete -f custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
+kubectl -n monitoring delete -f custom-metrics-apiserver-service-account.yaml
+kubectl -n monitoring delete -f custom-metrics-apiserver-service.yaml
+kubectl delete -f custom-metrics-apiservice.yaml
+kubectl delete -f custom-metrics-cluster-role.yaml
+kubectl delete -f custom-metrics-resource-reader-cluster-role.yaml
+kubectl delete -f hpa-custom-metrics-cluster-role-binding.yaml

manifests/metrics-server/auth-delegator.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-server:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system

manifests/metrics-server/auth-reader.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: metrics-server-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system

manifests/metrics-server/metrics-apiservice.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+spec:
+  service:
+    name: metrics-server
+    namespace: kube-system
+  group: metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100

manifests/metrics-server/metrics-server-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:metrics-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:metrics-server
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system

manifests/metrics-server/metrics-server-cluster-role.yaml

@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:metrics-server
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - "extensions"
+  resources:
+  - deployments
+  verbs:
+  - get
+  - list
+  - watch

manifests/metrics-server/metrics-server-deployment.yaml

@@ -0,0 +1,25 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    k8s-app: metrics-server
+spec:
+  selector:
+    matchLabels:
+      k8s-app: metrics-server
+  template:
+    metadata:
+      name: metrics-server
+      labels:
+        k8s-app: metrics-server
+    spec:
+      serviceAccountName: metrics-server
+      containers:
+      - name: metrics-server
+        image: gcr.io/google_containers/metrics-server-amd64:v0.2.0
+        imagePullPolicy: Always
+        command:
+        - /metrics-server
+        - --source=kubernetes.summary_api:''

manifests/metrics-server/metrics-server-service-account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metrics-server
+  namespace: kube-system

manifests/metrics-server/metrics-server-service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    kubernetes.io/name: "Metrics-server"
+spec:
+  selector:
+    k8s-app: metrics-server
+  ports:
+  - port: 443
+    protocol: TCP
+    targetPort: 443