From a24090932613f52b0dc40370a590679e6693b8a1 Mon Sep 17 00:00:00 2001
From: Matthias Loibl <mail@matthiasloibl.com>
Date: Tue, 13 Nov 2018 14:43:30 +0100
Subject: [PATCH 1/3] contrib/kube-prometheus: Update custom-metrics example to
 use our adapter

---
 ...r-auth-delegator-cluster-role-binding.yaml | 12 ---
 ...cs-apiserver-auth-reader-role-binding.yaml | 13 ---
 .../custom-metrics-apiserver-deployment.yaml  | 41 --------
 ...-resource-reader-cluster-role-binding.yaml |  6 +-
 ...tom-metrics-apiserver-service-account.yaml |  4 -
 .../custom-metrics-apiserver-service.yaml     | 10 --
 .../custom-metrics-apiservice.yaml            |  2 +-
 .../custom-metrics-configmap.yaml             | 98 +++++++++++++++++++
 ...-metrics-resource-reader-cluster-role.yaml | 14 ---
 experimental/custom-metrics-api/deploy.sh     | 13 ---
 experimental/custom-metrics-api/gencerts.sh   | 37 -------
 experimental/custom-metrics-api/teardown.sh   | 13 ---
 12 files changed, 102 insertions(+), 161 deletions(-)
 delete mode 100644 experimental/custom-metrics-api/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
 delete mode 100644 experimental/custom-metrics-api/custom-metrics-apiserver-auth-reader-role-binding.yaml
 delete mode 100644 experimental/custom-metrics-api/custom-metrics-apiserver-deployment.yaml
 delete mode 100644 experimental/custom-metrics-api/custom-metrics-apiserver-service-account.yaml
 delete mode 100644 experimental/custom-metrics-api/custom-metrics-apiserver-service.yaml
 create mode 100644 experimental/custom-metrics-api/custom-metrics-configmap.yaml
 delete mode 100644 experimental/custom-metrics-api/custom-metrics-resource-reader-cluster-role.yaml
 delete mode 100755 experimental/custom-metrics-api/deploy.sh
 delete mode 100755 experimental/custom-metrics-api/gencerts.sh
 delete mode 100755 experimental/custom-metrics-api/teardown.sh

diff --git a/experimental/custom-metrics-api/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml b/experimental/custom-metrics-api/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
deleted file mode 100644
index 8853bc1f..00000000
--- a/experimental/custom-metrics-api/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRoleBinding
-metadata:
-  name: custom-metrics:system:auth-delegator
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:auth-delegator
-subjects:
-- kind: ServiceAccount
-  name: custom-metrics-apiserver
-  namespace: monitoring
diff --git a/experimental/custom-metrics-api/custom-metrics-apiserver-auth-reader-role-binding.yaml b/experimental/custom-metrics-api/custom-metrics-apiserver-auth-reader-role-binding.yaml
deleted file mode 100644
index 682143cf..00000000
--- a/experimental/custom-metrics-api/custom-metrics-apiserver-auth-reader-role-binding.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: RoleBinding
-metadata:
-  name: custom-metrics-auth-reader
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: extension-apiserver-authentication-reader
-subjects:
-- kind: ServiceAccount
-  name: custom-metrics-apiserver
-  namespace: monitoring
diff --git a/experimental/custom-metrics-api/custom-metrics-apiserver-deployment.yaml b/experimental/custom-metrics-api/custom-metrics-apiserver-deployment.yaml
deleted file mode 100644
index e5b4beea..00000000
--- a/experimental/custom-metrics-api/custom-metrics-apiserver-deployment.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  labels:
-    app: custom-metrics-apiserver
-  name: custom-metrics-apiserver
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: custom-metrics-apiserver
-  template:
-    metadata:
-      labels:
-        app: custom-metrics-apiserver
-      name: custom-metrics-apiserver
-    spec:
-      serviceAccountName: custom-metrics-apiserver
-      containers:
-      - name: custom-metrics-apiserver
-        image: quay.io/coreos/k8s-prometheus-adapter-amd64:v0.2.0
-        args:
-        - /adapter
-        - --secure-port=6443
-        - --tls-cert-file=/var/run/serving-cert/serving.crt
-        - --tls-private-key-file=/var/run/serving-cert/serving.key
-        - --logtostderr=true
-        - --prometheus-url=http://prometheus-k8s.monitoring.svc:9090/
-        - --metrics-relist-interval=30s
-        - --rate-interval=5m
-        - --v=10
-        ports:
-        - containerPort: 6443
-        volumeMounts:
-        - mountPath: /var/run/serving-cert
-          name: volume-serving-cert
-          readOnly: true
-      volumes:
-      - name: volume-serving-cert
-        secret:
-          secretName: cm-adapter-serving-certs
diff --git a/experimental/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml b/experimental/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
index 0335c177..e2b1ca43 100644
--- a/experimental/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
+++ b/experimental/custom-metrics-api/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
@@ -1,12 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:
-  name: custom-metrics-resource-reader
+  name: custom-metrics-server-resources
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: custom-metrics-resource-reader
+  name: custom-metrics-server-resources
 subjects:
 - kind: ServiceAccount
-  name: custom-metrics-apiserver
+  name: prometheus-adapter
   namespace: monitoring
diff --git a/experimental/custom-metrics-api/custom-metrics-apiserver-service-account.yaml b/experimental/custom-metrics-api/custom-metrics-apiserver-service-account.yaml
deleted file mode 100644
index 29359409..00000000
--- a/experimental/custom-metrics-api/custom-metrics-apiserver-service-account.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: custom-metrics-apiserver
diff --git a/experimental/custom-metrics-api/custom-metrics-apiserver-service.yaml b/experimental/custom-metrics-api/custom-metrics-apiserver-service.yaml
deleted file mode 100644
index fb0addcb..00000000
--- a/experimental/custom-metrics-api/custom-metrics-apiserver-service.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: custom-metrics-apiserver
-spec:
-  ports:
-  - port: 443
-    targetPort: 6443
-  selector:
-    app: custom-metrics-apiserver
diff --git a/experimental/custom-metrics-api/custom-metrics-apiservice.yaml b/experimental/custom-metrics-api/custom-metrics-apiservice.yaml
index cfc2ee63..98f87495 100644
--- a/experimental/custom-metrics-api/custom-metrics-apiservice.yaml
+++ b/experimental/custom-metrics-api/custom-metrics-apiservice.yaml
@@ -4,7 +4,7 @@ metadata:
   name: v1beta1.custom.metrics.k8s.io
 spec:
   service:
-    name: custom-metrics-apiserver
+    name: prometheus-adapter
     namespace: monitoring
   group: custom.metrics.k8s.io
   version: v1beta1
diff --git a/experimental/custom-metrics-api/custom-metrics-configmap.yaml b/experimental/custom-metrics-api/custom-metrics-configmap.yaml
new file mode 100644
index 00000000..2e209cc3
--- /dev/null
+++ b/experimental/custom-metrics-api/custom-metrics-configmap.yaml
@@ -0,0 +1,98 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: adapter-config
+  namespace: monitoring
+data:
+  config.yaml: |
+    rules:
+    - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}'
+      seriesFilters: []
+      resources:
+        overrides:
+          namespace:
+            resource: namespace
+          pod_name:
+            resource: pod
+      name:
+        matches: ^container_(.*)_seconds_total$
+        as: ""
+      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[1m])) by (<<.GroupBy>>)
+    - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}'
+      seriesFilters:
+      - isNot: ^container_.*_seconds_total$
+      resources:
+        overrides:
+          namespace:
+            resource: namespace
+          pod_name:
+            resource: pod
+      name:
+        matches: ^container_(.*)_total$
+        as: ""
+      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[1m])) by (<<.GroupBy>>)
+    - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}'
+      seriesFilters:
+      - isNot: ^container_.*_total$
+      resources:
+        overrides:
+          namespace:
+            resource: namespace
+          pod_name:
+            resource: pod
+      name:
+        matches: ^container_(.*)$
+        as: ""
+      metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}) by (<<.GroupBy>>)
+    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
+      seriesFilters:
+      - isNot: .*_total$
+      resources:
+        template: <<.Resource>>
+      name:
+        matches: ""
+        as: ""
+      metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
+      seriesFilters:
+      - isNot: .*_seconds_total
+      resources:
+        template: <<.Resource>>
+      name:
+        matches: ^(.*)_total$
+        as: ""
+      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
+    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
+      seriesFilters: []
+      resources:
+        template: <<.Resource>>
+      name:
+        matches: ^(.*)_seconds_total$
+        as: ""
+      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
+    resourceRules:
+      cpu:
+        containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
+        nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[1m])) by (<<.GroupBy>>)
+        resources:
+          overrides:
+            node:
+              resource: node
+            namespace:
+              resource: namespace
+            pod_name:
+              resource: pod
+        containerLabel: container_name
+      memory:
+        containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+        nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
+        resources:
+          overrides:
+            node:
+              resource: node
+            namespace:
+              resource: namespace
+            pod_name:
+              resource: pod
+        containerLabel: container_name
+      window: 1m
diff --git a/experimental/custom-metrics-api/custom-metrics-resource-reader-cluster-role.yaml b/experimental/custom-metrics-api/custom-metrics-resource-reader-cluster-role.yaml
deleted file mode 100644
index a5ad7604..00000000
--- a/experimental/custom-metrics-api/custom-metrics-resource-reader-cluster-role.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRole
-metadata:
-  name: custom-metrics-resource-reader
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - namespaces
-  - pods
-  - services
-  verbs:
-  - get
-  - list
diff --git a/experimental/custom-metrics-api/deploy.sh b/experimental/custom-metrics-api/deploy.sh
deleted file mode 100755
index 2255c7fd..00000000
--- a/experimental/custom-metrics-api/deploy.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-kubectl create -f custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
-kubectl create -f custom-metrics-apiserver-auth-reader-role-binding.yaml
-kubectl -n monitoring create -f cm-adapter-serving-certs.yaml
-kubectl -n monitoring create -f custom-metrics-apiserver-deployment.yaml
-kubectl create -f custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
-kubectl -n monitoring create -f custom-metrics-apiserver-service-account.yaml
-kubectl -n monitoring create -f custom-metrics-apiserver-service.yaml
-kubectl create -f custom-metrics-apiservice.yaml
-kubectl create -f custom-metrics-cluster-role.yaml
-kubectl create -f custom-metrics-resource-reader-cluster-role.yaml
-kubectl create -f hpa-custom-metrics-cluster-role-binding.yaml
diff --git a/experimental/custom-metrics-api/gencerts.sh b/experimental/custom-metrics-api/gencerts.sh
deleted file mode 100755
index a8f5539d..00000000
--- a/experimental/custom-metrics-api/gencerts.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-# exit immediately when a command fails
-set -e
-# only exit with zero if all commands of the pipeline exit successfully
-set -o pipefail
-# error on unset variables
-set -u
-
-# Detect if we are on mac or should use GNU base64 options
-case $(uname) in
-        Darwin)
-            b64_opts='-b=0'
-            ;; 
-        *)
-            b64_opts='--wrap=0'
-esac
-
-go get -v -u github.com/cloudflare/cfssl/cmd/...
-
-export PURPOSE=metrics
-openssl req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout ${PURPOSE}-ca.key -out ${PURPOSE}-ca.crt -subj "/CN=ca"
-echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","'${PURPOSE}'"]}}}' > "${PURPOSE}-ca-config.json"
-
-export SERVICE_NAME=custom-metrics-apiserver
-export ALT_NAMES='"custom-metrics-apiserver.monitoring","custom-metrics-apiserver.monitoring.svc"'
-echo "{\"CN\":\"${SERVICE_NAME}\", \"hosts\": [${ALT_NAMES}], \"key\": {\"algo\": \"rsa\",\"size\": 2048}}" | \
-       	cfssl gencert -ca=metrics-ca.crt -ca-key=metrics-ca.key -config=metrics-ca-config.json - | cfssljson -bare apiserver
-
-cat <<-EOF > cm-adapter-serving-certs.yaml
-apiVersion: v1
-kind: Secret
-metadata:
-  name: cm-adapter-serving-certs
-data:
-  serving.crt: $(base64 ${b64_opts} < apiserver.pem)
-  serving.key: $(base64 ${b64_opts} < apiserver-key.pem)
-EOF
diff --git a/experimental/custom-metrics-api/teardown.sh b/experimental/custom-metrics-api/teardown.sh
deleted file mode 100755
index 4797de1c..00000000
--- a/experimental/custom-metrics-api/teardown.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-
-kubectl delete -f custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
-kubectl delete -f custom-metrics-apiserver-auth-reader-role-binding.yaml
-kubectl -n monitoring delete -f cm-adapter-serving-certs.yaml
-kubectl -n monitoring delete -f custom-metrics-apiserver-deployment.yaml
-kubectl delete -f custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
-kubectl -n monitoring delete -f custom-metrics-apiserver-service-account.yaml
-kubectl -n monitoring delete -f custom-metrics-apiserver-service.yaml
-kubectl delete -f custom-metrics-apiservice.yaml
-kubectl delete -f custom-metrics-cluster-role.yaml
-kubectl delete -f custom-metrics-resource-reader-cluster-role.yaml
-kubectl delete -f hpa-custom-metrics-cluster-role-binding.yaml

From d4089d7125bd765b2077b7a31589831b6f88006e Mon Sep 17 00:00:00 2001
From: Matthias Loibl <mail@matthiasloibl.com>
Date: Tue, 13 Nov 2018 14:43:53 +0100
Subject: [PATCH 2/3] contrib/kube-prometheus: Add sample-app for showing
 custom metrics with HPA

Use interval in ServiceMonitor endpoint

Reintroduce updated dpeloy.sh & teardown.sh
---
 experimental/custom-metrics-api/deploy.sh     |  7 ++
 .../custom-metrics-api/sample-app.yaml        | 67 +++++++++++++++++++
 experimental/custom-metrics-api/teardown.sh   |  7 ++
 3 files changed, 81 insertions(+)
 create mode 100644 experimental/custom-metrics-api/deploy.sh
 create mode 100644 experimental/custom-metrics-api/sample-app.yaml
 create mode 100644 experimental/custom-metrics-api/teardown.sh

diff --git a/experimental/custom-metrics-api/deploy.sh b/experimental/custom-metrics-api/deploy.sh
new file mode 100644
index 00000000..a7324831
--- /dev/null
+++ b/experimental/custom-metrics-api/deploy.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash	
+
+kubectl apply -n monitoring custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
+kubectl apply -n monitoring custom-metrics-apiservice.yaml
+kubectl apply -n monitoring custom-metrics-cluster-role.yaml
+kubectl apply -n monitoring custom-metrics-configmap.yaml
+kubectl apply -n monitoring hpa-custom-metrics-cluster-role-binding.yaml
diff --git a/experimental/custom-metrics-api/sample-app.yaml b/experimental/custom-metrics-api/sample-app.yaml
new file mode 100644
index 00000000..470887c6
--- /dev/null
+++ b/experimental/custom-metrics-api/sample-app.yaml
@@ -0,0 +1,67 @@
+kind: ServiceMonitor
+apiVersion: monitoring.coreos.com/v1
+metadata:
+  name: sample-app
+  labels:
+    app: sample-app
+spec:
+  selector:
+    matchLabels:
+      app: sample-app
+  endpoints: 
+  - port: http
+    interval: 5s
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: sample-app
+  labels:
+    app: sample-app
+spec:
+  ports:
+  - name: http
+    port: 8080
+    targetPort: 8080
+  selector:
+    app: sample-app
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sample-app
+  labels:
+    app: sample-app
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: sample-app
+  template:
+    metadata:
+      labels:
+        app: sample-app
+    spec:
+      containers:
+      - image: luxas/autoscale-demo:v0.1.2
+        name: metrics-provider
+        ports:
+        - name: http
+          containerPort: 8080
+---
+kind: HorizontalPodAutoscaler
+apiVersion: autoscaling/v2beta1
+metadata:
+  name: sample-app
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: sample-app
+  minReplicas: 1
+  maxReplicas: 10
+  metrics:
+  - type: Pods
+    pods:
+      metricName: http_requests
+      targetAverageValue: 500m
diff --git a/experimental/custom-metrics-api/teardown.sh b/experimental/custom-metrics-api/teardown.sh
new file mode 100644
index 00000000..2287c799
--- /dev/null
+++ b/experimental/custom-metrics-api/teardown.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash	
+
+kubectl delete -n monitoring custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
+kubectl delete -n monitoring custom-metrics-apiservice.yaml
+kubectl delete -n monitoring custom-metrics-cluster-role.yaml
+kubectl delete -n monitoring custom-metrics-configmap.yaml
+kubectl delete -n monitoring hpa-custom-metrics-cluster-role-binding.yaml

From 1b495e7bcb7713eb9b058dc9f98a0a6d283dd026 Mon Sep 17 00:00:00 2001
From: Matthias Loibl <mail@matthiasloibl.com>
Date: Wed, 14 Nov 2018 16:19:05 +0100
Subject: [PATCH 3/3] contrib/kube-prometheus: Update README.md for
 custom-metrics-api

---
 experimental/custom-metrics-api/README.md   | 18 ++++++++++++++----
 experimental/custom-metrics-api/deploy.sh   |  2 +-
 experimental/custom-metrics-api/teardown.sh |  2 +-
 3 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/experimental/custom-metrics-api/README.md b/experimental/custom-metrics-api/README.md
index 91375a42..c5c7102c 100644
--- a/experimental/custom-metrics-api/README.md
+++ b/experimental/custom-metrics-api/README.md
@@ -1,11 +1,21 @@
 # Custom Metrics API
 
-The custom metrics API allows the HPA v2 to scale on arbirary metrics.
+The custom metrics API allows the HPA v2 to scale based on arbirary metrics.
 
-This directory contains an example deployment of the custom metrics API adapter using Prometheus as the backing monitoring system.
+This directory contains an example deployment which extends the Prometheus Adapter, deployed with kube-prometheus, serve the [Custom Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/custom-metrics-api.md) by talking to Prometheus running inside the cluster.
 
-In order to deploy the custom metrics adapter for Prometheus you need to generate TLS certficates used to serve the API. An example of how these could be generated can be found in `./gencerts.sh`, note that this is _not_ recommended to be used in production. You need to employ a secure PKI strategy, this is merely an example to get started and try it out quickly.
+Make sure you have the Prometheus Adapter up and running in the `monitoring` namespace.
 
-Once the generated `Secret` with the certificates is in place, you can deploy everything in the `monitoring` namespace using `./deploy.sh`.
+You can deploy everything in the `monitoring` namespace using `./deploy.sh`.
 
 When you're done, you can teardown using the `./teardown.sh` script.
+
+### Sample App
+
+Additionally, this directory contains a sample app that uses the [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale the Deployment's replicas of Pods up and down as needed.  
+Deploy this app by running `kubectl apply -f sample-app.yaml`. 
+Make the app accessible on your system, for example by using `kubectl -n monitoring port-forward svc/sample-app 8080`. Next you need to put some load on its http endpoints. 
+
+A tool like [hey](https://github.com/rakyll/hey) is helpful for doing so: `hey -c 20 -n 100000000 http://localhost:8080/metrics`
+
+There is an even more detailed information on this sample app at [luxas/kubeadm-workshop](https://github.com/luxas/kubeadm-workshop#deploying-the-prometheus-operator-for-monitoring-services-in-the-cluster).
diff --git a/experimental/custom-metrics-api/deploy.sh b/experimental/custom-metrics-api/deploy.sh
index a7324831..1ac74878 100644
--- a/experimental/custom-metrics-api/deploy.sh
+++ b/experimental/custom-metrics-api/deploy.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash	
+#!/usr/bin/env bash
 
 kubectl apply -n monitoring custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
 kubectl apply -n monitoring custom-metrics-apiservice.yaml
diff --git a/experimental/custom-metrics-api/teardown.sh b/experimental/custom-metrics-api/teardown.sh
index 2287c799..a62f685e 100644
--- a/experimental/custom-metrics-api/teardown.sh
+++ b/experimental/custom-metrics-api/teardown.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash	
+#!/usr/bin/env bash
 
 kubectl delete -n monitoring custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
 kubectl delete -n monitoring custom-metrics-apiservice.yaml