Merge pull request #1238 from brancz/kube-prometheus-fixes

Various kube-prometheus fixes

hack/cluster-monitoring/deploy

@@ -1,10 +1,18 @@
 #!/usr/bin/env bash
+# exit immediately when a command fails
+set -e
+# only exit with zero if all commands of the pipeline exit successfully
+set -o pipefail
+# error on unset variables
+set -u
+# print each command before executing it
+set -x
 
 manifest_prefix=${1-.}
 
 kubectl create namespace monitoring
 
-kubectl apply -f ${manifest_prefix}/manifests/prometheus-operator/
+find ${manifest_prefix}/manifests/prometheus-operator/ -type f ! -name prometheus-operator-service-monitor.yaml -exec kubectl apply -f {} \;
 
 # Wait for CRDs to be ready.
 printf "Waiting for Operator to register custom resource definitions..."
@@ -16,9 +24,15 @@ until kubectl get prometheuses.monitoring.coreos.com > /dev/null 2>&1; do sleep
 until kubectl get alertmanagers.monitoring.coreos.com > /dev/null 2>&1; do sleep 1; printf "."; done
 echo "done!"
 
+# need to ensure that ServiceMonitors are registered before we can create the prometheus-operator ServiceMonitor
+kubectl apply -f ${manifest_prefix}/manifests/prometheus-operator/prometheus-operator-service-monitor.yaml
+
 kubectl apply -f ${manifest_prefix}/manifests/node-exporter/
 kubectl apply -f ${manifest_prefix}/manifests/kube-state-metrics/
-kubectl apply -f ${manifest_prefix}/manifests/grafana/
+find ${manifest_prefix}/manifests/grafana/ -type f ! -name grafana-dashboard-definitions.yaml -exec kubectl apply -f {} \;
+
+# kubectl apply wants to put the previous version in an annotation, which is too large, therefore create instead of apply
+kubectl create -f ${manifest_prefix}/manifests/grafana/grafana-dashboard-definitions.yaml
 kubectl apply -f ${manifest_prefix}/manifests/prometheus-k8s/
 kubectl apply -f ${manifest_prefix}/manifests/alertmanager-main/
 

jsonnet/prometheus/prometheus-k8s-role-binding-config.libsonnet

@@ -1,5 +1,5 @@
 local prometheusNamespaceRoleBinding = import "prometheus-namespace-role-binding.libsonnet";
 
 {
-    new(namespace):: prometheusNamespaceRoleBinding.new(namespace, namespace, "prometheus-k8s-config")
+    new(namespace):: prometheusNamespaceRoleBinding.new(namespace, namespace, "prometheus-k8s-config", "prometheus-k8s")
 }

jsonnet/prometheus/prometheus-k8s-role-binding-default.libsonnet

@@ -1,5 +1,5 @@
 local prometheusNamespaceRoleBinding = import "prometheus-namespace-role-binding.libsonnet";
 
 {
-    new(namespace):: prometheusNamespaceRoleBinding.new(namespace, "default", "prometheus-k8s")
+    new(namespace):: prometheusNamespaceRoleBinding.new(namespace, "default", "prometheus-k8s", "prometheus-k8s")
 }

jsonnet/prometheus/prometheus-k8s-role-binding-kube-system.libsonnet

@@ -1,5 +1,5 @@
 local prometheusNamespaceRoleBinding = import "prometheus-namespace-role-binding.libsonnet";
 
 {
-    new(namespace):: prometheusNamespaceRoleBinding.new(namespace, "kube-system", "prometheus-k8s")
+    new(namespace):: prometheusNamespaceRoleBinding.new(namespace, "kube-system", "prometheus-k8s", "prometheus-k8s")
 }

jsonnet/prometheus/prometheus-k8s-role-binding-namespace.libsonnet

@@ -1,5 +1,5 @@
 local prometheusNamespaceRoleBinding = import "prometheus-namespace-role-binding.libsonnet";
 
 {
-    new(namespace):: prometheusNamespaceRoleBinding.new(namespace, namespace, "prometheus-k8s")
+    new(namespace):: prometheusNamespaceRoleBinding.new(namespace, namespace, "prometheus-k8s", "prometheus-k8s")
 }

jsonnet/prometheus/prometheus-k8s-rules.libsonnet

@@ -4,5 +4,6 @@ local configMap = k.core.v1.configMap;
 {
     new(namespace, ruleFiles)::
         configMap.new("prometheus-k8s-rules", ruleFiles) +
+          configMap.mixin.metadata.withLabels({role: "alert-rules", prometheus: "k8s"}) +
           configMap.mixin.metadata.withNamespace(namespace)
 }

jsonnet/prometheus/prometheus-namespace-role-binding.libsonnet

@@ -2,12 +2,12 @@ local k = import "ksonnet.beta.3/k.libsonnet";
 local roleBinding = k.rbac.v1.roleBinding;
 
 {
-  new(serviceAccountNamespace, namespace, name)::
+  new(serviceAccountNamespace, namespace, roleName, serviceAccountName)::
     roleBinding.new() +
-      roleBinding.mixin.metadata.withName(name) +
+      roleBinding.mixin.metadata.withName(roleName) +
       roleBinding.mixin.metadata.withNamespace(namespace) +
       roleBinding.mixin.roleRef.withApiGroup("rbac.authorization.k8s.io") +
-      roleBinding.mixin.roleRef.withName(name) +
+      roleBinding.mixin.roleRef.withName(roleName) +
       roleBinding.mixin.roleRef.mixinInstance({kind: "Role"}) +
-      roleBinding.withSubjects([{kind: "ServiceAccount", name: name, namespace: serviceAccountNamespace}])
+      roleBinding.withSubjects([{kind: "ServiceAccount", name: serviceAccountName, namespace: serviceAccountNamespace}])
 }

manifests/prometheus-k8s/prometheus-k8s-role-binding-config.yaml

@@ -9,5 +9,5 @@ roleRef:
   name: prometheus-k8s-config
 subjects:
 - kind: ServiceAccount
-  name: prometheus-k8s-config
+  name: prometheus-k8s
   namespace: monitoring

manifests/prometheus-k8s/prometheus-k8s-rules.yaml

@@ -583,5 +583,8 @@ data:
     rejected\n"
 kind: ConfigMap
 metadata:
+  labels:
+    prometheus: k8s
+    role: alert-rules
   name: prometheus-k8s-rules
   namespace: monitoring