kube-prometheus: generate manifests without kubectl

For `--dry-run` to work with kubectl a Kubernetes cluster's apiserver is
actually used, which is unnecessary for generating these manifests. This
approach also allows further customization, such as adding labels to the
generated manifests.

hack/cluster-monitoring/deploy

@@ -32,6 +32,12 @@ kctl apply -f manifests/prometheus/prometheus-k8s-service.yaml
 kctl apply -f manifests/alertmanager/alertmanager-config.yaml
 kctl apply -f manifests/alertmanager/alertmanager-service.yaml
 
+# unfortunately statefulsets cannot be changed except for their replica count
+# so we need to make sure that the rule files are created before we create the
+# prometheus resource so it can properly discover the rule files when creating
+# the statefulset
+sleep 5
+
 # `kubectl apply` is currently not working for third party resources so we are
 # using `kubectl create` here for the time being.
 # (https://github.com/kubernetes/kubernetes/issues/29542)

hack/scripts/generate-alertmanager-config-secret.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+cat <<-EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: alertmanager-main
+data:
+  alertmanager.yaml: $(cat assets/alertmanager/alertmanager.yaml | base64 --wrap=0)
+EOF
+

hack/scripts/generate-dashboards-configmap.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+cat <<-EOF
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboards
+data:
+EOF
+
+for f in assets/grafana/*
+do
+  echo "  $(basename $f): |+"
+  cat $f | sed "s/^/    /g"
+done

hack/scripts/generate-manifests.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
 # Generate Alert Rules ConfigMap
-kubectl create configmap --dry-run=true prometheus-k8s-rules --from-file=assets/prometheus/rules/ -oyaml > manifests/prometheus/prometheus-k8s-rules.yaml
+hack/scripts/generate-rules-configmap.sh > manifests/prometheus/prometheus-k8s-rules.yaml
 
 # Generate Dashboard ConfigMap
-kubectl create configmap --dry-run=true grafana-dashboards --from-file=assets/grafana/ -oyaml > manifests/grafana/grafana-dashboards.yaml
+hack/scripts/generate-dashboards-configmap.sh > manifests/grafana/grafana-dashboards.yaml
 
 # Generate Secret for Alertmanager config
-kubectl create secret generic alertmanager-main --dry-run --from-file=assets/alertmanager/alertmanager.yaml -oyaml > manifests/alertmanager/alertmanager-config.yaml
+hack/scripts/generate-alertmanager-config-secret.sh > manifests/alertmanager/alertmanager-config.yaml
 

hack/scripts/generate-rules-configmap.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+cat <<-EOF
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-k8s-rules
+  labels:
+    role: prometheus-rulefiles
+    prometheus: k8s
+data:
+EOF
+
+for f in assets/prometheus/rules/*.rules
+do
+  echo "  $(basename $f): |+"
+  cat $f | sed "s/^/    /g"
+done