Add roadmap, fix manifests, improve hack scripts

README.md

@@ -24,7 +24,6 @@ controller, the [node_exporter](https://github.com/prometheus/node_exporter),
 
 ```bash
 export KUBECONFIG=<path>          # defaults to "~/.kube/config"
-export KUBE_NAMESPACE=<ns>        # defaults to "default"
 hack/cluster-monitoring/deploy
 ```
 
@@ -39,9 +38,9 @@ To tear it all down again, run:
 hack/cluster-monitoring/teardown
 ```
 
-__All services in the manifest still contain the `prometheus.io/scrape = true` annotations. It is not
+*All services in the manifest still contain the `prometheus.io/scrape = true` annotations. It is not
 used by the Prometheus controller. They remain for convential deployments as in
-[this example configuration](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml).__
+[this example configuration](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml).*
 
 ## Monitoring custom services
 
@@ -64,4 +63,12 @@ Teardown:
 hack/example-service-monitoring/teardown
 ```
 
+## Roadmap
+
+* Incorporate [Alertmanager controller](https://github.com/coreos/kube-alertmanager-controller)
+* Grafana controller that dynamically discovers and deploys dashboards from ConfigMaps
+* Collection of base alerting for cluster monitoring
+* KPM/Helm packages to easily provide production-ready cluster-monitoring setup (essentially contents of `hack/cluster-monitoring`)
+* Add meta-monitoring to default cluster monitoring setup
+
 

hack/cluster-monitoring/deploy

@@ -1,14 +1,23 @@
 #!/usr/bin/env bash
 
+set -o xtrace
+
 if [ -z "${KUBECONFIG}" ]; then
     KUBECONFIG=~/.kube/config
 fi
 
-if [ -z "${KUBE_NAMESPACE}" ]; then
-    KUBE_NAMESPACE=default
-fi  
+kubectl --kubeconfig="$KUBECONFIG" create namespace monitoring
 
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" create -f manifests/controllers/prometheus-controller.yaml
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" create -f manifests/exporters
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" create -f manifests/grafana
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" create -f manifests/prometheus
+kctl() {
+    kubectl --kubeconfig="$KUBECONFIG" -n "monitoring" "$@"
+}
+
+kctl apply -f manifests/controllers/prometheus-controller.yaml
+
+# Wait for TPRs to be ready.
+until kctl get servicemonitor; do sleep 0.1; done
+until kctl get prometheus; do sleep 0.1; done
+
+kctl apply -f manifests/exporters
+kctl apply -f manifests/grafana
+kctl apply -f manifests/prometheus

hack/cluster-monitoring/teardown

@@ -4,11 +4,15 @@ if [ -z "${KUBECONFIG}" ]; then
     KUBECONFIG=~/.kube/config
 fi
 
-if [ -z "${KUBE_NAMESPACE}" ]; then
-    KUBE_NAMESPACE=default
-fi  
+kctl() {
+    kubectl --kubeconfig="$KUBECONFIG" -n "monitoring" "$@"
+}
 
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" delete -f manifests/exporters
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" delete -f manifests/grafana
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" delete -f manifests/prometheus
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" delete -f manifests/controllers/prometheus-controller.yaml
+kctl delete -f manifests/exporters
+kctl delete -f manifests/grafana
+kctl delete -f manifests/prometheus
+
+# Hack: wait a bit to let the controller delete the deployed Prometheus server.
+sleep 5
+
+kctl delete -f manifests/controllers/prometheus-controller.yaml

hack/example-service-monitoring/teardown

@@ -4,4 +4,4 @@ if [ -z "${KUBECONFIG}" ]; then
     KUBECONFIG=~/.kube/config
 fi
 
-kubectl --kubeconfig="$KUBECONFIG" -n "$KUBE_NAMESPACE" delete -f manifests/examples/example-app
+kubectl --kubeconfig="$KUBECONFIG" delete -f manifests/examples/example-app

manifests/controllers/alertmanager-controller.yaml

@@ -21,6 +21,3 @@ spec:
            limits:
              cpu: 200m
              memory: 100Mi
-      imagePullSecrets:
-      - name: coreos-quay
-

manifests/controllers/prometheus-controller.yaml

@@ -21,6 +21,3 @@ spec:
            limits:
              cpu: 200m
              memory: 100Mi
-      imagePullSecrets:
-      - name: coreos-quay
-      

manifests/examples/example-app/prometheus-frontend-svc.yaml

@@ -12,4 +12,5 @@ spec:
     protocol: TCP
     targetPort: web
   selector:
-    prometheus.coreos.com: prometheus-frontend
+    prometheus.coreos.com/type: prometheus
+    prometheus.coreos.com/name: prometheus-frontend

manifests/grafana/grafana-depl.yaml

@@ -54,6 +54,4 @@ spec:
       - name: grafana-dashboards
         configMap:
           name: grafana-dashboards
-      imagePullSecrets:
-      - name: coreos-quay
 

manifests/grafana/grafana-svc.yaml

@@ -7,6 +7,7 @@ metadata:
   annotations:
     prometheus.io/scrape: 'true'
 spec:
+  type: NodePort
   ports:
   - name: web
     port: 3000
@@ -14,4 +15,3 @@ spec:
     nodePort: 30902
   selector:
     app: grafana
-  type: NodePort

manifests/prometheus/prometheus-k8s-cm.yaml

@@ -7,10 +7,6 @@ data:
     global:
       evaluation_interval: 30s
 
-    # Add your etcd scrape config here. We cannot default here as etcd is a
-    # prerequisite for Kubernetes.
-    # TODO(fabxc): potentially make this configurable via KPM in the future.
-
     scrape_configs:
     - job_name: kubelets
       scrape_interval: 20s

manifests/prometheus/prometheus-k8s-svc.yaml

@@ -11,4 +11,5 @@ spec:
     protocol: TCP
     targetPort: web
   selector:
-    prometheus.coreos.com: prometheus-k8s
+    prometheus.coreos.com/type: prometheus
+    prometheus.coreos.com/name: prometheus-k8s