prometheus: Introduce RuleFile Custom Resource Definition

This patch introduces a new Custom Resource Definition to the
Prometheus Operator - the Rule CRD. It addresses two main
needs:

1. Prometheus (alerting and recording) Rule validation during creation time
via Kubernetes Custom Resource Definition validation.

2. Life-cycle management of Prometheus application Rules alongside the
application itself, inside the applications Kubernetes namespace, not
necessarily the namespace of the scraping Prometheus instance.

A user defines Prometheus alerting and recording Rules via a Kubernetes
Custom Resource Definition. These Custom Resource Definitions can be
fully validated by the Kubernetes API server during creation time via
automatically generated OpenAPI specifications. Instead of the
restriction of a Prometheus instance to only select Rule definitions
inside its own namespace, the Prometheus specification is extended to
also specify namespaces to look for Rule Custom Resource Definitions
outside its own namespace.

---

Dependent technical changes:

- prometheus: Use github.com/jimmidyson/configmap-reload to reload rules

- prometheus: Remove Prometheus Statefulset deletion function. Starting
with K8s >=1.8 this is handled via OwnerReferences.

- prometheus: Do not add rule files checksum to Prometheus configuration
secret

- prometheus: Update StatefulSet only on relevant changes. Instead of
updating the Prometheus StatefulSet on every `sync()` run, only update
it if the input parameters to `makeStatefulSet` change.  Enforce this
via a checksum of the parameters which is saved inside the annotations
of the statefulset.

- e2e/prometheus: Check how often resources (Secret, ConfigMap,
Prometheus CRD, Service) are updated to enforce that Prometheus Operator
only updated created resources if necessary.

- contrib/prometheus-config-reloader: Remove logic to retriev K8s
ConfigMaps. These are mounted into the pod right away now.

Makefile

@@ -3,14 +3,15 @@ image:
 
 generate: image
 	@echo ">> Compiling assets and generating Kubernetes manifests"
-	docker run --rm -u=$(shell id -u $(USER)):$(shell id -g $(USER)) -v $(shell dirname $(dir $(abspath $(dir $$PWD)))):/go/src/github.com/coreos/prometheus-operator/ --workdir /go/src/github.com/coreos/prometheus-operator/contrib/kube-prometheus po-jsonnet make crdtojsonnet generate-raw
+	docker run --rm -u=$(shell id -u $(USER)):$(shell id -g $(USER)) -v $(shell dirname $(dir $(abspath $(dir $$PWD)))):/go/src/github.com/coreos/prometheus-operator/ --workdir /go/src/github.com/coreos/prometheus-operator/contrib/kube-prometheus po-jsonnet make generate-raw
 
 crdtojsonnet:
 	cat ../../example/prometheus-operator-crd/alertmanager.crd.yaml | gojsontoyaml -yamltojson > jsonnet/kube-prometheus/prometheus-operator/alertmanager-crd.libsonnet
 	cat ../../example/prometheus-operator-crd/prometheus.crd.yaml | gojsontoyaml -yamltojson > jsonnet/kube-prometheus/prometheus-operator/prometheus-crd.libsonnet
 	cat ../../example/prometheus-operator-crd/servicemonitor.crd.yaml | gojsontoyaml -yamltojson > jsonnet/kube-prometheus/prometheus-operator/servicemonitor-crd.libsonnet
+	cat ../../example/prometheus-operator-crd/rulefile.crd.yaml | gojsontoyaml -yamltojson > jsonnet/kube-prometheus/prometheus-operator/rulefile-crd.libsonnet
 
-generate-raw:
+generate-raw: crdtojsonnet
 	jb install
 	./build.sh
 

README.md

@@ -86,6 +86,10 @@ set -x
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+# Make sure to start with a clean 'manifests' dir
+rm -rf manifests
+mkdir manifests
+
                                                # optional, but we would like to generate yaml, not json
 jsonnet -J vendor -m manifests example.jsonnet | xargs -I{} sh -c 'cat $1 | gojsontoyaml > $1.yaml; rm -f $1' -- {}
 

build.sh

@@ -4,6 +4,10 @@ set -x
 # only exit with zero if all commands of the pipeline exit successfully
 set -o pipefail
 
+# Make sure to start with a clean 'manifests' dir
+rm -rf manifests
+mkdir manifests
+
                                                # optional, but we would like to generate yaml, not json
 jsonnet -J vendor -m manifests example.jsonnet | xargs -I{} sh -c 'cat $1 | gojsontoyaml > $1.yaml; rm -f $1' -- {}
 

manifests/0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml

@@ -666,7 +666,7 @@ spec:
                               description: Specify whether the ConfigMap must be defined
                               type: boolean
                         prefix:
-                          description: An optional identifer to prepend to each key
+                          description: An optional identifier to prepend to each key
                             in the ConfigMap. Must be a C_IDENTIFIER.
                           type: string
                         secretRef:
@@ -1120,6 +1120,14 @@ spec:
                         description: Whether this container has a read-only root filesystem.
                           Default is false.
                         type: boolean
+                      runAsGroup:
+                        description: The GID to run the entrypoint of the container
+                          process. Uses runtime default if unset. May also be set
+                          in PodSecurityContext.  If set in both SecurityContext and
+                          PodSecurityContext, the value specified in SecurityContext
+                          takes precedence.
+                        format: int64
+                        type: integer
                       runAsNonRoot:
                         description: Indicates that the container must run as a non-root
                           user. If true, the Kubelet will validate the image at runtime
@@ -1231,8 +1239,7 @@ spec:
                           description: mountPropagation determines how mounts are
                             propagated from the host to container and the other way
                             around. When not set, MountPropagationHostToContainer
-                            is used. This field is alpha in 1.8 and can be reworked
+                            is used. This field is beta in 1.10.
-                            or removed in a future release.
                           type: string
                         name:
                           description: This must match the Name of a Volume.
@@ -1614,6 +1621,13 @@ spec:
                     If unset, the Kubelet will not modify the ownership and permissions of any volume.
                   format: int64
                   type: integer
+                runAsGroup:
+                  description: The GID to run the entrypoint of the container process.
+                    Uses runtime default if unset. May also be set in SecurityContext.  If
+                    set in both SecurityContext and PodSecurityContext, the value
+                    specified in SecurityContext takes precedence for that container.
+                  format: int64
+                  type: integer
                 runAsNonRoot:
                   description: Indicates that the container must run as a non-root
                     user. If true, the Kubelet will validate the image at runtime

manifests/0prometheus-operator-0prometheusCustomResourceDefinition.yaml

@@ -27,6 +27,21 @@ spec:
           description: 'Specification of the desired behavior of the Prometheus cluster.
             More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status'
           properties:
+            additionalAlertManagerConfigs:
+              description: SecretKeySelector selects a key of a Secret.
+              properties:
+                key:
+                  description: The key of the secret to select from.  Must be a valid
+                    secret key.
+                  type: string
+                name:
+                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                  type: string
+                optional:
+                  description: Specify whether the Secret or it's key must be defined
+                  type: boolean
+              required:
+              - key
             additionalScrapeConfigs:
               description: SecretKeySelector selects a key of a Secret.
               properties:
@@ -734,7 +749,7 @@ spec:
                               description: Specify whether the ConfigMap must be defined
                               type: boolean
                         prefix:
-                          description: An optional identifer to prepend to each key
+                          description: An optional identifier to prepend to each key
                             in the ConfigMap. Must be a C_IDENTIFIER.
                           type: string
                         secretRef:
@@ -1188,6 +1203,14 @@ spec:
                         description: Whether this container has a read-only root filesystem.
                           Default is false.
                         type: boolean
+                      runAsGroup:
+                        description: The GID to run the entrypoint of the container
+                          process. Uses runtime default if unset. May also be set
+                          in PodSecurityContext.  If set in both SecurityContext and
+                          PodSecurityContext, the value specified in SecurityContext
+                          takes precedence.
+                        format: int64
+                        type: integer
                       runAsNonRoot:
                         description: Indicates that the container must run as a non-root
                           user. If true, the Kubelet will validate the image at runtime
@@ -1299,8 +1322,7 @@ spec:
                           description: mountPropagation determines how mounts are
                             propagated from the host to container and the other way
                             around. When not set, MountPropagationHostToContainer
-                            is used. This field is alpha in 1.8 and can be reworked
+                            is used. This field is beta in 1.10.
-                            or removed in a future release.
                           type: string
                         name:
                           description: This must match the Name of a Volume.
@@ -1877,6 +1899,90 @@ spec:
                 the server serves requests under a different route prefix. For example
                 for use with `kubectl proxy`.
               type: string
+            ruleFileNamespaceSelector:
+              description: A label selector is a label query over a set of resources.
+                The result of matchLabels and matchExpressions are ANDed. An empty
+                label selector matches all objects. A null label selector matches
+                no objects.
+              properties:
+                matchExpressions:
+                  description: matchExpressions is a list of label selector requirements.
+                    The requirements are ANDed.
+                  items:
+                    description: A label selector requirement is a selector that contains
+                      values, a key, and an operator that relates the key and values.
+                    properties:
+                      key:
+                        description: key is the label key that the selector applies
+                          to.
+                        type: string
+                      operator:
+                        description: operator represents a key's relationship to a
+                          set of values. Valid operators are In, NotIn, Exists and
+                          DoesNotExist.
+                        type: string
+                      values:
+                        description: values is an array of string values. If the operator
+                          is In or NotIn, the values array must be non-empty. If the
+                          operator is Exists or DoesNotExist, the values array must
+                          be empty. This array is replaced during a strategic merge
+                          patch.
+                        items:
+                          type: string
+                        type: array
+                    required:
+                    - key
+                    - operator
+                  type: array
+                matchLabels:
+                  description: matchLabels is a map of {key,value} pairs. A single
+                    {key,value} in the matchLabels map is equivalent to an element
+                    of matchExpressions, whose key field is "key", the operator is
+                    "In", and the values array contains only "value". The requirements
+                    are ANDed.
+                  type: object
+            ruleFileSelector:
+              description: A label selector is a label query over a set of resources.
+                The result of matchLabels and matchExpressions are ANDed. An empty
+                label selector matches all objects. A null label selector matches
+                no objects.
+              properties:
+                matchExpressions:
+                  description: matchExpressions is a list of label selector requirements.
+                    The requirements are ANDed.
+                  items:
+                    description: A label selector requirement is a selector that contains
+                      values, a key, and an operator that relates the key and values.
+                    properties:
+                      key:
+                        description: key is the label key that the selector applies
+                          to.
+                        type: string
+                      operator:
+                        description: operator represents a key's relationship to a
+                          set of values. Valid operators are In, NotIn, Exists and
+                          DoesNotExist.
+                        type: string
+                      values:
+                        description: values is an array of string values. If the operator
+                          is In or NotIn, the values array must be non-empty. If the
+                          operator is Exists or DoesNotExist, the values array must
+                          be empty. This array is replaced during a strategic merge
+                          patch.
+                        items:
+                          type: string
+                        type: array
+                    required:
+                    - key
+                    - operator
+                  type: array
+                matchLabels:
+                  description: matchLabels is a map of {key,value} pairs. A single
+                    {key,value} in the matchLabels map is equivalent to an element
+                    of matchExpressions, whose key field is "key", the operator is
+                    "In", and the values array contains only "value". The requirements
+                    are ANDed.
+                  type: object
             ruleSelector:
               description: A label selector is a label query over a set of resources.
                 The result of matchLabels and matchExpressions are ANDed. An empty
@@ -1948,6 +2054,13 @@ spec:
                     If unset, the Kubelet will not modify the ownership and permissions of any volume.
                   format: int64
                   type: integer
+                runAsGroup:
+                  description: The GID to run the entrypoint of the container process.
+                    Uses runtime default if unset. May also be set in SecurityContext.  If
+                    set in both SecurityContext and PodSecurityContext, the value
+                    specified in SecurityContext takes precedence for that container.
+                  format: int64
+                  type: integer
                 runAsNonRoot:
                   description: Indicates that the container must run as a non-root
                     user. If true, the Kubelet will validate the image at runtime

manifests/prometheus-rules.yaml

@@ -16,8 +16,9 @@ data:
     \ - \"expr\": |\n      sum by (namespace, label_name) (\n        sum(kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"})
     by (namespace, pod)\n      * on (namespace, pod) group_left(label_name)\n        label_replace(kube_pod_labels{job=\"kube-state-metrics\"},
     \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n      )\n    \"record\": \"namespace_name:kube_pod_container_resource_requests_memory_bytes:sum\"\n
-    \ - \"expr\": |\n      sum by (namespace, label_name) (\n        sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"})
+    \ - \"expr\": |\n      sum by (namespace, label_name) (\n        sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"}
-    by (namespace, pod)\n      * on (namespace, pod) group_left(label_name)\n        label_replace(kube_pod_labels{job=\"kube-state-metrics\"},
+    and on(pod) kube_pod_status_scheduled{condition=\"true\"}) by (namespace, pod)\n
+    \     * on (namespace, pod) group_left(label_name)\n        label_replace(kube_pod_labels{job=\"kube-state-metrics\"},
     \"pod_name\", \"$1\", \"pod\", \"(.*)\")\n      )\n    \"record\": \"namespace_name:kube_pod_container_resource_requests_cpu_cores:sum\"\n-
     \"name\": \"node.rules\"\n  \"rules\": \n  - \"expr\": \"sum(min(kube_pod_info)
     by (node))\"\n    \"record\": \":kube_pod_info_node_count:\"\n  - \"expr\": |\n