prometheus: Introduce RuleFile Custom Resource Definition
This patch introduces a new Custom Resource Definition to the Prometheus Operator - the Rule CRD. It addresses two main needs: 1. Prometheus (alerting and recording) Rule validation during creation time via Kubernetes Custom Resource Definition validation. 2. Life-cycle management of Prometheus application Rules alongside the application itself, inside the applications Kubernetes namespace, not necessarily the namespace of the scraping Prometheus instance. A user defines Prometheus alerting and recording Rules via a Kubernetes Custom Resource Definition. These Custom Resource Definitions can be fully validated by the Kubernetes API server during creation time via automatically generated OpenAPI specifications. Instead of the restriction of a Prometheus instance to only select Rule definitions inside its own namespace, the Prometheus specification is extended to also specify namespaces to look for Rule Custom Resource Definitions outside its own namespace. --- Dependent technical changes: - prometheus: Use github.com/jimmidyson/configmap-reload to reload rules - prometheus: Remove Prometheus Statefulset deletion function. Starting with K8s >=1.8 this is handled via OwnerReferences. - prometheus: Do not add rule files checksum to Prometheus configuration secret - prometheus: Update StatefulSet only on relevant changes. Instead of updating the Prometheus StatefulSet on every `sync()` run, only update it if the input parameters to `makeStatefulSet` change. Enforce this via a checksum of the parameters which is saved inside the annotations of the statefulset. - e2e/prometheus: Check how often resources (Secret, ConfigMap, Prometheus CRD, Service) are updated to enforce that Prometheus Operator only updated created resources if necessary. - contrib/prometheus-config-reloader: Remove logic to retriev K8s ConfigMaps. These are mounted into the pod right away now.
This commit is contained in:
Max Leonard Inden
5
Makefile
5
Makefile
@@ -3,14 +3,15 @@ image:
|
||||
|
||||
generate: image
|
||||
@echo ">> Compiling assets and generating Kubernetes manifests"
|
||||
docker run --rm -u=$(shell id -u $(USER)):$(shell id -g $(USER)) -v $(shell dirname $(dir $(abspath $(dir $$PWD)))):/go/src/github.com/coreos/prometheus-operator/ --workdir /go/src/github.com/coreos/prometheus-operator/contrib/kube-prometheus po-jsonnet make crdtojsonnet generate-raw
|
||||
docker run --rm -u=$(shell id -u $(USER)):$(shell id -g $(USER)) -v $(shell dirname $(dir $(abspath $(dir $$PWD)))):/go/src/github.com/coreos/prometheus-operator/ --workdir /go/src/github.com/coreos/prometheus-operator/contrib/kube-prometheus po-jsonnet make generate-raw
|
||||
|
||||
crdtojsonnet:
|
||||
cat ../../example/prometheus-operator-crd/alertmanager.crd.yaml | gojsontoyaml -yamltojson > jsonnet/kube-prometheus/prometheus-operator/alertmanager-crd.libsonnet
|
||||
cat ../../example/prometheus-operator-crd/prometheus.crd.yaml | gojsontoyaml -yamltojson > jsonnet/kube-prometheus/prometheus-operator/prometheus-crd.libsonnet
|
||||
cat ../../example/prometheus-operator-crd/servicemonitor.crd.yaml | gojsontoyaml -yamltojson > jsonnet/kube-prometheus/prometheus-operator/servicemonitor-crd.libsonnet
|
||||
cat ../../example/prometheus-operator-crd/rulefile.crd.yaml | gojsontoyaml -yamltojson > jsonnet/kube-prometheus/prometheus-operator/rulefile-crd.libsonnet
|
||||
|
||||
generate-raw:
|
||||
generate-raw: crdtojsonnet
|
||||
jb install
|
||||
./build.sh
|
||||
|
||||
|
||||
@@ -86,6 +86,10 @@ set -x
|
||||
# only exit with zero if all commands of the pipeline exit successfully
|
||||
set -o pipefail
|
||||
|
||||
# Make sure to start with a clean 'manifests' dir
|
||||
rm -rf manifests
|
||||
mkdir manifests
|
||||
|
||||
# optional, but we would like to generate yaml, not json
|
||||
jsonnet -J vendor -m manifests example.jsonnet | xargs -I{} sh -c 'cat $1 | gojsontoyaml > $1.yaml; rm -f $1' -- {}
|
||||
|
||||
|
||||
4
build.sh
4
build.sh
@@ -4,6 +4,10 @@ set -x
|
||||
# only exit with zero if all commands of the pipeline exit successfully
|
||||
set -o pipefail
|
||||
|
||||
# Make sure to start with a clean 'manifests' dir
|
||||
rm -rf manifests
|
||||
mkdir manifests
|
||||
|
||||
# optional, but we would like to generate yaml, not json
|
||||
jsonnet -J vendor -m manifests example.jsonnet | xargs -I{} sh -c 'cat $1 | gojsontoyaml > $1.yaml; rm -f $1' -- {}
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -666,7 +666,7 @@ spec:
|
||||
description: Specify whether the ConfigMap must be defined
|
||||
type: boolean
|
||||
prefix:
|
||||
description: An optional identifer to prepend to each key
|
||||
description: An optional identifier to prepend to each key
|
||||
in the ConfigMap. Must be a C_IDENTIFIER.
|
||||
type: string
|
||||
secretRef:
|
||||
@@ -1120,6 +1120,14 @@ spec:
|
||||
description: Whether this container has a read-only root filesystem.
|
||||
Default is false.
|
||||
type: boolean
|
||||
runAsGroup:
|
||||
description: The GID to run the entrypoint of the container
|
||||
process. Uses runtime default if unset. May also be set
|
||||
in PodSecurityContext. If set in both SecurityContext and
|
||||
PodSecurityContext, the value specified in SecurityContext
|
||||
takes precedence.
|
||||
format: int64
|
||||
type: integer
|
||||
runAsNonRoot:
|
||||
description: Indicates that the container must run as a non-root
|
||||
user. If true, the Kubelet will validate the image at runtime
|
||||
@@ -1231,8 +1239,7 @@ spec:
|
||||
description: mountPropagation determines how mounts are
|
||||
propagated from the host to container and the other way
|
||||
around. When not set, MountPropagationHostToContainer
|
||||
is used. This field is alpha in 1.8 and can be reworked
|
||||
or removed in a future release.
|
||||
is used. This field is beta in 1.10.
|
||||
type: string
|
||||
name:
|
||||
description: This must match the Name of a Volume.
|
||||
@@ -1614,6 +1621,13 @@ spec:
|
||||
If unset, the Kubelet will not modify the ownership and permissions of any volume.
|
||||
format: int64
|
||||
type: integer
|
||||
runAsGroup:
|
||||
description: The GID to run the entrypoint of the container process.
|
||||
Uses runtime default if unset. May also be set in SecurityContext. If
|
||||
set in both SecurityContext and PodSecurityContext, the value
|
||||
specified in SecurityContext takes precedence for that container.
|
||||
format: int64
|
||||
type: integer
|
||||
runAsNonRoot:
|
||||
description: Indicates that the container must run as a non-root
|
||||
user. If true, the Kubelet will validate the image at runtime
|
||||
|
||||
@@ -27,6 +27,21 @@ spec:
|
||||
description: 'Specification of the desired behavior of the Prometheus cluster.
|
||||
More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#spec-and-status'
|
||||
properties:
|
||||
additionalAlertManagerConfigs:
|
||||
description: SecretKeySelector selects a key of a Secret.
|
||||
properties:
|
||||
key:
|
||||
description: The key of the secret to select from. Must be a valid
|
||||
secret key.
|
||||
type: string
|
||||
name:
|
||||
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
|
||||
type: string
|
||||
optional:
|
||||
description: Specify whether the Secret or it's key must be defined
|
||||
type: boolean
|
||||
required:
|
||||
- key
|
||||
additionalScrapeConfigs:
|
||||
description: SecretKeySelector selects a key of a Secret.
|
||||
properties:
|
||||
@@ -734,7 +749,7 @@ spec:
|
||||
description: Specify whether the ConfigMap must be defined
|
||||
type: boolean
|
||||
prefix:
|
||||
description: An optional identifer to prepend to each key
|
||||
description: An optional identifier to prepend to each key
|
||||
in the ConfigMap. Must be a C_IDENTIFIER.
|
||||
type: string
|
||||
secretRef:
|
||||
@@ -1188,6 +1203,14 @@ spec:
|
||||
description: Whether this container has a read-only root filesystem.
|
||||
Default is false.
|
||||
type: boolean
|
||||
runAsGroup:
|
||||
description: The GID to run the entrypoint of the container
|
||||
process. Uses runtime default if unset. May also be set
|
||||
in PodSecurityContext. If set in both SecurityContext and
|
||||
PodSecurityContext, the value specified in SecurityContext
|
||||
takes precedence.
|
||||
format: int64
|
||||
type: integer
|
||||
runAsNonRoot:
|
||||
description: Indicates that the container must run as a non-root
|
||||
user. If true, the Kubelet will validate the image at runtime
|
||||
@@ -1299,8 +1322,7 @@ spec:
|
||||
description: mountPropagation determines how mounts are
|
||||
propagated from the host to container and the other way
|
||||
around. When not set, MountPropagationHostToContainer
|
||||
is used. This field is alpha in 1.8 and can be reworked
|
||||
or removed in a future release.
|
||||
is used. This field is beta in 1.10.
|
||||
type: string
|
||||
name:
|
||||
description: This must match the Name of a Volume.
|
||||
@@ -1877,6 +1899,90 @@ spec:
|
||||
the server serves requests under a different route prefix. For example
|
||||
for use with `kubectl proxy`.
|
||||
type: string
|
||||
ruleFileNamespaceSelector:
|
||||
description: A label selector is a label query over a set of resources.
|
||||
The result of matchLabels and matchExpressions are ANDed. An empty
|
||||
label selector matches all objects. A null label selector matches
|
||||
no objects.
|
||||
properties:
|
||||
matchExpressions:
|
||||
description: matchExpressions is a list of label selector requirements.
|
||||
The requirements are ANDed.
|
||||
items:
|
||||
description: A label selector requirement is a selector that contains
|
||||
values, a key, and an operator that relates the key and values.
|
||||
properties:
|
||||
key:
|
||||
description: key is the label key that the selector applies
|
||||
to.
|
||||
type: string
|
||||
operator:
|
||||
description: operator represents a key's relationship to a
|
||||
set of values. Valid operators are In, NotIn, Exists and
|
||||
DoesNotExist.
|
||||
type: string
|
||||
values:
|
||||
description: values is an array of string values. If the operator
|
||||
is In or NotIn, the values array must be non-empty. If the
|
||||
operator is Exists or DoesNotExist, the values array must
|
||||
be empty. This array is replaced during a strategic merge
|
||||
patch.
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: array
|
||||
matchLabels:
|
||||
description: matchLabels is a map of {key,value} pairs. A single
|
||||
{key,value} in the matchLabels map is equivalent to an element
|
||||
of matchExpressions, whose key field is "key", the operator is
|
||||
"In", and the values array contains only "value". The requirements
|
||||
are ANDed.
|
||||
type: object
|
||||
ruleFileSelector:
|
||||
description: A label selector is a label query over a set of resources.
|
||||
The result of matchLabels and matchExpressions are ANDed. An empty
|
||||
label selector matches all objects. A null label selector matches
|
||||
no objects.
|
||||
properties:
|
||||
matchExpressions:
|
||||
description: matchExpressions is a list of label selector requirements.
|
||||
The requirements are ANDed.
|
||||
items:
|
||||
description: A label selector requirement is a selector that contains
|
||||
values, a key, and an operator that relates the key and values.
|
||||
properties:
|
||||
key:
|
||||
description: key is the label key that the selector applies
|
||||
to.
|
||||
type: string
|
||||
operator:
|
||||
description: operator represents a key's relationship to a
|
||||
set of values. Valid operators are In, NotIn, Exists and
|
||||
DoesNotExist.
|
||||
type: string
|
||||
values:
|
||||
description: values is an array of string values. If the operator
|
||||
is In or NotIn, the values array must be non-empty. If the
|
||||
operator is Exists or DoesNotExist, the values array must
|
||||
be empty. This array is replaced during a strategic merge
|
||||
patch.
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: array
|
||||
matchLabels:
|
||||
description: matchLabels is a map of {key,value} pairs. A single
|
||||
{key,value} in the matchLabels map is equivalent to an element
|
||||
of matchExpressions, whose key field is "key", the operator is
|
||||
"In", and the values array contains only "value". The requirements
|
||||
are ANDed.
|
||||
type: object
|
||||
ruleSelector:
|
||||
description: A label selector is a label query over a set of resources.
|
||||
The result of matchLabels and matchExpressions are ANDed. An empty
|
||||
@@ -1948,6 +2054,13 @@ spec:
|
||||
If unset, the Kubelet will not modify the ownership and permissions of any volume.
|
||||
format: int64
|
||||
type: integer
|
||||
runAsGroup:
|
||||
description: The GID to run the entrypoint of the container process.
|
||||
Uses runtime default if unset. May also be set in SecurityContext. If
|
||||
set in both SecurityContext and PodSecurityContext, the value
|
||||
specified in SecurityContext takes precedence for that container.
|
||||
format: int64
|
||||
type: integer
|
||||
runAsNonRoot:
|
||||
description: Indicates that the container must run as a non-root
|
||||
user. If true, the Kubelet will validate the image at runtime
|
||||
|
||||
@@ -16,8 +16,9 @@ data:
|
||||
\ - \"expr\": |\n sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"})
|
||||
by (namespace, pod)\n * on (namespace, pod) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"},
|
||||
\"pod_name\", \"$1\", \"pod\", \"(.*)\")\n )\n \"record\": \"namespace_name:kube_pod_container_resource_requests_memory_bytes:sum\"\n
|
||||
\ - \"expr\": |\n sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"})
|
||||
by (namespace, pod)\n * on (namespace, pod) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"},
|
||||
\ - \"expr\": |\n sum by (namespace, label_name) (\n sum(kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"}
|
||||
and on(pod) kube_pod_status_scheduled{condition=\"true\"}) by (namespace, pod)\n
|
||||
\ * on (namespace, pod) group_left(label_name)\n label_replace(kube_pod_labels{job=\"kube-state-metrics\"},
|
||||
\"pod_name\", \"$1\", \"pod\", \"(.*)\")\n )\n \"record\": \"namespace_name:kube_pod_container_resource_requests_cpu_cores:sum\"\n-
|
||||
\"name\": \"node.rules\"\n \"rules\": \n - \"expr\": \"sum(min(kube_pod_info)
|
||||
by (node))\"\n \"record\": \":kube_pod_info_node_count:\"\n - \"expr\": |\n
|
||||
|
||||
Reference in New Issue
Block a user