From 714275f3b9620d744eea8fbcee1ac7bcced6dcb0 Mon Sep 17 00:00:00 2001 From: Sayak Mukhopadhyay Date: Thu, 24 Sep 2020 17:55:09 +0530 Subject: [PATCH 1/3] docs: Added documentation for monitoring all namespaces --- README.md | 30 ++++++++++++++++++++++++++++++ examples/all-namespaces.jsonnet | 18 ++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 examples/all-namespaces.jsonnet diff --git a/README.md b/README.md index 34f7c1d9..2699b072 100644 --- a/README.md +++ b/README.md @@ -648,6 +648,36 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + { > NOTE: make sure your service resources have the right labels (eg. `'app': 'myapp'`) applied. Prometheus uses kubernetes labels to discover resources inside the namespaces. +### Monitoring all namespaces + +In case you want to monitor all namespaces in a cluster, you can add the following mixin. Also, make sure to empty the namespaces defined in prometheus so that roleBindings are not created against them. + +[embedmd]:# (examples/all-namespaces.jsonnet) +```jsonnet +local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + + (import 'kube-prometheus/kube-prometheus-all-namespaces.libsonnet') + { + _config+:: { + namespace: 'monitoring', + + prometheus+:: { + namespaces: [], + }, + }, +}; + +{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + +{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } + +{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } + +{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + +{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + +{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } + +{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } +``` + +> NOTE: This configuration can potentially make your cluster insecure especially in a multi-tenant cluster. This is because this gives Prometheus visibility over the whole cluster which might not be expected in a scenario when certain namespaces are locked down for security reasons. + +Proceed with [creating ServiceMonitors for the services in the namespaces](#defining-the-servicemonitor-for-each-additional-namespace) you actually want to monitor + ### Static etcd configuration In order to configure a static etcd cluster to scrape there is a simple [kube-prometheus-static-etcd.libsonnet](jsonnet/kube-prometheus/kube-prometheus-static-etcd.libsonnet) mixin prepared - see [etcd.jsonnet](examples/etcd.jsonnet) for an example of how to use that mixin, and [Monitoring external etcd](docs/monitoring-external-etcd.md) for more information. diff --git a/examples/all-namespaces.jsonnet b/examples/all-namespaces.jsonnet new file mode 100644 index 00000000..24afe672 --- /dev/null +++ b/examples/all-namespaces.jsonnet @@ -0,0 +1,18 @@ +local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + + (import 'kube-prometheus/kube-prometheus-all-namespaces.libsonnet') + { + _config+:: { + namespace: 'monitoring', + + prometheus+:: { + namespaces: [], + }, + }, +}; + +{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + +{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } + +{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } + +{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + +{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + +{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } + +{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } From 5ffed9f9baf31a4f970e97ad2e53804750217425 Mon Sep 17 00:00:00 2001 From: Sayak Mukhopadhyay Date: Thu, 24 Sep 2020 18:15:16 +0530 Subject: [PATCH 2/3] Added item in the TOC --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2699b072..69bcb4fa 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ This stack is meant for cluster monitoring, so it is pre-configured to collect m - [Alertmanager configuration](#alertmanager-configuration) - [Adding additional namespaces to monitor](#adding-additional-namespaces-to-monitor) - [Defining the ServiceMonitor for each additional Namespace](#defining-the-servicemonitor-for-each-additional-namespace) + - [Monitoring all namespaces](#monitoring-all-namespaces) - [Static etcd configuration](#static-etcd-configuration) - [Pod Anti-Affinity](#pod-anti-affinity) - [Customizing Prometheus alerting/recording rules and Grafana dashboards](#customizing-prometheus-alertingrecording-rules-and-grafana-dashboards) From 3adc7528397e66f53de5c4716277a88194ad4bbc Mon Sep 17 00:00:00 2001 From: Sayak Mukhopadhyay Date: Thu, 24 Sep 2020 18:36:11 +0530 Subject: [PATCH 3/3] docs: Added documentation for limit stripping of containers --- README.md | 24 ++++++++++++++++++++++++ examples/strip-limits.jsonnet | 14 ++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 examples/strip-limits.jsonnet diff --git a/README.md b/README.md index 69bcb4fa..56843680 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ This stack is meant for cluster monitoring, so it is pre-configured to collect m - [Monitoring all namespaces](#monitoring-all-namespaces) - [Static etcd configuration](#static-etcd-configuration) - [Pod Anti-Affinity](#pod-anti-affinity) + - [Stripping container resource limits](#stripping-container-resource-limits) - [Customizing Prometheus alerting/recording rules and Grafana dashboards](#customizing-prometheus-alertingrecording-rules-and-grafana-dashboards) - [Exposing Prometheus/Alermanager/Grafana via Ingress](#exposing-prometheusalermanagergrafana-via-ingress) - [Minikube Example](#minikube-example) @@ -695,6 +696,29 @@ possible, one can include the [kube-prometheus-anti-affinity.libsonnet](jsonnet/ (import 'kube-prometheus/kube-prometheus-anti-affinity.libsonnet') ``` +### Stripping container resource limits + +Sometimes in small clusters, the CPU/memory limits can get high enough for alerts to be fired continuously. To prevent this, one can strip off the predefined limits. +To do that, one can import the following mixin + +[embedmd]:# (examples/strip-limits.jsonnet) +```jsonnet +local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + + (import 'kube-prometheus/kube-prometheus-strip-limits.libsonnet') + { + _config+:: { + namespace: 'monitoring', + }, +}; + +{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + +{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } + +{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } + +{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + +{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + +{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } + +{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } +``` + ### Customizing Prometheus alerting/recording rules and Grafana dashboards See [developing Prometheus rules and Grafana dashboards](docs/developing-prometheus-rules-and-grafana-dashboards.md) guide. diff --git a/examples/strip-limits.jsonnet b/examples/strip-limits.jsonnet new file mode 100644 index 00000000..5113da1b --- /dev/null +++ b/examples/strip-limits.jsonnet @@ -0,0 +1,14 @@ +local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') + + (import 'kube-prometheus/kube-prometheus-strip-limits.libsonnet') + { + _config+:: { + namespace: 'monitoring', + }, +}; + +{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } + +{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } + +{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } + +{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + +{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + +{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } + +{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) }