From 20f88eed106a772ced712d611df9a0de5cd1482d Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Wed, 30 Nov 2016 14:28:39 +0100 Subject: [PATCH 1/5] *: set lower resource requests --- manifests/examples/example-app/prometheus-frontend.yaml | 7 +++++++ manifests/prometheus/prometheus-k8s.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/manifests/examples/example-app/prometheus-frontend.yaml b/manifests/examples/example-app/prometheus-frontend.yaml index a367ae22..fe3c1cfe 100644 --- a/manifests/examples/example-app/prometheus-frontend.yaml +++ b/manifests/examples/example-app/prometheus-frontend.yaml @@ -11,3 +11,10 @@ spec: - selector: matchLabels: tier: frontend + resources: + requests: + # 2Gi is default, but won't schedule if you don't have a node with >2Gi + # memory. Modify based on your target and time-series count for + # production use. This value is mainly meant for demonstration/testing + # purposes. + memory: 400Mi diff --git a/manifests/prometheus/prometheus-k8s.yaml b/manifests/prometheus/prometheus-k8s.yaml index 0d13f1e1..5f04fbcc 100644 --- a/manifests/prometheus/prometheus-k8s.yaml +++ b/manifests/prometheus/prometheus-k8s.yaml @@ -6,3 +6,10 @@ metadata: prometheus: k8s spec: version: v1.3.0 + resources: + requests: + # 2Gi is default, but won't schedule if you don't have a node with >2Gi + # memory. Modify based on your target and time-series count for + # production use. This value is mainly meant for demonstration/testing + # purposes. + memory: 400Mi From 0c1bb5c1c9c3313a3790cf3f67f1964c4ed4e960 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Fri, 2 Dec 2016 13:51:45 +0100 Subject: [PATCH 2/5] manifests: remove prometheus.io/scrape annotations These annotations made sense in pre v1.3.0 Prometheus releases, however, with >=v1.3.0 and the Prometheus Operator these annotations are more confusing then helpful. --- README.md | 6 +----- manifests/etcd/etcd-bootkube-gce.yaml | 4 +--- manifests/etcd/etcd-bootkube-vagrant-multi.yaml | 4 +--- manifests/examples/example-app/example-app.yaml | 2 -- manifests/exporters/kube-state-metrics-svc.yaml | 2 -- manifests/exporters/node-exporter-svc.yaml | 2 -- manifests/grafana/grafana-svc.yaml | 4 +--- .../k8s/kube-controller-manager-bootkube-vagrant-multi.yaml | 2 -- manifests/k8s/kube-scheduler-bootkube-vagrant-multi.yaml | 2 -- 9 files changed, 4 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 04497988..4520d923 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,6 @@ To tear it all down again, run: hack/cluster-monitoring/teardown ``` -> All services in the manifest still contain the `prometheus.io/scrape = true` -> annotations. It is not used by the Prometheus Operator. They remain for -> pre Prometheus v1.3.0 deployments as in [this example configuration](https://github.com/prometheus/prometheus/blob/6703404cb431f57ca4c5097bc2762438d3c1968e/documentation/examples/prometheus-kubernetes.yml). - ## Monitoring custom services The example manifests in [/manifests/examples/example-app](/manifests/examples/example-app) @@ -133,4 +129,4 @@ default kubernetes 172.17.4.101:443 kube-system kube-controller-manager-prometheus-discovery 10.2.30.2:10252 1h kube-system kube-scheduler-prometheus-discovery 10.2.30.4:10251 1h monitoring etcd-k8s 172.17.4.51:2379 1h -``` \ No newline at end of file +``` diff --git a/manifests/etcd/etcd-bootkube-gce.yaml b/manifests/etcd/etcd-bootkube-gce.yaml index 542f5fe7..ee8f391c 100644 --- a/manifests/etcd/etcd-bootkube-gce.yaml +++ b/manifests/etcd/etcd-bootkube-gce.yaml @@ -5,8 +5,6 @@ metadata: labels: app: etcd etcd: k8s - annotations: - prometheus.io/scrape: 'true' spec: type: ClusterIP clusterIP: None @@ -29,4 +27,4 @@ subsets: ports: - name: api port: 2379 - protocol: TCP \ No newline at end of file + protocol: TCP diff --git a/manifests/etcd/etcd-bootkube-vagrant-multi.yaml b/manifests/etcd/etcd-bootkube-vagrant-multi.yaml index 9f956922..38cdc9c4 100644 --- a/manifests/etcd/etcd-bootkube-vagrant-multi.yaml +++ b/manifests/etcd/etcd-bootkube-vagrant-multi.yaml @@ -5,8 +5,6 @@ metadata: labels: app: etcd etcd: k8s - annotations: - prometheus.io/scrape: 'true' spec: type: ClusterIP clusterIP: None @@ -29,4 +27,4 @@ subsets: ports: - name: api port: 2379 - protocol: TCP \ No newline at end of file + protocol: TCP diff --git a/manifests/examples/example-app/example-app.yaml b/manifests/examples/example-app/example-app.yaml index 59a9b44d..adb6602b 100644 --- a/manifests/examples/example-app/example-app.yaml +++ b/manifests/examples/example-app/example-app.yaml @@ -4,8 +4,6 @@ metadata: name: example-app labels: tier: frontend - annotations: - prometheus.io/scrape: 'true' spec: selector: app: example-app diff --git a/manifests/exporters/kube-state-metrics-svc.yaml b/manifests/exporters/kube-state-metrics-svc.yaml index 3b996b88..8b68484b 100644 --- a/manifests/exporters/kube-state-metrics-svc.yaml +++ b/manifests/exporters/kube-state-metrics-svc.yaml @@ -1,8 +1,6 @@ apiVersion: v1 kind: Service metadata: - annotations: - prometheus.io/scrape: 'true' labels: app: kube-state-metrics name: kube-state-metrics diff --git a/manifests/exporters/node-exporter-svc.yaml b/manifests/exporters/node-exporter-svc.yaml index e9286e2b..f2d24a42 100644 --- a/manifests/exporters/node-exporter-svc.yaml +++ b/manifests/exporters/node-exporter-svc.yaml @@ -3,8 +3,6 @@ kind: Service metadata: labels: app: node-exporter - annotations: - prometheus.io/scrape: 'true' name: node-exporter spec: type: ClusterIP diff --git a/manifests/grafana/grafana-svc.yaml b/manifests/grafana/grafana-svc.yaml index 0fd4e87b..adb26233 100644 --- a/manifests/grafana/grafana-svc.yaml +++ b/manifests/grafana/grafana-svc.yaml @@ -4,8 +4,6 @@ metadata: name: grafana labels: app: grafana - annotations: - prometheus.io/scrape: 'true' spec: type: NodePort ports: @@ -14,4 +12,4 @@ spec: protocol: TCP nodePort: 30902 selector: - app: grafana \ No newline at end of file + app: grafana diff --git a/manifests/k8s/kube-controller-manager-bootkube-vagrant-multi.yaml b/manifests/k8s/kube-controller-manager-bootkube-vagrant-multi.yaml index 316076f3..2e6b4db9 100644 --- a/manifests/k8s/kube-controller-manager-bootkube-vagrant-multi.yaml +++ b/manifests/k8s/kube-controller-manager-bootkube-vagrant-multi.yaml @@ -4,8 +4,6 @@ metadata: name: kube-controller-manager-prometheus-discovery labels: k8s-app: kube-controller-manager - annotations: - prometheus.io/scrape: 'true' spec: selector: k8s-app: kube-controller-manager diff --git a/manifests/k8s/kube-scheduler-bootkube-vagrant-multi.yaml b/manifests/k8s/kube-scheduler-bootkube-vagrant-multi.yaml index 67360ec2..b79853d2 100644 --- a/manifests/k8s/kube-scheduler-bootkube-vagrant-multi.yaml +++ b/manifests/k8s/kube-scheduler-bootkube-vagrant-multi.yaml @@ -4,8 +4,6 @@ metadata: name: kube-scheduler-prometheus-discovery labels: k8s-app: kube-scheduler - annotations: - prometheus.io/scrape: 'true' spec: selector: k8s-app: kube-scheduler From d0c1fb522b5a50aab16b928d16f3a530025dde54 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Fri, 2 Dec 2016 13:57:12 +0100 Subject: [PATCH 3/5] manifests: add Alertmanager manifests and discovery The latest version of the Prometheus Operator requires Prometheus >=v1.4.0 for the Alertmanger discovery feature. --- .../alertmanager/alertmanager-config.yaml | 18 ++++++++++++++++++ .../alertmanager/alertmanager-service.yaml | 14 ++++++++++++++ manifests/alertmanager/alertmanager.yaml | 9 +++++++++ .../example-app/prometheus-frontend.yaml | 7 ++++++- manifests/prometheus/prometheus-k8s.yaml | 7 ++++++- 5 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 manifests/alertmanager/alertmanager-config.yaml create mode 100644 manifests/alertmanager/alertmanager-service.yaml create mode 100644 manifests/alertmanager/alertmanager.yaml diff --git a/manifests/alertmanager/alertmanager-config.yaml b/manifests/alertmanager/alertmanager-config.yaml new file mode 100644 index 00000000..8f7fce5b --- /dev/null +++ b/manifests/alertmanager/alertmanager-config.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: alertmanager-main +data: + alertmanager.yaml: |- + global: + resolve_timeout: 5m + route: + group_by: ['job'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'webhook' + receivers: + - name: 'webhook' + webhook_configs: + - url: 'http://alertmanagerwh:30500/' diff --git a/manifests/alertmanager/alertmanager-service.yaml b/manifests/alertmanager/alertmanager-service.yaml new file mode 100644 index 00000000..86599c35 --- /dev/null +++ b/manifests/alertmanager/alertmanager-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: alertmanager-main +spec: + type: NodePort + ports: + - name: web + nodePort: 30903 + port: 9093 + protocol: TCP + targetPort: web + selector: + alertmanager: alertmanager-main diff --git a/manifests/alertmanager/alertmanager.yaml b/manifests/alertmanager/alertmanager.yaml new file mode 100644 index 00000000..ce67f3bb --- /dev/null +++ b/manifests/alertmanager/alertmanager.yaml @@ -0,0 +1,9 @@ +apiVersion: "monitoring.coreos.com/v1alpha1" +kind: "Alertmanager" +metadata: + name: "alertmanager-main" + labels: + alertmanager: "main" +spec: + replicas: 3 + version: v0.5.1 diff --git a/manifests/examples/example-app/prometheus-frontend.yaml b/manifests/examples/example-app/prometheus-frontend.yaml index fe3c1cfe..80fd9e04 100644 --- a/manifests/examples/example-app/prometheus-frontend.yaml +++ b/manifests/examples/example-app/prometheus-frontend.yaml @@ -6,7 +6,7 @@ metadata: labels: prometheus: frontend spec: - version: v1.3.0 + version: v1.4.1 serviceMonitors: - selector: matchLabels: @@ -18,3 +18,8 @@ spec: # production use. This value is mainly meant for demonstration/testing # purposes. memory: 400Mi + alerting: + alertmanagers: + - namespace: monitoring + name: alertmanager-main + port: web diff --git a/manifests/prometheus/prometheus-k8s.yaml b/manifests/prometheus/prometheus-k8s.yaml index 5f04fbcc..602ff146 100644 --- a/manifests/prometheus/prometheus-k8s.yaml +++ b/manifests/prometheus/prometheus-k8s.yaml @@ -5,7 +5,7 @@ metadata: labels: prometheus: k8s spec: - version: v1.3.0 + version: v1.4.1 resources: requests: # 2Gi is default, but won't schedule if you don't have a node with >2Gi @@ -13,3 +13,8 @@ spec: # production use. This value is mainly meant for demonstration/testing # purposes. memory: 400Mi + alerting: + alertmanagers: + - namespace: monitoring + name: alertmanager-main + port: web From a40ca507c6f67a13fff7699b3cf679a0c960fe58 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Fri, 2 Dec 2016 14:01:02 +0100 Subject: [PATCH 4/5] manifests/k8s: add service for discovering kube-dns metrics The ports reflect the upstream kube-dns manifests of bootkube and kubernetes/kubernetes. --- assets/prometheus/prometheus.yaml | 2 +- .../k8s/kube-dns-bootkube-vagrant-multi.yaml | 20 +++++++++++++++++++ manifests/prometheus/prometheus-k8s-cm.yaml | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 manifests/k8s/kube-dns-bootkube-vagrant-multi.yaml diff --git a/assets/prometheus/prometheus.yaml b/assets/prometheus/prometheus.yaml index ea10a608..d48d5648 100644 --- a/assets/prometheus/prometheus.yaml +++ b/assets/prometheus/prometheus.yaml @@ -65,4 +65,4 @@ scrape_configs: regex: "kube-(.*)-prometheus-discovery" - action: keep source_labels: [__meta_kubernetes_endpoint_port_name] - regex: "prometheus" + regex: "prometheus.*" diff --git a/manifests/k8s/kube-dns-bootkube-vagrant-multi.yaml b/manifests/k8s/kube-dns-bootkube-vagrant-multi.yaml new file mode 100644 index 00000000..4ae570f8 --- /dev/null +++ b/manifests/k8s/kube-dns-bootkube-vagrant-multi.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-prometheus-discovery + labels: + k8s-app: kube-dns +spec: + selector: + k8s-app: kube-dns + type: ClusterIP + clusterIP: None + ports: + - name: prometheus-skydns + port: 10055 + targetPort: 10055 + protocol: TCP + - name: prometheus-dnsmasq + port: 10054 + targetPort: 10054 + protocol: TCP diff --git a/manifests/prometheus/prometheus-k8s-cm.yaml b/manifests/prometheus/prometheus-k8s-cm.yaml index 59c6389b..a8846b92 100644 --- a/manifests/prometheus/prometheus-k8s-cm.yaml +++ b/manifests/prometheus/prometheus-k8s-cm.yaml @@ -68,7 +68,7 @@ data: regex: "kube-(.*)-prometheus-discovery" - action: keep source_labels: [__meta_kubernetes_endpoint_port_name] - regex: "prometheus" + regex: "prometheus.*" kind: ConfigMap metadata: creationTimestamp: null From cbf86730947df46c03dc8a7d848c7b95d2e9d200 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Fri, 9 Dec 2016 17:09:06 -0800 Subject: [PATCH 5/5] manifests/exporters: update node-exporter version --- manifests/exporters/node-exporter-ds.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/exporters/node-exporter-ds.yaml b/manifests/exporters/node-exporter-ds.yaml index b2cceee9..0bbfe8eb 100644 --- a/manifests/exporters/node-exporter-ds.yaml +++ b/manifests/exporters/node-exporter-ds.yaml @@ -12,7 +12,7 @@ spec: hostNetwork: true hostPID: true containers: - - image: quay.io/prometheus/node-exporter:0.12.0 + - image: quay.io/prometheus/node-exporter:0.13.0 args: - "-collector.procfs=/host/proc" - "-collector.sysfs=/host/sys"