From b85b5b6bcf3a17e4d850b43d6aae4aee54f2416c Mon Sep 17 00:00:00 2001 From: Mike Bryant Date: Thu, 9 Mar 2017 21:23:55 +0000 Subject: [PATCH 1/3] Account for multiple copies of kube-state-metrics This can happen if you run multiple replicas, or if you redeploy kube-state-metrics. In either case, prometheus records multiple metrics with the instance ip in, and the dashboard fails. Use aggregation functions to get sensible output in either case --- assets/grafana/deployment-dashboard.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/assets/grafana/deployment-dashboard.json b/assets/grafana/deployment-dashboard.json index 69638d15..357bd8e6 100644 --- a/assets/grafana/deployment-dashboard.json +++ b/assets/grafana/deployment-dashboard.json @@ -302,7 +302,7 @@ "targets": [ { "refId": "A", - "expr": "kube_deployment_spec_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_spec_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "step": 600, "metric": "kube_deployment_spec_replicas" @@ -381,7 +381,7 @@ "targets": [ { "refId": "A", - "expr": "kube_deployment_status_replicas_available{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "min(kube_deployment_status_replicas_available{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "step": 600 } @@ -505,7 +505,7 @@ }, "targets": [ { - "expr": "kube_deployment_status_observed_generation{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_status_observed_generation{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "", "refId": "A", @@ -583,7 +583,7 @@ }, "targets": [ { - "expr": "kube_deployment_metadata_generation{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_metadata_generation{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "", "refId": "A", @@ -649,35 +649,35 @@ "steppedLine": false, "targets": [ { - "expr": "kube_deployment_status_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_status_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "current replicas", "refId": "A", "step": 30 }, { - "expr": "kube_deployment_status_replicas_available{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "min(kube_deployment_status_replicas_available{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "available", "refId": "B", "step": 30 }, { - "expr": "kube_deployment_status_replicas_unavailable{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_status_replicas_unavailable{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "unavailable", "refId": "C", "step": 30 }, { - "expr": "kube_deployment_status_replicas_updated{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "min(kube_deployment_status_replicas_updated{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "updated", "refId": "D", "step": 30 }, { - "expr": "kube_deployment_spec_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_spec_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "desired", "refId": "E", From 3a5b762cef5578a58077e2db691d219c56d6fa5a Mon Sep 17 00:00:00 2001 From: Mike Bryant Date: Thu, 9 Mar 2017 21:25:27 +0000 Subject: [PATCH 2/3] Update generated files --- manifests/grafana/grafana-dashboards.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/manifests/grafana/grafana-dashboards.yaml b/manifests/grafana/grafana-dashboards.yaml index f9757dc1..b9cfac08 100644 --- a/manifests/grafana/grafana-dashboards.yaml +++ b/manifests/grafana/grafana-dashboards.yaml @@ -1166,7 +1166,7 @@ data: "targets": [ { "refId": "A", - "expr": "kube_deployment_spec_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_spec_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "step": 600, "metric": "kube_deployment_spec_replicas" @@ -1245,7 +1245,7 @@ data: "targets": [ { "refId": "A", - "expr": "kube_deployment_status_replicas_available{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "min(kube_deployment_status_replicas_available{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "step": 600 } @@ -1369,7 +1369,7 @@ data: }, "targets": [ { - "expr": "kube_deployment_status_observed_generation{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_status_observed_generation{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "", "refId": "A", @@ -1447,7 +1447,7 @@ data: }, "targets": [ { - "expr": "kube_deployment_metadata_generation{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_metadata_generation{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "", "refId": "A", @@ -1513,35 +1513,35 @@ data: "steppedLine": false, "targets": [ { - "expr": "kube_deployment_status_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_status_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "current replicas", "refId": "A", "step": 30 }, { - "expr": "kube_deployment_status_replicas_available{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "min(kube_deployment_status_replicas_available{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "available", "refId": "B", "step": 30 }, { - "expr": "kube_deployment_status_replicas_unavailable{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_status_replicas_unavailable{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "unavailable", "refId": "C", "step": 30 }, { - "expr": "kube_deployment_status_replicas_updated{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "min(kube_deployment_status_replicas_updated{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "updated", "refId": "D", "step": 30 }, { - "expr": "kube_deployment_spec_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}", + "expr": "max(kube_deployment_spec_replicas{deployment=\"$deployment_name\",namespace=\"$deployment_namespace\"}) without (instance)", "intervalFactor": 2, "legendFormat": "desired", "refId": "E", From 6a52b78990df8007750173f485e7dde8df3db999 Mon Sep 17 00:00:00 2001 From: Mike Bryant Date: Fri, 10 Mar 2017 10:02:05 +0000 Subject: [PATCH 3/3] Make kube-state-metrics HA by default, now the dashboards support it --- manifests/exporters/kube-state-metrics-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/exporters/kube-state-metrics-deployment.yaml b/manifests/exporters/kube-state-metrics-deployment.yaml index 6ef971ce..3fec8cad 100644 --- a/manifests/exporters/kube-state-metrics-deployment.yaml +++ b/manifests/exporters/kube-state-metrics-deployment.yaml @@ -3,7 +3,7 @@ kind: Deployment metadata: name: kube-state-metrics spec: - replicas: 1 + replicas: 2 template: metadata: labels: