From 8e7d55d7954b700384285996d0417a53648e8684 Mon Sep 17 00:00:00 2001 From: Benoit Gagnon Date: Mon, 24 Jun 2019 11:39:47 -0400 Subject: [PATCH 1/4] Add mixin to strip cpu/memory limits (issue #72) --- .../kube-prometheus-strip-limits.libsonnet | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet diff --git a/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet new file mode 100644 index 00000000..90446e0d --- /dev/null +++ b/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet @@ -0,0 +1,56 @@ +// Strips spec.containers[].limits for certain containers +// https://github.com/coreos/kube-prometheus/issues/72 +{ + nodeExporter+: { + daemonset+: { + spec+: { + template+: { + spec+: { + local stripLimits(c) = + if std.count([ + 'node-exporter', + 'kube-rbac-proxy' + ], c.name) > 0 + then c + {resources+: {limits: {}}} + else c, + containers: std.map(stripLimits, super.containers), + }, + }, + }, + }, + }, + prometheusOperator+: { + deployment+: { + spec+: { + template+: { + spec+: { + local addArgs(c) = + if c.name == 'prometheus-operator' + then c + {args+: ['--config-reloader-cpu=0']} + else c, + containers: std.map(addArgs, super.containers), + }, + }, + }, + }, + }, + kubeStateMetrics+: { + deployment+: { + spec+: { + template+: { + spec+: { + local stripLimits(c) = + if std.count([ + 'kube-rbac-proxy-main', + 'kube-rbac-proxy-self', + 'addon-resizer' + ], c.name) > 0 + then c + {resources+: {limits: {}}} + else c, + containers: std.map(stripLimits, super.containers), + }, + }, + }, + }, + }, +} From ffc8832ef9c35b096d5d5b39bf265e5252cc2742 Mon Sep 17 00:00:00 2001 From: Benoit Gagnon Date: Mon, 24 Jun 2019 11:43:22 -0400 Subject: [PATCH 2/4] use std.setMember instead of std.count --- .../kube-prometheus-strip-limits.libsonnet | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet index 90446e0d..4ff5c788 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet @@ -7,10 +7,10 @@ template+: { spec+: { local stripLimits(c) = - if std.count([ - 'node-exporter', - 'kube-rbac-proxy' - ], c.name) > 0 + if std.setMember(c.name, [ + 'kube-rbac-proxy', + 'node-exporter', + ]) then c + {resources+: {limits: {}}} else c, containers: std.map(stripLimits, super.containers), @@ -40,11 +40,11 @@ template+: { spec+: { local stripLimits(c) = - if std.count([ - 'kube-rbac-proxy-main', - 'kube-rbac-proxy-self', - 'addon-resizer' - ], c.name) > 0 + if std.setMember(c.name, [ + 'addon-resizer', + 'kube-rbac-proxy-main', + 'kube-rbac-proxy-self', + ]) then c + {resources+: {limits: {}}} else c, containers: std.map(stripLimits, super.containers), From 16c22b72500750941d1f2d2e2b0eb91a5a416a8b Mon Sep 17 00:00:00 2001 From: Benoit Gagnon Date: Sat, 10 Aug 2019 14:31:28 -0400 Subject: [PATCH 3/4] move resources requests/limits to _config and simplify mixin --- .../kube-prometheus-strip-limits.libsonnet | 44 +++++-------------- .../kube-prometheus/kube-prometheus.libsonnet | 14 ++++++ .../kube-state-metrics.libsonnet | 12 ++--- .../node-exporter/node-exporter.libsonnet | 4 +- 4 files changed, 32 insertions(+), 42 deletions(-) diff --git a/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet index 4ff5c788..659a4d3b 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet @@ -1,21 +1,16 @@ // Strips spec.containers[].limits for certain containers // https://github.com/coreos/kube-prometheus/issues/72 { - nodeExporter+: { - daemonset+: { - spec+: { - template+: { - spec+: { - local stripLimits(c) = - if std.setMember(c.name, [ - 'kube-rbac-proxy', - 'node-exporter', - ]) - then c + {resources+: {limits: {}}} - else c, - containers: std.map(stripLimits, super.containers), - }, - }, + _config+:: { + resources+:: { + 'addon-resizer': { + limits: {}, + }, + 'kube-rbac-proxy': { + limits: {}, + }, + 'node-exporter': { + limits: {}, }, }, }, @@ -34,23 +29,4 @@ }, }, }, - kubeStateMetrics+: { - deployment+: { - spec+: { - template+: { - spec+: { - local stripLimits(c) = - if std.setMember(c.name, [ - 'addon-resizer', - 'kube-rbac-proxy-main', - 'kube-rbac-proxy-self', - ]) - then c + {resources+: {limits: {}}} - else c, - containers: std.map(stripLimits, super.containers), - }, - }, - }, - }, - }, } diff --git a/jsonnet/kube-prometheus/kube-prometheus.libsonnet b/jsonnet/kube-prometheus/kube-prometheus.libsonnet index 631423bd..80b63b0c 100644 --- a/jsonnet/kube-prometheus/kube-prometheus.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus.libsonnet @@ -104,6 +104,20 @@ local configMapList = k3.core.v1.configMapList; CoreDNS: $._config.coreDNSSelector, }, + resources+:: { + 'addon-resizer': { + requests: { cpu: '10m', memory: '30Mi' }, + limits: { cpu: '50m', memory: '30Mi' }, + }, + 'kube-rbac-proxy': { + requests: { cpu: '10m', memory: '20Mi' }, + limits: { cpu: '20m', memory: '40Mi' }, + }, + 'node-exporter': { + requests: { cpu: '102m', memory: '180Mi' }, + limits: { cpu: '250m', memory: '180Mi' }, + }, + }, prometheus+:: { rules: $.prometheusRules + $.prometheusAlerts, }, diff --git a/jsonnet/kube-prometheus/kube-state-metrics/kube-state-metrics.libsonnet b/jsonnet/kube-prometheus/kube-state-metrics/kube-state-metrics.libsonnet index 0e981974..86da57c5 100644 --- a/jsonnet/kube-prometheus/kube-state-metrics/kube-state-metrics.libsonnet +++ b/jsonnet/kube-prometheus/kube-state-metrics/kube-state-metrics.libsonnet @@ -140,8 +140,8 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; '--upstream=http://127.0.0.1:8081/', ]) + container.withPorts(containerPort.newNamed(8443, 'https-main',)) + - container.mixin.resources.withRequests({ cpu: '10m', memory: '20Mi' }) + - container.mixin.resources.withLimits({ cpu: '20m', memory: '40Mi' }); + container.mixin.resources.withRequests($._config.resources['kube-rbac-proxy'].requests) + + container.mixin.resources.withLimits($._config.resources['kube-rbac-proxy'].limits); local proxySelfMetrics = container.new('kube-rbac-proxy-self', $._config.imageRepos.kubeRbacProxy + ':' + $._config.versions.kubeRbacProxy) + @@ -152,8 +152,8 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; '--upstream=http://127.0.0.1:8082/', ]) + container.withPorts(containerPort.newNamed(9443, 'https-self',)) + - container.mixin.resources.withRequests({ cpu: '10m', memory: '20Mi' }) + - container.mixin.resources.withLimits({ cpu: '20m', memory: '40Mi' }); + container.mixin.resources.withRequests($._config.resources['kube-rbac-proxy'].requests) + + container.mixin.resources.withLimits($._config.resources['kube-rbac-proxy'].limits); local kubeStateMetrics = container.new('kube-state-metrics', $._config.imageRepos.kubeStateMetrics + ':' + $._config.versions.kubeStateMetrics) + @@ -192,8 +192,8 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; }, }, ]) + - container.mixin.resources.withRequests({ cpu: '10m', memory: '30Mi' }) + - container.mixin.resources.withLimits({ cpu: '50m', memory: '30Mi' }); + container.mixin.resources.withRequests($._config.resources['addon-resizer'].requests) + + container.mixin.resources.withLimits($._config.resources['addon-resizer'].limits); local c = [proxyClusterMetrics, proxySelfMetrics, kubeStateMetrics, addonResizer]; diff --git a/jsonnet/kube-prometheus/node-exporter/node-exporter.libsonnet b/jsonnet/kube-prometheus/node-exporter/node-exporter.libsonnet index a55ca26c..01215ef2 100644 --- a/jsonnet/kube-prometheus/node-exporter/node-exporter.libsonnet +++ b/jsonnet/kube-prometheus/node-exporter/node-exporter.libsonnet @@ -97,8 +97,8 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; '--collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$', ]) + container.withVolumeMounts([procVolumeMount, sysVolumeMount, rootVolumeMount]) + - container.mixin.resources.withRequests({ cpu: '102m', memory: '180Mi' }) + - container.mixin.resources.withLimits({ cpu: '250m', memory: '180Mi' }); + container.mixin.resources.withRequests($._config.resources['node-exporter'].requests) + + container.mixin.resources.withLimits($._config.resources['node-exporter'].requests); local ip = containerEnv.fromFieldPath('IP', 'status.podIP'); local proxy = From da644166a370ba65daee3944845914157431750b Mon Sep 17 00:00:00 2001 From: Benoit Gagnon Date: Sat, 10 Aug 2019 14:35:06 -0400 Subject: [PATCH 4/4] use proper operator so requests are not stripped --- .../kube-prometheus/kube-prometheus-strip-limits.libsonnet | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet b/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet index 659a4d3b..52660d7b 100644 --- a/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet +++ b/jsonnet/kube-prometheus/kube-prometheus-strip-limits.libsonnet @@ -3,13 +3,13 @@ { _config+:: { resources+:: { - 'addon-resizer': { + 'addon-resizer'+: { limits: {}, }, - 'kube-rbac-proxy': { + 'kube-rbac-proxy'+: { limits: {}, }, - 'node-exporter': { + 'node-exporter'+: { limits: {}, }, },