|
|
|
|
@@ -759,18 +759,18 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
|
|
|
|
|
-
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d]))
|
|
|
|
|
or
|
|
|
|
|
vector(0)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d]))
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d]))
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
@@ -786,18 +786,18 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
|
|
|
|
|
-
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h]))
|
|
|
|
|
or
|
|
|
|
|
vector(0)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h]))
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h]))
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
@@ -813,18 +813,18 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
|
|
|
|
|
-
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h]))
|
|
|
|
|
or
|
|
|
|
|
vector(0)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h]))
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h]))
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
@@ -840,18 +840,18 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
|
|
|
|
|
-
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m]))
|
|
|
|
|
or
|
|
|
|
|
vector(0)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m]))
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m]))
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
@@ -867,18 +867,18 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
|
|
|
|
|
-
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d]))
|
|
|
|
|
or
|
|
|
|
|
vector(0)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d]))
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d]))
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
@@ -894,18 +894,18 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
|
|
|
|
|
-
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m]))
|
|
|
|
|
or
|
|
|
|
|
vector(0)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m]))
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m]))
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
@@ -921,18 +921,18 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
|
|
|
|
|
-
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h]))
|
|
|
|
|
or
|
|
|
|
|
vector(0)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h]))
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h]))
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
@@ -948,9 +948,9 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
|
|
|
|
|
-
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d]))
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d]))
|
|
|
|
|
@@ -964,9 +964,9 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
|
|
|
|
|
-
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h]))
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h]))
|
|
|
|
|
@@ -980,9 +980,9 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
|
|
|
|
|
-
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h]))
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h]))
|
|
|
|
|
@@ -996,9 +996,9 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
|
|
|
|
|
-
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m]))
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m]))
|
|
|
|
|
@@ -1012,9 +1012,9 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
|
|
|
|
|
-
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d]))
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d]))
|
|
|
|
|
@@ -1028,9 +1028,9 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
|
|
|
|
|
-
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m]))
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m]))
|
|
|
|
|
@@ -1044,9 +1044,9 @@ spec:
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
|
|
|
|
|
-
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h]))
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h]))
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h]))
|
|
|
|
|
@@ -1059,17 +1059,17 @@ spec:
|
|
|
|
|
- name: kube-apiserver-histogram.rules
|
|
|
|
|
rules:
|
|
|
|
|
- expr: |
|
|
|
|
|
histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0
|
|
|
|
|
histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0
|
|
|
|
|
labels:
|
|
|
|
|
quantile: "0.99"
|
|
|
|
|
verb: read
|
|
|
|
|
record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
|
|
|
|
|
record: cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile
|
|
|
|
|
- expr: |
|
|
|
|
|
histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0
|
|
|
|
|
histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0
|
|
|
|
|
labels:
|
|
|
|
|
quantile: "0.99"
|
|
|
|
|
verb: write
|
|
|
|
|
record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
|
|
|
|
|
record: cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile
|
|
|
|
|
- interval: 3m
|
|
|
|
|
name: kube-apiserver-availability.rules
|
|
|
|
|
rules:
|
|
|
|
|
@@ -1087,39 +1087,39 @@ spec:
|
|
|
|
|
verb: write
|
|
|
|
|
record: code:apiserver_request_total:increase30d
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, verb, scope) (increase(apiserver_request_duration_seconds_count[1h]))
|
|
|
|
|
record: cluster_verb_scope:apiserver_request_duration_seconds_count:increase1h
|
|
|
|
|
sum by (cluster, verb, scope) (increase(apiserver_request_slo_duration_seconds_count[1h]))
|
|
|
|
|
record: cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_duration_seconds_count:increase1h[30d]) * 24 * 30)
|
|
|
|
|
record: cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d
|
|
|
|
|
sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h[30d]) * 24 * 30)
|
|
|
|
|
record: cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, verb, scope, le) (increase(apiserver_request_duration_seconds_bucket[1h]))
|
|
|
|
|
record: cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase1h
|
|
|
|
|
sum by (cluster, verb, scope, le) (increase(apiserver_request_slo_duration_seconds_bucket[1h]))
|
|
|
|
|
record: cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase1h
|
|
|
|
|
- expr: |
|
|
|
|
|
sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase1h[30d]) * 24 * 30)
|
|
|
|
|
record: cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d
|
|
|
|
|
sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase1h[30d]) * 24 * 30)
|
|
|
|
|
record: cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d
|
|
|
|
|
- expr: |
|
|
|
|
|
1 - (
|
|
|
|
|
(
|
|
|
|
|
# write too slow
|
|
|
|
|
sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
|
|
|
|
|
-
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"})
|
|
|
|
|
) +
|
|
|
|
|
(
|
|
|
|
|
# read too slow
|
|
|
|
|
sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~"LIST|GET"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~"LIST|GET"})
|
|
|
|
|
-
|
|
|
|
|
(
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"})
|
|
|
|
|
or
|
|
|
|
|
vector(0)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"})
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"})
|
|
|
|
|
)
|
|
|
|
|
) +
|
|
|
|
|
# errors
|
|
|
|
|
@@ -1132,19 +1132,19 @@ spec:
|
|
|
|
|
record: apiserver_request:availability30d
|
|
|
|
|
- expr: |
|
|
|
|
|
1 - (
|
|
|
|
|
sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~"LIST|GET"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~"LIST|GET"})
|
|
|
|
|
-
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
(
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"})
|
|
|
|
|
or
|
|
|
|
|
vector(0)
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"})
|
|
|
|
|
+
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"})
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
# errors
|
|
|
|
|
@@ -1159,9 +1159,9 @@ spec:
|
|
|
|
|
1 - (
|
|
|
|
|
(
|
|
|
|
|
# too slow
|
|
|
|
|
sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
|
|
|
|
|
-
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"})
|
|
|
|
|
sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"})
|
|
|
|
|
)
|
|
|
|
|
+
|
|
|
|
|
# errors
|
|
|
|
|
|