From 7af2e2e49a6d56ff762e47bc1fa4003070d2783d Mon Sep 17 00:00:00 2001 From: Yuri Nikolic Date: Mon, 25 Nov 2024 10:36:31 +0100 Subject: [PATCH] Jsonnet: fix KEDA autoscaling metric errors during rollouts Signed-off-by: Yuri Nikolic --- CHANGELOG.md | 2 +- ...g-custom-target-utilization-generated.yaml | 96 --- .../test-autoscaling-generated.yaml | 96 --- ...test-multi-zone-distributor-generated.yaml | 32 - ...-new-resource-scaled-object-generated.yaml | 768 ------------------ ...-zone-distributor-migration-generated.yaml | 48 -- operations/mimir/autoscaling.libsonnet | 26 +- 7 files changed, 2 insertions(+), 1066 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3061edafe..d0b7004db4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -545,7 +545,7 @@ * [ENHANCEMENT] Add `_config.autoscaling_querier_predictive_scaling_enabled` to scale querier based on inflight queries 7 days ago. #7775 * [ENHANCEMENT] Add support to autoscale ruler-querier replicas based on in-flight queries too (in addition to CPU and memory based scaling). #8060 #8188 * [ENHANCEMENT] Distributor: improved distributor HPA scaling metric to only take in account ready pods. This requires the metric `kube_pod_status_ready` to be available in the data source used by KEDA to query scaling metrics (configured via `_config.autoscaling_prometheus_url`). #8251 -* [BUGFIX] Guard against missing samples in KEDA queries. #7691 +* [BUGFIX] Guard against missing samples in KEDA queries. #7691 #10013 * [BUGFIX] Alertmanager: Set -server.http-idle-timeout to avoid EOF errors in ruler. #8192 ### Mimirtool diff --git a/operations/mimir-tests/test-autoscaling-custom-target-utilization-generated.yaml b/operations/mimir-tests/test-autoscaling-custom-target-utilization-generated.yaml index e6a039b701..579a36b47e 100644 --- a/operations/mimir-tests/test-autoscaling-custom-target-utilization-generated.yaml +++ b/operations/mimir-tests/test-autoscaling-custom-target-utilization-generated.yaml @@ -1974,14 +1974,6 @@ spec: max by (pod) (up{container="alertmanager",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="alertmanager",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "1780" name: cortex_alertmanager_cpu_hpa_default @@ -2008,14 +2000,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="alertmanager", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="alertmanager",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "9556302233" name: cortex_alertmanager_memory_hpa_default @@ -2062,14 +2046,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="distributor",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "1780" name: cortex_distributor_cpu_hpa_default @@ -2096,14 +2072,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="distributor",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "3058016714" name: cortex_distributor_memory_hpa_default @@ -2193,14 +2161,6 @@ spec: max by (pod) (up{container="query-frontend",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="query-frontend",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2225" name: query_frontend_cpu_hpa_default @@ -2227,14 +2187,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="query-frontend", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="query-frontend",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "559939584" name: query_frontend_memory_hpa_default @@ -2271,14 +2223,6 @@ spec: max by (pod) (up{container="ruler",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="ruler",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "890" name: ruler_cpu_hpa_default @@ -2305,14 +2249,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="ruler", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="ruler",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "5733781340" name: ruler_memory_hpa_default @@ -2349,14 +2285,6 @@ spec: max by (pod) (up{container="ruler-querier",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="ruler-querier",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "178" name: ruler_querier_cpu_hpa_default @@ -2383,14 +2311,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="ruler-querier", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="ruler-querier",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "955630223" name: ruler_querier_memory_hpa_default @@ -2435,14 +2355,6 @@ spec: max by (pod) (up{container="ruler-query-frontend",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="ruler-query-frontend",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "1780" name: ruler_query_frontend_cpu_hpa_default @@ -2469,14 +2381,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="ruler-query-frontend", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="ruler-query-frontend",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "559939584" name: ruler_query_frontend_memory_hpa_default diff --git a/operations/mimir-tests/test-autoscaling-generated.yaml b/operations/mimir-tests/test-autoscaling-generated.yaml index 7201cb9331..bebe99a6a5 100644 --- a/operations/mimir-tests/test-autoscaling-generated.yaml +++ b/operations/mimir-tests/test-autoscaling-generated.yaml @@ -1974,14 +1974,6 @@ spec: max by (pod) (up{container="alertmanager",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="alertmanager",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2000" name: cortex_alertmanager_cpu_hpa_default @@ -2008,14 +2000,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="alertmanager", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="alertmanager",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "10737418240" name: cortex_alertmanager_memory_hpa_default @@ -2062,14 +2046,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="distributor",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2000" name: cortex_distributor_cpu_hpa_default @@ -2096,14 +2072,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="distributor",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "3435973836" name: cortex_distributor_memory_hpa_default @@ -2193,14 +2161,6 @@ spec: max by (pod) (up{container="query-frontend",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="query-frontend",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "1875" name: query_frontend_cpu_hpa_default @@ -2227,14 +2187,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="query-frontend", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="query-frontend",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "629145600" name: query_frontend_memory_hpa_default @@ -2271,14 +2223,6 @@ spec: max by (pod) (up{container="ruler",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="ruler",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "1000" name: ruler_cpu_hpa_default @@ -2305,14 +2249,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="ruler", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="ruler",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "6442450944" name: ruler_memory_hpa_default @@ -2349,14 +2285,6 @@ spec: max by (pod) (up{container="ruler-querier",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="ruler-querier",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "200" name: ruler_querier_cpu_hpa_default @@ -2383,14 +2311,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="ruler-querier", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="ruler-querier",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "1073741824" name: ruler_querier_memory_hpa_default @@ -2435,14 +2355,6 @@ spec: max by (pod) (up{container="ruler-query-frontend",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="ruler-query-frontend",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2000" name: ruler_query_frontend_cpu_hpa_default @@ -2469,14 +2381,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="ruler-query-frontend", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="ruler-query-frontend",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "629145600" name: ruler_query_frontend_memory_hpa_default diff --git a/operations/mimir-tests/test-multi-zone-distributor-generated.yaml b/operations/mimir-tests/test-multi-zone-distributor-generated.yaml index fb14c67336..4f8cf6127a 100644 --- a/operations/mimir-tests/test-multi-zone-distributor-generated.yaml +++ b/operations/mimir-tests/test-multi-zone-distributor-generated.yaml @@ -2592,14 +2592,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"distributor-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="distributor",namespace="default",pod=~"distributor-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2000" name: cortex_distributor_zone_a_cpu_hpa_default @@ -2626,14 +2618,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor", namespace="default", reason="OOMKilled",pod=~"distributor-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="distributor",namespace="default",pod=~"distributor-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2147483648" name: cortex_distributor_zone_a_memory_hpa_default @@ -2680,14 +2664,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"distributor-zone-b.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="distributor",namespace="default",pod=~"distributor-zone-b.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2000" name: cortex_distributor_zone_b_cpu_hpa_default @@ -2714,14 +2690,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor", namespace="default", reason="OOMKilled",pod=~"distributor-zone-b.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="distributor",namespace="default",pod=~"distributor-zone-b.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2147483648" name: cortex_distributor_zone_b_memory_hpa_default diff --git a/operations/mimir-tests/test-new-resource-scaled-object-generated.yaml b/operations/mimir-tests/test-new-resource-scaled-object-generated.yaml index 8c7b8dfeae..92a936a1e1 100644 --- a/operations/mimir-tests/test-new-resource-scaled-object-generated.yaml +++ b/operations/mimir-tests/test-new-resource-scaled-object-generated.yaml @@ -1363,14 +1363,6 @@ spec: max by (pod) (up{container="test_false_false_0.7__",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_false_0.7__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -1398,14 +1390,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_false_0.7__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_false_0.7__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_0.7___memory_hpa_default @@ -1442,14 +1426,6 @@ spec: max by (pod) (up{container="test_false_false_0.7__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_false_0.7__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -1477,14 +1453,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_false_0.7__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_false_0.7__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_0.7__pod_zone_a.*_memory_hpa_default @@ -1521,14 +1489,6 @@ spec: max by (pod) (up{container="test_container",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -1556,14 +1516,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_0.7_test_container__memory_hpa_default @@ -1600,14 +1552,6 @@ spec: max by (pod) (up{container="test_container",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -1635,14 +1579,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_0.7_test_container_pod_zone_a.*_memory_hpa_default @@ -1679,14 +1615,6 @@ spec: max by (pod) (up{container="test_false_false_1.5__",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_false_1.5__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -1714,14 +1642,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_false_1.5__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_false_1.5__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_1.5___memory_hpa_default @@ -1758,14 +1678,6 @@ spec: max by (pod) (up{container="test_false_false_1.5__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_false_1.5__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -1793,14 +1705,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_false_1.5__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_false_1.5__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_1.5__pod_zone_a.*_memory_hpa_default @@ -1837,14 +1741,6 @@ spec: max by (pod) (up{container="test_container",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -1872,14 +1768,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_1.5_test_container__memory_hpa_default @@ -1916,14 +1804,6 @@ spec: max by (pod) (up{container="test_container",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -1951,14 +1831,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_1.5_test_container_pod_zone_a.*_memory_hpa_default @@ -1995,14 +1867,6 @@ spec: max by (pod) (up{container="test_false_false_1__",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_false_1__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: test_false_false_1___cpu_hpa_default @@ -2029,14 +1893,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_false_1__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_false_1__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_1___memory_hpa_default @@ -2073,14 +1929,6 @@ spec: max by (pod) (up{container="test_false_false_1__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_false_1__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: test_false_false_1__pod_zone_a.*_cpu_hpa_default @@ -2107,14 +1955,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_false_1__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_false_1__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_1__pod_zone_a.*_memory_hpa_default @@ -2151,14 +1991,6 @@ spec: max by (pod) (up{container="test_container",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: test_false_false_1_test_container__cpu_hpa_default @@ -2185,14 +2017,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_1_test_container__memory_hpa_default @@ -2229,14 +2053,6 @@ spec: max by (pod) (up{container="test_container",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: test_false_false_1_test_container_pod_zone_a.*_cpu_hpa_default @@ -2263,14 +2079,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_false_1_test_container_pod_zone_a.*_memory_hpa_default @@ -2307,14 +2115,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_true_0.7__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -2342,14 +2142,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_true_0.7__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_true_0.7__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_0.7___memory_hpa_default @@ -2386,14 +2178,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_true_0.7__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -2421,14 +2205,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_true_0.7__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_true_0.7__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_0.7__pod_zone_a.*_memory_hpa_default @@ -2465,14 +2241,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -2500,14 +2268,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_0.7_test_container__memory_hpa_default @@ -2544,14 +2304,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -2579,14 +2331,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_0.7_test_container_pod_zone_a.*_memory_hpa_default @@ -2623,14 +2367,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_true_1.5__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -2658,14 +2394,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_true_1.5__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_true_1.5__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_1.5___memory_hpa_default @@ -2702,14 +2430,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_true_1.5__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -2737,14 +2457,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_true_1.5__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_true_1.5__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_1.5__pod_zone_a.*_memory_hpa_default @@ -2781,14 +2493,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -2816,14 +2520,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_1.5_test_container__memory_hpa_default @@ -2860,14 +2556,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -2895,14 +2583,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_1.5_test_container_pod_zone_a.*_memory_hpa_default @@ -2939,14 +2619,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_true_1__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: test_false_true_1___cpu_hpa_default @@ -2973,14 +2645,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_true_1__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_true_1__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_1___memory_hpa_default @@ -3017,14 +2681,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_false_true_1__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: test_false_true_1__pod_zone_a.*_cpu_hpa_default @@ -3051,14 +2707,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_false_true_1__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_false_true_1__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_1__pod_zone_a.*_memory_hpa_default @@ -3095,14 +2743,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: test_false_true_1_test_container__cpu_hpa_default @@ -3129,14 +2769,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_1_test_container__memory_hpa_default @@ -3173,14 +2805,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: test_false_true_1_test_container_pod_zone_a.*_cpu_hpa_default @@ -3207,14 +2831,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: test_false_true_1_test_container_pod_zone_a.*_memory_hpa_default @@ -3251,14 +2867,6 @@ spec: max by (pod) (up{container="test_true_false_0.7__",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_false_0.7__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -3286,14 +2894,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_false_0.7__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_false_0.7__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_0.7___memory_hpa_default @@ -3330,14 +2930,6 @@ spec: max by (pod) (up{container="test_true_false_0.7__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_false_0.7__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -3365,14 +2957,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_false_0.7__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_false_0.7__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_0.7__pod_zone_a.*_memory_hpa_default @@ -3409,14 +2993,6 @@ spec: max by (pod) (up{container="test_container",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -3444,14 +3020,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_0.7_test_container__memory_hpa_default @@ -3488,14 +3056,6 @@ spec: max by (pod) (up{container="test_container",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -3523,14 +3083,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_0.7_test_container_pod_zone_a.*_memory_hpa_default @@ -3567,14 +3119,6 @@ spec: max by (pod) (up{container="test_true_false_1.5__",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_false_1.5__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -3602,14 +3146,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_false_1.5__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_false_1.5__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_1.5___memory_hpa_default @@ -3646,14 +3182,6 @@ spec: max by (pod) (up{container="test_true_false_1.5__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_false_1.5__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -3681,14 +3209,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_false_1.5__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_false_1.5__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_1.5__pod_zone_a.*_memory_hpa_default @@ -3725,14 +3245,6 @@ spec: max by (pod) (up{container="test_container",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -3760,14 +3272,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_1.5_test_container__memory_hpa_default @@ -3804,14 +3308,6 @@ spec: max by (pod) (up{container="test_container",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -3839,14 +3335,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_1.5_test_container_pod_zone_a.*_memory_hpa_default @@ -3883,14 +3371,6 @@ spec: max by (pod) (up{container="test_true_false_1__",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_false_1__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: cortex_test_true_false_1___cpu_hpa_default @@ -3917,14 +3397,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_false_1__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_false_1__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_1___memory_hpa_default @@ -3961,14 +3433,6 @@ spec: max by (pod) (up{container="test_true_false_1__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_false_1__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: cortex_test_true_false_1__pod_zone_a.*_cpu_hpa_default @@ -3995,14 +3459,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_false_1__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_false_1__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_1__pod_zone_a.*_memory_hpa_default @@ -4039,14 +3495,6 @@ spec: max by (pod) (up{container="test_container",namespace="default"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: cortex_test_true_false_1_test_container__cpu_hpa_default @@ -4073,14 +3521,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_1_test_container__memory_hpa_default @@ -4117,14 +3557,6 @@ spec: max by (pod) (up{container="test_container",namespace="default",pod=~"pod-zone-a.*"}) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: cortex_test_true_false_1_test_container_pod_zone_a.*_cpu_hpa_default @@ -4151,14 +3583,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_false_1_test_container_pod_zone_a.*_memory_hpa_default @@ -4195,14 +3619,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_true_0.7__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -4230,14 +3646,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_true_0.7__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_true_0.7__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_0.7___memory_hpa_default @@ -4274,14 +3682,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_true_0.7__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -4309,14 +3709,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_true_0.7__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_true_0.7__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_0.7__pod_zone_a.*_memory_hpa_default @@ -4353,14 +3745,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -4388,14 +3772,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_0.7_test_container__memory_hpa_default @@ -4432,14 +3808,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 0.70 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -4467,14 +3835,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_0.7_test_container_pod_zone_a.*_memory_hpa_default @@ -4511,14 +3871,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_true_1.5__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -4546,14 +3898,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_true_1.5__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_true_1.5__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_1.5___memory_hpa_default @@ -4590,14 +3934,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_true_1.5__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -4625,14 +3961,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_true_1.5__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_true_1.5__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_1.5__pod_zone_a.*_memory_hpa_default @@ -4669,14 +3997,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -4704,14 +4024,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_1.5_test_container__memory_hpa_default @@ -4748,14 +4060,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) * 1.50 serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" @@ -4783,14 +4087,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_1.5_test_container_pod_zone_a.*_memory_hpa_default @@ -4827,14 +4123,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_true_1__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: cortex_test_true_true_1___cpu_hpa_default @@ -4861,14 +4149,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_true_1__", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_true_1__",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_1___memory_hpa_default @@ -4905,14 +4185,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_true_true_1__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: cortex_test_true_true_1__pod_zone_a.*_cpu_hpa_default @@ -4939,14 +4211,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_true_true_1__pod-zone-a.*", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_true_true_1__pod-zone-a.*",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_1__pod_zone_a.*_memory_hpa_default @@ -4983,14 +4247,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: cortex_test_true_true_1_test_container__cpu_hpa_default @@ -5017,14 +4273,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_1_test_container__memory_hpa_default @@ -5061,14 +4309,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"pod-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "8000" name: cortex_test_true_true_1_test_container_pod_zone_a.*_cpu_hpa_default @@ -5095,14 +4335,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="test_container", namespace="default", reason="OOMKilled",pod=~"pod-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="test_container",namespace="default",pod=~"pod-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "858993459" name: cortex_test_true_true_1_test_container_pod_zone_a.*_memory_hpa_default diff --git a/operations/mimir-tests/test-single-to-multi-zone-distributor-migration-generated.yaml b/operations/mimir-tests/test-single-to-multi-zone-distributor-migration-generated.yaml index 13bd426422..0d654fcd23 100644 --- a/operations/mimir-tests/test-single-to-multi-zone-distributor-migration-generated.yaml +++ b/operations/mimir-tests/test-single-to-multi-zone-distributor-migration-generated.yaml @@ -2723,14 +2723,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod!~"distributor-zone.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="distributor",namespace="default",pod!~"distributor-zone.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2000" name: cortex_distributor_cpu_hpa_default @@ -2757,14 +2749,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor", namespace="default", reason="OOMKilled",pod!~"distributor-zone.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="distributor",namespace="default",pod!~"distributor-zone.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2147483648" name: cortex_distributor_memory_hpa_default @@ -2811,14 +2795,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"distributor-zone-a.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="distributor",namespace="default",pod=~"distributor-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2000" name: cortex_distributor_zone_a_cpu_hpa_default @@ -2845,14 +2821,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor", namespace="default", reason="OOMKilled",pod=~"distributor-zone-a.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="distributor",namespace="default",pod=~"distributor-zone-a.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2147483648" name: cortex_distributor_zone_a_memory_hpa_default @@ -2899,14 +2867,6 @@ spec: max by (pod) (min_over_time(kube_pod_status_ready{namespace="default",condition="true",pod=~"distributor-zone-b.*"}[1m])) > 0 )[15m:] ) * 1000 - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="distributor",namespace="default",pod=~"distributor-zone-b.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2000" name: cortex_distributor_zone_b_cpu_hpa_default @@ -2933,14 +2893,6 @@ spec: max by (pod) (kube_pod_container_status_last_terminated_reason{container="distributor", namespace="default", reason="OOMKilled",pod=~"distributor-zone-b.*"}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="distributor",namespace="default",pod=~"distributor-zone-b.*"}[1m] - )[15m:1m] - ) >= 15 - ) serverAddress: http://prometheus.default:9090/prometheus threshold: "2147483648" name: cortex_distributor_zone_b_memory_hpa_default diff --git a/operations/mimir/autoscaling.libsonnet b/operations/mimir/autoscaling.libsonnet index 53745fb88b..4c2fd54f90 100644 --- a/operations/mimir/autoscaling.libsonnet +++ b/operations/mimir/autoscaling.libsonnet @@ -299,19 +299,6 @@ )[15m:] ) * 1000 ||| - ) + ( - // Ensures that it only returns a result if all expected samples were present for the CPU metric over - // the last 15 minutes. - ||| - and - count ( - count_over_time( - present_over_time( - container_cpu_usage_seconds_total{container="%(container)s",namespace="%(namespace)s"%(extra_matchers)s}[1m] - )[15m:1m] - ) >= 15 - ) - ||| ), local memoryHPAQuery(with_ready_trigger) = @@ -351,10 +338,7 @@ ) ||| ) + ( - // The first section of the query adds pods that were terminated due to an OOM in the memory calculation. - // - // The second section of the query ensures that it only returns a result if all expected samples were - // present for the memory metric over the last 15 minutes. + // Add pods that were terminated due to an OOM in the memory calculation. ||| + sum( @@ -365,14 +349,6 @@ max by (pod) (kube_pod_container_status_last_terminated_reason{container="%(container)s", namespace="%(namespace)s", reason="OOMKilled"%(extra_matchers)s}) or vector(0) ) - and - count ( - count_over_time( - present_over_time( - container_memory_working_set_bytes{container="%(container)s",namespace="%(namespace)s"%(extra_matchers)s}[1m] - )[15m:1m] - ) >= 15 - ) ||| ),