From 4a1f571014b85abe6d1ac3284122517269111982 Mon Sep 17 00:00:00 2001 From: David Grant Date: Fri, 22 Nov 2024 15:34:20 -0800 Subject: [PATCH 1/4] Use resident memory for the etcd memory alert. --- operations/mimir-mixin/alerts/alerts.libsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operations/mimir-mixin/alerts/alerts.libsonnet b/operations/mimir-mixin/alerts/alerts.libsonnet index f66af7829a..75c7e1ca78 100644 --- a/operations/mimir-mixin/alerts/alerts.libsonnet +++ b/operations/mimir-mixin/alerts/alerts.libsonnet @@ -875,7 +875,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; alert: 'EtcdAllocatingTooMuchMemory', expr: ||| ( - container_memory_working_set_bytes{container="etcd"} + container_memory_rss{container="etcd"} / ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.65 @@ -894,7 +894,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; alert: 'EtcdAllocatingTooMuchMemory', expr: ||| ( - container_memory_working_set_bytes{container="etcd"} + container_memory_rss{container="etcd"} / ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.8 From 58671381c72bfa7f5481a654651e69c620d6b80d Mon Sep 17 00:00:00 2001 From: David Grant Date: Fri, 22 Nov 2024 15:43:26 -0800 Subject: [PATCH 2/4] Changelog entry. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index baa8b3b5e6..13c2edab1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,7 @@ * [BUGFIX] Alerts: Fix autoscaling metrics joins in `MimirAutoscalerNotActive` when series churn. #9412 * [BUGFIX] Alerts: Exclude failed cache "add" operations from alerting since failures are expected in normal operation. #9658 * [BUGFIX] Alerts: Exclude read-only replicas from `IngesterInstanceHasNoTenants` alert. #9843 +* [BUGFIX] Alerts: Use resident set memory for the `EtcdAllocatingTooMuchMemory` alert so that ephemeral file cache memory doesn't cause the alert to misfire. #9997 ### Jsonnet From 668d6fbea75d364123785099b8f8590bd9e58237 Mon Sep 17 00:00:00 2001 From: David Grant Date: Fri, 22 Nov 2024 15:44:38 -0800 Subject: [PATCH 3/4] Rebuild mixin yamls. --- operations/mimir-mixin-compiled-baremetal/alerts.yaml | 4 ++-- operations/mimir-mixin-compiled/alerts.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/operations/mimir-mixin-compiled-baremetal/alerts.yaml b/operations/mimir-mixin-compiled-baremetal/alerts.yaml index 9283e7e8ee..d4776cf7c2 100644 --- a/operations/mimir-mixin-compiled-baremetal/alerts.yaml +++ b/operations/mimir-mixin-compiled-baremetal/alerts.yaml @@ -583,7 +583,7 @@ groups: runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#etcdallocatingtoomuchmemory expr: | ( - container_memory_working_set_bytes{container="etcd"} + container_memory_rss{container="etcd"} / ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.65 @@ -597,7 +597,7 @@ groups: runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#etcdallocatingtoomuchmemory expr: | ( - container_memory_working_set_bytes{container="etcd"} + container_memory_rss{container="etcd"} / ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.8 diff --git a/operations/mimir-mixin-compiled/alerts.yaml b/operations/mimir-mixin-compiled/alerts.yaml index be782a551d..275dd27111 100644 --- a/operations/mimir-mixin-compiled/alerts.yaml +++ b/operations/mimir-mixin-compiled/alerts.yaml @@ -593,7 +593,7 @@ groups: runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#etcdallocatingtoomuchmemory expr: | ( - container_memory_working_set_bytes{container="etcd"} + container_memory_rss{container="etcd"} / ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.65 @@ -607,7 +607,7 @@ groups: runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#etcdallocatingtoomuchmemory expr: | ( - container_memory_working_set_bytes{container="etcd"} + container_memory_rss{container="etcd"} / ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.8 From b499db54b4b03a12d7effb6f8da5bea94f824d11 Mon Sep 17 00:00:00 2001 From: David Grant Date: Fri, 22 Nov 2024 15:48:27 -0800 Subject: [PATCH 4/4] And rebuild helm alerts. --- .../templates/metamonitoring/mixin-alerts.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml index 929e6927b5..8f5d18333d 100644 --- a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml +++ b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml @@ -605,7 +605,7 @@ spec: runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#etcdallocatingtoomuchmemory expr: | ( - container_memory_working_set_bytes{container="etcd"} + container_memory_rss{container="etcd"} / ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.65 @@ -619,7 +619,7 @@ spec: runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#etcdallocatingtoomuchmemory expr: | ( - container_memory_working_set_bytes{container="etcd"} + container_memory_rss{container="etcd"} / ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.8