From a1937fd4394051034e2d26c68e8a4e56cea431d7 Mon Sep 17 00:00:00 2001 From: Evan Anderson Date: Tue, 5 Dec 2023 11:39:41 -0800 Subject: [PATCH 1/2] Change write correctness counter to be a delta Signed-off-by: Evan Anderson --- terraform/gcp/modules/monitoring/prober/prober_alerts.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/terraform/gcp/modules/monitoring/prober/prober_alerts.tf b/terraform/gcp/modules/monitoring/prober/prober_alerts.tf index da52197f6..d4d5abdfa 100644 --- a/terraform/gcp/modules/monitoring/prober/prober_alerts.tf +++ b/terraform/gcp/modules/monitoring/prober/prober_alerts.tf @@ -199,7 +199,7 @@ resource "google_monitoring_alert_policy" "prober_verification" { condition_threshold { aggregations { alignment_period = "60s" - per_series_aligner = "ALIGN_SUM" + per_series_aligner = "ALIGN_DELTA" } comparison = "COMPARISON_GT" @@ -211,6 +211,8 @@ resource "google_monitoring_alert_policy" "prober_verification" { count = "1" percent = "0" } + + evaluation_missing_data = "EVALUATION_MISSING_DATA_INACTIVE" } display_name = "Kubernetes Container - prometheus/verification" From 5d304667c5190bdd2549fcf815f8ea26fbbe278f Mon Sep 17 00:00:00 2001 From: Evan Anderson Date: Tue, 5 Dec 2023 12:00:10 -0800 Subject: [PATCH 2/2] Revert "Change write correctness counter to be a delta" It turns out that the value we have is a gauge, not a counter. When we try use a counter, we don't detect the edge from "not collected" to "1", so the alert doesn't work. This reverts commit a1937fd4394051034e2d26c68e8a4e56cea431d7. Signed-off-by: Evan Anderson --- terraform/gcp/modules/monitoring/prober/prober_alerts.tf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/terraform/gcp/modules/monitoring/prober/prober_alerts.tf b/terraform/gcp/modules/monitoring/prober/prober_alerts.tf index d4d5abdfa..da52197f6 100644 --- a/terraform/gcp/modules/monitoring/prober/prober_alerts.tf +++ b/terraform/gcp/modules/monitoring/prober/prober_alerts.tf @@ -199,7 +199,7 @@ resource "google_monitoring_alert_policy" "prober_verification" { condition_threshold { aggregations { alignment_period = "60s" - per_series_aligner = "ALIGN_DELTA" + per_series_aligner = "ALIGN_SUM" } comparison = "COMPARISON_GT" @@ -211,8 +211,6 @@ resource "google_monitoring_alert_policy" "prober_verification" { count = "1" percent = "0" } - - evaluation_missing_data = "EVALUATION_MISSING_DATA_INACTIVE" } display_name = "Kubernetes Container - prometheus/verification"