From f13cff5d9a428cb69a9bdd564954c38bc8ada314 Mon Sep 17 00:00:00 2001
From: Colleen Murphy <cmurphy@users.noreply.github.com>
Date: Wed, 3 Jul 2024 06:56:34 -0700
Subject: [PATCH] Add alert for cloud-sql-proxy connection failures (#1162)

Add an alert to report on connection errors coming from the
cloud-sql-proxy sidecars in the trillian and rekor deployments. This
should detect when the proxy service account user has lost its
permissions to access the sql instance.

Signed-off-by: Colleen Murphy <colleenmurphy@google.com>
---
 .../gcp/modules/monitoring/infra/alerts.tf    | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/terraform/gcp/modules/monitoring/infra/alerts.tf b/terraform/gcp/modules/monitoring/infra/alerts.tf
index 849d6849d..0fc7ae519 100644
--- a/terraform/gcp/modules/monitoring/infra/alerts.tf
+++ b/terraform/gcp/modules/monitoring/infra/alerts.tf
@@ -275,6 +275,52 @@ resource "google_monitoring_alert_policy" "cloud_sql_disk_utilization" {
 }
 
 
+### Cloud SQL Proxy Alerts
+
+# Cloud SQL Proxy Connection Failures
+resource "google_monitoring_alert_policy" "cloudsqlconn_connection_failure" {
+  # In the absence of data, incident will auto-close in 7 days
+  alert_strategy {
+    auto_close = "604800s"
+  }
+
+  combiner = "OR"
+
+  # Connection failures are greater than 0
+  conditions {
+    condition_threshold {
+      aggregations {
+        alignment_period   = "60s"
+        per_series_aligner = "ALIGN_RATE"
+      }
+
+      comparison      = "COMPARISON_GT"
+      duration        = "300s"
+      filter          = "metric.type=\"prometheus.googleapis.com/cloudsqlconn_dial_failure_count/counter\" resource.type=\"prometheus_target\""
+      threshold_value = "0"
+
+      trigger {
+        count   = "1"
+        percent = "0"
+      }
+    }
+
+    display_name = "Cloud SQL Proxy connections failing"
+  }
+
+  display_name = "Cloud SQL Proxy connections failing"
+
+  documentation {
+    content   = "Cloud SQL Proxy connections have been failing for at least 5 minutes.\n"
+    mime_type = "text/markdown"
+  }
+
+  enabled               = "true"
+  notification_channels = local.notification_channels
+  project               = var.project_id
+}
+
+
 ### KMS Alerts
 
 resource "google_monitoring_alert_policy" "kms_read_request_alert" {