fix: opsgenie integration healthcheck (#379)

Update to an API entpoint that the bot's Team API key can access.

Move the API call into the try block so that a request failure is
properly identified as a failed healthcheck.

Make the alarms trigger for all `ERROR`, `Exception` and `WARNING` strings.

app/integrations/opsgenie.py

@@ -44,14 +44,14 @@ def create_alert(description):
 def healthcheck():
     """Check if the bot can interact with the Opsgenie API."""
     healthy = False
-    content = api_get_request(
-        "https://api.opsgenie.com/v2/account",
-        {"name": "GenieKey", "token": OPSGENIE_KEY},
-    )
     try:
+        content = api_get_request(
+            "https://api.opsgenie.com/v1/services",
+            {"name": "GenieKey", "token": OPSGENIE_KEY},
+        )
         result = json.loads(content)
-        logging.info(f"OpsGenie healthcheck result: {result}")
         healthy = "data" in result
+        logging.info(f"OpsGenie healthcheck result: {result}")
     except Exception as error:
         logging.error(f"OpsGenie healthcheck failed: {error}")
     return healthy

app/jobs/scheduled_tasks.py

@@ -28,10 +28,14 @@ def scheduler_heartbeat():
 
 def integration_healthchecks():
     logging.info("Running integration healthchecks ...")
-    healthchecks = [opsgenie.healthcheck]
-    for healthcheck in healthchecks:
+    healthchecks = {
+        "opsgenie": opsgenie.healthcheck,
+    }
+    for key, healthcheck in healthchecks.items():
         if not healthcheck():
-            logging.error(f"Integration {healthcheck.__name__} is unhealthy 💀")
+            logging.error(f"Integration {key} is unhealthy 💀")
+        else:
+            logging.info(f"Integration {key} is healthy 🌈")
 
 
 def run_continuously(interval=1):

terraform/alarms.tf

@@ -1,6 +1,6 @@
 resource "aws_cloudwatch_log_metric_filter" "sre_bot_error" {
   name           = local.error_logged
-  pattern        = "\"ERROR:slack_bolt.App\""
+  pattern        = "?ERROR ?Exception"
   log_group_name = local.api_cloudwatch_log_group
 
   metric_transformation {
@@ -29,7 +29,7 @@ resource "aws_cloudwatch_metric_alarm" "sre_bot_error" {
 
 resource "aws_cloudwatch_log_metric_filter" "sre_bot_warning" {
   name           = local.warning_logged
-  pattern        = "\"WARNING:slack_bolt.App\""
+  pattern        = "WARNING"
   log_group_name = local.api_cloudwatch_log_group
 
   metric_transformation {