Add email sending functionality for callback email

- Updated the callback email migration script with the latest changes from content
- Added a method to send the callback failure email to service owners
- Stubbed a method to query cloudwatch so we can determine if callbacks for the service have failed at least 5 times in a 30 minute time period before we send the email to the service owner

app/celery/service_callback_tasks.py

@@ -2,10 +2,12 @@
 
 from flask import current_app
 from notifications_utils.statsd_decorators import statsd
-from requests import HTTPError, RequestException, request
+from requests import HTTPError, RequestException, TimeoutError, InvalidURL, request
 
 from app import notify_celery, signer_complaint, signer_delivery_status
 from app.config import QueueNames
+from app.models import Service
+from app.service.sender import send_notification_to_service_users
 
 
 @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
@@ -80,6 +82,15 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
         current_app.logger.warning(
             f"{function_name} request failed for notification_id: {notification_id} and url: {service_callback_url}. exc: {e}"
         )
+
+        # TODO: Instate once we monitor alarms to determine how often this happens and we implement
+        #       check_cloudwatch_for_callback_failures(), otherwise we risk flooding the service
+        #       owner's inbox with callback failure email notifications.
+
+        # if isinstance(e, TimeoutError) or isinstance(e, InvalidURL) or e.response.status_code == 500:
+        #     if check_cloudwatch_for_callback_failures():
+        #         send_email_callback_failure_email(current_app.service)
+
         # Retry if the response status code is server-side or 429 (too many requests).
         if not isinstance(e, HTTPError) or e.response.status_code >= 500 or e.response.status_code == 429:
             try:
@@ -88,3 +99,55 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
                 current_app.logger.warning(
                     f"Retry: {function_name} has retried the max num of times for callback url {service_callback_url} and notification_id: {notification_id}"
                 )
+
+def send_email_callback_failure_email(service: Service):
+    send_notification_to_service_users(
+        service_id=service.id,
+        template_id=current_app.config["CALLBACK_FAILURE_TEMPLATE_ID"],
+        personalisation={
+            "service_name": service.name,
+            "contact_url": f"{current_app.config['ADMIN_BASE_URL']}/contact",
+            "callback_doc_url": f"{current_app.config['DOCUMENTATION_DOAMIN']}/en/callbacks.html"
+        },
+        include_user_fields=["name"],
+    )
+
+
+def check_cloudwatch_for_callback_failures():
+    """
+        TODO: Use boto3 to check cloudwatch for callback failures
+        https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/logs/client/start_query.html
+
+        Check if a service has failed 5 callbacks in a 30 minute time period
+
+        ----------------
+
+        import boto3
+        from datetime import datetime, timedelta
+        import time
+
+        client = boto3.client('logs')
+
+        query = "TODO"
+
+        log_group = 'TODO'
+
+        start_query_response = client.start_query(
+            logGroupName=log_group,
+            startTime=int((datetime.today() - timedelta(minutes=30)).timestamp()),
+            endTime=int(datetime.now().timestamp()),
+            queryString=query,
+        )
+
+        query_id = start_query_response['queryId']
+
+        response = None
+
+        while response == None or response['status'] == 'Running':
+            print('Waiting for query to complete ...')
+            time.sleep(1)
+            response = client.get_query_results(
+                queryId=query_id
+            )
+
+    """

migrations/versions/0453_add_callback_failure_email.py

@@ -40,7 +40,9 @@ def upgrade():
             "(2) Your service is using a proxy that we cannot access.",
             "(3) We’re able to reach your service, but it responds with errors.",
             "",
-            "It’s important to check your callback service is running, check your callback service’s logs for errors and repair any errors in your logs. Once you’ve taken these steps, request confirmation that your callbacks are working again by contacting us.",
+            "It’s important to check your callback service is running, check your callback service’s logs for errors and repair any errors in your logs. To find your callback configuration, sign into your account, visit the API integration page for “((service_name))” and select callbacks.",
+            "",
+            "Once you’ve taken these steps, request confirmation that your callbacks are working again by [contacting us](((contact_url))). For more information, you can also access our [API documentation on callbacks](((callback_docs_url))).",
             "",
             "The GC Notify team",
             "[[/en]]",