diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py index 9296958f85..47aaa4d11c 100644 --- a/app/celery/service_callback_tasks.py +++ b/app/celery/service_callback_tasks.py @@ -6,6 +6,10 @@ from app import notify_celery, signer_complaint, signer_delivery_status from app.config import QueueNames +from app.models import Service + +# Uncomment when we implement email sending for callback failures +# from requests.exceptions import InvalidURL, Timeout @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300) @@ -68,7 +72,7 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token, "Content-Type": "application/json", "Authorization": f"Bearer {token}", }, - timeout=5, + timeout=1, ) current_app.logger.info( @@ -80,11 +84,73 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token, current_app.logger.warning( f"{function_name} request failed for notification_id: {notification_id} and url: {service_callback_url}. exc: {e}" ) + + # TODO: Instate once we monitor alarms to determine how often this happens and we implement + # check_cloudwatch_for_callback_failures(), otherwise we risk flooding the service + # owner's inbox with callback failure email notifications. + + # if isinstance(e, Timeout) or isinstance(e, InvalidURL) or e.response.status_code == 500: + # if check_cloudwatch_for_callback_failures(): + # send_email_callback_failure_email(current_app.service) + # Retry if the response status code is server-side or 429 (too many requests). if not isinstance(e, HTTPError) or e.response.status_code >= 500 or e.response.status_code == 429: try: self.retry(queue=QueueNames.CALLBACKS_RETRY) except self.MaxRetriesExceededError: current_app.logger.warning( - "Retry: {function_name} has retried the max num of times for callback url {service_callback_url} and notification_id: {notification_id}" + f"Retry: {function_name} has retried the max num of times for callback url {service_callback_url} and notification_id: {notification_id} for service: {current_app.service.id}" ) + + +def send_email_callback_failure_email(service: Service): + service.send_notification_to_service_users( + service_id=service.id, + template_id=current_app.config["CALLBACK_FAILURE_TEMPLATE_ID"], + personalisation={ + "service_name": service.name, + "contact_url": f"{current_app.config['ADMIN_BASE_URL']}/contact", + "callback_doc_url": f"{current_app.config['DOCUMENTATION_DOAMIN']}/en/callbacks.html", + }, + include_user_fields=["name"], + ) + + +def check_cloudwatch_for_callback_failures(): + """ + TODO: Use boto3 to check cloudwatch for callback failures + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/logs/client/start_query.html + + Check if a service has failed 5 callbacks in a 30 minute time period + + ---------------- + + import boto3 + from datetime import datetime, timedelta + import time + + client = boto3.client('logs') + + query = "TODO" + + log_group = 'TODO' + + start_query_response = client.start_query( + logGroupName=log_group, + startTime=int((datetime.today() - timedelta(minutes=30)).timestamp()), + endTime=int(datetime.now().timestamp()), + queryString=query, + ) + + query_id = start_query_response['queryId'] + + response = None + + while response == None or response['status'] == 'Running': + print('Waiting for query to complete ...') + time.sleep(1) + response = client.get_query_results( + queryId=query_id + ) + + """ diff --git a/app/config.py b/app/config.py index fcb6aa4297..b170e77add 100644 --- a/app/config.py +++ b/app/config.py @@ -299,50 +299,52 @@ class Config(object): CHECK_PROXY_HEADER = False # Notify's notifications templates - NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553" - NOTIFY_USER_ID = "6af522d0-2915-4e52-83a3-3690455a5fe6" - INVITATION_EMAIL_TEMPLATE_ID = "4f46df42-f795-4cc4-83bb-65ca312f49cc" - SMS_CODE_TEMPLATE_ID = "36fb0730-6259-4da1-8a80-c8de22ad4246" - EMAIL_2FA_TEMPLATE_ID = "299726d2-dba6-42b8-8209-30e1d66ea164" - EMAIL_MAGIC_LINK_TEMPLATE_ID = "6e97fd09-6da0-4cc8-829d-33cf5b818103" - NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = "ece42649-22a8-4d06-b87f-d52d5d3f0a27" - PASSWORD_RESET_TEMPLATE_ID = "474e9242-823b-4f99-813d-ed392e7f1201" - FORCED_PASSWORD_RESET_TEMPLATE_ID = "e9a65a6b-497b-42f2-8f43-1736e43e13b3" + ACCOUNT_CHANGE_TEMPLATE_ID = "5b39e16a-9ff8-487c-9bfb-9e06bdb70f36" ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = "0880fbb1-a0c6-46f0-9a8e-36c986381ceb" + APIKEY_REVOKE_TEMPLATE_ID = "a0a4e7b8-8a6a-4eaa-9f4e-9c3a5b2dbcf3" + BRANDING_REQUEST_TEMPLATE_ID = "7d423d9e-e94e-4118-879d-d52f383206ae" + CALLBACK_FAILURE_TEMPLATE_ID = "d8d580f4-86b3-4ba4-9d7c-263a630af354" CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = "eb4d9930-87ab-4aef-9bce-786762687884" - SERVICE_NOW_LIVE_TEMPLATE_ID = "618185c6-3636-49cd-b7d2-6f6f5eb3bdde" - ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = "203566f0-d835-47c5-aa06-932439c86573" - TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = "c73f1d71-4049-46d5-a647-d013bdeca3f0" - TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = "8a31520f-4751-4789-8ea1-fe54496725eb" - REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = "a42f1d17-9404-46d5-a647-d013bdfca3e1" - MOU_SIGNER_RECEIPT_TEMPLATE_ID = "4fd2e43c-309b-4e50-8fb8-1955852d9d71" - MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = "c20206d5-bf03-4002-9a90-37d5032d9e84" - MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = "522b6657-5ca5-4368-a294-6b527703bd0b" - MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = "d0e66c4c-0c50-43f0-94f5-f85b613202d4" + CONTACT_FORM_DIRECT_EMAIL_TEMPLATE_ID = "b04beb4a-8408-4280-9a5c-6a046b6f7704" CONTACT_US_TEMPLATE_ID = "8ea9b7a0-a824-4dd3-a4c3-1f508ed20a69" - ACCOUNT_CHANGE_TEMPLATE_ID = "5b39e16a-9ff8-487c-9bfb-9e06bdb70f36" - BRANDING_REQUEST_TEMPLATE_ID = "7d423d9e-e94e-4118-879d-d52f383206ae" - NO_REPLY_TEMPLATE_ID = "86950840-6da4-4865-841b-16028110e980" - NEAR_DAILY_LIMIT_TEMPLATE_ID = "5d3e4322-4ee6-457a-a710-c48755f6b643" - REACHED_DAILY_LIMIT_TEMPLATE_ID = "fd29f796-fcdc-471b-a0d4-0093880d9173" + DAILY_EMAIL_LIMIT_UPDATED_TEMPLATE_ID = "97dade64-ea8d-460f-8a34-900b74ee5eb0" DAILY_LIMIT_UPDATED_TEMPLATE_ID = "b3c766e6-be32-4edf-b8db-0f04ef404edc" - NEAR_DAILY_SMS_LIMIT_TEMPLATE_ID = "a796568f-a89b-468e-b635-8105554301b9" - REACHED_DAILY_SMS_LIMIT_TEMPLATE_ID = "a646e614-c527-4f94-a955-ed7185d577f4" DAILY_SMS_LIMIT_UPDATED_TEMPLATE_ID = "6ec12dd0-680a-4073-8d58-91d17cc8442f" - CONTACT_FORM_DIRECT_EMAIL_TEMPLATE_ID = "b04beb4a-8408-4280-9a5c-6a046b6f7704" - NEAR_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "9aa60ad7-2d7f-46f0-8cbe-2bac3d4d77d8" - REACHED_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "ee036547-e51b-49f1-862b-10ea982cfceb" - DAILY_EMAIL_LIMIT_UPDATED_TEMPLATE_ID = "97dade64-ea8d-460f-8a34-900b74ee5eb0" - APIKEY_REVOKE_TEMPLATE_ID = "a0a4e7b8-8a6a-4eaa-9f4e-9c3a5b2dbcf3" + DEFAULT_TEMPLATE_CATEGORY_LOW = "0dda24c2-982a-4f44-9749-0e38b2607e89" + DEFAULT_TEMPLATE_CATEGORY_MEDIUM = "f75d6706-21b7-437e-b93a-2c0ab771e28e" + DEFAULT_TEMPLATE_CATEGORY_HIGH = "c4f87d7c-a55b-4c0f-91fe-e56c65bb1871" + EMAIL_2FA_TEMPLATE_ID = "299726d2-dba6-42b8-8209-30e1d66ea164" + EMAIL_MAGIC_LINK_TEMPLATE_ID = "6e97fd09-6da0-4cc8-829d-33cf5b818103" + FORCED_PASSWORD_RESET_TEMPLATE_ID = "e9a65a6b-497b-42f2-8f43-1736e43e13b3" + HEARTBEAT_TEMPLATE_EMAIL_HIGH = "276da251-3103-49f3-9054-cbf6b5d74411" HEARTBEAT_TEMPLATE_EMAIL_LOW = "73079cb9-c169-44ea-8cf4-8d397711cc9d" HEARTBEAT_TEMPLATE_EMAIL_MEDIUM = "c75c4539-3014-4c4c-96b5-94d326758a74" - HEARTBEAT_TEMPLATE_EMAIL_HIGH = "276da251-3103-49f3-9054-cbf6b5d74411" + HEARTBEAT_TEMPLATE_SMS_HIGH = "4969a9e9-ddfd-476e-8b93-6231e6f1be4a" HEARTBEAT_TEMPLATE_SMS_LOW = "ab3a603b-d602-46ea-8c83-e05cb280b950" HEARTBEAT_TEMPLATE_SMS_MEDIUM = "a48b54ce-40f6-4e4a-abe8-1e2fa389455b" HEARTBEAT_TEMPLATE_SMS_HIGH = "4969a9e9-ddfd-476e-8b93-6231e6f1be4a" - DEFAULT_TEMPLATE_CATEGORY_LOW = "0dda24c2-982a-4f44-9749-0e38b2607e89" - DEFAULT_TEMPLATE_CATEGORY_MEDIUM = "f75d6706-21b7-437e-b93a-2c0ab771e28e" - DEFAULT_TEMPLATE_CATEGORY_HIGH = "c4f87d7c-a55b-4c0f-91fe-e56c65bb1871" + INVITATION_EMAIL_TEMPLATE_ID = "4f46df42-f795-4cc4-83bb-65ca312f49cc" + MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = "d0e66c4c-0c50-43f0-94f5-f85b613202d4" + MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = "522b6657-5ca5-4368-a294-6b527703bd0b" + MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = "c20206d5-bf03-4002-9a90-37d5032d9e84" + MOU_SIGNER_RECEIPT_TEMPLATE_ID = "4fd2e43c-309b-4e50-8fb8-1955852d9d71" + NEAR_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "9aa60ad7-2d7f-46f0-8cbe-2bac3d4d77d8" + NEAR_DAILY_LIMIT_TEMPLATE_ID = "5d3e4322-4ee6-457a-a710-c48755f6b643" + NEAR_DAILY_SMS_LIMIT_TEMPLATE_ID = "a796568f-a89b-468e-b635-8105554301b9" + NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = "ece42649-22a8-4d06-b87f-d52d5d3f0a27" + NO_REPLY_TEMPLATE_ID = "86950840-6da4-4865-841b-16028110e980" + NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553" + NOTIFY_USER_ID = "6af522d0-2915-4e52-83a3-3690455a5fe6" + ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = "203566f0-d835-47c5-aa06-932439c86573" + PASSWORD_RESET_TEMPLATE_ID = "474e9242-823b-4f99-813d-ed392e7f1201" + REACHED_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "ee036547-e51b-49f1-862b-10ea982cfceb" + REACHED_DAILY_LIMIT_TEMPLATE_ID = "fd29f796-fcdc-471b-a0d4-0093880d9173" + REACHED_DAILY_SMS_LIMIT_TEMPLATE_ID = "a646e614-c527-4f94-a955-ed7185d577f4" + REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = "a42f1d17-9404-46d5-a647-d013bdfca3e1" + SERVICE_NOW_LIVE_TEMPLATE_ID = "618185c6-3636-49cd-b7d2-6f6f5eb3bdde" + SMS_CODE_TEMPLATE_ID = "36fb0730-6259-4da1-8a80-c8de22ad4246" + TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = "c73f1d71-4049-46d5-a647-d013bdeca3f0" + TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = "8a31520f-4751-4789-8ea1-fe54496725eb" # Allowed service IDs able to send HTML through their templates. ALLOW_HTML_SERVICE_IDS: List[str] = [id.strip() for id in os.getenv("ALLOW_HTML_SERVICE_IDS", "").split(",")] diff --git a/migrations/versions/0453_add_callback_failure_email.py b/migrations/versions/0453_add_callback_failure_email.py new file mode 100644 index 0000000000..cba644bb28 --- /dev/null +++ b/migrations/versions/0453_add_callback_failure_email.py @@ -0,0 +1,114 @@ +""" +Revision ID: 0458_add_callback_failure_email +Revises: 0457_update_categories +Create Date: 2024-07-30 15:51:00 +""" +from datetime import datetime + +from alembic import op +from flask import current_app + +revision = "0458_add_callback_failure_email" +down_revision = "0457_update_categories" + +callback_failure_template_id = current_app.config["CALLBACK_FAILURE_TEMPLATE_ID"] + + +def upgrade(): + template_insert = """ + INSERT INTO templates (id, name, template_type, created_at, content, archived, service_id, subject, + created_by_id, version, process_type, hidden) + VALUES('{}', '{}', '{}', '{}', '{}', False, '{}', '{}', '{}', 1, '{}', false) + """ + template_history_insert = """ + INSERT INTO templates_history (id, name, template_type, created_at, content, archived, service_id, subject, + created_by_id, version, process_type, hidden) + VALUES ('{}', '{}', '{}', '{}', '{}', False, '{}', '{}', '{}', 1, '{}', false) + """ + + callback_failure_content = "\n".join( + [ + "[[fr]]", + "(la version française suit)", + "[[/fr]]", + "", + "[[en]]", + "Hello ((name)),", + "", + "The callbacks for “((service_name))” are not working. This could mean that:", + "", + "(1) Your callback service is down.", + "(2) Your service is using a proxy that we cannot access.", + "(3) We’re able to reach your service, but it responds with errors.", + "", + "It’s important to check your callback service is running, check your callback service’s logs for errors and repair any errors in your logs. To find your callback configuration, sign into your account, visit the API integration page for “((service_name))” and select callbacks.", + "", + "Once you’ve taken these steps, request confirmation that your callbacks are working again by [contacting us](((contact_url))). For more information, you can also access our [API documentation on callbacks](((callback_docs_url))).", + "", + "The GC Notify team", + "[[/en]]", + "", + "---", + "", + "[[fr]]", + "Bonjour ((name)),", + "", + "Les rappels pour « ((service_name)) » ne fonctionnent pas. Cela pourrait signifier que :" "", + "(1) Votre service de rappel est hors service.", + "(2) Votre service utilise un proxy auquel nous ne pouvons pas accéder.", + "(3) Nous parvenons à joindre votre service, mais il répond avec des erreurs.", + "", + "Il est important de vérifier que votre service de rappel fonctionne, de vérifier les journaux de votre service de rappel pour détecter des erreurs et de corriger toute erreur dans vos journaux. Pour trouver votre configuration de rappel, connectez-vous à votre compte, visitez la page d’intégration API pour « ((service_name)) » et sélectionnez rappels.", + "", + "Une fois ces étapes effectuées, demandez une confirmation que vos rappels fonctionnent à nouveau en nous contactant. Pour plus d’informations, vous pouvez également consulter notre documentation API sur les rappels.", + "", + "L’équipe GC Notify", + "[[/fr]]", + ] + ) + + templates = [ + { + "id": callback_failure_template_id, + "name": "Callback failures EMAIL", + "subject": "Your callbacks are not working | Vos rappels ne fonctionnent pas", + "content": callback_failure_content, + } + ] + + for template in templates: + op.execute( + sqltext=template_insert.format( + template["id"], + template["name"], + "email", + datetime.utcnow(), + template["content"], + current_app.config["NOTIFY_SERVICE_ID"], + template["subject"], + current_app.config["NOTIFY_USER_ID"], + "normal", + ) + ) + + op.execute( + template_history_insert.format( + template["id"], + template["name"], + "email", + datetime.utcnow(), + template["content"], + current_app.config["NOTIFY_SERVICE_ID"], + template["subject"], + current_app.config["NOTIFY_USER_ID"], + "normal", + ) + ) + + +def downgrade(): + op.execute("DELETE FROM notifications WHERE template_id = '{}'".format(callback_failure_template_id)) + op.execute("DELETE FROM notification_history WHERE template_id = '{}'".format(callback_failure_template_id)) + op.execute("DELETE FROM template_redacted WHERE template_id = '{}'".format(callback_failure_template_id)) + op.execute("DELETE FROM templates_history WHERE id = '{}'".format(callback_failure_template_id)) + op.execute("DELETE FROM templates WHERE id = '{}'".format(callback_failure_template_id))