Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add callback failure warning email #2190

Closed
wants to merge 12 commits into from
68 changes: 67 additions & 1 deletion app/celery/service_callback_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@

from app import notify_celery, signer_complaint, signer_delivery_status
from app.config import QueueNames
from app.models import Service

# Uncomment when we implement email sending for callback failures
# from requests.exceptions import InvalidURL, Timeout


@notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
Expand Down Expand Up @@ -80,11 +84,73 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
current_app.logger.warning(
f"{function_name} request failed for notification_id: {notification_id} and url: {service_callback_url}. exc: {e}"
)

# TODO: Instate once we monitor alarms to determine how often this happens and we implement
# check_cloudwatch_for_callback_failures(), otherwise we risk flooding the service
# owner's inbox with callback failure email notifications.

# if isinstance(e, Timeout) or isinstance(e, InvalidURL) or e.response.status_code == 500:
# if check_cloudwatch_for_callback_failures():
# send_email_callback_failure_email(current_app.service)

# Retry if the response status code is server-side or 429 (too many requests).
if not isinstance(e, HTTPError) or e.response.status_code >= 500 or e.response.status_code == 429:
try:
self.retry(queue=QueueNames.CALLBACKS_RETRY)
except self.MaxRetriesExceededError:
current_app.logger.warning(
"Retry: {function_name} has retried the max num of times for callback url {service_callback_url} and notification_id: {notification_id}"
f"Retry: {function_name} has retried the max num of times for callback url {service_callback_url} and notification_id: {notification_id} for service: {current_app.service.id}"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For the limits feature, we use an expiring redis key to ensure only one email is sent within a particular time period. For example, if they reach the threshold, we send the email then set this key in redis to true. Then if they hit the threshold again within the time period the redis key will still be true and we will check it before sending another email.

)


def send_email_callback_failure_email(service: Service):
service.send_notification_to_service_users(
service_id=service.id,
template_id=current_app.config["CALLBACK_FAILURE_TEMPLATE_ID"],
personalisation={
"service_name": service.name,
"contact_url": f"{current_app.config['ADMIN_BASE_URL']}/contact",
"callback_doc_url": f"{current_app.config['DOCUMENTATION_DOAMIN']}/en/callbacks.html",
},
include_user_fields=["name"],
)


def check_cloudwatch_for_callback_failures():
"""
TODO: Use boto3 to check cloudwatch for callback failures
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/logs/client/start_query.html

Check if a service has failed 5 callbacks in a 30 minute time period

----------------

import boto3
from datetime import datetime, timedelta
import time

client = boto3.client('logs')

query = "TODO"

log_group = 'TODO'

start_query_response = client.start_query(
logGroupName=log_group,
startTime=int((datetime.today() - timedelta(minutes=30)).timestamp()),
endTime=int(datetime.now().timestamp()),
queryString=query,
)

query_id = start_query_response['queryId']

response = None

while response == None or response['status'] == 'Running':
print('Waiting for query to complete ...')
time.sleep(1)
response = client.get_query_results(
queryId=query_id
)

"""
65 changes: 33 additions & 32 deletions app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,47 +299,48 @@ class Config(object):
CHECK_PROXY_HEADER = False

# Notify's notifications templates
NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553"
NOTIFY_USER_ID = "6af522d0-2915-4e52-83a3-3690455a5fe6"
INVITATION_EMAIL_TEMPLATE_ID = "4f46df42-f795-4cc4-83bb-65ca312f49cc"
SMS_CODE_TEMPLATE_ID = "36fb0730-6259-4da1-8a80-c8de22ad4246"
EMAIL_2FA_TEMPLATE_ID = "299726d2-dba6-42b8-8209-30e1d66ea164"
EMAIL_MAGIC_LINK_TEMPLATE_ID = "6e97fd09-6da0-4cc8-829d-33cf5b818103"
NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = "ece42649-22a8-4d06-b87f-d52d5d3f0a27"
PASSWORD_RESET_TEMPLATE_ID = "474e9242-823b-4f99-813d-ed392e7f1201"
FORCED_PASSWORD_RESET_TEMPLATE_ID = "e9a65a6b-497b-42f2-8f43-1736e43e13b3"
ACCOUNT_CHANGE_TEMPLATE_ID = "5b39e16a-9ff8-487c-9bfb-9e06bdb70f36"
ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = "0880fbb1-a0c6-46f0-9a8e-36c986381ceb"
APIKEY_REVOKE_TEMPLATE_ID = "a0a4e7b8-8a6a-4eaa-9f4e-9c3a5b2dbcf3"
BRANDING_REQUEST_TEMPLATE_ID = "7d423d9e-e94e-4118-879d-d52f383206ae"
CALLBACK_FAILURE_TEMPLATE_ID = "d8d580f4-86b3-4ba4-9d7c-263a630af354"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Amongst the rearranging, this env var is the one I added.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Easier to read, thank you 🔤

CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = "eb4d9930-87ab-4aef-9bce-786762687884"
SERVICE_NOW_LIVE_TEMPLATE_ID = "618185c6-3636-49cd-b7d2-6f6f5eb3bdde"
ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = "203566f0-d835-47c5-aa06-932439c86573"
TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = "c73f1d71-4049-46d5-a647-d013bdeca3f0"
TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = "8a31520f-4751-4789-8ea1-fe54496725eb"
REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = "a42f1d17-9404-46d5-a647-d013bdfca3e1"
MOU_SIGNER_RECEIPT_TEMPLATE_ID = "4fd2e43c-309b-4e50-8fb8-1955852d9d71"
MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = "c20206d5-bf03-4002-9a90-37d5032d9e84"
MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = "522b6657-5ca5-4368-a294-6b527703bd0b"
MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = "d0e66c4c-0c50-43f0-94f5-f85b613202d4"
CONTACT_FORM_DIRECT_EMAIL_TEMPLATE_ID = "b04beb4a-8408-4280-9a5c-6a046b6f7704"
CONTACT_US_TEMPLATE_ID = "8ea9b7a0-a824-4dd3-a4c3-1f508ed20a69"
ACCOUNT_CHANGE_TEMPLATE_ID = "5b39e16a-9ff8-487c-9bfb-9e06bdb70f36"
BRANDING_REQUEST_TEMPLATE_ID = "7d423d9e-e94e-4118-879d-d52f383206ae"
NO_REPLY_TEMPLATE_ID = "86950840-6da4-4865-841b-16028110e980"
NEAR_DAILY_LIMIT_TEMPLATE_ID = "5d3e4322-4ee6-457a-a710-c48755f6b643"
REACHED_DAILY_LIMIT_TEMPLATE_ID = "fd29f796-fcdc-471b-a0d4-0093880d9173"
DAILY_EMAIL_LIMIT_UPDATED_TEMPLATE_ID = "97dade64-ea8d-460f-8a34-900b74ee5eb0"
DAILY_LIMIT_UPDATED_TEMPLATE_ID = "b3c766e6-be32-4edf-b8db-0f04ef404edc"
NEAR_DAILY_SMS_LIMIT_TEMPLATE_ID = "a796568f-a89b-468e-b635-8105554301b9"
REACHED_DAILY_SMS_LIMIT_TEMPLATE_ID = "a646e614-c527-4f94-a955-ed7185d577f4"
DAILY_SMS_LIMIT_UPDATED_TEMPLATE_ID = "6ec12dd0-680a-4073-8d58-91d17cc8442f"
CONTACT_FORM_DIRECT_EMAIL_TEMPLATE_ID = "b04beb4a-8408-4280-9a5c-6a046b6f7704"
NEAR_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "9aa60ad7-2d7f-46f0-8cbe-2bac3d4d77d8"
REACHED_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "ee036547-e51b-49f1-862b-10ea982cfceb"
DAILY_EMAIL_LIMIT_UPDATED_TEMPLATE_ID = "97dade64-ea8d-460f-8a34-900b74ee5eb0"
APIKEY_REVOKE_TEMPLATE_ID = "a0a4e7b8-8a6a-4eaa-9f4e-9c3a5b2dbcf3"
EMAIL_2FA_TEMPLATE_ID = "299726d2-dba6-42b8-8209-30e1d66ea164"
EMAIL_MAGIC_LINK_TEMPLATE_ID = "6e97fd09-6da0-4cc8-829d-33cf5b818103"
FORCED_PASSWORD_RESET_TEMPLATE_ID = "e9a65a6b-497b-42f2-8f43-1736e43e13b3"
HEARTBEAT_TEMPLATE_EMAIL_HIGH = "276da251-3103-49f3-9054-cbf6b5d74411"
HEARTBEAT_TEMPLATE_EMAIL_LOW = "73079cb9-c169-44ea-8cf4-8d397711cc9d"
HEARTBEAT_TEMPLATE_EMAIL_MEDIUM = "c75c4539-3014-4c4c-96b5-94d326758a74"
HEARTBEAT_TEMPLATE_EMAIL_HIGH = "276da251-3103-49f3-9054-cbf6b5d74411"
HEARTBEAT_TEMPLATE_SMS_HIGH = "4969a9e9-ddfd-476e-8b93-6231e6f1be4a"

Check warning

Code scanning / CodeQL

Variable defined multiple times Warning

This assignment to 'HEARTBEAT_TEMPLATE_SMS_HIGH' is unnecessary as it is
redefined
before this value is used.
HEARTBEAT_TEMPLATE_SMS_LOW = "ab3a603b-d602-46ea-8c83-e05cb280b950"
HEARTBEAT_TEMPLATE_SMS_MEDIUM = "a48b54ce-40f6-4e4a-abe8-1e2fa389455b"
HEARTBEAT_TEMPLATE_SMS_HIGH = "4969a9e9-ddfd-476e-8b93-6231e6f1be4a"
INVITATION_EMAIL_TEMPLATE_ID = "4f46df42-f795-4cc4-83bb-65ca312f49cc"
MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = "d0e66c4c-0c50-43f0-94f5-f85b613202d4"
MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = "522b6657-5ca5-4368-a294-6b527703bd0b"
MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = "c20206d5-bf03-4002-9a90-37d5032d9e84"
MOU_SIGNER_RECEIPT_TEMPLATE_ID = "4fd2e43c-309b-4e50-8fb8-1955852d9d71"
NEAR_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "9aa60ad7-2d7f-46f0-8cbe-2bac3d4d77d8"
NEAR_DAILY_LIMIT_TEMPLATE_ID = "5d3e4322-4ee6-457a-a710-c48755f6b643"
NEAR_DAILY_SMS_LIMIT_TEMPLATE_ID = "a796568f-a89b-468e-b635-8105554301b9"
NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = "ece42649-22a8-4d06-b87f-d52d5d3f0a27"
NO_REPLY_TEMPLATE_ID = "86950840-6da4-4865-841b-16028110e980"
NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553"
NOTIFY_USER_ID = "6af522d0-2915-4e52-83a3-3690455a5fe6"
ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = "203566f0-d835-47c5-aa06-932439c86573"
PASSWORD_RESET_TEMPLATE_ID = "474e9242-823b-4f99-813d-ed392e7f1201"
REACHED_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "ee036547-e51b-49f1-862b-10ea982cfceb"
REACHED_DAILY_LIMIT_TEMPLATE_ID = "fd29f796-fcdc-471b-a0d4-0093880d9173"
REACHED_DAILY_SMS_LIMIT_TEMPLATE_ID = "a646e614-c527-4f94-a955-ed7185d577f4"
REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = "a42f1d17-9404-46d5-a647-d013bdfca3e1"
SERVICE_NOW_LIVE_TEMPLATE_ID = "618185c6-3636-49cd-b7d2-6f6f5eb3bdde"
SMS_CODE_TEMPLATE_ID = "36fb0730-6259-4da1-8a80-c8de22ad4246"
TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = "c73f1d71-4049-46d5-a647-d013bdeca3f0"
TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = "8a31520f-4751-4789-8ea1-fe54496725eb"

# Allowed service IDs able to send HTML through their templates.
ALLOW_HTML_SERVICE_IDS: List[str] = [id.strip() for id in os.getenv("ALLOW_HTML_SERVICE_IDS", "").split(",")]
Expand Down
114 changes: 114 additions & 0 deletions migrations/versions/0453_add_callback_failure_email.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
"""
Revision ID: 0453_add_callback_failure_email
Revises: 0452_set_pgaudit_config
Create Date: 2024-06-04 16:09:00
"""
from datetime import datetime

from alembic import op
from flask import current_app

revision = "0453_add_callback_failure_email"
down_revision = "0452_set_pgaudit_config"

callback_failure_template_id = current_app.config["CALLBACK_FAILURE_TEMPLATE_ID"]


def upgrade():
template_insert = """
INSERT INTO templates (id, name, template_type, created_at, content, archived, service_id, subject,
created_by_id, version, process_type, hidden)
VALUES('{}', '{}', '{}', '{}', '{}', False, '{}', '{}', '{}', 1, '{}', false)
"""
template_history_insert = """
INSERT INTO templates_history (id, name, template_type, created_at, content, archived, service_id, subject,
created_by_id, version, process_type, hidden)
VALUES ('{}', '{}', '{}', '{}', '{}', False, '{}', '{}', '{}', 1, '{}', false)
"""

callback_failure_content = "\n".join(
[
"[[fr]]",
"(la version française suit)",
"[[/fr]]",
"",
"[[en]]",
"Hello ((name)),",
"",
"The callbacks for “((service_name))” are not working. This could mean that:",
"",
"(1) Your callback service is down.",
"(2) Your service is using a proxy that we cannot access.",
"(3) We’re able to reach your service, but it responds with errors.",
"",
"It’s important to check your callback service is running, check your callback service’s logs for errors and repair any errors in your logs. To find your callback configuration, sign into your account, visit the API integration page for “((service_name))” and select callbacks.",
"",
"Once you’ve taken these steps, request confirmation that your callbacks are working again by [contacting us](((contact_url))). For more information, you can also access our [API documentation on callbacks](((callback_docs_url))).",
"",
"The GC Notify team",
"[[/en]]",
"",
"---",
"",
"[[fr]]",
"Bonjour ((name)),",
"",
"Les rappels pour « ((service_name)) » ne fonctionnent pas. Cela pourrait signifier que :" "",
whabanks marked this conversation as resolved.
Show resolved Hide resolved
"(1) Votre service de rappel est hors service.",
"(2) Votre service utilise un proxy auquel nous ne pouvons pas accéder.",
"(3) Nous parvenons à joindre votre service, mais il répond avec des erreurs.",
"",
"Il est important de vérifier que votre service de rappel fonctionne, de vérifier les journaux de votre service de rappel pour détecter des erreurs et de corriger toute erreur dans vos journaux. Pour trouver votre configuration de rappel, connectez-vous à votre compte, visitez la page d’intégration API pour « ((service_name)) » et sélectionnez rappels.",
"",
"Une fois ces étapes effectuées, demandez une confirmation que vos rappels fonctionnent à nouveau en nous contactant. Pour plus d’informations, vous pouvez également consulter notre documentation API sur les rappels.",
"",
"L’équipe GC Notify",
"[[/fr]]",
]
)

templates = [
{
"id": callback_failure_template_id,
"name": "Callback failures EMAIL",
"subject": "Your callbacks are not working | Vos rappels ne fonctionnent pas",
"content": callback_failure_content,
}
]

for template in templates:
op.execute(
sqltext=template_insert.format(
template["id"],
template["name"],
"email",
datetime.utcnow(),
template["content"],
current_app.config["NOTIFY_SERVICE_ID"],
template["subject"],
current_app.config["NOTIFY_USER_ID"],
"normal",
)
)

op.execute(
template_history_insert.format(
template["id"],
template["name"],
"email",
datetime.utcnow(),
template["content"],
current_app.config["NOTIFY_SERVICE_ID"],
template["subject"],
current_app.config["NOTIFY_USER_ID"],
"normal",
)
)


def downgrade():
op.execute("DELETE FROM notifications WHERE template_id = '{}'".format(callback_failure_template_id))
op.execute("DELETE FROM notification_history WHERE template_id = '{}'".format(callback_failure_template_id))
op.execute("DELETE FROM template_redacted WHERE template_id = '{}'".format(callback_failure_template_id))
op.execute("DELETE FROM templates_history WHERE id = '{}'".format(callback_failure_template_id))
op.execute("DELETE FROM templates WHERE id = '{}'".format(callback_failure_template_id))
Loading