Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add callback failure warning email #2190

Closed
wants to merge 12 commits into from
2 changes: 1 addition & 1 deletion app/celery/service_callback_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,5 +86,5 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token,
self.retry(queue=QueueNames.CALLBACKS_RETRY)
except self.MaxRetriesExceededError:
current_app.logger.warning(
"Retry: {function_name} has retried the max num of times for callback url {service_callback_url} and notification_id: {notification_id}"
f"Retry: {function_name} has retried the max num of times for callback url {service_callback_url} and notification_id: {notification_id}"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Turns out we have not been properly capturing callback failure logs for a while now.

)
68 changes: 36 additions & 32 deletions app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,47 +299,48 @@ class Config(object):
CHECK_PROXY_HEADER = False

# Notify's notifications templates
NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553"
NOTIFY_USER_ID = "6af522d0-2915-4e52-83a3-3690455a5fe6"
INVITATION_EMAIL_TEMPLATE_ID = "4f46df42-f795-4cc4-83bb-65ca312f49cc"
SMS_CODE_TEMPLATE_ID = "36fb0730-6259-4da1-8a80-c8de22ad4246"
EMAIL_2FA_TEMPLATE_ID = "299726d2-dba6-42b8-8209-30e1d66ea164"
EMAIL_MAGIC_LINK_TEMPLATE_ID = "6e97fd09-6da0-4cc8-829d-33cf5b818103"
NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = "ece42649-22a8-4d06-b87f-d52d5d3f0a27"
PASSWORD_RESET_TEMPLATE_ID = "474e9242-823b-4f99-813d-ed392e7f1201"
FORCED_PASSWORD_RESET_TEMPLATE_ID = "e9a65a6b-497b-42f2-8f43-1736e43e13b3"
ACCOUNT_CHANGE_TEMPLATE_ID = "5b39e16a-9ff8-487c-9bfb-9e06bdb70f36"
ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = "0880fbb1-a0c6-46f0-9a8e-36c986381ceb"
APIKEY_REVOKE_TEMPLATE_ID = "a0a4e7b8-8a6a-4eaa-9f4e-9c3a5b2dbcf3"
BRANDING_REQUEST_TEMPLATE_ID = "7d423d9e-e94e-4118-879d-d52f383206ae"
CALLBACK_FAILURE_TEMPLATE_ID = "d8d580f4-86b3-4ba4-9d7c-263a630af354"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Amongst the rearranging, this env var is the one I added.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Easier to read, thank you 🔤

CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = "eb4d9930-87ab-4aef-9bce-786762687884"
SERVICE_NOW_LIVE_TEMPLATE_ID = "618185c6-3636-49cd-b7d2-6f6f5eb3bdde"
ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = "203566f0-d835-47c5-aa06-932439c86573"
TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = "c73f1d71-4049-46d5-a647-d013bdeca3f0"
TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = "8a31520f-4751-4789-8ea1-fe54496725eb"
REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = "a42f1d17-9404-46d5-a647-d013bdfca3e1"
MOU_SIGNER_RECEIPT_TEMPLATE_ID = "4fd2e43c-309b-4e50-8fb8-1955852d9d71"
MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = "c20206d5-bf03-4002-9a90-37d5032d9e84"
MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = "522b6657-5ca5-4368-a294-6b527703bd0b"
MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = "d0e66c4c-0c50-43f0-94f5-f85b613202d4"
CONTACT_FORM_DIRECT_EMAIL_TEMPLATE_ID = "b04beb4a-8408-4280-9a5c-6a046b6f7704"
CONTACT_US_TEMPLATE_ID = "8ea9b7a0-a824-4dd3-a4c3-1f508ed20a69"
ACCOUNT_CHANGE_TEMPLATE_ID = "5b39e16a-9ff8-487c-9bfb-9e06bdb70f36"
BRANDING_REQUEST_TEMPLATE_ID = "7d423d9e-e94e-4118-879d-d52f383206ae"
NO_REPLY_TEMPLATE_ID = "86950840-6da4-4865-841b-16028110e980"
NEAR_DAILY_LIMIT_TEMPLATE_ID = "5d3e4322-4ee6-457a-a710-c48755f6b643"
REACHED_DAILY_LIMIT_TEMPLATE_ID = "fd29f796-fcdc-471b-a0d4-0093880d9173"
DAILY_EMAIL_LIMIT_UPDATED_TEMPLATE_ID = "97dade64-ea8d-460f-8a34-900b74ee5eb0"
DAILY_LIMIT_UPDATED_TEMPLATE_ID = "b3c766e6-be32-4edf-b8db-0f04ef404edc"
NEAR_DAILY_SMS_LIMIT_TEMPLATE_ID = "a796568f-a89b-468e-b635-8105554301b9"
REACHED_DAILY_SMS_LIMIT_TEMPLATE_ID = "a646e614-c527-4f94-a955-ed7185d577f4"
DAILY_SMS_LIMIT_UPDATED_TEMPLATE_ID = "6ec12dd0-680a-4073-8d58-91d17cc8442f"
CONTACT_FORM_DIRECT_EMAIL_TEMPLATE_ID = "b04beb4a-8408-4280-9a5c-6a046b6f7704"
NEAR_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "9aa60ad7-2d7f-46f0-8cbe-2bac3d4d77d8"
REACHED_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "ee036547-e51b-49f1-862b-10ea982cfceb"
DAILY_EMAIL_LIMIT_UPDATED_TEMPLATE_ID = "97dade64-ea8d-460f-8a34-900b74ee5eb0"
APIKEY_REVOKE_TEMPLATE_ID = "a0a4e7b8-8a6a-4eaa-9f4e-9c3a5b2dbcf3"
EMAIL_2FA_TEMPLATE_ID = "299726d2-dba6-42b8-8209-30e1d66ea164"
EMAIL_MAGIC_LINK_TEMPLATE_ID = "6e97fd09-6da0-4cc8-829d-33cf5b818103"
FORCED_PASSWORD_RESET_TEMPLATE_ID = "e9a65a6b-497b-42f2-8f43-1736e43e13b3"
HEARTBEAT_TEMPLATE_EMAIL_HIGH = "276da251-3103-49f3-9054-cbf6b5d74411"
HEARTBEAT_TEMPLATE_EMAIL_LOW = "73079cb9-c169-44ea-8cf4-8d397711cc9d"
HEARTBEAT_TEMPLATE_EMAIL_MEDIUM = "c75c4539-3014-4c4c-96b5-94d326758a74"
HEARTBEAT_TEMPLATE_EMAIL_HIGH = "276da251-3103-49f3-9054-cbf6b5d74411"
HEARTBEAT_TEMPLATE_SMS_HIGH = "4969a9e9-ddfd-476e-8b93-6231e6f1be4a"

Check warning

Code scanning / CodeQL

Variable defined multiple times Warning

This assignment to 'HEARTBEAT_TEMPLATE_SMS_HIGH' is unnecessary as it is
redefined
before this value is used.
HEARTBEAT_TEMPLATE_SMS_LOW = "ab3a603b-d602-46ea-8c83-e05cb280b950"
HEARTBEAT_TEMPLATE_SMS_MEDIUM = "a48b54ce-40f6-4e4a-abe8-1e2fa389455b"
HEARTBEAT_TEMPLATE_SMS_HIGH = "4969a9e9-ddfd-476e-8b93-6231e6f1be4a"
INVITATION_EMAIL_TEMPLATE_ID = "4f46df42-f795-4cc4-83bb-65ca312f49cc"
MOU_NOTIFY_TEAM_ALERT_TEMPLATE_ID = "d0e66c4c-0c50-43f0-94f5-f85b613202d4"
MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = "522b6657-5ca5-4368-a294-6b527703bd0b"
MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = "c20206d5-bf03-4002-9a90-37d5032d9e84"
MOU_SIGNER_RECEIPT_TEMPLATE_ID = "4fd2e43c-309b-4e50-8fb8-1955852d9d71"
NEAR_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "9aa60ad7-2d7f-46f0-8cbe-2bac3d4d77d8"
NEAR_DAILY_LIMIT_TEMPLATE_ID = "5d3e4322-4ee6-457a-a710-c48755f6b643"
NEAR_DAILY_SMS_LIMIT_TEMPLATE_ID = "a796568f-a89b-468e-b635-8105554301b9"
NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = "ece42649-22a8-4d06-b87f-d52d5d3f0a27"
NO_REPLY_TEMPLATE_ID = "86950840-6da4-4865-841b-16028110e980"
NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553"
NOTIFY_USER_ID = "6af522d0-2915-4e52-83a3-3690455a5fe6"
ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = "203566f0-d835-47c5-aa06-932439c86573"
PASSWORD_RESET_TEMPLATE_ID = "474e9242-823b-4f99-813d-ed392e7f1201"
REACHED_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "ee036547-e51b-49f1-862b-10ea982cfceb"
REACHED_DAILY_LIMIT_TEMPLATE_ID = "fd29f796-fcdc-471b-a0d4-0093880d9173"
REACHED_DAILY_SMS_LIMIT_TEMPLATE_ID = "a646e614-c527-4f94-a955-ed7185d577f4"
REPLY_TO_EMAIL_ADDRESS_VERIFICATION_TEMPLATE_ID = "a42f1d17-9404-46d5-a647-d013bdfca3e1"
SERVICE_NOW_LIVE_TEMPLATE_ID = "618185c6-3636-49cd-b7d2-6f6f5eb3bdde"
SMS_CODE_TEMPLATE_ID = "36fb0730-6259-4da1-8a80-c8de22ad4246"
TEAM_MEMBER_EDIT_EMAIL_TEMPLATE_ID = "c73f1d71-4049-46d5-a647-d013bdeca3f0"
TEAM_MEMBER_EDIT_MOBILE_TEMPLATE_ID = "8a31520f-4751-4789-8ea1-fe54496725eb"

# Allowed service IDs able to send HTML through their templates.
ALLOW_HTML_SERVICE_IDS: List[str] = [id.strip() for id in os.getenv("ALLOW_HTML_SERVICE_IDS", "").split(",")]
Expand Down Expand Up @@ -564,6 +565,9 @@ class Config(object):
SRE_USER_NAME = "SRE_CLIENT_USER"
SRE_CLIENT_SECRET = os.getenv("SRE_CLIENT_SECRET")

# TODO: Adjust when actual threshold is defined
CALLBACK_FAILURE_THRESHOLD_PERCENTAGE = 0.5

@classmethod
def get_sensitive_config(cls) -> list[str]:
"List of config keys that contain sensitive information"
Expand Down
108 changes: 108 additions & 0 deletions migrations/versions/0453_add_callback_failure_email.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
"""
Revision ID: 0453_add_callback_failure_email
Revises: 0452_set_pgaudit_config
Create Date: 2024-06-04 16:09:00
"""
from datetime import datetime

from alembic import op
from flask import current_app

revision = "0453_add_callback_failure_email"
down_revision = "0452_set_pgaudit_config"

callback_failure_template_id = current_app.config["CALLBACK_FAILURE_TEMPLATE_ID"]

def upgrade():
template_insert = """
INSERT INTO templates (id, name, template_type, created_at, content, archived, service_id, subject,
created_by_id, version, process_type, hidden)
VALUES('{}', '{}', '{}', '{}', '{}', False, '{}', '{}', '{}', 1, '{}', false)
"""
template_history_insert = """
INSERT INTO templates_history (id, name, template_type, created_at, content, archived, service_id, subject,
created_by_id, version, process_type, hidden)
VALUES ('{}', '{}', '{}', '{}', '{}', False, '{}', '{}', '{}', 1, '{}', false)
"""

callback_failure_content = "\n".join(
[
"[[fr]]",
"(la version française suit)",
"[[/fr]]",
"",
"[[en]]",
"Hello ((name)),",
"",
"The callbacks for “((service_name))” are not working. This could mean that:",
"",
"(1) Your callback service is down.",
"(2) Your service is using a proxy that we cannot access.",
"(3) We’re able to reach your service, but it responds with errors.",
"",
"It’s important to check your callback service is running, check your callback service’s logs for errors and repair any errors in your logs. Once you’ve taken these steps, request confirmation that your callbacks are working again by contacting us.",
"",
"The GC Notify team",
"[[/en]]",
"",
"---",
"",
"[[fr]]",
"Bonjour ((name)),",
"",
"Les rappels pour « ((service_name)) » ne fonctionnent pas. Cela pourrait signifier que :"
"",
Fixed Show fixed Hide fixed
Fixed Show fixed Hide fixed
"Votre service de rappel est hors service.",
"Votre service utilise un proxy auquel nous ne pouvons pas accéder.",
"Nous parvenons à joindre votre service, mais il répond avec des erreurs.",
"Il est important de vérifier que votre service de rappel fonctionne, de consulter les journaux de votre service de rappel pour y détecter des erreurs et de corriger les erreurs dans vos journaux. Une fois ces étapes effectuées, veuillez demander une confirmation que vos rappels fonctionnent à nouveau en nous contactant.",
"",
"L’équipe GC Notify",
"[[/fr]]",
]
)

templates = [
{
"id": callback_failure_template_id,
"name": "Callback failures EMAIL",
"subject": "Your callbacks are not working | Vos rappels ne fonctionnent pas",
"content": callback_failure_content,
}
]

for template in templates:
op.execute(
sqltext=template_insert.format(
template["id"],
template["name"],
"email",
datetime.utcnow(),
template["content"],
current_app.config["NOTIFY_SERVICE_ID"],
template["subject"],
current_app.config["NOTIFY_USER_ID"],
"normal",
)
)

op.execute(
template_history_insert.format(
template["id"],
template["name"],
"email",
datetime.utcnow(),
template["content"],
current_app.config["NOTIFY_SERVICE_ID"],
template["subject"],
current_app.config["NOTIFY_USER_ID"],
"normal",
)
)

def downgrade():
op.execute("DELETE FROM notifications WHERE template_id = '{}'".format(callback_failure_template_id))
op.execute("DELETE FROM notification_history WHERE template_id = '{}'".format(callback_failure_template_id))
op.execute("DELETE FROM template_redacted WHERE template_id = '{}'".format(callback_failure_template_id))
op.execute("DELETE FROM templates_history WHERE id = '{}'".format(callback_failure_template_id))
op.execute("DELETE FROM templates WHERE id = '{}'".format(callback_failure_template_id))
Loading