Overriding retry policy for SMS high priority

app/delivery/send_to_providers.py

@@ -60,6 +60,7 @@ def send_sms_to_provider(notification):
         inactive_service_failure(notification=notification)
         return
 
+    # If the notification was not sent already, the status should be created.
     if notification.status == "created":
         provider = provider_to_use(
             SMS_TYPE,
@@ -215,6 +216,8 @@ def send_email_to_provider(notification: Notification):
     if not service.active:
         inactive_service_failure(notification=notification)
         return
+
+    # If the notification was not sent already, the status should be created.
     if notification.status == "created":
         provider = provider_to_use(EMAIL_TYPE, notification.id)
 

app/notifications/__init__.py

@@ -0,0 +1,26 @@
+from app.models import BULK, NORMAL, PRIORITY
+
+# Default retry policy for all notifications.
+RETRY_POLICY_DEFAULT = {
+    "max_retries": 48,
+    "interval_start": 300,
+    "interval_step": 300,
+    "interval_max": 300,
+    "retry_errors": None,
+}
+
+# Retry policy for high priority notifications.
+RETRY_POLICY_HIGH = {
+    "max_retries": 48,
+    "interval_start": 25,
+    "interval_step": 25,
+    "interval_max": 25,
+    "retry_errors": None,
+}
+
+# Retry policies for each notification priority lanes.
+RETRY_POLICIES = {
+    BULK: RETRY_POLICY_DEFAULT,
+    NORMAL: RETRY_POLICY_DEFAULT,
+    PRIORITY: RETRY_POLICY_HIGH,
+}

app/notifications/process_notifications.py

@@ -1,6 +1,6 @@
 import uuid
 from datetime import datetime
-from typing import List
+from typing import Any, Dict, List
 
 from flask import current_app
 from notifications_utils.clients import redis
@@ -35,6 +35,7 @@
     ScheduledNotification,
     Service,
 )
+from app.notifications import RETRY_POLICIES, RETRY_POLICY_DEFAULT
 from app.types import VerifiedNotification
 from app.utils import get_delivery_queue_for_template, get_template_instance
 from app.v2.errors import BadRequestError
@@ -210,7 +211,11 @@ def db_save_and_send_notification(notification: Notification):
 
     deliver_task = choose_deliver_task(notification)
     try:
-        deliver_task.apply_async([str(notification.id)], queue=notification.queue_name)
+        deliver_task.apply_async(
+            [str(notification.id)],
+            queue=notification.queue_name,
+            **build_delivery_task_params(notification.notification_type, notification.template.process_type),
+        )
     except Exception:
         dao_delete_notifications_by_id(notification.id)
         raise
@@ -219,6 +224,24 @@ def db_save_and_send_notification(notification: Notification):
     )
 
 
+def build_delivery_task_params(notification_type: str, notification_process_type: str) -> Dict[str, Any]:
+    """
+    Build task params for the sending parameter tasks.
+
+    If the notification is a high priority SMS, set the retry policy to retry every 25 seconds
+    else fall back to the default retry policy of retrying every 5 minutes.
+    """
+    params: dict[str, Any] = {}
+    params["retry"] = True
+    # Overring the retry policy is only supported for SMS for now;
+    # email support coming later.
+    if notification_type == SMS_TYPE:
+        params["retry_policy"] = RETRY_POLICIES[notification_process_type]
+    else:
+        params["retry_policy"] = RETRY_POLICY_DEFAULT
+    return params
+
+
 def choose_queue(notification, research_mode, queue=None) -> QueueNames:
     if research_mode or notification.key_type == KEY_TYPE_TEST:
         queue = QueueNames.RESEARCH_MODE

tests/app/notifications/test_process_notification.py

@@ -1,5 +1,6 @@
 import datetime
 import uuid
+from typing import Any, Dict
 from unittest.mock import call
 
 import pytest
@@ -14,13 +15,20 @@
 from app.config import QueueNames
 from app.dao.service_sms_sender_dao import dao_update_service_sms_sender
 from app.models import (
+    BULK,
+    EMAIL_TYPE,
     LETTER_TYPE,
+    NORMAL,
+    PRIORITY,
+    SMS_TYPE,
     Notification,
     NotificationHistory,
     ScheduledNotification,
     Template,
 )
+from app.notifications import RETRY_POLICY_DEFAULT
 from app.notifications.process_notifications import (
+    build_delivery_task_params,
     choose_queue,
     create_content_for_notification,
     db_save_and_send_notification,
@@ -1051,7 +1059,9 @@ def test_db_save_and_send_notification_sends_to_queue(
 
         db_save_and_send_notification(notification=notification)
 
-        mocked.assert_called_once_with([str(notification.id)], queue=expected_queue)
+        mocked.assert_called_once_with(
+            [str(notification.id)], queue=expected_queue, retry=True, retry_policy=RETRY_POLICY_DEFAULT
+        )
 
     def test_db_save_and_send_notification_throws_exception_deletes_notification(
         self, sample_template, sample_api_key, sample_job, mocker
@@ -1080,11 +1090,34 @@ def test_db_save_and_send_notification_throws_exception_deletes_notification(
 
         with pytest.raises(Boto3Error):
             db_save_and_send_notification(notification)
-        mocked.assert_called_once_with([(str(notification.id))], queue=QueueNames.SEND_SMS_MEDIUM)
+        mocked.assert_called_once_with(
+            [(str(notification.id))], queue=QueueNames.SEND_SMS_MEDIUM, retry=True, retry_policy=RETRY_POLICY_DEFAULT
+        )
 
         assert Notification.query.count() == 0
         assert NotificationHistory.query.count() == 0
 
+    @pytest.mark.parametrize(
+        ("notification_type, process_type, expected_retry, expected_retry_period"),
+        [
+            (EMAIL_TYPE, BULK, 48, 300),
+            (EMAIL_TYPE, NORMAL, 48, 300),
+            (EMAIL_TYPE, PRIORITY, 48, 300),
+            (SMS_TYPE, BULK, 48, 300),
+            (SMS_TYPE, NORMAL, 48, 300),
+            (SMS_TYPE, PRIORITY, 48, 26),
+        ],
+    )
+    def test_delivery_task_parameters(self, notification_type, process_type, expected_retry, expected_retry_period):
+        params: Dict[str, Any] = build_delivery_task_params(notification_type, process_type)
+        assert params["retry"] is True
+
+        retry_policy: Dict[str, Any] = params["retry_policy"]
+        assert retry_policy["max_retries"] == expected_retry
+        assert retry_policy["interval_start"] == expected_retry_period
+        assert retry_policy["interval_step"] == expected_retry_period
+        assert retry_policy["interval_max"] == expected_retry_period
+
     def test_db_save_and_send_notification_throws_exception_when_missing_template(self, sample_api_key, mocker):
         mocker.patch("app.celery.provider_tasks.deliver_sms.apply_async")
         assert Notification.query.count() == 0