From d5ce6a27fb854c839c7e83207fe1279dd4f21807 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Tue, 24 Oct 2023 09:07:10 -0400
Subject: [PATCH 01/27] Overriding retry policy for SMS high priority

---
 app/notifications/__init__.py              | 26 ++++++++++++++++++++++
 app/notifications/process_notifications.py | 21 ++++++++++++++++-
 2 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/app/notifications/__init__.py b/app/notifications/__init__.py
index e69de29bb2..bfc56cafbd 100644
--- a/app/notifications/__init__.py
+++ b/app/notifications/__init__.py
@@ -0,0 +1,26 @@
+from app.models import BULK, NORMAL, PRIORITY
+
+# Default retry policy for all notifications.
+RETRY_POLICY_DEFAULT = {
+    "max_retries": 48,
+    "interval_start": 300,
+    "interval_step": 300,
+    "interval_max": 300,
+    "retry_errors": None,
+}
+
+# Retry policy for high priority notifications.
+RETRY_POLICY_HIGH = {
+    "max_retries": 48,
+    "interval_start": 25,
+    "interval_step": 25,
+    "interval_max": 25,
+    "retry_errors": None,
+}
+
+# Retry policies for each notification priority lanes.
+RETRY_POLICIES = {
+    BULK: RETRY_POLICY_DEFAULT,
+    NORMAL: RETRY_POLICY_DEFAULT,
+    PRIORITY: RETRY_POLICY_HIGH,
+}
diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index ab37b57dd2..c689b7b4a8 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -35,6 +35,7 @@
     ScheduledNotification,
     Service,
 )
+from app.notifications import RETRY_POLICY_DEFAULT, RETRY_POLICIES
 from app.types import VerifiedNotification
 from app.utils import get_delivery_queue_for_template, get_template_instance
 from app.v2.errors import BadRequestError
@@ -210,7 +211,7 @@ def db_save_and_send_notification(notification: Notification):
 
     deliver_task = choose_deliver_task(notification)
     try:
-        deliver_task.apply_async([str(notification.id)], queue=notification.queue_name)
+        deliver_task.apply_async([str(notification.id)], queue=notification.queue_name, **build_task_params(notification))
     except Exception:
         dao_delete_notifications_by_id(notification.id)
         raise
@@ -219,6 +220,24 @@ def db_save_and_send_notification(notification: Notification):
     )
 
 
+def build_task_params(notification: Notification):
+    """
+    Build task params for the sending parameter tasks.
+
+    If the notification is a high priority SMS, set the retry policy to retry every 25 seconds
+    else fall back to the default retry policy of retrying every 5 minutes.
+    """
+    params = {}
+    params['retry'] = True
+    # Overring the retry policy is only supported for SMS for now;
+    # email support coming later.
+    if notification.notification_type == SMS_TYPE:
+        params['retry_policy'] = RETRY_POLICIES[notification.template.process_type]
+    else:
+        params['retry_policy'] = RETRY_POLICY_DEFAULT
+    return params
+
+
 def choose_queue(notification, research_mode, queue=None) -> QueueNames:
     if research_mode or notification.key_type == KEY_TYPE_TEST:
         queue = QueueNames.RESEARCH_MODE

From 1099d8537dab822099d40595cdf2bfe045848e99 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Tue, 24 Oct 2023 16:23:24 -0400
Subject: [PATCH 02/27] Added comments + format

---
 app/delivery/send_to_providers.py          | 3 +++
 app/notifications/process_notifications.py | 6 +++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py
index e39872bcb0..294c2c5fac 100644
--- a/app/delivery/send_to_providers.py
+++ b/app/delivery/send_to_providers.py
@@ -60,6 +60,7 @@ def send_sms_to_provider(notification):
         inactive_service_failure(notification=notification)
         return
 
+    # If the notification was not sent already, the status should be created.
     if notification.status == "created":
         provider = provider_to_use(
             SMS_TYPE,
@@ -215,6 +216,8 @@ def send_email_to_provider(notification: Notification):
     if not service.active:
         inactive_service_failure(notification=notification)
         return
+
+    # If the notification was not sent already, the status should be created.
     if notification.status == "created":
         provider = provider_to_use(EMAIL_TYPE, notification.id)
 
diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index c689b7b4a8..2d8c3f9d04 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -228,13 +228,13 @@ def build_task_params(notification: Notification):
     else fall back to the default retry policy of retrying every 5 minutes.
     """
     params = {}
-    params['retry'] = True
+    params["retry"] = True
     # Overring the retry policy is only supported for SMS for now;
     # email support coming later.
     if notification.notification_type == SMS_TYPE:
-        params['retry_policy'] = RETRY_POLICIES[notification.template.process_type]
+        params["retry_policy"] = RETRY_POLICIES[notification.template.process_type]
     else:
-        params['retry_policy'] = RETRY_POLICY_DEFAULT
+        params["retry_policy"] = RETRY_POLICY_DEFAULT
     return params
 
 

From 990f3399b9b2487bdece77b265ed27ed885677d1 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Tue, 24 Oct 2023 16:40:54 -0400
Subject: [PATCH 03/27] Fixed ordering

---
 app/notifications/process_notifications.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index 2d8c3f9d04..68d2784f25 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -35,7 +35,7 @@
     ScheduledNotification,
     Service,
 )
-from app.notifications import RETRY_POLICY_DEFAULT, RETRY_POLICIES
+from app.notifications import RETRY_POLICIES, RETRY_POLICY_DEFAULT
 from app.types import VerifiedNotification
 from app.utils import get_delivery_queue_for_template, get_template_instance
 from app.v2.errors import BadRequestError

From 89731a3a9f6eded7990d4d0bfea3efe4d081ac1a Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Wed, 25 Oct 2023 10:10:07 -0400
Subject: [PATCH 04/27] mypy: Added typing to the untyped dict

---
 app/notifications/process_notifications.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index 68d2784f25..f1ce7573a2 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -227,7 +227,7 @@ def build_task_params(notification: Notification):
     If the notification is a high priority SMS, set the retry policy to retry every 25 seconds
     else fall back to the default retry policy of retrying every 5 minutes.
     """
-    params = {}
+    params: dict[str, any] = {}
     params["retry"] = True
     # Overring the retry policy is only supported for SMS for now;
     # email support coming later.

From 00457b78a994193085967e29c299a166ad612b1a Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Wed, 25 Oct 2023 10:32:39 -0400
Subject: [PATCH 05/27] mypy again

---
 app/notifications/process_notifications.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index f1ce7573a2..39f0c7af86 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -1,6 +1,6 @@
 import uuid
 from datetime import datetime
-from typing import List
+from typing import Any, List
 
 from flask import current_app
 from notifications_utils.clients import redis
@@ -227,7 +227,7 @@ def build_task_params(notification: Notification):
     If the notification is a high priority SMS, set the retry policy to retry every 25 seconds
     else fall back to the default retry policy of retrying every 5 minutes.
     """
-    params: dict[str, any] = {}
+    params: dict[str, Any] = {}
     params["retry"] = True
     # Overring the retry policy is only supported for SMS for now;
     # email support coming later.

From 4804bbb46160986050691557488e97b4d0ffdd6a Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Wed, 25 Oct 2023 18:06:27 -0400
Subject: [PATCH 06/27] Fix existing test with new changes

---
 tests/app/notifications/test_process_notification.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index 7202ff8353..1552a6ee91 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -20,6 +20,7 @@
     ScheduledNotification,
     Template,
 )
+from app.notifications import RETRY_POLICY_DEFAULT
 from app.notifications.process_notifications import (
     choose_queue,
     create_content_for_notification,
@@ -1051,7 +1052,9 @@ def test_db_save_and_send_notification_sends_to_queue(
 
         db_save_and_send_notification(notification=notification)
 
-        mocked.assert_called_once_with([str(notification.id)], queue=expected_queue)
+        mocked.assert_called_once_with(
+            [str(notification.id)], queue=expected_queue, retry=True, retry_policy=RETRY_POLICY_DEFAULT
+        )
 
     def test_db_save_and_send_notification_throws_exception_deletes_notification(
         self, sample_template, sample_api_key, sample_job, mocker

From 2d95aa01637b9a4ded1508b5c2bc891b750a4d11 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Wed, 25 Oct 2023 19:59:23 -0400
Subject: [PATCH 07/27] Fix existing test with new changes

---
 tests/app/notifications/test_process_notification.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index 1552a6ee91..b27e82a665 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -1083,7 +1083,9 @@ def test_db_save_and_send_notification_throws_exception_deletes_notification(
 
         with pytest.raises(Boto3Error):
             db_save_and_send_notification(notification)
-        mocked.assert_called_once_with([(str(notification.id))], queue=QueueNames.SEND_SMS_MEDIUM)
+        mocked.assert_called_once_with(
+            [(str(notification.id))], queue=QueueNames.SEND_SMS_MEDIUM, retry=True, retry_policy=RETRY_POLICY_DEFAULT
+        )
 
         assert Notification.query.count() == 0
         assert NotificationHistory.query.count() == 0

From 2dc05b873a2175d860792b23e15c914df022f6e6 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Thu, 26 Oct 2023 16:21:46 -0400
Subject: [PATCH 08/27] Refactoring: rename method

---
 app/notifications/process_notifications.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index 30059296b7..5d919d69f4 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -211,7 +211,7 @@ def db_save_and_send_notification(notification: Notification):
 
     deliver_task = choose_deliver_task(notification)
     try:
-        deliver_task.apply_async([str(notification.id)], queue=notification.queue_name, **build_task_params(notification))
+        deliver_task.apply_async([str(notification.id)], queue=notification.queue_name, **build_delivery_task_params(notification))
     except Exception:
         dao_delete_notifications_by_id(notification.id)
         raise
@@ -220,7 +220,7 @@ def db_save_and_send_notification(notification: Notification):
     )
 
 
-def build_task_params(notification: Notification):
+def build_delivery_task_params(notification: Notification):
     """
     Build task params for the sending parameter tasks.
 

From 77dd00fd8ea33fd7f06ccc50a732001a03f9d4c8 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Thu, 26 Oct 2023 16:30:05 -0400
Subject: [PATCH 09/27] Refactoring: granular params

---
 app/notifications/process_notifications.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index 5d919d69f4..204574bc8d 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -211,7 +211,11 @@ def db_save_and_send_notification(notification: Notification):
 
     deliver_task = choose_deliver_task(notification)
     try:
-        deliver_task.apply_async([str(notification.id)], queue=notification.queue_name, **build_delivery_task_params(notification))
+        deliver_task.apply_async(
+            [str(notification.id)],
+            queue=notification.queue_name,
+            **build_delivery_task_params(notification.notification_type, notification.template.process_type),
+        )
     except Exception:
         dao_delete_notifications_by_id(notification.id)
         raise
@@ -220,7 +224,7 @@ def db_save_and_send_notification(notification: Notification):
     )
 
 
-def build_delivery_task_params(notification: Notification):
+def build_delivery_task_params(notification_type: str, notification_process_type: str):
     """
     Build task params for the sending parameter tasks.
 
@@ -231,8 +235,8 @@ def build_delivery_task_params(notification: Notification):
     params["retry"] = True
     # Overring the retry policy is only supported for SMS for now;
     # email support coming later.
-    if notification.notification_type == SMS_TYPE:
-        params["retry_policy"] = RETRY_POLICIES[notification.template.process_type]
+    if notification_type == SMS_TYPE:
+        params["retry_policy"] = RETRY_POLICIES[notification_process_type]
     else:
         params["retry_policy"] = RETRY_POLICY_DEFAULT
     return params

From d80087a28c4f430c59bdf41da4fe01cbb8a2174d Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Thu, 26 Oct 2023 16:52:36 -0400
Subject: [PATCH 10/27] Added test around params building for delivery tasks

---
 app/notifications/process_notifications.py    |  4 +--
 .../test_process_notification.py              | 28 +++++++++++++++++++
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index 204574bc8d..ce979d9b71 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -1,6 +1,6 @@
 import uuid
 from datetime import datetime
-from typing import Any, List
+from typing import Any, Dict, List
 
 from flask import current_app
 from notifications_utils.clients import redis
@@ -224,7 +224,7 @@ def db_save_and_send_notification(notification: Notification):
     )
 
 
-def build_delivery_task_params(notification_type: str, notification_process_type: str):
+def build_delivery_task_params(notification_type: str, notification_process_type: str) -> Dict[str, Any]:
     """
     Build task params for the sending parameter tasks.
 
diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index 6b4f4f7467..25aa093630 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -1,5 +1,6 @@
 import datetime
 import uuid
+from typing import Any, Dict
 from unittest.mock import call
 
 import pytest
@@ -14,7 +15,12 @@
 from app.config import QueueNames
 from app.dao.service_sms_sender_dao import dao_update_service_sms_sender
 from app.models import (
+    BULK,
+    EMAIL_TYPE,
     LETTER_TYPE,
+    NORMAL,
+    PRIORITY,
+    SMS_TYPE,
     Notification,
     NotificationHistory,
     ScheduledNotification,
@@ -22,6 +28,7 @@
 )
 from app.notifications import RETRY_POLICY_DEFAULT
 from app.notifications.process_notifications import (
+    build_delivery_task_params,
     choose_queue,
     create_content_for_notification,
     db_save_and_send_notification,
@@ -1090,6 +1097,27 @@ def test_db_save_and_send_notification_throws_exception_deletes_notification(
         assert Notification.query.count() == 0
         assert NotificationHistory.query.count() == 0
 
+    @pytest.mark.parametrize(
+        ("notification_type, process_type, expected_retry, expected_retry_period"),
+        [
+            (EMAIL_TYPE, BULK, 48, 300),
+            (EMAIL_TYPE, NORMAL, 48, 300),
+            (EMAIL_TYPE, PRIORITY, 48, 300),
+            (SMS_TYPE, BULK, 48, 300),
+            (SMS_TYPE, NORMAL, 48, 300),
+            (SMS_TYPE, PRIORITY, 48, 26),
+        ],
+    )
+    def test_delivery_task_parameters(self, notification_type, process_type, expected_retry, expected_retry_period):
+        params: Dict[str, Any] = build_delivery_task_params(notification_type, process_type)
+        assert params["retry"] is True
+
+        retry_policy: Dict[str, Any] = params["retry_policy"]
+        assert retry_policy["max_retries"] == expected_retry
+        assert retry_policy["interval_start"] == expected_retry_period
+        assert retry_policy["interval_step"] == expected_retry_period
+        assert retry_policy["interval_max"] == expected_retry_period
+
     def test_db_save_and_send_notification_throws_exception_when_missing_template(self, sample_api_key, mocker):
         mocker.patch("app.celery.provider_tasks.deliver_sms.apply_async")
         assert Notification.query.count() == 0

From eaebf1167d92cfa0652dc70f894ea35cbab6d98f Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Thu, 26 Oct 2023 16:58:57 -0400
Subject: [PATCH 11/27] Changed test for correct value of retry period for sms
 high

---
 tests/app/notifications/test_process_notification.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index 25aa093630..20b0a6962f 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -1105,7 +1105,7 @@ def test_db_save_and_send_notification_throws_exception_deletes_notification(
             (EMAIL_TYPE, PRIORITY, 48, 300),
             (SMS_TYPE, BULK, 48, 300),
             (SMS_TYPE, NORMAL, 48, 300),
-            (SMS_TYPE, PRIORITY, 48, 26),
+            (SMS_TYPE, PRIORITY, 48, 25),
         ],
     )
     def test_delivery_task_parameters(self, notification_type, process_type, expected_retry, expected_retry_period):

From cba4269b9c479999aa0757e0adaee2118b57b0e3 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Fri, 27 Oct 2023 13:50:05 -0400
Subject: [PATCH 12/27] Introducing feature flag for celery retry policies

---
 app/config.py                              | 14 +++++++++-----
 app/notifications/process_notifications.py |  3 +++
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/app/config.py b/app/config.py
index bf7014caac..f087c69eaa 100644
--- a/app/config.py
+++ b/app/config.py
@@ -573,7 +573,6 @@ class Config(object):
     CSV_BULK_REDIRECT_THRESHOLD = os.getenv("CSV_BULK_REDIRECT_THRESHOLD", 200)
 
     # Endpoint of Cloudwatch agent running as a side car in EKS listening for embedded metrics
-    FF_CLOUDWATCH_METRICS_ENABLED = env.bool("FF_CLOUDWATCH_METRICS_ENABLED", False)
     CLOUDWATCH_AGENT_EMF_PORT = 25888
     CLOUDWATCH_AGENT_ENDPOINT = os.getenv("CLOUDWATCH_AGENT_ENDPOINT", f"tcp://{STATSD_HOST}:{CLOUDWATCH_AGENT_EMF_PORT}")
 
@@ -582,14 +581,15 @@ class Config(object):
     BR_WARNING_PERCENTAGE = 0.05
     BR_CRITICAL_PERCENTAGE = 0.1
 
-    FF_SALESFORCE_CONTACT = env.bool("FF_SALESFORCE_CONTACT", False)
-
     # Feature flags for bounce rate
     # Timestamp in epoch milliseconds to seed the bounce rate. We will seed data for (24, the below config) included.
     FF_BOUNCE_RATE_SEED_EPOCH_MS = os.getenv("FF_BOUNCE_RATE_SEED_EPOCH_MS", False)
-
+    # Feature flag to enable custom retry policies such as lowering retry period for certain priority lanes.
+    FF_CELERY_CUSTOM_RETRY_POLICIES = env.bool("FF_CELERY_CUSTOM_RETRY_POLICIES", False)
+    FF_CLOUDWATCH_METRICS_ENABLED = env.bool("FF_CLOUDWATCH_METRICS_ENABLED", False)
     # Feature flags for email_daily_limit
     FF_EMAIL_DAILY_LIMIT = env.bool("FF_EMAIL_DAILY_LIMIT", False)
+    FF_SALESFORCE_CONTACT = env.bool("FF_SALESFORCE_CONTACT", False)
 
     @classmethod
     def get_sensitive_config(cls) -> list[str]:
@@ -654,6 +654,8 @@ class Development(Config):
 
     API_HOST_NAME = "http://localhost:6011"
     API_RATE_LIMIT_ENABLED = True
+    
+    FF_CELERY_CUSTOM_RETRY_POLICIES = True
 
 
 class Test(Development):
@@ -686,10 +688,12 @@ class Test(Development):
     API_HOST_NAME = "http://localhost:6011"
 
     TEMPLATE_PREVIEW_API_HOST = "http://localhost:9999"
-    FF_EMAIL_DAILY_LIMIT = False
     CRM_GITHUB_PERSONAL_ACCESS_TOKEN = "test-token"
     CRM_ORG_LIST_URL = "https://test-url.com"
 
+    FF_CELERY_CUSTOM_RETRY_POLICIES = True
+    FF_EMAIL_DAILY_LIMIT = False
+
 
 class Production(Config):
     NOTIFY_EMAIL_DOMAIN = os.getenv("NOTIFY_EMAIL_DOMAIN", "notification.canada.ca")
diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index ce979d9b71..390cc43d1d 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -231,6 +231,9 @@ def build_delivery_task_params(notification_type: str, notification_process_type
     If the notification is a high priority SMS, set the retry policy to retry every 25 seconds
     else fall back to the default retry policy of retrying every 5 minutes.
     """
+    if current_app.config["FF_CELERY_CUSTOM_RETRY_POLICIES"]:
+        return {}
+
     params: dict[str, Any] = {}
     params["retry"] = True
     # Overring the retry policy is only supported for SMS for now;

From 0a5d38869fc6acec2207121828cf8d3d4e754cee Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Fri, 27 Oct 2023 13:51:12 -0400
Subject: [PATCH 13/27] Changed FF name for custom task celery params

---
 app/config.py                              | 8 ++++----
 app/notifications/process_notifications.py | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/config.py b/app/config.py
index f087c69eaa..0481b5af82 100644
--- a/app/config.py
+++ b/app/config.py
@@ -585,7 +585,7 @@ class Config(object):
     # Timestamp in epoch milliseconds to seed the bounce rate. We will seed data for (24, the below config) included.
     FF_BOUNCE_RATE_SEED_EPOCH_MS = os.getenv("FF_BOUNCE_RATE_SEED_EPOCH_MS", False)
     # Feature flag to enable custom retry policies such as lowering retry period for certain priority lanes.
-    FF_CELERY_CUSTOM_RETRY_POLICIES = env.bool("FF_CELERY_CUSTOM_RETRY_POLICIES", False)
+    FF_CELERY_CUSTOM_TASK_PARAMS = env.bool("FF_CELERY_CUSTOM_TASK_PARAMS", False)
     FF_CLOUDWATCH_METRICS_ENABLED = env.bool("FF_CLOUDWATCH_METRICS_ENABLED", False)
     # Feature flags for email_daily_limit
     FF_EMAIL_DAILY_LIMIT = env.bool("FF_EMAIL_DAILY_LIMIT", False)
@@ -654,8 +654,8 @@ class Development(Config):
 
     API_HOST_NAME = "http://localhost:6011"
     API_RATE_LIMIT_ENABLED = True
-    
-    FF_CELERY_CUSTOM_RETRY_POLICIES = True
+
+    FF_CELERY_CUSTOM_TASK_PARAMS = True
 
 
 class Test(Development):
@@ -691,7 +691,7 @@ class Test(Development):
     CRM_GITHUB_PERSONAL_ACCESS_TOKEN = "test-token"
     CRM_ORG_LIST_URL = "https://test-url.com"
 
-    FF_CELERY_CUSTOM_RETRY_POLICIES = True
+    FF_CELERY_CUSTOM_TASK_PARAMS = True
     FF_EMAIL_DAILY_LIMIT = False
 
 
diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index 390cc43d1d..17083d9a07 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -231,7 +231,7 @@ def build_delivery_task_params(notification_type: str, notification_process_type
     If the notification is a high priority SMS, set the retry policy to retry every 25 seconds
     else fall back to the default retry policy of retrying every 5 minutes.
     """
-    if current_app.config["FF_CELERY_CUSTOM_RETRY_POLICIES"]:
+    if current_app.config["FF_CELERY_CUSTOM_TASK_PARAMS"]:
         return {}
 
     params: dict[str, Any] = {}

From 5a144f87127a7b6e7ad36f07da815a5ba9499f4a Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Fri, 27 Oct 2023 13:54:42 -0400
Subject: [PATCH 14/27] Adding extension to center editor on kb cursor

---
 .devcontainer/devcontainer.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 3d4181650c..9db4d34d72 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -26,6 +26,7 @@
 				"GitHub.copilot",
 				"GitHub.copilot-labs",
 				"googlecloudtools.cloudcode",
+				"kaiwood.center-editor-window",
 				"matangover.mypy",
 				"ms-azuretools.vscode-docker",
 				"ms-ossdata.vscode-postgresql",

From 74834f7ac19d36a23e0f783786be2f694277c129 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Mon, 30 Oct 2023 11:43:22 -0400
Subject: [PATCH 15/27] Forgot the test config to activate env var

---
 pytest.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pytest.ini b/pytest.ini
index 3cb2c63d3e..0ab0190e73 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -15,6 +15,7 @@ env =
     ASSET_DOMAIN=assets.notification.canada.ca
     NOTIFY_EMAIL_DOMAIN=notification.canada.ca
     AWS_EMF_ENVIRONMENT=local
+    D:FF_CELERY_CUSTOM_TASK_PARAMS=True
     D:FF_SALESFORCE_CONTACT=True
     D:FF_CLOUDWATCH_METRICS_ENABLED=True
     D:REDIS_URL=redis://localhost:6380

From 9cc04c23fd6d6aea188f6fdd888573fe3aa1e646 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Mon, 30 Oct 2023 11:58:43 -0400
Subject: [PATCH 16/27] Use constants in tests

---
 .../app/notifications/test_process_notification.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index 20b0a6962f..d9fa7bb07b 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -26,7 +26,7 @@
     ScheduledNotification,
     Template,
 )
-from app.notifications import RETRY_POLICY_DEFAULT
+from app.notifications import RETRY_POLICY_DEFAULT, RETRY_POLICY_HIGH
 from app.notifications.process_notifications import (
     build_delivery_task_params,
     choose_queue,
@@ -1100,12 +1100,12 @@ def test_db_save_and_send_notification_throws_exception_deletes_notification(
     @pytest.mark.parametrize(
         ("notification_type, process_type, expected_retry, expected_retry_period"),
         [
-            (EMAIL_TYPE, BULK, 48, 300),
-            (EMAIL_TYPE, NORMAL, 48, 300),
-            (EMAIL_TYPE, PRIORITY, 48, 300),
-            (SMS_TYPE, BULK, 48, 300),
-            (SMS_TYPE, NORMAL, 48, 300),
-            (SMS_TYPE, PRIORITY, 48, 25),
+            (EMAIL_TYPE, BULK, RETRY_POLICY_DEFAULT["max_retries"], RETRY_POLICY_DEFAULT["interval_step"]),
+            (EMAIL_TYPE, NORMAL, RETRY_POLICY_DEFAULT["max_retries"], RETRY_POLICY_DEFAULT["interval_step"]),
+            (EMAIL_TYPE, PRIORITY, RETRY_POLICY_DEFAULT["max_retries"], RETRY_POLICY_DEFAULT["interval_step"]),
+            (SMS_TYPE, BULK, RETRY_POLICY_DEFAULT["max_retries"], RETRY_POLICY_DEFAULT["interval_step"]),
+            (SMS_TYPE, NORMAL, RETRY_POLICY_DEFAULT["max_retries"], RETRY_POLICY_DEFAULT["interval_step"]),
+            (SMS_TYPE, PRIORITY, RETRY_POLICY_HIGH["max_retries"], RETRY_POLICY_HIGH["interval_step"]),
         ],
     )
     def test_delivery_task_parameters(self, notification_type, process_type, expected_retry, expected_retry_period):

From b51f63f861ffb3b8577219686404ef80185f41a7 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Mon, 30 Oct 2023 12:19:44 -0400
Subject: [PATCH 17/27] Reversing env var config enabling logic (but same
 config)

---
 app/config.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/config.py b/app/config.py
index 0481b5af82..58e1f727d2 100644
--- a/app/config.py
+++ b/app/config.py
@@ -585,7 +585,7 @@ class Config(object):
     # Timestamp in epoch milliseconds to seed the bounce rate. We will seed data for (24, the below config) included.
     FF_BOUNCE_RATE_SEED_EPOCH_MS = os.getenv("FF_BOUNCE_RATE_SEED_EPOCH_MS", False)
     # Feature flag to enable custom retry policies such as lowering retry period for certain priority lanes.
-    FF_CELERY_CUSTOM_TASK_PARAMS = env.bool("FF_CELERY_CUSTOM_TASK_PARAMS", False)
+    FF_CELERY_CUSTOM_TASK_PARAMS = env.bool("FF_CELERY_CUSTOM_TASK_PARAMS", True)
     FF_CLOUDWATCH_METRICS_ENABLED = env.bool("FF_CLOUDWATCH_METRICS_ENABLED", False)
     # Feature flags for email_daily_limit
     FF_EMAIL_DAILY_LIMIT = env.bool("FF_EMAIL_DAILY_LIMIT", False)
@@ -655,8 +655,6 @@ class Development(Config):
     API_HOST_NAME = "http://localhost:6011"
     API_RATE_LIMIT_ENABLED = True
 
-    FF_CELERY_CUSTOM_TASK_PARAMS = True
-
 
 class Test(Development):
     NOTIFY_EMAIL_DOMAIN = os.getenv("NOTIFY_EMAIL_DOMAIN", "notification.canada.ca")
@@ -691,7 +689,6 @@ class Test(Development):
     CRM_GITHUB_PERSONAL_ACCESS_TOKEN = "test-token"
     CRM_ORG_LIST_URL = "https://test-url.com"
 
-    FF_CELERY_CUSTOM_TASK_PARAMS = True
     FF_EMAIL_DAILY_LIMIT = False
 
 
@@ -710,6 +707,7 @@ class Production(Config):
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = False
     CRONITOR_ENABLED = False
+    FF_CELERY_CUSTOM_TASK_PARAMS = False
 
 
 class Staging(Production):

From dc81e0d922e4d31fc6e9d491968c6c2463f4ed6e Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Mon, 30 Oct 2023 13:30:44 -0400
Subject: [PATCH 18/27] Arf got trick by Python conditional logic

---
 app/notifications/process_notifications.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index 17083d9a07..fd2b6a1a62 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -231,7 +231,7 @@ def build_delivery_task_params(notification_type: str, notification_process_type
     If the notification is a high priority SMS, set the retry policy to retry every 25 seconds
     else fall back to the default retry policy of retrying every 5 minutes.
     """
-    if current_app.config["FF_CELERY_CUSTOM_TASK_PARAMS"]:
+    if current_app.config["FF_CELERY_CUSTOM_TASK_PARAMS"] is False:
         return {}
 
     params: dict[str, Any] = {}

From d36e65fcf42e389708454f22fc17633bfb393c48 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Mon, 30 Oct 2023 13:49:22 -0400
Subject: [PATCH 19/27] Print that env var because it aint working

---
 tests/app/notifications/test_process_notification.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index d9fa7bb07b..3fcc713d04 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -1109,6 +1109,10 @@ def test_db_save_and_send_notification_throws_exception_deletes_notification(
         ],
     )
     def test_delivery_task_parameters(self, notification_type, process_type, expected_retry, expected_retry_period):
+        from flask import current_app
+        ff_value = current_app.config["FF_CELERY_CUSTOM_TASK_PARAMS"]
+        print(f"test_process_notification: FF_CELERY_CUSTOM_TASK_PARAMS={ff_value}")
+
         params: Dict[str, Any] = build_delivery_task_params(notification_type, process_type)
         assert params["retry"] is True
 

From 2fd7be1e880b9250fd5b13f5d3c2eb0507e3297f Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Mon, 30 Oct 2023 14:15:21 -0400
Subject: [PATCH 20/27] Format

---
 tests/app/notifications/test_process_notification.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index 3fcc713d04..6d81c0b40c 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -1110,6 +1110,7 @@ def test_db_save_and_send_notification_throws_exception_deletes_notification(
     )
     def test_delivery_task_parameters(self, notification_type, process_type, expected_retry, expected_retry_period):
         from flask import current_app
+
         ff_value = current_app.config["FF_CELERY_CUSTOM_TASK_PARAMS"]
         print(f"test_process_notification: FF_CELERY_CUSTOM_TASK_PARAMS={ff_value}")
 

From 7737bfca657d7b6027e46c6c38f7449dd0c8e413 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Mon, 30 Oct 2023 16:39:46 -0400
Subject: [PATCH 21/27] Removing print debug statement

---
 tests/app/notifications/test_process_notification.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index 6d81c0b40c..d9fa7bb07b 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -1109,11 +1109,6 @@ def test_db_save_and_send_notification_throws_exception_deletes_notification(
         ],
     )
     def test_delivery_task_parameters(self, notification_type, process_type, expected_retry, expected_retry_period):
-        from flask import current_app
-
-        ff_value = current_app.config["FF_CELERY_CUSTOM_TASK_PARAMS"]
-        print(f"test_process_notification: FF_CELERY_CUSTOM_TASK_PARAMS={ff_value}")
-
         params: Dict[str, Any] = build_delivery_task_params(notification_type, process_type)
         assert params["retry"] is True
 

From 77b436fe7aad2513b951fdb1e146cd26bbb8e481 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Sun, 5 Nov 2023 22:02:59 -0500
Subject: [PATCH 22/27] Apply low retry period for the sms high priority lane
 on celery retry send

---
 .vscode/launch.json                           |  2 +-
 app/celery/provider_tasks.py                  |  5 +-
 app/notifications/__init__.py                 | 46 ++++++++++++++++++-
 app/notifications/process_notifications.py    | 25 +---------
 .../test_process_notification.py              |  7 ++-
 5 files changed, 57 insertions(+), 28 deletions(-)

diff --git a/.vscode/launch.json b/.vscode/launch.json
index bb86be77fc..628e4cc52f 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -43,7 +43,7 @@
                 "-l",
                 "DEBUG",
                 "-Q",
-                "database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-email-tasks,service-callbacks,delivery-receipts",
+                "database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,send-email-tasks,service-callbacks,delivery-receipts",
             ]
         },
         {
diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py
index bdd9cfe3c6..1d3dac31e0 100644
--- a/app/celery/provider_tasks.py
+++ b/app/celery/provider_tasks.py
@@ -15,6 +15,7 @@
     NotificationTechnicalFailureException,
 )
 from app.models import NOTIFICATION_TECHNICAL_FAILURE
+from app.notifications import build_retry_task_params
 from app.notifications.callbacks import _check_and_queue_callback_task
 
 
@@ -112,6 +113,7 @@ def _deliver_sms(self, notification_id):
         notification = notifications_dao.get_notification_by_id(notification_id)
         if not notification:
             raise NoResultFound()
+        # raise Exception("Trigger artificial Celery retry")
         send_to_providers.send_sms_to_provider(notification)
     except InvalidUrlException:
         current_app.logger.error(f"Cannot send notification {notification_id}, got an invalid direct file url.")
@@ -127,7 +129,8 @@ def _deliver_sms(self, notification_id):
                 # Once the previous retry failed, log the exception and this time,
                 # retry with the default delay.
                 current_app.logger.exception("SMS notification delivery for id: {} failed".format(notification_id))
-                self.retry(queue=QueueNames.RETRY)
+                self.retry(**build_retry_task_params(notification.notification_type, notification.template.process_type))
+                # self.retry(kwargs=build_retry_task_params(notification.notification_type, notification.template.process_type))
         except self.MaxRetriesExceededError:
             message = (
                 "RETRY FAILED: Max retries reached. The task send_sms_to_provider failed for notification {}. "
diff --git a/app/notifications/__init__.py b/app/notifications/__init__.py
index bfc56cafbd..effd5c8c62 100644
--- a/app/notifications/__init__.py
+++ b/app/notifications/__init__.py
@@ -1,4 +1,7 @@
-from app.models import BULK, NORMAL, PRIORITY
+from app.config import QueueNames
+from app.models import BULK, NORMAL, PRIORITY, SMS_TYPE
+from flask import current_app
+from typing import Any, Dict
 
 # Default retry policy for all notifications.
 RETRY_POLICY_DEFAULT = {
@@ -24,3 +27,44 @@
     NORMAL: RETRY_POLICY_DEFAULT,
     PRIORITY: RETRY_POLICY_HIGH,
 }
+
+
+def build_delivery_task_params(notification_type: str, notification_process_type: str) -> Dict[str, Any]:
+    """
+    Build task params for the sending parameter tasks.
+
+    If the notification is a high priority SMS, set the retry policy to retry every 25 seconds
+    else fall back to the default retry policy of retrying every 5 minutes.
+    """
+    if current_app.config["FF_CELERY_CUSTOM_TASK_PARAMS"] is False:
+        return {}
+
+    params: dict[str, Any] = {}
+    params["retry"] = True
+    # Overring the retry policy is only supported for SMS for now;
+    # email support coming later.
+    if notification_type == SMS_TYPE:
+        params["retry_policy"] = RETRY_POLICIES[notification_process_type]
+    else:
+        params["retry_policy"] = RETRY_POLICY_DEFAULT
+    return params
+
+
+def build_retry_task_params(notification_type: str, notification_process_type: str) -> Dict[str, Any]:
+    """
+    Build task params for the sending parameter retry tasks.
+
+    If the notification is a high priority SMS, set the retry policy to retry every 25 seconds
+    else fall back to the default retry policy of retrying every 5 minutes.
+    """
+    params: dict[str, Any] = {"queue": QueueNames.RETRY}
+    if current_app.config["FF_CELERY_CUSTOM_TASK_PARAMS"] is False:
+        return params
+
+    # Overring the retry policy is only supported for SMS for now;
+    # email support coming later.
+    if notification_type == SMS_TYPE:
+        params["countdown"] = RETRY_POLICIES[notification_process_type]["interval_step"]
+    else:
+        params["countdown"] = RETRY_POLICY_DEFAULT["interval_step"]
+    return params
diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index fd2b6a1a62..1da7aaba53 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -1,6 +1,6 @@
 import uuid
 from datetime import datetime
-from typing import Any, Dict, List
+from typing import List
 
 from flask import current_app
 from notifications_utils.clients import redis
@@ -35,7 +35,7 @@
     ScheduledNotification,
     Service,
 )
-from app.notifications import RETRY_POLICIES, RETRY_POLICY_DEFAULT
+from app.notifications import build_delivery_task_params
 from app.types import VerifiedNotification
 from app.utils import get_delivery_queue_for_template, get_template_instance
 from app.v2.errors import BadRequestError
@@ -224,27 +224,6 @@ def db_save_and_send_notification(notification: Notification):
     )
 
 
-def build_delivery_task_params(notification_type: str, notification_process_type: str) -> Dict[str, Any]:
-    """
-    Build task params for the sending parameter tasks.
-
-    If the notification is a high priority SMS, set the retry policy to retry every 25 seconds
-    else fall back to the default retry policy of retrying every 5 minutes.
-    """
-    if current_app.config["FF_CELERY_CUSTOM_TASK_PARAMS"] is False:
-        return {}
-
-    params: dict[str, Any] = {}
-    params["retry"] = True
-    # Overring the retry policy is only supported for SMS for now;
-    # email support coming later.
-    if notification_type == SMS_TYPE:
-        params["retry_policy"] = RETRY_POLICIES[notification_process_type]
-    else:
-        params["retry_policy"] = RETRY_POLICY_DEFAULT
-    return params
-
-
 def choose_queue(notification, research_mode, queue=None) -> QueueNames:
     if research_mode or notification.key_type == KEY_TYPE_TEST:
         queue = QueueNames.RESEARCH_MODE
diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index d9fa7bb07b..f5771d806e 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -26,9 +26,12 @@
     ScheduledNotification,
     Template,
 )
-from app.notifications import RETRY_POLICY_DEFAULT, RETRY_POLICY_HIGH
-from app.notifications.process_notifications import (
+from app.notifications import (
+    RETRY_POLICY_DEFAULT,
+    RETRY_POLICY_HIGH,
     build_delivery_task_params,
+)
+from app.notifications.process_notifications import (
     choose_queue,
     create_content_for_notification,
     db_save_and_send_notification,

From 687b790d5274653027731ac6f0bc074f5b10e8b9 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Sun, 5 Nov 2023 22:56:00 -0500
Subject: [PATCH 23/27] Refactoring retry periods for sms high priority

---
 app/celery/provider_tasks.py                  |  1 -
 app/notifications/__init__.py                 | 56 ++++---------------
 app/notifications/process_notifications.py    |  2 -
 .../test_process_notification.py              | 39 +++++--------
 4 files changed, 25 insertions(+), 73 deletions(-)

diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py
index 1d3dac31e0..f284a78028 100644
--- a/app/celery/provider_tasks.py
+++ b/app/celery/provider_tasks.py
@@ -130,7 +130,6 @@ def _deliver_sms(self, notification_id):
                 # retry with the default delay.
                 current_app.logger.exception("SMS notification delivery for id: {} failed".format(notification_id))
                 self.retry(**build_retry_task_params(notification.notification_type, notification.template.process_type))
-                # self.retry(kwargs=build_retry_task_params(notification.notification_type, notification.template.process_type))
         except self.MaxRetriesExceededError:
             message = (
                 "RETRY FAILED: Max retries reached. The task send_sms_to_provider failed for notification {}. "
diff --git a/app/notifications/__init__.py b/app/notifications/__init__.py
index effd5c8c62..b94d9f7419 100644
--- a/app/notifications/__init__.py
+++ b/app/notifications/__init__.py
@@ -3,52 +3,16 @@
 from flask import current_app
 from typing import Any, Dict
 
-# Default retry policy for all notifications.
-RETRY_POLICY_DEFAULT = {
-    "max_retries": 48,
-    "interval_start": 300,
-    "interval_step": 300,
-    "interval_max": 300,
-    "retry_errors": None,
+# Default retry periods for sending notifications.
+RETRY_DEFAULT = 300
+RETRY_HIGH = 25
+
+RETRY_PERIODS = {
+    BULK: RETRY_DEFAULT,
+    NORMAL: RETRY_DEFAULT,
+    PRIORITY: RETRY_HIGH,
 }
 
-# Retry policy for high priority notifications.
-RETRY_POLICY_HIGH = {
-    "max_retries": 48,
-    "interval_start": 25,
-    "interval_step": 25,
-    "interval_max": 25,
-    "retry_errors": None,
-}
-
-# Retry policies for each notification priority lanes.
-RETRY_POLICIES = {
-    BULK: RETRY_POLICY_DEFAULT,
-    NORMAL: RETRY_POLICY_DEFAULT,
-    PRIORITY: RETRY_POLICY_HIGH,
-}
-
-
-def build_delivery_task_params(notification_type: str, notification_process_type: str) -> Dict[str, Any]:
-    """
-    Build task params for the sending parameter tasks.
-
-    If the notification is a high priority SMS, set the retry policy to retry every 25 seconds
-    else fall back to the default retry policy of retrying every 5 minutes.
-    """
-    if current_app.config["FF_CELERY_CUSTOM_TASK_PARAMS"] is False:
-        return {}
-
-    params: dict[str, Any] = {}
-    params["retry"] = True
-    # Overring the retry policy is only supported for SMS for now;
-    # email support coming later.
-    if notification_type == SMS_TYPE:
-        params["retry_policy"] = RETRY_POLICIES[notification_process_type]
-    else:
-        params["retry_policy"] = RETRY_POLICY_DEFAULT
-    return params
-
 
 def build_retry_task_params(notification_type: str, notification_process_type: str) -> Dict[str, Any]:
     """
@@ -64,7 +28,7 @@ def build_retry_task_params(notification_type: str, notification_process_type: s
     # Overring the retry policy is only supported for SMS for now;
     # email support coming later.
     if notification_type == SMS_TYPE:
-        params["countdown"] = RETRY_POLICIES[notification_process_type]["interval_step"]
+        params["countdown"] = RETRY_PERIODS[notification_process_type]
     else:
-        params["countdown"] = RETRY_POLICY_DEFAULT["interval_step"]
+        params["countdown"] = RETRY_DEFAULT
     return params
diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index 1da7aaba53..56a2dc98d3 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -35,7 +35,6 @@
     ScheduledNotification,
     Service,
 )
-from app.notifications import build_delivery_task_params
 from app.types import VerifiedNotification
 from app.utils import get_delivery_queue_for_template, get_template_instance
 from app.v2.errors import BadRequestError
@@ -214,7 +213,6 @@ def db_save_and_send_notification(notification: Notification):
         deliver_task.apply_async(
             [str(notification.id)],
             queue=notification.queue_name,
-            **build_delivery_task_params(notification.notification_type, notification.template.process_type),
         )
     except Exception:
         dao_delete_notifications_by_id(notification.id)
diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index f5771d806e..04f4c60d59 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -26,11 +26,7 @@
     ScheduledNotification,
     Template,
 )
-from app.notifications import (
-    RETRY_POLICY_DEFAULT,
-    RETRY_POLICY_HIGH,
-    build_delivery_task_params,
-)
+from app.notifications import RETRY_PERIODS, build_retry_task_params
 from app.notifications.process_notifications import (
     choose_queue,
     create_content_for_notification,
@@ -1062,9 +1058,7 @@ def test_db_save_and_send_notification_sends_to_queue(
 
         db_save_and_send_notification(notification=notification)
 
-        mocked.assert_called_once_with(
-            [str(notification.id)], queue=expected_queue, retry=True, retry_policy=RETRY_POLICY_DEFAULT
-        )
+        mocked.assert_called_once_with([str(notification.id)], queue=expected_queue)
 
     def test_db_save_and_send_notification_throws_exception_deletes_notification(
         self, sample_template, sample_api_key, sample_job, mocker
@@ -1101,25 +1095,22 @@ def test_db_save_and_send_notification_throws_exception_deletes_notification(
         assert NotificationHistory.query.count() == 0
 
     @pytest.mark.parametrize(
-        ("notification_type, process_type, expected_retry, expected_retry_period"),
+        ("notification_type, process_type, expected_retry_period"),
         [
-            (EMAIL_TYPE, BULK, RETRY_POLICY_DEFAULT["max_retries"], RETRY_POLICY_DEFAULT["interval_step"]),
-            (EMAIL_TYPE, NORMAL, RETRY_POLICY_DEFAULT["max_retries"], RETRY_POLICY_DEFAULT["interval_step"]),
-            (EMAIL_TYPE, PRIORITY, RETRY_POLICY_DEFAULT["max_retries"], RETRY_POLICY_DEFAULT["interval_step"]),
-            (SMS_TYPE, BULK, RETRY_POLICY_DEFAULT["max_retries"], RETRY_POLICY_DEFAULT["interval_step"]),
-            (SMS_TYPE, NORMAL, RETRY_POLICY_DEFAULT["max_retries"], RETRY_POLICY_DEFAULT["interval_step"]),
-            (SMS_TYPE, PRIORITY, RETRY_POLICY_HIGH["max_retries"], RETRY_POLICY_HIGH["interval_step"]),
+            (EMAIL_TYPE, BULK, RETRY_PERIODS[BULK]),
+            (EMAIL_TYPE, NORMAL, RETRY_PERIODS[NORMAL]),
+            (EMAIL_TYPE, PRIORITY, RETRY_PERIODS[NORMAL]),
+            (SMS_TYPE, BULK, RETRY_PERIODS[BULK]),
+            (SMS_TYPE, NORMAL, RETRY_PERIODS[NORMAL]),
+            (SMS_TYPE, PRIORITY, RETRY_PERIODS[PRIORITY]),
         ],
     )
-    def test_delivery_task_parameters(self, notification_type, process_type, expected_retry, expected_retry_period):
-        params: Dict[str, Any] = build_delivery_task_params(notification_type, process_type)
-        assert params["retry"] is True
-
-        retry_policy: Dict[str, Any] = params["retry_policy"]
-        assert retry_policy["max_retries"] == expected_retry
-        assert retry_policy["interval_start"] == expected_retry_period
-        assert retry_policy["interval_step"] == expected_retry_period
-        assert retry_policy["interval_max"] == expected_retry_period
+    def test_retry_task_parameters(self, notify_api, notification_type, process_type, expected_retry_period):
+        with notify_api.app_context():
+            params: Dict[str, Any] = build_retry_task_params(notification_type, process_type)
+
+        assert params["queue"] == QueueNames.RETRY
+        assert params["countdown"] == expected_retry_period
 
     def test_db_save_and_send_notification_throws_exception_when_missing_template(self, sample_api_key, mocker):
         mocker.patch("app.celery.provider_tasks.deliver_sms.apply_async")

From ad01cfde935cc6e4729bb7a13d27b7e44ec746df Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Sun, 5 Nov 2023 23:04:20 -0500
Subject: [PATCH 24/27] Fix test

---
 tests/app/notifications/test_process_notification.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index 04f4c60d59..c9a627439e 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -1087,9 +1087,7 @@ def test_db_save_and_send_notification_throws_exception_deletes_notification(
 
         with pytest.raises(Boto3Error):
             db_save_and_send_notification(notification)
-        mocked.assert_called_once_with(
-            [(str(notification.id))], queue=QueueNames.SEND_SMS_MEDIUM, retry=True, retry_policy=RETRY_POLICY_DEFAULT
-        )
+        mocked.assert_called_once_with([(str(notification.id))], queue=QueueNames.SEND_SMS_MEDIUM)
 
         assert Notification.query.count() == 0
         assert NotificationHistory.query.count() == 0

From c82785fb8a999e17e14406dc2ac85841613e02b6 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Sun, 5 Nov 2023 23:08:58 -0500
Subject: [PATCH 25/27] Fix sort

---
 app/notifications/__init__.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/notifications/__init__.py b/app/notifications/__init__.py
index b94d9f7419..1ec9036155 100644
--- a/app/notifications/__init__.py
+++ b/app/notifications/__init__.py
@@ -1,7 +1,9 @@
+from typing import Any, Dict
+
+from flask import current_app
+
 from app.config import QueueNames
 from app.models import BULK, NORMAL, PRIORITY, SMS_TYPE
-from flask import current_app
-from typing import Any, Dict
 
 # Default retry periods for sending notifications.
 RETRY_DEFAULT = 300

From 57161f18cea542ce3cd28469dd478d998b361a5e Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Mon, 6 Nov 2023 13:45:28 -0500
Subject: [PATCH 26/27] Added timonwong.shellcheck shell checker to
 devcontainer

---
 .devcontainer/devcontainer.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 9db4d34d72..90716e9912 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -36,6 +36,7 @@
 				"mtxr.sqltools",
 				"mtxr.sqltools-driver-pg",
 				"pmbenjamin.vscode-snyk",
+				"timonwong.shellcheck",
 				"usernamehw.errorlens",
 				"visualstudioexptteam.vscodeintellicode",
 				"wenfangdu.jump",

From d52c17328f17262fabb68f59a32a0350a8b3f906 Mon Sep 17 00:00:00 2001
From: Jimmy Royer <jimleroyer@gmail.com>
Date: Mon, 6 Nov 2023 16:19:19 -0500
Subject: [PATCH 27/27] Removed manually raised exception only meant for
 testing

---
 app/celery/provider_tasks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py
index f284a78028..bfdc671a2e 100644
--- a/app/celery/provider_tasks.py
+++ b/app/celery/provider_tasks.py
@@ -113,7 +113,6 @@ def _deliver_sms(self, notification_id):
         notification = notifications_dao.get_notification_by_id(notification_id)
         if not notification:
             raise NoResultFound()
-        # raise Exception("Trigger artificial Celery retry")
         send_to_providers.send_sms_to_provider(notification)
     except InvalidUrlException:
         current_app.logger.error(f"Cannot send notification {notification_id}, got an invalid direct file url.")