From 9f9fd5e885d1b1c834a99b23a46490c67c9da354 Mon Sep 17 00:00:00 2001 From: Stephen Astels Date: Fri, 6 Oct 2023 15:45:19 -0400 Subject: [PATCH 1/3] send emails with separate email queues --- app/config.py | 18 +++++--- app/notifications/process_notifications.py | 4 +- scripts/run_celery_local.sh | 2 +- scripts/run_celery_no_sending.sh | 10 +++++ scripts/run_celery_no_sms_sending.sh | 2 +- scripts/run_celery_send_email.sh | 10 +++++ tests/app/celery/test_scheduled_tasks.py | 2 +- tests/app/celery/test_tasks.py | 42 ++++++++++--------- .../rest/test_send_notification.py | 12 +++--- .../test_process_notification.py | 6 +-- .../service/test_send_one_off_notification.py | 4 +- tests/app/test_config.py | 5 ++- tests/app/test_utils.py | 6 +-- 13 files changed, 77 insertions(+), 46 deletions(-) create mode 100755 scripts/run_celery_no_sending.sh create mode 100755 scripts/run_celery_send_email.sh diff --git a/app/config.py b/app/config.py index 0d48aa7c73..f66b95ca85 100644 --- a/app/config.py +++ b/app/config.py @@ -10,7 +10,6 @@ from kombu import Exchange, Queue from notifications_utils import logging -# from app.models import EMAIL_TYPE, SMS_TYPE, Priorities from celery.schedules import crontab env = Env() @@ -88,6 +87,11 @@ class QueueNames(object): SEND_SMS_MEDIUM = "send-sms-medium" SEND_SMS_LOW = "send-sms-low" + # Queues for sending all emails. + SEND_EMAIL_HIGH = "send-email-high" + SEND_EMAIL_MEDIUM = "send-email-medium" + SEND_EMAIL_LOW = "send-email-low" + # TODO: Delete this queue once we verify that it is not used anymore. SEND_SMS = "send-sms-tasks" @@ -95,8 +99,7 @@ class QueueNames(object): # we have a limit to send per second and hence, needs to be throttled. SEND_THROTTLED_SMS = "send-throttled-sms-tasks" - # The queue to send emails by default, normal priority. - # TODO: Deprecate to favor priority queues instead, i.e. bulk, normal, priority. + # TODO: Delete this queue once we verify that it is not used anymore. SEND_EMAIL = "send-email-tasks" # The research mode queue for notifications that are tested by users trying @@ -131,9 +134,9 @@ class QueueNames(object): Priorities.HIGH: SEND_SMS_HIGH, }, "email": { - Priorities.LOW: BULK, - Priorities.MEDIUM: SEND_EMAIL, - Priorities.HIGH: PRIORITY, + Priorities.LOW: SEND_EMAIL_LOW, + Priorities.MEDIUM: SEND_EMAIL_MEDIUM, + Priorities.HIGH: SEND_EMAIL_HIGH, }, "letter": { Priorities.LOW: BULK, @@ -157,6 +160,9 @@ def all_queues(): QueueNames.SEND_SMS_LOW, QueueNames.SEND_SMS, QueueNames.SEND_THROTTLED_SMS, + QueueNames.SEND_EMAIL_HIGH, + QueueNames.SEND_EMAIL_MEDIUM, + QueueNames.SEND_EMAIL_LOW, QueueNames.SEND_EMAIL, QueueNames.RESEARCH_MODE, QueueNames.REPORTING, diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index ab37b57dd2..c3425b576d 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -230,7 +230,7 @@ def choose_queue(notification, research_mode, queue=None) -> QueueNames: queue = QueueNames.SEND_SMS_MEDIUM if notification.notification_type == EMAIL_TYPE: if not queue: - queue = QueueNames.SEND_EMAIL + queue = QueueNames.SEND_EMAIL_MEDIUM if notification.notification_type == LETTER_TYPE: if not queue: queue = QueueNames.CREATE_LETTERS_PDF @@ -264,7 +264,7 @@ def send_notification_to_queue(notification, research_mode, queue=None): queue = QueueNames.SEND_SMS_MEDIUM if notification.notification_type == EMAIL_TYPE: if not queue or queue == QueueNames.NORMAL: - queue = QueueNames.SEND_EMAIL + queue = QueueNames.SEND_EMAIL_MEDIUM deliver_task = provider_tasks.deliver_email if notification.notification_type == LETTER_TYPE: if not queue or queue == QueueNames.NORMAL: diff --git a/scripts/run_celery_local.sh b/scripts/run_celery_local.sh index b9e37240e3..069387e4b0 100755 --- a/scripts/run_celery_local.sh +++ b/scripts/run_celery_local.sh @@ -7,4 +7,4 @@ set -e echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" -celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=${CELERY_CONCURRENCY-4} -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-tasks,service-callbacks,delivery-receipts +celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=${CELERY_CONCURRENCY-4} -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,send-email-tasks,service-callbacks,delivery-receipts diff --git a/scripts/run_celery_no_sending.sh b/scripts/run_celery_no_sending.sh new file mode 100755 index 0000000000..37669f15b9 --- /dev/null +++ b/scripts/run_celery_no_sending.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# runs celery with all celery queues except send-throttled-sms-tasks, send-sms-tasks, send-sms-high, send-sms-medium, or send-sms-low + +set -e + +echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" + +celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=${CELERY_CONCURRENCY-4} -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks, +service-callbacks,delivery-receipts diff --git a/scripts/run_celery_no_sms_sending.sh b/scripts/run_celery_no_sms_sending.sh index 7a16de2111..0639cc03c9 100755 --- a/scripts/run_celery_no_sms_sending.sh +++ b/scripts/run_celery_no_sms_sending.sh @@ -6,4 +6,4 @@ set -e echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" -celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=${CELERY_CONCURRENCY-4} -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-email-tasks,service-callbacks,delivery-receipts +celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=${CELERY_CONCURRENCY-4} -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-email-tasks,send-email-high,send-email-medium,send-email-low,service-callbacks,delivery-receipts diff --git a/scripts/run_celery_send_email.sh b/scripts/run_celery_send_email.sh new file mode 100755 index 0000000000..410e0a3423 --- /dev/null +++ b/scripts/run_celery_send_email.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# runs celery with only the send-email-* queues + +set -e + +echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" + +# TODO: we shouldn't be using the send-email-tasks queue anymore - once we verify this we can remove it +celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=${CELERY_CONCURRENCY-4} -Q send-email-tasks,send-email-high,send-email-medium,send-email-low diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 97fd8ece82..cceace4f30 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -363,7 +363,7 @@ def test_replay_created_notifications(notify_db_session, sample_service, mocker) save_notification(create_notification(template=email_template, created_at=datetime.utcnow(), status="created")) replay_created_notifications() - email_delivery_queue.assert_called_once_with([str(old_email.id)], queue="send-email-tasks") + email_delivery_queue.assert_called_once_with([str(old_email.id)], queue=QueueNames.SEND_EMAIL_MEDIUM) sms_delivery_queue.assert_called_once_with([str(old_sms.id)], queue=QueueNames.SEND_SMS_MEDIUM) diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index cbe9f78a16..1b4f5b4e31 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -920,7 +920,7 @@ def test_process_rows_sends_save_task( "to": "recip", "row_number": "row_num", "personalisation": {"foo": "bar"}, - "queue": QueueNames.SEND_SMS_MEDIUM if template_type == SMS_TYPE else "send-{}-tasks".format(template_type), + "queue": QueueNames.SEND_SMS_MEDIUM if template_type == SMS_TYPE else QueueNames.SEND_EMAIL_MEDIUM, "client_reference": reference, "sender_id": str(sender_id) if sender_id else None, }, @@ -930,12 +930,12 @@ def test_process_rows_sends_save_task( @pytest.mark.parametrize( "csv_bulk_threshold, template_process_type, expected_queue", [ - (1_000, PRIORITY, "priority-tasks"), # keep priority when no thresholds are met - (1, PRIORITY, "bulk-tasks"), # autoswitch to bulk queue if bulk threshold is met, even if in priority. - (1, NORMAL, "bulk-tasks"), # autoswitch to bulk queue if bulk threshold is met. - (1_000, NORMAL, "send-email-tasks"), # keep normal priority - (1, BULK, "bulk-tasks"), # keep bulk priority - (1_000, BULK, "send-email-tasks"), # autoswitch to normal queue if normal threshold is met. + (1_000, PRIORITY, QueueNames.SEND_EMAIL_HIGH), # keep priority when no thresholds are met + (1, PRIORITY, QueueNames.SEND_EMAIL_LOW), # autoswitch to bulk queue if bulk threshold is met, even if in priority. + (1, NORMAL, QueueNames.SEND_EMAIL_LOW), # autoswitch to bulk queue if bulk threshold is met. + (1_000, NORMAL, QueueNames.SEND_EMAIL_MEDIUM), # keep normal priority + (1, BULK, QueueNames.SEND_EMAIL_LOW), # keep bulk priority + (1_000, BULK, QueueNames.SEND_EMAIL_MEDIUM), # autoswitch to normal queue if normal threshold is met. ], ) def test_should_redirect_email_job_to_queue_depending_on_csv_threshold( @@ -1103,7 +1103,7 @@ def test_process_rows_works_without_key_type( "to": "recip", "row_number": "row_num", "personalisation": {"foo": "bar"}, - "queue": QueueNames.SEND_SMS_MEDIUM if template_type == SMS_TYPE else "send-{}-tasks".format(template_type), + "queue": QueueNames.SEND_SMS_MEDIUM if template_type == SMS_TYPE else QueueNames.SEND_EMAIL_MEDIUM, "sender_id": str(sender_id) if sender_id else None, "client_reference": reference, }, @@ -1582,7 +1582,7 @@ def test_save_emails_should_use_redis_cache_to_retrieve_service_and_template_whe assert persisted_notification.personalisation == {"name": "Jo"} assert persisted_notification._personalisation == signer_personalisation.sign({"name": "Jo"}) assert persisted_notification.notification_type == "email" - mocked_deliver_email.assert_called_once_with([str(persisted_notification.id)], queue="send-email-tasks") + mocked_deliver_email.assert_called_once_with([str(persisted_notification.id)], queue=QueueNames.SEND_EMAIL_MEDIUM) if sender_id: mocked_get_sender_id.assert_called_once_with(persisted_notification.service_id, sender_id) @@ -1633,9 +1633,11 @@ def test_should_put_save_email_task_in_research_mode_queue_if_research_mode_serv [str(persisted_notification.id)], queue="research-mode-tasks" ) - @pytest.mark.parametrize("process_type", ["priority", "bulk"]) + @pytest.mark.parametrize( + "process_type,expected_queue", [("priority", QueueNames.SEND_EMAIL_HIGH), ("bulk", QueueNames.SEND_EMAIL_LOW)] + ) def test_should_route_save_email_task_to_appropriate_queue_according_to_template_process_type( - self, notify_db_session, mocker, process_type + self, notify_db_session, mocker, process_type, expected_queue ): service = create_service() template = create_template(service=service, template_type="email", process_type=process_type) @@ -1648,14 +1650,12 @@ def test_should_route_save_email_task_to_appropriate_queue_according_to_template save_emails(service.id, [signer_notification.sign(notification)], notification_id) persisted_notification = Notification.query.one() - provider_tasks.deliver_email.apply_async.assert_called_once_with( - [str(persisted_notification.id)], queue=f"{process_type}-tasks" - ) + provider_tasks.deliver_email.apply_async.assert_called_once_with([str(persisted_notification.id)], queue=expected_queue) def test_should_route_save_email_task_to_bulk_on_large_csv_file(self, notify_db_session, mocker): service = create_service() template = create_template(service=service, template_type="email", process_type="normal") - notification = _notification_json(template, to="test@test.com", queue="bulk-tasks") + notification = _notification_json(template, to="test@test.com", queue=QueueNames.SEND_EMAIL_LOW) mocker.patch("app.celery.provider_tasks.deliver_email.apply_async") @@ -1664,7 +1664,9 @@ def test_should_route_save_email_task_to_bulk_on_large_csv_file(self, notify_db_ save_emails(service.id, [signer_notification.sign(notification)], notification_id) persisted_notification = Notification.query.one() - provider_tasks.deliver_email.apply_async.assert_called_once_with([str(persisted_notification.id)], queue="bulk-tasks") + provider_tasks.deliver_email.apply_async.assert_called_once_with( + [str(persisted_notification.id)], queue=QueueNames.SEND_EMAIL_LOW + ) def test_should_use_email_template_and_persist( self, notify_api, sample_email_template_with_placeholders, sample_api_key, mocker @@ -1705,7 +1707,7 @@ def test_should_use_email_template_and_persist( assert persisted_notification.notification_type == "email" provider_tasks.deliver_email.apply_async.assert_called_once_with( - [str(persisted_notification.id)], queue="send-email-tasks" + [str(persisted_notification.id)], queue=QueueNames.SEND_EMAIL_MEDIUM ) mock_over_daily_limit.assert_called_once_with("normal", sample_email_template_with_placeholders.service) @@ -1734,7 +1736,7 @@ def test_save_email_should_use_template_version_from_job_not_latest(self, sample assert not persisted_notification.sent_by assert persisted_notification.notification_type == "email" provider_tasks.deliver_email.apply_async.assert_called_once_with( - [str(persisted_notification.id)], queue="send-email-tasks" + [str(persisted_notification.id)], queue=QueueNames.SEND_EMAIL_MEDIUM ) def test_should_use_email_template_subject_placeholders(self, sample_email_template_with_placeholders, mocker): @@ -1756,7 +1758,7 @@ def test_should_use_email_template_subject_placeholders(self, sample_email_templ assert not persisted_notification.reference assert persisted_notification.notification_type == "email" provider_tasks.deliver_email.apply_async.assert_called_once_with( - [str(persisted_notification.id)], queue="send-email-tasks" + [str(persisted_notification.id)], queue=QueueNames.SEND_EMAIL_MEDIUM ) def test_save_email_uses_the_reply_to_text_when_provided(self, sample_email_template, mocker): @@ -1813,7 +1815,7 @@ def test_should_use_email_template_and_persist_without_personalisation(self, sam assert not persisted_notification.reference assert persisted_notification.notification_type == "email" provider_tasks.deliver_email.apply_async.assert_called_once_with( - [str(persisted_notification.id)], queue="send-email-tasks" + [str(persisted_notification.id)], queue=QueueNames.SEND_EMAIL_MEDIUM ) def test_save_email_should_go_to_retry_queue_if_database_errors(self, sample_email_template, mocker): diff --git a/tests/app/notifications/rest/test_send_notification.py b/tests/app/notifications/rest/test_send_notification.py index 02933dc80a..b1f797f25a 100644 --- a/tests/app/notifications/rest/test_send_notification.py +++ b/tests/app/notifications/rest/test_send_notification.py @@ -132,7 +132,7 @@ def test_send_notification_with_placeholders_replaced(notify_api, sample_email_t notification_id = response_data["notification"]["id"] data.update({"template_version": sample_email_template_with_placeholders.version}) - mocked.assert_called_once_with([notification_id], queue="send-email-tasks") + mocked.assert_called_once_with([notification_id], queue=QueueNames.SEND_EMAIL_MEDIUM) assert response.status_code == 201 assert response_data["body"] == "Hello Jo\nThis is an email from GOV.UK" assert response_data["subject"] == "Jo" @@ -370,7 +370,7 @@ def test_should_allow_valid_email_notification(notify_api, sample_email_template response_data = json.loads(response.get_data(as_text=True))["data"] notification_id = response_data["notification"]["id"] app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with( - [notification_id], queue="send-email-tasks" + [notification_id], queue=QueueNames.SEND_EMAIL_MEDIUM ) assert response.status_code == 201 @@ -563,7 +563,7 @@ def test_should_send_email_if_team_api_key_and_a_service_user(client, sample_ema headers=[("Content-Type", "application/json"), auth_header], ) - app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with([fake_uuid], queue="send-email-tasks") + app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with([fake_uuid], queue=QueueNames.SEND_EMAIL_MEDIUM) assert response.status_code == 201 @@ -660,7 +660,7 @@ def test_should_send_sms_if_team_api_key_and_a_service_user(client, sample_templ @pytest.mark.parametrize( "template_type,queue_name", - [(SMS_TYPE, QueueNames.SEND_SMS_MEDIUM), (EMAIL_TYPE, "send-email-tasks")], + [(SMS_TYPE, QueueNames.SEND_SMS_MEDIUM), (EMAIL_TYPE, QueueNames.SEND_EMAIL_MEDIUM)], ) def test_should_persist_notification( client, @@ -710,7 +710,7 @@ def test_should_persist_notification( @pytest.mark.parametrize( "template_type,queue_name", - [(SMS_TYPE, QueueNames.SEND_SMS_MEDIUM), (EMAIL_TYPE, "send-email-tasks")], + [(SMS_TYPE, QueueNames.SEND_SMS_MEDIUM), (EMAIL_TYPE, QueueNames.SEND_EMAIL_MEDIUM)], ) def test_should_delete_notification_and_return_error_if_sqs_fails( client, @@ -1028,7 +1028,7 @@ def test_send_notification_uses_appropriate_queue_when_template_has_process_type if notification_type == SMS_TYPE: expected_queue = QueueNames.SEND_SMS_HIGH if process_type == "priority" else QueueNames.SEND_SMS_LOW else: - expected_queue = f"{process_type}-tasks" + expected_queue = QueueNames.SEND_EMAIL_HIGH if process_type == "priority" else QueueNames.SEND_EMAIL_LOW mocked.assert_called_once_with([notification_id], queue=expected_queue) diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py index 7202ff8353..b7a6d33fa6 100644 --- a/tests/app/notifications/test_process_notification.py +++ b/tests/app/notifications/test_process_notification.py @@ -497,7 +497,7 @@ class TestSendNotificationQueue: "deliver_throttled_sms", ), (False, None, "sms", "normal", None, QueueNames.SEND_SMS_MEDIUM, "deliver_sms"), - (False, None, "email", "normal", None, "send-email-tasks", "deliver_email"), + (False, None, "email", "normal", None, QueueNames.SEND_EMAIL_MEDIUM, "deliver_email"), (False, None, "sms", "team", None, QueueNames.SEND_SMS_MEDIUM, "deliver_sms"), ( False, @@ -677,7 +677,7 @@ class TestChooseQueue: "send-throttled-sms-tasks", ), (False, None, "sms", "normal", None, QueueNames.SEND_SMS_MEDIUM), - (False, None, "email", "normal", None, "send-email-tasks"), + (False, None, "email", "normal", None, QueueNames.SEND_EMAIL_MEDIUM), (False, None, "sms", "team", None, QueueNames.SEND_SMS_MEDIUM), ( False, @@ -975,7 +975,7 @@ def test_db_save_and_send_notification_saves_to_db(self, client, sample_template "deliver_throttled_sms", ), ("sms", "normal", None, QueueNames.SEND_SMS_MEDIUM, "deliver_sms"), - ("email", "normal", None, "send-email-tasks", "deliver_email"), + ("email", "normal", None, QueueNames.SEND_EMAIL_MEDIUM, "deliver_email"), ("sms", "team", None, QueueNames.SEND_SMS_MEDIUM, "deliver_sms"), ("sms", "test", None, "research-mode-tasks", "deliver_sms"), ( diff --git a/tests/app/service/test_send_one_off_notification.py b/tests/app/service/test_send_one_off_notification.py index 91827402e1..4b8de0890c 100644 --- a/tests/app/service/test_send_one_off_notification.py +++ b/tests/app/service/test_send_one_off_notification.py @@ -194,7 +194,7 @@ def test_send_one_off_notification_honors_research_mode(notify_db_session, persi @pytest.mark.parametrize( "process_type, expected_queue", - [("priority", QueueNames.PRIORITY), ("bulk", QueueNames.SEND_EMAIL), ("normal", QueueNames.SEND_EMAIL)], + [("priority", QueueNames.SEND_EMAIL_HIGH), ("bulk", QueueNames.SEND_EMAIL_MEDIUM), ("normal", QueueNames.SEND_EMAIL_MEDIUM)], ) def test_send_one_off_email_notification_honors_process_type( notify_db_session, persist_mock, celery_mock, process_type, expected_queue @@ -382,7 +382,7 @@ def test_send_one_off_notification_should_add_email_reply_to_text_for_notificati notification_id = send_one_off_notification(service_id=sample_email_template.service.id, post_data=data) notification = Notification.query.get(notification_id["id"]) - celery_mock.assert_called_once_with(notification=notification, research_mode=False, queue=QueueNames.SEND_EMAIL) + celery_mock.assert_called_once_with(notification=notification, research_mode=False, queue=QueueNames.SEND_EMAIL_MEDIUM) assert notification.reply_to_text == reply_to_email.email_address diff --git a/tests/app/test_config.py b/tests/app/test_config.py index 3dbffe39a4..5b25f4b093 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -27,7 +27,7 @@ def reload_config(): def test_queue_names_all_queues_correct(): # Need to ensure that all_queues() only returns queue names used in API queues = QueueNames.all_queues() - assert len(queues) == 20 + assert len(queues) == 23 assert set( [ QueueNames.PRIORITY, @@ -43,6 +43,9 @@ def test_queue_names_all_queues_correct(): QueueNames.SEND_SMS_LOW, QueueNames.SEND_THROTTLED_SMS, QueueNames.SEND_EMAIL, + QueueNames.SEND_EMAIL_HIGH, + QueueNames.SEND_EMAIL_MEDIUM, + QueueNames.SEND_EMAIL_LOW, QueueNames.RESEARCH_MODE, QueueNames.REPORTING, QueueNames.JOBS, diff --git a/tests/app/test_utils.py b/tests/app/test_utils.py index eab8af777a..2d504c2050 100644 --- a/tests/app/test_utils.py +++ b/tests/app/test_utils.py @@ -148,9 +148,9 @@ def test_get_limit_reset_time_et(): (SMS_TYPE, "normal", QueueNames.SEND_SMS_MEDIUM), (SMS_TYPE, "priority", QueueNames.SEND_SMS_HIGH), (SMS_TYPE, "bulk", QueueNames.SEND_SMS_LOW), - (EMAIL_TYPE, "normal", QueueNames.SEND_EMAIL), - (EMAIL_TYPE, "priority", QueueNames.PRIORITY), - (EMAIL_TYPE, "bulk", QueueNames.BULK), + (EMAIL_TYPE, "normal", QueueNames.SEND_EMAIL_MEDIUM), + (EMAIL_TYPE, "priority", QueueNames.SEND_EMAIL_HIGH), + (EMAIL_TYPE, "bulk", QueueNames.SEND_EMAIL_LOW), ], ) def test_get_delivery_queue_for_template(sample_service, template_type, process_type, expected_queue): From 6e0c512f0d57faaf6558acc85b1f3e7d25eccc10 Mon Sep 17 00:00:00 2001 From: Stephen Astels Date: Fri, 6 Oct 2023 16:39:40 -0400 Subject: [PATCH 2/3] tweak --- README.md | 8 ++------ app/config.py | 10 +++++----- scripts/run_celery_no_sending.sh | 2 +- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 66fdd02615..26e1a18155 100644 --- a/README.md +++ b/README.md @@ -110,11 +110,7 @@ file. Copy that file to `.env` and customize it to your needs. ## To run the queues ``` -scripts/run_celery.sh -``` - -``` -scripts/run_celery_sms.sh +scripts/run_celery_local.sh ``` ``` @@ -179,7 +175,7 @@ To help debug full code paths of emails and SMS, we have a special email and pho set in the application's configuration. As it stands at the moment these are the following: | Notification Type | Test destination | -|-------------------|--------------------------| +| ----------------- | ------------------------ | | Email | internal.test@cds-snc.ca | | SMS | +16135550123 | diff --git a/app/config.py b/app/config.py index f66b95ca85..4a11187c90 100644 --- a/app/config.py +++ b/app/config.py @@ -87,11 +87,6 @@ class QueueNames(object): SEND_SMS_MEDIUM = "send-sms-medium" SEND_SMS_LOW = "send-sms-low" - # Queues for sending all emails. - SEND_EMAIL_HIGH = "send-email-high" - SEND_EMAIL_MEDIUM = "send-email-medium" - SEND_EMAIL_LOW = "send-email-low" - # TODO: Delete this queue once we verify that it is not used anymore. SEND_SMS = "send-sms-tasks" @@ -99,6 +94,11 @@ class QueueNames(object): # we have a limit to send per second and hence, needs to be throttled. SEND_THROTTLED_SMS = "send-throttled-sms-tasks" + # Queues for sending all emails. + SEND_EMAIL_HIGH = "send-email-high" + SEND_EMAIL_MEDIUM = "send-email-medium" + SEND_EMAIL_LOW = "send-email-low" + # TODO: Delete this queue once we verify that it is not used anymore. SEND_EMAIL = "send-email-tasks" diff --git a/scripts/run_celery_no_sending.sh b/scripts/run_celery_no_sending.sh index 37669f15b9..8c7fad7d5b 100755 --- a/scripts/run_celery_no_sending.sh +++ b/scripts/run_celery_no_sending.sh @@ -1,6 +1,6 @@ #!/bin/sh -# runs celery with all celery queues except send-throttled-sms-tasks, send-sms-tasks, send-sms-high, send-sms-medium, or send-sms-low +# runs celery with all celery queues except send-throttled-sms-tasks, send-sms-* and send-email-* set -e From 070e90d60accc7d603602a304bbe41ec70347ba5 Mon Sep 17 00:00:00 2001 From: Stephen Astels Date: Wed, 11 Oct 2023 15:56:50 -0400 Subject: [PATCH 3/3] better name for script --- scripts/{run_celery_no_sending.sh => run_celery_core_tasks.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{run_celery_no_sending.sh => run_celery_core_tasks.sh} (100%) diff --git a/scripts/run_celery_no_sending.sh b/scripts/run_celery_core_tasks.sh similarity index 100% rename from scripts/run_celery_no_sending.sh rename to scripts/run_celery_core_tasks.sh