remove unused queues (#2148)

cds-snc · Apr 2, 2024 · 7c19d9f · 7c19d9f
1 parent 51377ad
commit 7c19d9f
Show file tree

Hide file tree

Showing 9 changed files with 12 additions and 28 deletions.
diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py
@@ -42,10 +42,10 @@ def deliver_throttled_sms(self, notification_id):
 
 # Celery rate limits are per worker instance and not a global rate limit.
 # https://docs.celeryproject.org/en/stable/userguide/tasks.html#Task.rate_limit
-# This task is dispatched through the `send-sms-tasks` queue.
-# This queue is consumed by 6 Celery instances with 4 workers in production.
-# The maximum throughput is therefore 6 instances * 4 workers = 24 tasks per second
-# if we set rate_limit="1/s" on the Celery task
+# We currently set rate_limit="1/s" on the Celery task and 4 workers per pod, and so a limit of 4 tasks per second per pod.
+# The number of pods is controlled by the Kubernetes HPA and scales up and down with demand.
+# Currently in production we have 3 celery-sms-send-primary pods, and up to 20 celery-sms-send-scalable pods
+# This means we can send up to 92 messages per second.
 @notify_celery.task(
     bind=True,
     name="deliver_sms",

diff --git a/app/config.py b/app/config.py
@@ -78,18 +78,11 @@ class QueueNames(object):
     # A queue for the tasks associated with the batch saving
     NOTIFY_CACHE = "notifiy-cache-tasks"
 
-    # For normal send of notifications. This is relatively normal volume and flushed
-    # pretty quickly.
-    SEND_NORMAL_QUEUE = "send-{}-tasks"  # notification type to be filled in the queue name
-
     # Queues for sending all SMS, except long dedicated numbers.
     SEND_SMS_HIGH = "send-sms-high"
     SEND_SMS_MEDIUM = "send-sms-medium"
     SEND_SMS_LOW = "send-sms-low"
 
-    # TODO: Delete this queue once we verify that it is not used anymore.
-    SEND_SMS = "send-sms-tasks"
-
     # Primarily used for long dedicated numbers sent from us-west-2 upon which
     # we have a limit to send per second and hence, needs to be throttled.
     SEND_THROTTLED_SMS = "send-throttled-sms-tasks"
@@ -99,9 +92,6 @@ class QueueNames(object):
     SEND_EMAIL_MEDIUM = "send-email-medium"
     SEND_EMAIL_LOW = "send-email-low"
 
-    # TODO: Delete this queue once we verify that it is not used anymore.
-    SEND_EMAIL = "send-email-tasks"
-
     # The research mode queue for notifications that are tested by users trying
     # out Notify.
     RESEARCH_MODE = "research-mode-tasks"
@@ -158,12 +148,10 @@ def all_queues():
             QueueNames.SEND_SMS_HIGH,
             QueueNames.SEND_SMS_MEDIUM,
             QueueNames.SEND_SMS_LOW,
-            QueueNames.SEND_SMS,
             QueueNames.SEND_THROTTLED_SMS,
             QueueNames.SEND_EMAIL_HIGH,
             QueueNames.SEND_EMAIL_MEDIUM,
             QueueNames.SEND_EMAIL_LOW,
-            QueueNames.SEND_EMAIL,
             QueueNames.RESEARCH_MODE,
             QueueNames.REPORTING,
             QueueNames.JOBS,

diff --git a/scripts/run_celery.ps1 b/scripts/run_celery.ps1
@@ -1,3 +1,3 @@
 $ENV:FORKED_BY_MULTIPROCESSING=1
 
-celery --app run_celery worker --pidfile="$env:TEMP\celery.pid" --pool=solo --loglevel=DEBUG --concurrency=1 -Q "database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-email-tasks,service-callbacks,delivery-receipts"
+celery --app run_celery worker --pidfile="$env:TEMP\celery.pid" --pool=solo --loglevel=DEBUG --concurrency=1 -Q "database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-high,send-sms-medium,send-sms-low,service-callbacks,delivery-receipts"
diff --git a/scripts/run_celery.sh b/scripts/run_celery.sh
@@ -6,4 +6,4 @@ set -e
 
 echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}"
 
-celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-email-tasks,service-callbacks,delivery-receipts
+celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-high,send-sms-medium,send-sms-low,service-callbacks,delivery-receipts
diff --git a/scripts/run_celery_local.sh b/scripts/run_celery_local.sh
@@ -7,4 +7,4 @@ set -e
 
 echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}"
 
-celery -A run_celery.notify_celery worker --beat --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,send-email-tasks,service-callbacks,delivery-receipts
+celery -A run_celery.notify_celery worker --beat --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,service-callbacks,delivery-receipts
diff --git a/scripts/run_celery_no_sms_sending.sh b/scripts/run_celery_no_sms_sending.sh
@@ -3,7 +3,7 @@
 set -e
 
 # Runs celery with all celery queues except send-throttled-sms-tasks, 
-# send-sms-tasks, send-sms-high, send-sms-medium, or send-sms-low.
+# send-sms-high, send-sms-medium, or send-sms-low.
 
 # Check and see if this is running in K8s and if so, wait for cloudwatch agent
 if [ -n "${STATSD_HOST}" ]; then
@@ -28,4 +28,4 @@ fi
 
 echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}"
 
-celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-email-tasks,send-email-high,send-email-medium,send-email-low,service-callbacks,delivery-receipts
+celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-email-high,send-email-medium,send-email-low,service-callbacks,delivery-receipts
diff --git a/scripts/run_celery_send_email.sh b/scripts/run_celery_send_email.sh
@@ -6,5 +6,4 @@ set -e
 
 echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}"
 
-# TODO: we shouldn't be using the send-email-tasks queue anymore, once we verify this we can remove it
-celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q send-email-tasks,send-email-high,send-email-medium,send-email-low
+celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q send-email-high,send-email-medium,send-email-low
diff --git a/scripts/run_celery_send_sms.sh b/scripts/run_celery_send_sms.sh
@@ -6,5 +6,4 @@ set -e
 
 echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}"
 
-# TODO: we shouldn't be using the send-sms-tasks queue anymore - once we verify this we can remove it
-celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low
+celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q send-sms-high,send-sms-medium,send-sms-low
diff --git a/tests/app/test_config.py b/tests/app/test_config.py
@@ -27,7 +27,7 @@ def reload_config():
 def test_queue_names_all_queues_correct():
     # Need to ensure that all_queues() only returns queue names used in API
     queues = QueueNames.all_queues()
-    assert len(queues) == 23
+    assert len(queues) == 21
     assert set(
         [
             QueueNames.PRIORITY,
@@ -37,12 +37,10 @@ def test_queue_names_all_queues_correct():
             QueueNames.PRIORITY_DATABASE,
             QueueNames.NORMAL_DATABASE,
             QueueNames.BULK_DATABASE,
-            QueueNames.SEND_SMS,
             QueueNames.SEND_SMS_HIGH,
             QueueNames.SEND_SMS_MEDIUM,
             QueueNames.SEND_SMS_LOW,
             QueueNames.SEND_THROTTLED_SMS,
-            QueueNames.SEND_EMAIL,
             QueueNames.SEND_EMAIL_HIGH,
             QueueNames.SEND_EMAIL_MEDIUM,
             QueueNames.SEND_EMAIL_LOW,
Original file line number	Diff line number	Diff line change
Expand Up		@@ -6,4 +6,4 @@ set -e

		echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}"

		celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-email-tasks,service-callbacks,delivery-receipts
		celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-high,send-sms-medium,send-sms-low,service-callbacks,delivery-receipts
Original file line number	Diff line number	Diff line change
Expand Up		@@ -7,4 +7,4 @@ set -e

		echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}"

		celery -A run_celery.notify_celery worker --beat --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,send-email-tasks,service-callbacks,delivery-receipts
		celery -A run_celery.notify_celery worker --beat --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,service-callbacks,delivery-receipts