diff --git a/.vscode/launch.json b/.vscode/launch.json index 628e4cc52f..711f43a9c6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -43,7 +43,7 @@ "-l", "DEBUG", "-Q", - "database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,send-email-tasks,service-callbacks,delivery-receipts", + "database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,service-callbacks-retry,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,send-email-tasks,service-callbacks,delivery-receipts", ] }, { diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py index d5b413922f..c3a560b175 100644 --- a/app/celery/service_callback_tasks.py +++ b/app/celery/service_callback_tasks.py @@ -67,7 +67,7 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token, "Content-Type": "application/json", "Authorization": "Bearer {}".format(token), }, - timeout=60, + timeout=5, ) current_app.logger.info( "{} sending {} to {}, response {}".format( @@ -86,7 +86,7 @@ def _send_data_to_service_callback_api(self, data, service_callback_url, token, ) if not isinstance(e, HTTPError) or e.response.status_code >= 500: try: - self.retry(queue=QueueNames.RETRY) + self.retry(queue=QueueNames.CALLBACKS_RETRY) except self.MaxRetriesExceededError: current_app.logger.warning( "Retry: {} has retried the max num of times for callback url {} and notification_id: {}".format( diff --git a/app/config.py b/app/config.py index b8f8f8a3ec..b9b82dedb4 100644 --- a/app/config.py +++ b/app/config.py @@ -107,6 +107,7 @@ class QueueNames(object): PROCESS_FTP = "process-ftp-tasks" CREATE_LETTERS_PDF = "create-letters-pdf-tasks" CALLBACKS = "service-callbacks" + CALLBACKS_RETRY = "service-callbacks-retry" # Queue for letters, unused by CDS at this time as we don't use these. LETTERS = "letter-tasks" @@ -156,6 +157,7 @@ def all_queues(): QueueNames.REPORTING, QueueNames.JOBS, QueueNames.RETRY, + QueueNames.CALLBACKS_RETRY, QueueNames.NOTIFY, # QueueNames.CREATE_LETTERS_PDF, QueueNames.CALLBACKS, diff --git a/scripts/run_celery.ps1 b/scripts/run_celery.ps1 index 16203f1d16..b35cb71e94 100644 --- a/scripts/run_celery.ps1 +++ b/scripts/run_celery.ps1 @@ -1,3 +1,3 @@ $ENV:FORKED_BY_MULTIPROCESSING=1 -celery --app run_celery worker --pidfile="$env:TEMP\celery.pid" --pool=solo --loglevel=DEBUG --concurrency=1 -Q "database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-high,send-sms-medium,send-sms-low,service-callbacks,delivery-receipts" +celery --app run_celery worker --pidfile="$env:TEMP\celery.pid" --pool=solo --loglevel=DEBUG --concurrency=1 -Q "database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-high,send-sms-medium,send-sms-low,service-callbacks,service-callbacks-retry,delivery-receipts" diff --git a/scripts/run_celery.sh b/scripts/run_celery.sh index 2cf10f82fd..6d83d67054 100755 --- a/scripts/run_celery.sh +++ b/scripts/run_celery.sh @@ -6,4 +6,4 @@ set -e echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" -celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-high,send-sms-medium,send-sms-low,service-callbacks,delivery-receipts +celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-high,send-sms-medium,send-sms-low,service-callbacks,service-callbacks-retry,delivery-receipts diff --git a/scripts/run_celery_core_tasks.sh b/scripts/run_celery_core_tasks.sh index e109bce78d..060af2ad37 100755 --- a/scripts/run_celery_core_tasks.sh +++ b/scripts/run_celery_core_tasks.sh @@ -7,4 +7,4 @@ set -e echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" -celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,service-callbacks,delivery-receipts +celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,service-callbacks,service-callbacks-retry,delivery-receipts diff --git a/scripts/run_celery_local.sh b/scripts/run_celery_local.sh index 58a914e2ab..9eb29f2658 100755 --- a/scripts/run_celery_local.sh +++ b/scripts/run_celery_local.sh @@ -7,4 +7,4 @@ set -e echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" -celery -A run_celery.notify_celery worker --beat --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,service-callbacks,delivery-receipts +celery -A run_celery.notify_celery worker --beat --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,service-callbacks,service-callbacks-retry,delivery-receipts diff --git a/scripts/run_celery_no_sms_sending.sh b/scripts/run_celery_no_sms_sending.sh index 6c86bc2758..53546088b1 100755 --- a/scripts/run_celery_no_sms_sending.sh +++ b/scripts/run_celery_no_sms_sending.sh @@ -28,4 +28,4 @@ fi echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" -celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-email-high,send-email-medium,send-email-low,service-callbacks,delivery-receipts +celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-email-high,send-email-medium,send-email-low,service-callbacks,service-callbacks-retry,delivery-receipts diff --git a/tests/app/celery/test_service_callback_tasks.py b/tests/app/celery/test_service_callback_tasks.py index 265734b39e..a8259ccd29 100644 --- a/tests/app/celery/test_service_callback_tasks.py +++ b/tests/app/celery/test_service_callback_tasks.py @@ -111,7 +111,7 @@ def test__send_data_to_service_callback_api_retries_if_request_returns_500_with_ send_delivery_status_to_service(notification.id, signed_status_update=signed_data) assert mocked.call_count == 1 - assert mocked.call_args[1]["queue"] == "retry-tasks" + assert mocked.call_args[1]["queue"] == "service-callbacks-retry" @pytest.mark.parametrize("notification_type", ["email", "letter", "sms"]) diff --git a/tests/app/test_config.py b/tests/app/test_config.py index a722e1ea42..e27ba129d1 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -27,7 +27,7 @@ def reload_config(): def test_queue_names_all_queues_correct(): # Need to ensure that all_queues() only returns queue names used in API queues = QueueNames.all_queues() - assert len(queues) == 21 + assert len(queues) == 22 assert set( [ QueueNames.PRIORITY, @@ -48,6 +48,7 @@ def test_queue_names_all_queues_correct(): QueueNames.REPORTING, QueueNames.JOBS, QueueNames.RETRY, + QueueNames.CALLBACKS_RETRY, QueueNames.NOTIFY, # QueueNames.CREATE_LETTERS_PDF, QueueNames.CALLBACKS,