From 1310f6eda9121481d2db7036d210d1e9e323a074 Mon Sep 17 00:00:00 2001 From: Ben Larabie Date: Fri, 6 Oct 2023 08:53:35 -0400 Subject: [PATCH 1/3] Adding a check to wait for cwagent before starting --- scripts/run_celery.sh | 21 +++++++++++++++++++++ scripts/run_celery_sms.sh | 20 ++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/scripts/run_celery.sh b/scripts/run_celery.sh index 5bc21274c0..ad8635791b 100755 --- a/scripts/run_celery.sh +++ b/scripts/run_celery.sh @@ -2,8 +2,29 @@ # runs celery with all celery queues except the throtted sms queue +init() +{ + # Wait for cwagent to become available. + + while : + do + if nc -vz $STATSD_HOST 25888; then + echo "CWAgent is Ready." + break; + else + echo "Waiting for CWAgent to become ready." + sleep 1 + fi + done +} + set -e +# Check and see if this is running in K8s and if so, wait for cloudwatch agent +if [[ -z "${STATSD_HOST}" ]]; then + init +fi + echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=${CELERY_CONCURRENCY-4} -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-email-tasks,service-callbacks,delivery-receipts diff --git a/scripts/run_celery_sms.sh b/scripts/run_celery_sms.sh index 52f727123a..583fab7127 100755 --- a/scripts/run_celery_sms.sh +++ b/scripts/run_celery_sms.sh @@ -2,6 +2,26 @@ # runs celery with only the throttled sms sending queue +init() +{ + # Wait for cwagent to become available. + while : + do + if nc -vz $STATSD_HOST 25888; then + echo "CWAgent is Ready." + break; + else + echo "Waiting for CWAgent to become ready." + sleep 1 + fi + done +} + set -e +# Check and see if this is running in K8s and if so, wait for cloudwatch agent +if [[ -z "${STATSD_HOST}" ]]; then + init +fi + celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=1 -Q send-throttled-sms-tasks From 3b26ffdf20010fc29b13078e1f97441f9d0b2a9f Mon Sep 17 00:00:00 2001 From: Ben Larabie Date: Wed, 11 Oct 2023 08:16:40 -0400 Subject: [PATCH 2/3] Adding cwagent check to nosms script --- scripts/run_celery_no_sms_sending.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/scripts/run_celery_no_sms_sending.sh b/scripts/run_celery_no_sms_sending.sh index 7a16de2111..f0669c7d39 100755 --- a/scripts/run_celery_no_sms_sending.sh +++ b/scripts/run_celery_no_sms_sending.sh @@ -1,9 +1,31 @@ #!/bin/sh +# runs celery with only the throttled sms sending queue + +init() +{ + # Wait for cwagent to become available. + while : + do + if nc -vz $STATSD_HOST 25888; then + echo "CWAgent is Ready." + break; + else + echo "Waiting for CWAgent to become ready." + sleep 1 + fi + done +} + # runs celery with all celery queues except send-throttled-sms-tasks, send-sms-tasks, send-sms-high, send-sms-medium, or send-sms-low set -e +# Check and see if this is running in K8s and if so, wait for cloudwatch agent +if [[ -z "${STATSD_HOST}" ]]; then + init +fi + echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=${CELERY_CONCURRENCY-4} -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-email-tasks,service-callbacks,delivery-receipts From 5533bae94615b018c4eeabf99f20bffc98e9a874 Mon Sep 17 00:00:00 2001 From: Ben Larabie Date: Wed, 11 Oct 2023 10:13:18 -0400 Subject: [PATCH 3/3] adding init script to ANOTHER script --- scripts/run_celery_send_sms.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/run_celery_send_sms.sh b/scripts/run_celery_send_sms.sh index 1b8ea7de50..e6c32a5d75 100755 --- a/scripts/run_celery_send_sms.sh +++ b/scripts/run_celery_send_sms.sh @@ -2,8 +2,27 @@ # runs celery with only the send-sms-* queues +init() +{ + # Wait for cwagent to become available. + while : + do + if nc -vz $STATSD_HOST 25888; then + echo "CWAgent is Ready." + break; + else + echo "Waiting for CWAgent to become ready." + sleep 1 + fi + done +} + set -e +if [[ -z "${STATSD_HOST}" ]]; then + init +fi + echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" # TODO: we shouldn't be using the send-sms-tasks queue anymore - once we verify this we can remove it