From 5b8602ff7680959ab6361863454fb15b200fa8bb Mon Sep 17 00:00:00 2001 From: Ben Larabie Date: Wed, 11 Oct 2023 14:43:13 -0400 Subject: [PATCH 1/2] Reworking the celery init on k8s so I can see logging (#1999) * Reworking the celery init on k8s so I can see logging * Removing the redirect to stderr which was put there by mistake --- scripts/run_celery.sh | 14 ++++---------- scripts/run_celery_no_sms_sending.sh | 19 +++++++------------ scripts/run_celery_send_sms.sh | 12 +++--------- scripts/run_celery_sms.sh | 14 ++++---------- 4 files changed, 18 insertions(+), 41 deletions(-) diff --git a/scripts/run_celery.sh b/scripts/run_celery.sh index ad8635791b..2b3d4a1bc0 100755 --- a/scripts/run_celery.sh +++ b/scripts/run_celery.sh @@ -2,10 +2,11 @@ # runs celery with all celery queues except the throtted sms queue -init() -{ - # Wait for cwagent to become available. +set -e +# Check and see if this is running in K8s and if so, wait for cloudwatch agent +if [[ -z "${STATSD_HOST}" ]]; then + echo "Initializing... Waiting for CWAgent to become ready." while : do if nc -vz $STATSD_HOST 25888; then @@ -16,13 +17,6 @@ init() sleep 1 fi done -} - -set -e - -# Check and see if this is running in K8s and if so, wait for cloudwatch agent -if [[ -z "${STATSD_HOST}" ]]; then - init fi echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" diff --git a/scripts/run_celery_no_sms_sending.sh b/scripts/run_celery_no_sms_sending.sh index f0669c7d39..f0fe622681 100755 --- a/scripts/run_celery_no_sms_sending.sh +++ b/scripts/run_celery_no_sms_sending.sh @@ -2,9 +2,13 @@ # runs celery with only the throttled sms sending queue -init() -{ - # Wait for cwagent to become available. +# runs celery with all celery queues except send-throttled-sms-tasks, send-sms-tasks, send-sms-high, send-sms-medium, or send-sms-low + +set -e + +# Check and see if this is running in K8s and if so, wait for cloudwatch agent +if [[ -z "${STATSD_HOST}" ]]; then + echo "Initializing... Waiting for CWAgent to become ready." while : do if nc -vz $STATSD_HOST 25888; then @@ -15,15 +19,6 @@ init() sleep 1 fi done -} - -# runs celery with all celery queues except send-throttled-sms-tasks, send-sms-tasks, send-sms-high, send-sms-medium, or send-sms-low - -set -e - -# Check and see if this is running in K8s and if so, wait for cloudwatch agent -if [[ -z "${STATSD_HOST}" ]]; then - init fi echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" diff --git a/scripts/run_celery_send_sms.sh b/scripts/run_celery_send_sms.sh index e6c32a5d75..ee0101cdd4 100755 --- a/scripts/run_celery_send_sms.sh +++ b/scripts/run_celery_send_sms.sh @@ -1,10 +1,10 @@ #!/bin/sh # runs celery with only the send-sms-* queues +set -e -init() -{ - # Wait for cwagent to become available. +if [[ -z "${STATSD_HOST}" ]]; then + echo "Initializing... Waiting for CWAgent to become ready." while : do if nc -vz $STATSD_HOST 25888; then @@ -15,12 +15,6 @@ init() sleep 1 fi done -} - -set -e - -if [[ -z "${STATSD_HOST}" ]]; then - init fi echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}" diff --git a/scripts/run_celery_sms.sh b/scripts/run_celery_sms.sh index 583fab7127..0afca890d2 100755 --- a/scripts/run_celery_sms.sh +++ b/scripts/run_celery_sms.sh @@ -2,9 +2,10 @@ # runs celery with only the throttled sms sending queue -init() -{ - # Wait for cwagent to become available. +set -e + +if [[ -z "${STATSD_HOST}" ]]; then + echo "Initializing... Waiting for CWAgent to become ready." while : do if nc -vz $STATSD_HOST 25888; then @@ -15,13 +16,6 @@ init() sleep 1 fi done -} - -set -e - -# Check and see if this is running in K8s and if so, wait for cloudwatch agent -if [[ -z "${STATSD_HOST}" ]]; then - init fi celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=1 -Q send-throttled-sms-tasks From bdba8439f6020e858756ffa5f1e7537f0cbb90fc Mon Sep 17 00:00:00 2001 From: "sre-read-write[bot]" <92993749+sre-read-write[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 19:24:54 +0000 Subject: [PATCH 2/2] chore: synced file(s) with cds-snc/site-reliability-engineering (#1998) * chore: synced local '.github/workflows/s3-backup.yml' with remote 'tools/sre_file_sync/s3-backup.yml' * chore: synced local '.github/workflows/ossf-scorecard.yml' with remote 'tools/sre_file_sync/ossf-scorecard.yml' * chore: synced local '.github/workflows/export_github_data.yml' with remote 'tools/sre_file_sync/export_github_data.yml' --------- Co-authored-by: sre-read-write[bot] <92993749+sre-read-write[bot]@users.noreply.github.com> --- .github/workflows/export_github_data.yml | 2 +- .github/workflows/ossf-scorecard.yml | 4 ++-- .github/workflows/s3-backup.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/export_github_data.yml b/.github/workflows/export_github_data.yml index d37b731ecc..941e3ba2fb 100644 --- a/.github/workflows/export_github_data.yml +++ b/.github/workflows/export_github_data.yml @@ -14,7 +14,7 @@ jobs: DNS_PROXY_FORWARDTOSENTINEL: "true" DNS_PROXY_LOGANALYTICSWORKSPACEID: ${{ secrets.LOG_ANALYTICS_WORKSPACE_ID }} DNS_PROXY_LOGANALYTICSSHAREDKEY: ${{ secrets.LOG_ANALYTICS_WORKSPACE_KEY }} - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: Export Data uses: cds-snc/github-repository-metadata-exporter@main with: diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index 6d9b6c11f0..c85904fcad 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -20,12 +20,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@b687393d2370bdf6d960ea972ff690c9ed797189 + uses: ossf/scorecard-action@2fa1e2fa153141e2950c7e1299ed05e2081ead0c with: results_file: ossf-results.json results_format: json diff --git a/.github/workflows/s3-backup.yml b/.github/workflows/s3-backup.yml index 65899b02a3..61e9aab068 100644 --- a/.github/workflows/s3-backup.yml +++ b/.github/workflows/s3-backup.yml @@ -10,12 +10,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 with: fetch-depth: 0 # retrieve all history - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 + uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 with: aws-access-key-id: ${{ secrets.AWS_S3_BACKUP_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_S3_BACKUP_SECRET_ACCESS_KEY }}