Merge branch 'main' into sms-soak-test

.env.example

@@ -1,6 +1,7 @@
 NOTIFY_ENVIRONMENT=development
 
 ADMIN_CLIENT_SECRET=dev-notify-secret-key
+SRE_CLIENT_SECRET=dev-notify-secret-key
 SECRET_KEY=dev-notify-secret-key
 DANGEROUS_SALT=dev-notify-salt
 

.github/workflows/scripts/run-shellcheck.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run --rm -v "$PWD:/mnt" koalaman/shellcheck:v0.9.0 -P ./bin/ -x ./scripts/*.sh

.github/workflows/shellcheck.yml

@@ -0,0 +1,14 @@
+name: Shellcheck
+on:
+  push:
+    paths:
+      - "**/*.sh"
+
+jobs:
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Shellcheck
+      run: |
+        .github/workflows/scripts/run-shellcheck.sh

app/__init__.py

@@ -169,11 +169,12 @@ def register_notify_blueprint(application, blueprint, auth_function, prefix=None
 
 def register_blueprint(application):
     from app.accept_invite.rest import accept_invite
-    from app.api_key.rest import api_key_blueprint
+    from app.api_key.rest import api_key_blueprint, sre_tools_blueprint
     from app.authentication.auth import (
         requires_admin_auth,
         requires_auth,
         requires_no_auth,
+        requires_sre_auth,
     )
     from app.billing.rest import billing_blueprint
     from app.complaint.complaint_rest import complaint_blueprint
@@ -233,6 +234,8 @@ def register_blueprint(application):
 
     register_notify_blueprint(application, api_key_blueprint, requires_admin_auth, "/api-key")
 
+    register_notify_blueprint(application, sre_tools_blueprint, requires_sre_auth, "/sre-tools")
+
     register_notify_blueprint(application, letter_job, requires_admin_auth)
 
     register_notify_blueprint(application, letter_callback_blueprint, requires_no_auth)

app/api_key/rest.py

@@ -19,6 +19,9 @@
 api_key_blueprint = Blueprint("api_key", __name__)
 register_errors(api_key_blueprint)
 
+sre_tools_blueprint = Blueprint("sre_tools", __name__)
+register_errors(sre_tools_blueprint)
+
 
 @api_key_blueprint.route("/<uuid:api_key_id>/summary-statistics", methods=["GET"])
 def get_api_key_stats(api_key_id):
@@ -103,58 +106,68 @@ def send_api_key_revokation_email(service_id, api_key_name, api_key_information)
     return
 
 
-@api_key_blueprint.route("/revoke-api-keys", methods=["POST"])
+@sre_tools_blueprint.route("/api-key-revoke", methods=["POST"])
 def revoke_api_keys():
     """
-    We take a list of api keys and revoke them. The data is of the form:
-    [
-        {
-            "token": "NMIfyYncKcRALEXAMPLE",
-            "type": "mycompany_api_token",
-            "url": "https://github.com/octocat/Hello-World/blob/12345600b9cbe38a219f39a9941c9319b600c002/foo/bar.txt",
-            "source": "content",
-        }
-    ]
-
-    The function does 3 things:
-    1. Finds the api key by the token
-    2. Revokes the api key
+    This method accepts a single api key and revokes it. The data is of the form:
+    {
+        "token": "gcntfy-key-name-uuid-uuid",
+        "type": "mycompany_api_token",
+        "url": "https://github.com/octocat/Hello-World/blob/12345600b9cbe38a219f39a9941c9319b600c002/foo/bar.txt",
+        "source": "content",
+    }
+
+    The function does 4 things:
+    1. Finds the api key by API key itself
+    2. Revokes the API key
     3. Saves the source and url into the compromised_key_info field
-    4. Sends the service owners of the api key an email notification indicating that the key has been revoked
+    4. TODO: Sends the service owners of the api key an email notification indicating that the key has been revoked
     """
     try:
-        data = request.get_json()
+        api_key_data = request.get_json()
+        # check for correct payload
+        if (
+            isinstance(api_key_data, list)
+            or api_key_data.get("token") is None
+            or api_key_data.get("type") is None
+            or api_key_data.get("url") is None
+            or api_key_data.get("source") is None
+        ):
+            raise InvalidRequest("Invalid payload", status_code=400)
     except werkzeug.exceptions.BadRequest as errors:
         raise InvalidRequest(errors, status_code=400)
 
     # Step 1
-    for api_key_data in data:
-        try:
-            # take last 36 chars of string so that it works even if the full key is provided.
-            api_key_token = api_key_data["token"][-36:]
-            api_key = get_api_key_by_secret(api_key_token)
-        except Exception:
-            current_app.logger.error(f"API key not found for token {api_key_data['type']}")
-            continue  # skip to next api key
-
-        # Step 2
-        expire_api_key(api_key.service_id, api_key.id)
-
-        current_app.logger.info("Expired api key {} for service {}".format(api_key.id, api_key.service_id))
-
-        # Step 3
-        update_compromised_api_key_info(
-            api_key.service_id,
-            api_key.id,
-            {
-                "time_of_revocation": str(datetime.utcnow()),
-                "type": api_key_data["type"],
-                "url": api_key_data["url"],
-                "source": api_key_data["source"],
-            },
+    try:
+        # take last 36 chars of string so that it works even if the full key is provided.
+        api_key_token = api_key_data["token"][-36:]
+        api_key = get_api_key_by_secret(api_key_token)
+    except Exception:
+        current_app.logger.error(
+            "Revoke api key: API key not found for token {}".format(api_key_data["token"])
+            if api_key_data.get("token")
+            else "Revoke api key: no token provided"
         )
+        raise InvalidRequest("Invalid request", status_code=400)
+
+    # Step 2
+    expire_api_key(api_key.service_id, api_key.id)
 
-        # Step 4
-        send_api_key_revokation_email(api_key.service_id, api_key.name, api_key_data)
+    current_app.logger.info("Expired api key {} for service {}".format(api_key.id, api_key.service_id))
+
+    # Step 3
+    update_compromised_api_key_info(
+        api_key.service_id,
+        api_key.id,
+        {
+            "time_of_revocation": str(datetime.utcnow()),
+            "type": api_key_data["type"],
+            "url": api_key_data["url"],
+            "source": api_key_data["source"],
+        },
+    )
+
+    # Step 4
+    send_api_key_revokation_email(api_key.service_id, api_key.name, api_key_data)
 
     return jsonify(result="ok"), 201

app/authentication/auth.py

@@ -93,6 +93,21 @@ def requires_admin_auth():
         raise AuthError("Unauthorized, admin authentication token required", 401)
 
 
+def requires_sre_auth():
+    request_helper.check_proxy_header_before_request()
+
+    auth_type, auth_token = get_auth_token(request)
+    if auth_type != JWT_AUTH_TYPE:
+        raise AuthError("Invalid scheme: can only use JWT for sre authentication", 401)
+    client = __get_token_issuer(auth_token)
+
+    if client == current_app.config.get("SRE_USER_NAME"):
+        g.service_id = current_app.config.get("SRE_USER_NAME")
+        return handle_admin_key(auth_token, current_app.config.get("SRE_CLIENT_SECRET"))
+    else:
+        raise AuthError("Unauthorized, sre authentication token required", 401)
+
+
 def requires_auth():
     request_helper.check_proxy_header_before_request()
 

app/config.py

@@ -592,6 +592,10 @@ class Config(object):
     FF_EMAIL_DAILY_LIMIT = env.bool("FF_EMAIL_DAILY_LIMIT", False)
     FF_SALESFORCE_CONTACT = env.bool("FF_SALESFORCE_CONTACT", False)
 
+    # SRE Tools auth keys
+    SRE_USER_NAME = "SRE_CLIENT_USER"
+    SRE_CLIENT_SECRET = os.getenv("SRE_CLIENT_SECRET")
+
     @classmethod
     def get_sensitive_config(cls) -> list[str]:
         "List of config keys that contain sensitive information"
@@ -612,6 +616,7 @@ def get_sensitive_config(cls) -> list[str]:
             "SALESFORCE_SECURITY_TOKEN",
             "TEMPLATE_PREVIEW_API_KEY",
             "DOCUMENT_DOWNLOAD_API_KEY",
+            "SRE_CLIENT_SECRET",
         ]
 
     @classmethod
@@ -639,6 +644,7 @@ class Development(Config):
     ADMIN_CLIENT_SECRET = os.getenv("ADMIN_CLIENT_SECRET", "dev-notify-secret-key")
     SECRET_KEY = env.list("SECRET_KEY", ["dev-notify-secret-key"])
     DANGEROUS_SALT = os.getenv("DANGEROUS_SALT", "dev-notify-salt ")
+    SRE_CLIENT_SECRET = os.getenv("SRE_CLIENT_SECRET", "dev-notify-secret-key")
 
     NOTIFY_ENVIRONMENT = "development"
     NOTIFICATION_QUEUE_PREFIX = os.getenv("NOTIFICATION_QUEUE_PREFIX", "notification-canada-ca")
@@ -709,7 +715,6 @@ class Production(Config):
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = False
     CRONITOR_ENABLED = False
-    FF_CELERY_CUSTOM_TASK_PARAMS = False
 
 
 class Staging(Production):

scripts/cwcheck.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Check and see if this is running in K8s and if so, wait for cloudwatch agent
+if [ -n "${STATSD_HOST}" ]; then
+    echo "Initializing... Waiting for CWAgent to become ready within the next 30 seconds."
+    timeout=30
+    while [ $timeout -gt 0 ]; do
+        if nc -vz "$STATSD_HOST" 25888; then
+            echo "CWAgent is Ready."
+            break
+        else
+            echo "Waiting for CWAgent to become ready."
+            sleep 1
+            timeout=$((timeout - 1))
+        fi
+    done
+
+    if [ $timeout -eq 0 ]; then
+        echo "Timeout reached. CWAgent did not become ready in 30 seconds."
+        exit 1
+    fi
+fi

scripts/run_celery.sh

@@ -1,25 +1,9 @@
 #!/bin/sh
 
-# runs celery with all celery queues except the throtted sms queue
-
 set -e
 
-# Check and see if this is running in K8s and if so, wait for cloudwatch agent
-if [[ ! -z "${STATSD_HOST}" ]]; then
-
-    echo "Initializing... Waiting for CWAgent to become ready."
-    while :
-    do
-        if  nc -vz $STATSD_HOST 25888; then
-            echo "CWAgent is Ready."
-            break;
-        else
-            echo "Waiting for CWAgent to become ready."
-            sleep 1
-        fi
-    done
-fi
+# Runs celery with all celery queues except the throtted sms queue.
 
 echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}"
 
-celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=${CELERY_CONCURRENCY-4} -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-email-tasks,service-callbacks,delivery-receipts
+celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-email-tasks,service-callbacks,delivery-receipts

scripts/run_celery_beat.sh

@@ -1,23 +1,7 @@
 #!/bin/sh
 
-# runs the celery beat process. This runs the periodic tasks
-
 set -e
 
-# Check and see if this is running in K8s and if so, wait for cloudwatch agent
-if [[ ! -z "${STATSD_HOST}" ]]; then
-
-    echo "Initializing... Waiting for CWAgent to become ready."
-    while :
-    do
-        if  nc -vz $STATSD_HOST 25888; then
-            echo "CWAgent is Ready."
-            break;
-        else
-            echo "Waiting for CWAgent to become ready."
-            sleep 1
-        fi
-    done
-fi
+# Runs the celery beat process, i.e the Celery periodic tasks.
 
 celery -A run_celery.notify_celery beat --loglevel=INFO

scripts/run_celery_core_tasks.sh

@@ -1,25 +1,10 @@
 #!/bin/sh
 
-# runs celery with all celery queues except send-throttled-sms-tasks, send-sms-* and send-email-*
-
 set -e
 
-# Check and see if this is running in K8s and if so, wait for cloudwatch agent
-if [[ ! -z "${STATSD_HOST}" ]]; then
-
-    echo "Initializing... Waiting for CWAgent to become ready."
-    while :
-    do
-        if  nc -vz $STATSD_HOST 25888; then
-            echo "CWAgent is Ready."
-            break;
-        else
-            echo "Waiting for CWAgent to become ready."
-            sleep 1
-        fi
-    done
-fi
+# Runs celery with all celery queues except send-throttled-sms-tasks, 
+# send-sms-* and send-email-*.
 
 echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}"
 
-celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=${CELERY_CONCURRENCY-4} -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,service-callbacks,delivery-receipts
+celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,service-callbacks,delivery-receipts

scripts/run_celery_exit.sh

@@ -10,6 +10,7 @@ function get_celery_pids {
   # and keep only these PIDs
 
   set +o pipefail # so grep returning no matches does not premature fail pipe
+  # shellcheck disable=SC2009 # We don't want to bother re-writing this to use pgrep
   APP_PIDS=$(ps aux --sort=start_time | grep 'celery worker' | grep 'bin/celery' | head -1 | awk '{print $2}')
   set -o pipefail # pipefail should be set everywhere else
 }
@@ -18,12 +19,12 @@ function send_signal_to_celery_processes {
   # refresh pids to account for the case that some workers may have terminated but others not
   get_celery_pids
   # send signal to all remaining apps
-  echo ${APP_PIDS} | tr -d '\n' | tr -s ' ' | xargs echo "Sending signal ${1} to processes with pids: " >> /proc/1/fd/1
+  echo "${APP_PIDS}" | tr -d '\n' | tr -s ' ' | xargs echo "Sending signal ${1} to processes with pids: " >> /proc/1/fd/1
   echo "We will send ${1} signal" >> /proc/1/fd/1
   for value in ${APP_PIDS}
   do
-    echo kill -s ${1} $value
-    kill -s ${1} $value
+    echo kill -s "${1}" "$value"
+    kill -s "${1}" "$value"
   done
   #echo ${APP_PIDS} | xargs kill -s ${1}
 }
@@ -59,6 +60,7 @@ function on_exit {
     echo "exit function is running with wait time of 9s" >> /proc/1/fd/1
     get_celery_pids
     ensure_celery_is_running
+    # shellcheck disable=SC2219 # We could probably rewrite it as `((wait_time++)) || true` but I haven't tested and I assume this works as is
     let wait_time=wait_time+1
     sleep 1
   done

scripts/run_celery_local.sh

@@ -7,4 +7,4 @@ set -e
 
 echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}"
 
-celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=${CELERY_CONCURRENCY-4} -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,send-email-tasks,service-callbacks,delivery-receipts
+celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,send-email-tasks,service-callbacks,delivery-receipts