Merge branch 'main' into task/add-cypress-to-ci

.github/workflows/build_and_push_performance_test.yml

@@ -20,7 +20,7 @@ jobs:
       images: ${{ steps.filter.outputs.changes }}
     steps:
       - name: Checkout
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
       - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
         id: filter
@@ -41,7 +41,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
       - name: Build container
         run: |
@@ -61,7 +61,7 @@ jobs:
 
       - name: Login to ECR
         id: login-ecr
-        uses: aws-actions/amazon-ecr-login@2f9f10ea3fa2eed41ac443fee8bfbd059af2d0a4 # v1.6.0
+        uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d # v1.7.0
 
       - name: Push containers to ECR
         run: |

.github/workflows/codeql.yml

@@ -24,18 +24,18 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@29b1f65c5e92e24fe6b6647da1eaabe529cec70f # v2.3.3
+        uses: github/codeql-action/init@a09933a12a80f87b87005513f0abb1494c27a716 # v2.21.4
         with:
           languages: ${{ matrix.language }}
           queries: +security-and-quality
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@29b1f65c5e92e24fe6b6647da1eaabe529cec70f # v2.3.3
+        uses: github/codeql-action/autobuild@a09933a12a80f87b87005513f0abb1494c27a716 # v2.21.4
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@29b1f65c5e92e24fe6b6647da1eaabe529cec70f # v2.3.3
+        uses: github/codeql-action/analyze@a09933a12a80f87b87005513f0abb1494c27a716 # v2.21.4
         with:
           category: "/language:${{ matrix.language }}"

.github/workflows/docker-vulnerability-scan.yml

@@ -27,7 +27,7 @@ jobs:
 
       - name: Login to ECR
         id: login-ecr
-        uses: aws-actions/amazon-ecr-login@2f9f10ea3fa2eed41ac443fee8bfbd059af2d0a4 # v1.6.0
+        uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d # v1.7.0
         with:
           registry-type: public
 
@@ -62,7 +62,7 @@ jobs:
 
       - name: Login to ECR
         id: login-ecr
-        uses: aws-actions/amazon-ecr-login@2f9f10ea3fa2eed41ac443fee8bfbd059af2d0a4 # v1.6.0
+        uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d # v1.7.0
 
       - name: Docker vulnerability scan
         uses: cds-snc/security-tools/.github/actions/docker-scan@cfec0943e40dbb78cee115bbbe89dc17f07b7a0f # v2.1.3

.github/workflows/docker.yaml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Build and push
     steps:
-    - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+    - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
     - name: Install AWS CLI
       run: |
         curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
@@ -43,7 +43,7 @@ jobs:
 
     - name: Login to ECR
       id: login-ecr
-      uses: aws-actions/amazon-ecr-login@2f9f10ea3fa2eed41ac443fee8bfbd059af2d0a4 # v1.6.0
+      uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d # v1.7.0
       with:
         registry-type: public
 

.github/workflows/lambda_production.yml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
       - name: Configure AWS credentials
         id: aws-creds
@@ -43,7 +43,7 @@ jobs:
 
       - name: Login to ECR
         id: login-ecr
-        uses: aws-actions/amazon-ecr-login@2f9f10ea3fa2eed41ac443fee8bfbd059af2d0a4 # v1.6.0
+        uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d # v1.7.0
 
       - name: Push containers to ECR
         run: |

.github/workflows/lambda_staging.yml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
       - name: Configure AWS credentials
         id: aws-creds
@@ -39,7 +39,7 @@ jobs:
 
       - name: Login to ECR
         id: login-ecr
-        uses: aws-actions/amazon-ecr-login@2f9f10ea3fa2eed41ac443fee8bfbd059af2d0a4 # v1.6.0
+        uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d # v1.7.0
 
       - name: Push containers to ECR
         run: |

.github/workflows/performance.yml

@@ -9,9 +9,9 @@ jobs:
     steps:
       - name: Install libcurl
         run: sudo apt-get update && sudo apt-get install libssl-dev libcurl4-openssl-dev
-      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
       - name: Set up Python 3.10
-        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
           python-version: '3.10'
       - name: Upgrade pip

.github/workflows/s3-backup.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
 
     - name: Checkout
-      uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+      uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
       with:
         fetch-depth: 0 # retrieve all history
 

.github/workflows/test.yaml

@@ -18,9 +18,9 @@ jobs:
     steps:
     - name: Install libcurl
       run: sudo apt-get update && sudo apt-get install libssl-dev libcurl4-openssl-dev
-    - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+    - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
     - name: Set up Python 3.10
-      uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0
+      uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
       with:
         python-version: '3.10'
     - name: Upgrade pip
@@ -67,7 +67,7 @@ jobs:
       run: |
         cp -f .env.example .env
     - name: Checks for new endpoints against AWS WAF rules
-      uses: cds-snc/notification-utils/.github/actions/[email protected].4
+      uses: cds-snc/notification-utils/.github/actions/waffles@415cd22db72ea1bcc56b7904f984cc7de369b7df # 52.0.6
       with:
         app-loc: '/github/workspace'
         app-libs: '/github/workspace/env/site-packages'

.gitignore

@@ -30,6 +30,7 @@ var/
 /cache
 newrelic-layer.zip
 smoketest.sh
+smoketest-prod.sh
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

Makefile

@@ -50,17 +50,13 @@ smoke-test:
 run: ## Run the web app
 	flask run -p 6011 --host=0.0.0.0
 
-.PHONY: run-celery
-run-celery: ## Run the celery workers
-	./scripts/run_celery.sh
+.PHONY: run-celery-local
+run-celery-local: ## Run the celery workers with all the queues
+	./scripts/run_celery_local.sh
 
-.PHONY: run-celery-clean
-run-celery-clean: ## Run the celery workers but filter out common scheduled tasks
-	./scripts/run_celery.sh 2>&1 >/dev/null | grep -Ev 'beat|in-flight-to-inbox|run-scheduled-jobs|check-job-status'
-
-.PHONY: run-celery-sms
-run-celery-sms: ## run the celery workers for sms from dedicated numbers
-	./scripts/run_celery_sms.sh
+.PHONY: run-celery-local-filtered
+run-celery-local-filtered: ## Run the celery workers with all queues but filter out common scheduled tasks
+	./scripts/run_celery_local.sh 2>&1 >/dev/null | grep -Ev 'beat|in-flight-to-inbox|run-scheduled-jobs|check-job-status'
 
 .PHONY: run-celery-beat
 run-celery-beat: ## Run the celery beat

app/api_key/rest.py

@@ -1,12 +1,20 @@
-from flask import Blueprint, jsonify
+from datetime import datetime
+
+import werkzeug
+from flask import Blueprint, current_app, jsonify, request
 
 from app import DATETIME_FORMAT
+from app.dao.api_key_dao import (
+    expire_api_key,
+    get_api_key_by_secret,
+    update_compromised_api_key_info,
+)
 from app.dao.fact_notification_status_dao import (
     get_api_key_ranked_by_notifications_created,
     get_last_send_for_api_key,
     get_total_notifications_sent_for_api_key,
 )
-from app.errors import register_errors
+from app.errors import InvalidRequest, register_errors
 
 api_key_blueprint = Blueprint("api_key", __name__)
 register_errors(api_key_blueprint)
@@ -59,3 +67,94 @@ def get_api_keys_ranked(n_days_back):
             }
         )
     return jsonify(data=data)
+
+
+def send_api_key_revokation_email(service_id, api_key_name, api_key_information):
+    """
+    TODO: this function if not ready yet. It needs a template to be created.
+    email = email_data_request_schema.load(request.get_json())
+
+    users_to_send_to = dao_fetch_active_users_for_service(service_id)
+
+    template = dao_get_template_by_id(current_app.config["API_KEY_REVOKED_TEMPLATE_ID"])  # this template currently doesn't exist
+    service = Service.query.get(current_app.config["NOTIFY_SERVICE_ID"])
+    users_service = Service.query.get(service_id)
+    for user_to_send_to in users_to_send_to:
+        saved_notification = persist_notification(
+            template_id=template.id,
+            template_version=template.version,
+            recipient=email["email"],
+            service=service,
+            personalisation={
+                "user_name": user_to_send_to.name,
+                "api_key_name": api_key_name,
+                "service_name": users_service.name,
+                "api_key_information": api_key_information,
+            },
+            notification_type=template.template_type,
+            api_key_id=None,
+            key_type=KEY_TYPE_NORMAL,
+            reply_to_text=service.get_default_reply_to_email_address(),
+        )
+
+        send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
+
+    """
+    return
+
+
+@api_key_blueprint.route("/revoke-api-keys", methods=["POST"])
+def revoke_api_keys():
+    """
+    We take a list of api keys and revoke them. The data is of the form:
+    [
+        {
+            "token": "NMIfyYncKcRALEXAMPLE",
+            "type": "mycompany_api_token",
+            "url": "https://github.com/octocat/Hello-World/blob/12345600b9cbe38a219f39a9941c9319b600c002/foo/bar.txt",
+            "source": "content",
+        }
+    ]
+
+    The function does 3 things:
+    1. Finds the api key by the token
+    2. Revokes the api key
+    3. Saves the source and url into the compromised_key_info field
+    4. Sends the service owners of the api key an email notification indicating that the key has been revoked
+    """
+    try:
+        data = request.get_json()
+    except werkzeug.exceptions.BadRequest as errors:
+        raise InvalidRequest(errors, status_code=400)
+
+    # Step 1
+    for api_key_data in data:
+        try:
+            # take last 36 chars of string so that it works even if the full key is provided.
+            api_key_token = api_key_data["token"][-36:]
+            api_key = get_api_key_by_secret(api_key_token)
+        except Exception:
+            current_app.logger.error(f"API key not found for token {api_key_data['type']}")
+            continue  # skip to next api key
+
+        # Step 2
+        expire_api_key(api_key.service_id, api_key.id)
+
+        current_app.logger.info("Expired api key {} for service {}".format(api_key.id, api_key.service_id))
+
+        # Step 3
+        update_compromised_api_key_info(
+            api_key.service_id,
+            api_key.id,
+            {
+                "time_of_revocation": str(datetime.utcnow()),
+                "type": api_key_data["type"],
+                "url": api_key_data["url"],
+                "source": api_key_data["source"],
+            },
+        )
+
+        # Step 4
+        send_api_key_revokation_email(api_key.service_id, api_key.name, api_key_data)
+
+    return jsonify(result="ok"), 201

app/celery/process_ses_receipts_tasks.py

@@ -79,13 +79,11 @@ def process_ses_results(self, response):
 
         statsd_client.incr("callback.ses.{}".format(notification_status))
 
-        # Record bounces and notifications in Redis
-        if current_app.config["FF_BOUNCE_RATE_BACKEND"]:
-            if notification_status == NOTIFICATION_PERMANENT_FAILURE:
-                bounce_rate_client.set_sliding_hard_bounce(notification.service_id, str(notification.id))
-                current_app.logger.info(
-                    f"Setting total hard bounce notifications for service {notification.service.id} with notification {notification.id} in REDIS"
-                )
+        if notification_status == NOTIFICATION_PERMANENT_FAILURE:
+            bounce_rate_client.set_sliding_hard_bounce(notification.service_id, str(notification.id))
+            current_app.logger.info(
+                f"Setting total hard bounce notifications for service {notification.service.id} with notification {notification.id} in REDIS"
+            )
 
         if notification.sent_at:
             statsd_client.timing_with_dates("callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at)

app/celery/scheduled_tasks.py

@@ -190,7 +190,7 @@ def check_job_status():
 
     # temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
     # if they haven't been re-processed in time.
-    job_ids: List[str] = []
+    job_ids: List[str] = []  # type: ignore
     for job in jobs_not_complete_after_30_minutes:
         job.job_status = JOB_STATUS_ERROR
         dao_update_job(job)
@@ -383,5 +383,5 @@ def beat_inbox_sms_priority():
 
     while list_of_sms_notifications:
         save_smss.apply_async((None, list_of_sms_notifications, receipt_id_sms), queue=QueueNames.PRIORITY_DATABASE)
-        current_app.logger.info(f"Batch saving with Bulk Priority: SMS receipt {receipt_id_sms} sent to in-flight.")
+        current_app.logger.info(f"Batch saving with Priority: SMS receipt {receipt_id_sms} sent to in-flight.")
         receipt_id_sms, list_of_sms_notifications = sms_priority.poll()