Merge branch 'main' into enlarge-db-script

.github/workflows/codeql.yml

@@ -27,15 +27,15 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@1500a131381b66de0c52ac28abb13cd79f4b7ecc # v2.22.12
+        uses: github/codeql-action/init@8b7fcbfac2aae0e6c24d9f9ebd5830b1290b18e4 # v2.23.0
         with:
           languages: ${{ matrix.language }}
           queries: +security-and-quality
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@1500a131381b66de0c52ac28abb13cd79f4b7ecc # v2.22.12
+        uses: github/codeql-action/autobuild@8b7fcbfac2aae0e6c24d9f9ebd5830b1290b18e4 # v2.23.0
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@1500a131381b66de0c52ac28abb13cd79f4b7ecc # v2.22.12
+        uses: github/codeql-action/analyze@8b7fcbfac2aae0e6c24d9f9ebd5830b1290b18e4 # v2.23.0
         with:
           category: "/language:${{ matrix.language }}"

.github/workflows/performance.yml

@@ -16,7 +16,7 @@ jobs:
           python-version: '3.10'
       - name: Upgrade pip
         run: python -m pip install --upgrade pip
-      - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+      - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}

.github/workflows/test.yaml

@@ -25,7 +25,7 @@ jobs:
         python-version: '3.10'
     - name: Upgrade pip
       run: python -m pip install --upgrade pip
-    - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+    - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}

app/api_key/rest.py

@@ -121,7 +121,8 @@ def revoke_api_keys():
 
     # Step 1
     try:
-        api_key_token = api_key_data["token"]
+        # take last 36 chars of string so that it works even if the full key is provided.
+        api_key_token = api_key_data["token"][-36:]
         api_key = get_api_key_by_secret(api_key_token)
     except Exception:
         current_app.logger.error(

app/authentication/auth.py

@@ -152,8 +152,19 @@ def requires_auth():
 
 
 def _auth_by_api_key(auth_token):
+    # TODO: uncomment this when the grace period for the token prefix is over
+    # orig_token = auth_token
+
     try:
+        # take last 36 chars of string so that it works even if the full key is provided.
+        auth_token = auth_token[-36:]
         api_key = get_api_key_by_secret(auth_token)
+
+        # TODO: uncomment this when the grace period for the token prefix is over
+        # check for token prefix
+        # if current_app.config["API_KEY_PREFIX"] not in orig_token:
+        #     raise AuthError("Invalid token: you must re-generate your API key to continue using GC Notify", 403, service_id=api_key.service.id, api_key_id=api_key.id)
+
     except NoResultFound:
         raise AuthError("Invalid token: API key not found", 403)
     _auth_with_api_key(api_key, api_key.service)

app/celery/celery.py

@@ -48,6 +48,7 @@ def init_app(self, app):
                 "beat_schedule": app.config["CELERYBEAT_SCHEDULE"],
                 "imports": app.config["CELERY_IMPORTS"],
                 "task_serializer": app.config["CELERY_TASK_SERIALIZER"],
+                "enable_utc": app.config["CELERY_ENABLE_UTC"],
                 "timezone": app.config["CELERY_TIMEZONE"],
                 "broker_transport_options": app.config["BROKER_TRANSPORT_OPTIONS"],
                 "task_queues": app.config["CELERY_QUEUES"],

app/config.py

@@ -359,7 +359,7 @@ class Config(object):
         "queue_name_prefix": NOTIFICATION_QUEUE_PREFIX,
     }
     CELERY_ENABLE_UTC = True
-    CELERY_TIMEZONE = os.getenv("TIMEZONE", "America/Toronto")
+    CELERY_TIMEZONE = os.getenv("TIMEZONE", "UTC")
     CELERY_ACCEPT_CONTENT = ["json"]
     CELERY_TASK_SERIALIZER = "json"
     CELERY_IMPORTS = (

app/dao/api_key_dao.py

@@ -76,30 +76,13 @@ def update_compromised_api_key_info(service_id, api_key_id, compromised_info):
     db.session.add(api_key)
 
 
-def get_api_key_by_secret(secret, service_id=None):
-    # Check the first part of the secret is the gc prefix
-    if current_app.config["API_KEY_PREFIX"] != secret[: len(current_app.config["API_KEY_PREFIX"])]:
-        raise NoResultFound()
-
-    # Check if the remaining part of the secret is a the valid api key
-    token = secret[-36:]
-    signed_with_all_keys = signer_api_key.sign_with_all_keys(str(token))
+def get_api_key_by_secret(secret):
+    signed_with_all_keys = signer_api_key.sign_with_all_keys(str(secret))
     for signed_secret in signed_with_all_keys:
         try:
-            api_key = db.on_reader().query(ApiKey).filter_by(_secret=signed_secret).options(joinedload("service")).one()
+            return db.on_reader().query(ApiKey).filter_by(_secret=signed_secret).options(joinedload("service")).one()
         except NoResultFound:
             pass
-
-    # Check the middle portion of the secret is the valid service id
-    if api_key.service_id:
-        if len(secret) >= 79:
-            service_id_from_token = str(secret[-73:-37])
-            if str(api_key.service_id) != service_id_from_token:
-                raise NoResultFound()
-        else:
-            raise NoResultFound()
-    if api_key:
-        return api_key
     raise NoResultFound()
 
 

app/delivery/send_to_providers.py

@@ -1,7 +1,6 @@
 import base64
 import os
 import re
-import urllib.request
 from datetime import datetime
 from typing import Dict
 from uuid import UUID
@@ -17,6 +16,8 @@
     SMSMessageTemplate,
 )
 from unidecode import unidecode
+from urllib3 import PoolManager
+from urllib3.util import Retry
 
 from app import bounce_rate_client, clients, document_download_client, statsd_client
 from app.celery.research_mode_tasks import send_email_response, send_sms_response
@@ -231,27 +232,29 @@ def send_email_to_provider(notification: Notification):
             check_file_url(personalisation_data[key]["document"], notification.id)
             sending_method = personalisation_data[key]["document"].get("sending_method")
             direct_file_url = personalisation_data[key]["document"]["direct_file_url"]
+            filename = personalisation_data[key]["document"].get("filename")
+            mime_type = personalisation_data[key]["document"].get("mime_type")
             document_id = personalisation_data[key]["document"]["id"]
             scan_verdict_response = document_download_client.check_scan_verdict(service.id, document_id, sending_method)
             check_for_malware_errors(scan_verdict_response.status_code, notification)
             current_app.logger.info(f"scan_verdict for document_id {document_id} is {scan_verdict_response.json()}")
             if sending_method == "attach":
                 try:
-                    req = urllib.request.Request(direct_file_url)
-                    with urllib.request.urlopen(req) as response:
-                        buffer = response.read()
-                        filename = personalisation_data[key]["document"].get("filename")
-                        mime_type = personalisation_data[key]["document"].get("mime_type")
-                        attachments.append(
-                            {
-                                "name": filename,
-                                "data": buffer,
-                                "mime_type": mime_type,
-                            }
-                        )
+                    retries = Retry(total=5)
+                    http = PoolManager(retries=retries)
+
+                    response = http.request("GET", url=direct_file_url)
+                    attachments.append(
+                        {
+                            "name": filename,
+                            "data": response.data,
+                            "mime_type": mime_type,
+                        }
+                    )
                 except Exception as e:
                     current_app.logger.error(f"Could not download and attach {direct_file_url}\nException: {e}")
-                del personalisation_data[key]
+                    del personalisation_data[key]
+
             else:
                 personalisation_data[key] = personalisation_data[key]["document"]["url"]
 

app/models.py

@@ -918,6 +918,7 @@ class ApiKey(BaseModel, Versioned):
     created_by = db.relationship("User")
     created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey("users.id"), index=True, nullable=False)
     compromised_key_info = db.Column(JSONB(none_as_null=True), nullable=True, default={})
+    last_used_timestamp = db.Column(db.DateTime, index=False, unique=False, nullable=True, default=None)
 
     __table_args__ = (
         Index(

migrations/versions/0443_add_apikey_last_used_column.py

@@ -0,0 +1,22 @@
+"""
+Revision ID: 0443_add_apikey_last_used_column
+Revises: 0442_add_heartbeat_templates
+Create Date: 2022-09-21 00:00:00
+"""
+from datetime import datetime
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0443_add_apikey_last_used_column"
+down_revision = "0442_add_heartbeat_templates"
+
+
+def upgrade():
+    op.add_column("api_keys", sa.Column("last_used_timestamp", sa.DateTime(), nullable=True))
+    op.add_column("api_keys_history", sa.Column("last_used_timestamp", sa.DateTime(), nullable=True))
+
+
+def downgrade():
+    op.drop_column("api_keys", "last_used_timestamp")
+    op.drop_column("api_keys_history", "last_used_timestamp")