Merge branch 'main' into feat/rework-callback-logging

.devcontainer/Dockerfile

@@ -2,7 +2,7 @@ FROM mcr.microsoft.com/vscode/devcontainers/python:0-3.10@sha256:ef9cc483a593c95
 
 ARG KUBENS_VERSION="0.9.4"
 ARG OCTANT_VERSION="0.25.1"
-ENV POETRY_VERSION="1.3.2"
+ENV POETRY_VERSION="1.7.1"
 
 # Install packages
 RUN apt-get update \

.devcontainer/scripts/notify-dev-entrypoint.sh

@@ -34,8 +34,10 @@ cd /workspace
 echo -e "fpath+=/.zfunc" >> ~/.zshrc
 echo -e "autoload -Uz compinit && compinit"
 
-pip install poetry==${POETRY_VERSION} \
-  && poetry --version
+pip install poetry==${POETRY_VERSION}
+export PATH=$PATH:/home/vscode/.local/bin/
+which poetry
+poetry --version
 
 # Initialize poetry autocompletions
 mkdir ~/.zfunc

.github/workflows/build_and_push_performance_test.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
+      - uses: dorny/paths-filter@7267a8516b6f92bdb098633497bad573efdbf271 # v2.12.0
         id: filter
         with:
           filters: |

.github/workflows/codeql.yml

@@ -27,15 +27,15 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@1500a131381b66de0c52ac28abb13cd79f4b7ecc # v2.22.12
+        uses: github/codeql-action/init@2f93e4319b2f04a2efc38fa7f78bd681bc3f7b2f # v2.23.2
         with:
           languages: ${{ matrix.language }}
           queries: +security-and-quality
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@1500a131381b66de0c52ac28abb13cd79f4b7ecc # v2.22.12
+        uses: github/codeql-action/autobuild@2f93e4319b2f04a2efc38fa7f78bd681bc3f7b2f # v2.23.2
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@1500a131381b66de0c52ac28abb13cd79f4b7ecc # v2.22.12
+        uses: github/codeql-action/analyze@2f93e4319b2f04a2efc38fa7f78bd681bc3f7b2f # v2.23.2
         with:
           category: "/language:${{ matrix.language }}"

.github/workflows/ossf-scorecard.yml

@@ -25,7 +25,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@75cb7af1033cfb77c9fc7d8abc30420008f558f4
+        uses: ossf/scorecard-action@0ae0fb3a2ca18a43d6dea9c07cfb9bd01d17eae1
         with:
           results_file: ossf-results.json
           results_format: json

.github/workflows/performance.yml

@@ -16,7 +16,7 @@ jobs:
           python-version: '3.10'
       - name: Upgrade pip
         run: python -m pip install --upgrade pip
-      - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+      - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}

.github/workflows/test.yaml

@@ -25,18 +25,18 @@ jobs:
         python-version: '3.10'
     - name: Upgrade pip
       run: python -m pip install --upgrade pip
-    - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+    - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
         restore-keys: |
           ${{ runner.os }}-pip-
     - name: Install poetry
       env:
-        POETRY_VERSION: "1.3.2"
+        POETRY_VERSION: "1.7.1"
       run: pip install poetry==${POETRY_VERSION} && poetry --version
     - name: Check poetry.lock aligns with pyproject.toml
-      run: poetry lock --check
+      run: poetry check --lock
     - name: Install requirements
       run: poetry install --with test
     - name: Run tests
@@ -67,7 +67,7 @@ jobs:
       run: |
         cp -f .env.example .env
     - name: Checks for new endpoints against AWS WAF rules
-      uses: cds-snc/notification-utils/.github/actions/waffles@69b6eee0c715fa4c3fe9ac48e4ccd6e3d4c1e239 # 52.0.18
+      uses: cds-snc/notification-utils/.github/actions/waffles@06a40db6286f525fe3551e029418458d33342592 # 52.1.0
       with:
         app-loc: '/github/workspace'
         app-libs: '/github/workspace/env/site-packages'

.gitignore

@@ -72,10 +72,9 @@ target/
 # Mac
 *.DS_Store
 environment.sh
-.envrc
-.env
-.env_*
-.env.lambda
+.env*
+!.env.devcontainer
+!.env.example
 
 celerybeat-schedule
 celerybeat-schedule.*

Makefile

@@ -25,7 +25,7 @@ freeze-requirements:
 
 .PHONY: test-requirements
 test-requirements:
-	poetry lock --check
+	poetry check --lock
 
 .PHONY: coverage
 coverage: venv ## Create coverage report

app/celery/celery.py

@@ -48,6 +48,7 @@ def init_app(self, app):
                 "beat_schedule": app.config["CELERYBEAT_SCHEDULE"],
                 "imports": app.config["CELERY_IMPORTS"],
                 "task_serializer": app.config["CELERY_TASK_SERIALIZER"],
+                "enable_utc": app.config["CELERY_ENABLE_UTC"],
                 "timezone": app.config["CELERY_TIMEZONE"],
                 "broker_transport_options": app.config["BROKER_TRANSPORT_OPTIONS"],
                 "task_queues": app.config["CELERY_QUEUES"],

app/celery/tasks.py

@@ -78,7 +78,6 @@
     persist_notifications,
     send_notification_to_queue,
 )
-from app.notifications.validators import check_service_over_daily_message_limit
 from app.types import VerifiedNotification
 from app.utils import get_csv_max_rows, get_delivery_queue_for_template
 from app.v2.errors import (
@@ -300,8 +299,6 @@ def save_smss(self, service_id: Optional[str], signed_notifications: List[Signed
     current_app.logger.debug(f"Sending following sms notifications to AWS: {notification_id_queue.keys()}")
     for notification_obj in saved_notifications:
         try:
-            if not current_app.config["FF_EMAIL_DAILY_LIMIT"]:
-                check_service_over_daily_message_limit(notification_obj.key_type, service)
             queue = notification_id_queue.get(notification_obj.id) or get_delivery_queue_for_template(template)
             send_notification_to_queue(
                 notification_obj,
@@ -423,8 +420,6 @@ def try_to_send_notifications_to_queue(notification_id_queue, service, saved_not
     research_mode = service.research_mode  # type: ignore
     for notification_obj in saved_notifications:
         try:
-            if not current_app.config["FF_EMAIL_DAILY_LIMIT"]:
-                check_service_over_daily_message_limit(notification_obj.key_type, service)
             queue = notification_id_queue.get(notification_obj.id) or get_delivery_queue_for_template(template)
             send_notification_to_queue(
                 notification_obj,

app/config.py

@@ -359,7 +359,7 @@ class Config(object):
         "queue_name_prefix": NOTIFICATION_QUEUE_PREFIX,
     }
     CELERY_ENABLE_UTC = True
-    CELERY_TIMEZONE = os.getenv("TIMEZONE", "America/Toronto")
+    CELERY_TIMEZONE = os.getenv("TIMEZONE", "UTC")
     CELERY_ACCEPT_CONTENT = ["json"]
     CELERY_TASK_SERIALIZER = "json"
     CELERY_IMPORTS = (
@@ -438,87 +438,54 @@ class Config(object):
         # app/celery/nightly_tasks.py
         "timeout-sending-notifications": {
             "task": "timeout-sending-notifications",
-            "schedule": crontab(hour=0, minute=5),
+            "schedule": crontab(hour=5, minute=5),  # 00:05 EST in UTC
             "options": {"queue": QueueNames.PERIODIC},
         },
         "create-nightly-billing": {
             "task": "create-nightly-billing",
-            "schedule": crontab(hour=0, minute=15),
+            "schedule": crontab(hour=5, minute=15),  # 00:15 EST in UTC
             "options": {"queue": QueueNames.REPORTING},
         },
         "create-nightly-notification-status": {
             "task": "create-nightly-notification-status",
-            "schedule": crontab(hour=0, minute=30),  # after 'timeout-sending-notifications'
+            "schedule": crontab(hour=5, minute=30),  # 00:30 EST in UTC, after 'timeout-sending-notifications'
             "options": {"queue": QueueNames.REPORTING},
         },
         "delete-sms-notifications": {
             "task": "delete-sms-notifications",
-            "schedule": crontab(hour=4, minute=15),  # after 'create-nightly-notification-status'
+            "schedule": crontab(hour=9, minute=15),  # 4:15 EST in UTC,  after 'create-nightly-notification-status'
             "options": {"queue": QueueNames.PERIODIC},
         },
         "delete-email-notifications": {
             "task": "delete-email-notifications",
-            "schedule": crontab(hour=4, minute=30),  # after 'create-nightly-notification-status'
+            "schedule": crontab(hour=9, minute=30),  # 4:30 EST in UTC, after 'create-nightly-notification-status'
             "options": {"queue": QueueNames.PERIODIC},
         },
         "delete-letter-notifications": {
             "task": "delete-letter-notifications",
-            "schedule": crontab(hour=4, minute=45),  # after 'create-nightly-notification-status'
+            "schedule": crontab(hour=9, minute=45),  # 4:45 EST in UTC, after 'create-nightly-notification-status'
             "options": {"queue": QueueNames.PERIODIC},
         },
         "delete-inbound-sms": {
             "task": "delete-inbound-sms",
-            "schedule": crontab(hour=1, minute=40),
+            "schedule": crontab(hour=6, minute=40),  # 1:40 EST in UTC
             "options": {"queue": QueueNames.PERIODIC},
         },
         "send-daily-performance-platform-stats": {
             "task": "send-daily-performance-platform-stats",
-            "schedule": crontab(hour=2, minute=0),
+            "schedule": crontab(hour=7, minute=0),  # 2:00 EST in UTC
             "options": {"queue": QueueNames.PERIODIC},
         },
         "remove_transformed_dvla_files": {
             "task": "remove_transformed_dvla_files",
-            "schedule": crontab(hour=3, minute=40),
+            "schedule": crontab(hour=8, minute=40),  # 3:40 EST in UTC
             "options": {"queue": QueueNames.PERIODIC},
         },
         "remove_sms_email_jobs": {
             "task": "remove_sms_email_jobs",
-            "schedule": crontab(hour=4, minute=0),
+            "schedule": crontab(hour=9, minute=0),  # 4:00 EST in UTC
             "options": {"queue": QueueNames.PERIODIC},
         },
-        # 'remove_letter_jobs': {
-        # 'task': 'remove_letter_jobs',
-        # 'schedule': crontab(hour=4, minute=20),
-        #  since we mark jobs as archived
-        # 'options': {'queue': QueueNames.PERIODIC},
-        # },
-        # 'check-templated-letter-state': {
-        # 'task': 'check-templated-letter-state',
-        # 'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0),
-        # 'options': {'queue': QueueNames.PERIODIC}
-        # },
-        # 'check-precompiled-letter-state': {
-        # 'task': 'check-precompiled-letter-state',
-        # 'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
-        # 'options': {'queue': QueueNames.PERIODIC}
-        # },
-        # 'raise-alert-if-letter-notifications-still-sending': {
-        # 'task': 'raise-alert-if-letter-notifications-still-sending',
-        # 'schedule': crontab(hour=16, minute=30),
-        # 'options': {'queue': QueueNames.PERIODIC}
-        # },
-        # The collate-letter-pdf does assume it is called in an hour that BST does not make a
-        # difference to the truncate date which translates to the filename to process
-        # 'collate-letter-pdfs-for-day': {
-        # 'task': 'collate-letter-pdfs-for-day',
-        # 'schedule': crontab(hour=17, minute=50),
-        # 'options': {'queue': QueueNames.PERIODIC}
-        # },
-        # 'raise-alert-if-no-letter-ack-file': {
-        # 'task': 'raise-alert-if-no-letter-ack-file',
-        # 'schedule': crontab(hour=23, minute=00),
-        # 'options': {'queue': QueueNames.PERIODIC}
-        # },
     }
     CELERY_QUEUES: List[Any] = []
     CELERY_DELIVER_SMS_RATE_LIMIT = os.getenv("CELERY_DELIVER_SMS_RATE_LIMIT", "1/s")
@@ -595,8 +562,6 @@ class Config(object):
     # Feature flag to enable custom retry policies such as lowering retry period for certain priority lanes.
     FF_CELERY_CUSTOM_TASK_PARAMS = env.bool("FF_CELERY_CUSTOM_TASK_PARAMS", True)
     FF_CLOUDWATCH_METRICS_ENABLED = env.bool("FF_CLOUDWATCH_METRICS_ENABLED", False)
-    # Feature flags for email_daily_limit
-    FF_EMAIL_DAILY_LIMIT = env.bool("FF_EMAIL_DAILY_LIMIT", False)
     FF_SALESFORCE_CONTACT = env.bool("FF_SALESFORCE_CONTACT", False)
 
     # SRE Tools auth keys
@@ -704,8 +669,6 @@ class Test(Development):
     CRM_ORG_LIST_URL = "https://test-url.com"
     FAILED_LOGIN_LIMIT = 0
 
-    FF_EMAIL_DAILY_LIMIT = False
-
 
 class Production(Config):
     NOTIFY_EMAIL_DOMAIN = os.getenv("NOTIFY_EMAIL_DOMAIN", "notification.canada.ca")

app/dao/api_key_dao.py

@@ -68,6 +68,13 @@ def expire_api_key(service_id, api_key_id):
     db.session.add(api_key)
 
 
+@transactional
+def update_last_used_api_key(api_key_id, last_used=None) -> None:
+    api_key = ApiKey.query.filter_by(id=api_key_id).one()
+    api_key.last_used_timestamp = last_used if last_used else datetime.utcnow()
+    db.session.add(api_key)
+
+
 @transactional
 @version_class(ApiKey)
 def update_compromised_api_key_info(service_id, api_key_id, compromised_info):

app/dao/fact_notification_status_dao.py

@@ -1,7 +1,6 @@
 from datetime import datetime, time, timedelta
 
 from flask import current_app
-from notifications_utils.timezones import convert_local_timezone_to_utc
 from sqlalchemy import Date, case, func
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.sql.expression import extract, literal
@@ -39,8 +38,8 @@
 
 
 def fetch_notification_status_for_day(process_day, service_id=None):
-    start_date = convert_local_timezone_to_utc(datetime.combine(process_day, time.min))
-    end_date = convert_local_timezone_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
+    start_date = datetime.combine(process_day, time.min)
+    end_date = datetime.combine(process_day + timedelta(days=1), time.min)
     # use notification_history if process day is older than 7 days
     # this is useful if we need to rebuild the ft_billing table for a date older than 7 days ago.
     current_app.logger.info("Fetch ft_notification_status for {} to {}".format(start_date, end_date))
@@ -239,13 +238,19 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id):
 
 
 def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, by_template=False, limit_days=7):
-    ft_start_date = utc_midnight_n_days_ago(limit_days)
+    if limit_days == 1:
+        ft_start_date = utc_midnight_n_days_ago(limit_days - 1)
+        # For daily stats, service limits reset at 12:00am UTC each night, so we need to fetch the data from 12:00 UTC to now
+        start = utc_midnight_n_days_ago(0)
+        end = datetime.utcnow()
+    else:
+        ft_start_date = utc_midnight_n_days_ago(limit_days)
 
-    # The nightly task that populates ft_notification_status counts collects notifications from
-    # 5AM the day before to 5AM of the current day. So we need to match that timeframe when
-    # we fetch notifications for the current day.
-    start = (tz_aware_midnight_n_days_ago(1) + timedelta(hours=5)).replace(minute=0, second=0, microsecond=0)
-    end = (tz_aware_midnight_n_days_ago(0) + timedelta(hours=5)).replace(minute=0, second=0, microsecond=0)
+        # The nightly task that populates ft_notification_status counts collects notifications from
+        # 5AM the day before to 5AM of the current day. So we need to match that timeframe when
+        # we fetch notifications for the current day.
+        start = (tz_aware_midnight_n_days_ago(1) + timedelta(hours=5)).replace(minute=0, second=0, microsecond=0)
+        end = (tz_aware_midnight_n_days_ago(0) + timedelta(hours=5)).replace(minute=0, second=0, microsecond=0)
 
     stats_for_7_days = db.session.query(
         FactNotificationStatus.notification_type.label("notification_type"),
@@ -336,24 +341,21 @@ def get_total_notifications_sent_for_api_key(api_key_id):
 
 def get_last_send_for_api_key(api_key_id):
     """
+    SELECT last_used_timestamp as last_notification_created
+    FROM api_keys
+    WHERE id = 'api_key_id';
+
+    If last_used_timestamp is null, then check notifications table/ or notification_history.
     SELECT max(created_at) as last_notification_created
     FROM notifications
     WHERE api_key_id = 'api_key_id'
     GROUP BY api_key_id;
     """
-    notification_table = (
-        db.session.query(func.max(Notification.created_at).label("last_notification_created"))
-        .filter(Notification.api_key_id == api_key_id)
-        .all()
+    # Fetch last_used_timestamp from api_keys table
+    api_key_table = (
+        db.session.query(ApiKey.last_used_timestamp.label("last_notification_created")).filter(ApiKey.id == api_key_id).all()
     )
-    if not notification_table[0][0]:
-        notification_table = (
-            db.session.query(func.max(NotificationHistory.created_at).label("last_notification_created"))
-            .filter(NotificationHistory.api_key_id == api_key_id)
-            .all()
-        )
-        notification_table = [] if notification_table[0][0] is None else notification_table
-    return notification_table
+    return [] if api_key_table[0][0] is None else api_key_table
 
 
 def get_api_key_ranked_by_notifications_created(n_days_back):