Merge branch 'main' into fix/provide-secret-fallback

.github/workflows/build_and_push_performance_test.yml

@@ -11,7 +11,7 @@ on:
 
 env:
   GITHUB_SHA: ${{ github.sha }}
-  REGISTRY: ${{ secrets.AWS_ACCOUNT }}.dkr.ecr.ca-central-1.amazonaws.com/notify
+  REGISTRY: ${{ secrets.STAGING_AWS_ACCOUNT_ID }}.dkr.ecr.ca-central-1.amazonaws.com/notify
 
 jobs:
   changes:
@@ -55,8 +55,8 @@ jobs:
         id: aws-creds
         uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
           aws-region: ca-central-1
 
       - name: Login to ECR

.github/workflows/docker-vulnerability-scan.yml

@@ -47,13 +47,13 @@ jobs:
     runs-on: ubuntu-latest
     env:
       DOCKERFILE_PATH: "ci/Dockerfile.lambda"
-      DOCKER_IMAGE: "${{ secrets.PRODUCTION_API_LAMBDA_ECR_ACCOUNT }}.dkr.ecr.ca-central-1.amazonaws.com/notify/api-lambda"
+      DOCKER_IMAGE: "${{ secrets.PRODUCTION_AWS_ACCOUNT_ID }}.dkr.ecr.ca-central-1.amazonaws.com/notify/api-lambda"
 
     steps:
       - name: Configure credentials to Notify account using OIDC
         uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:
-          role-to-assume: arn:aws:iam::${{ secrets.PRODUCTION_API_LAMBDA_ECR_ACCOUNT }}:role/notification-api-apply
+          role-to-assume: arn:aws:iam::${{ secrets.PRODUCTION_AWS_ACCOUNT_ID }}:role/notification-api-apply
           role-session-name: NotifyApiGitHubActions
           aws-region: "ca-central-1"
 

.github/workflows/docker.yaml

@@ -9,8 +9,7 @@ env:
   DOCKER_ORG: public.ecr.aws/v6b8u5o6
   DOCKER_SLUG: public.ecr.aws/v6b8u5o6/notify-api
   KUBECTL_VERSION: '1.23.6'
-  WORKFLOW_PAT: ${{ secrets.WORKFLOW_GITHUB_PAT }}
-  OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+  OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN_STAGING }}
 
 permissions:
   id-token: write   # This is required for requesting the OIDC JWT
@@ -59,8 +58,8 @@ jobs:
     - name: Configure AWS credentials
       uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
       with:
-        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
         aws-region: ca-central-1
 
     - name: Install OpenVPN
@@ -89,7 +88,6 @@ jobs:
       uses: "kota65535/github-openvpn-connect-action@cd2ed8a90cc7b060dc4e001143e811b5f7ea0af5"
       with:
         config_file: /var/tmp/staging.ovpn
-        client_key: ${{ secrets.STAGING_OVPN_CLIENT_KEY }}
         echo_config: false       
 
     - name: Configure kubeconfig

.github/workflows/lambda_production.yml

@@ -7,7 +7,7 @@ on:
     branches: [main]
 
 env:
-  REGISTRY: ${{ secrets.PRODUCTION_API_LAMBDA_ECR_ACCOUNT }}.dkr.ecr.ca-central-1.amazonaws.com/notify
+  REGISTRY: ${{ secrets.PRODUCTION_AWS_ACCOUNT_ID }}.dkr.ecr.ca-central-1.amazonaws.com/notify
 
 jobs:
   build-and-push:
@@ -25,8 +25,8 @@ jobs:
         id: aws-creds
         uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:
-          aws-access-key-id: ${{ secrets.PRODUCTION_ECR_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.PRODUCTION_ECR_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.PRODUCTION_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.PRODUCTION_AWS_SECRET_ACCESS_KEY }}
           aws-region: ca-central-1
 
       - name: Set Docker image tag

.github/workflows/lambda_staging.yml

@@ -7,7 +7,7 @@ on:
     branches: [main]
 
 env:
-  REGISTRY: ${{ secrets.STAGING_API_LAMBDA_ECR_ACCOUNT }}.dkr.ecr.ca-central-1.amazonaws.com/notify
+  REGISTRY: ${{ secrets.STAGING_AWS_ACCOUNT_ID }}.dkr.ecr.ca-central-1.amazonaws.com/notify
 
 jobs:
   build-push-and-deploy:
@@ -25,8 +25,8 @@ jobs:
         id: aws-creds
         uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:
-          aws-access-key-id: ${{ secrets.STAGING_ECR_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.STAGING_ECR_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
           aws-region: ca-central-1
 
       - name: Build container

app/celery/tasks.py

@@ -39,6 +39,9 @@
     put_batch_saving_bulk_processed,
 )
 from app.config import Config, Priorities, QueueNames
+from app.dao.fact_notification_status_dao import (
+    fetch_notification_status_totals_for_service_by_fiscal_year,
+)
 from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
 from app.dao.jobs_dao import dao_get_in_progress_jobs, dao_get_job_by_id, dao_update_job
 from app.dao.notifications_dao import (
@@ -52,11 +55,9 @@
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
-from app.dao.services_dao import (
-    dao_fetch_service_by_id,
-    fetch_todays_total_message_count,
-)
+from app.dao.services_dao import dao_fetch_service_by_id
 from app.dao.templates_dao import dao_get_template_by_id
+from app.email_limit_utils import fetch_todays_email_count
 from app.encryption import SignedNotification
 from app.exceptions import DVLAException
 from app.models import (
@@ -81,8 +82,9 @@
     persist_notifications,
     send_notification_to_queue,
 )
+from app.sms_fragment_utils import fetch_todays_requested_sms_count
 from app.types import VerifiedNotification
-from app.utils import get_csv_max_rows, get_delivery_queue_for_template
+from app.utils import get_csv_max_rows, get_delivery_queue_for_template, get_fiscal_year
 from app.v2.errors import (
     LiveServiceTooManyRequestsError,
     LiveServiceTooManySMSRequestsError,
@@ -205,15 +207,38 @@ def process_rows(rows: List, template: Template, job: Job, service: Service):
 
 
 def __sending_limits_for_job_exceeded(service, job: Job, job_id):
-    total_sent = fetch_todays_total_message_count(service.id)
+    error_message = None
+
+    if job.template.template_type == SMS_TYPE:
+        total_post_send = fetch_todays_requested_sms_count(service.id) + job.notification_count
+        total_sent_this_fiscal = fetch_notification_status_totals_for_service_by_fiscal_year(
+            service.id, get_fiscal_year(datetime.utcnow()), notification_type=SMS_TYPE
+        )
+        send_exceeds_annual_limit = (total_post_send + total_sent_this_fiscal) > service.sms_annual_limit
+        send_exceeds_daily_limit = total_post_send > service.sms_daily_limit
+
+        if send_exceeds_annual_limit and current_app.config["FF_ANNUAL_LIMIT"]:
+            error_message = f"SMS annual limit of {service.sms_annual_limit} would be exceeded if job {job_id} is sent. Job size: {job.notification_count} Total SMS sent this fiscal + job size: {total_post_send + total_sent_this_fiscal} Over by: {total_post_send + total_sent_this_fiscal - service.sms_annual_limit}"
+        elif send_exceeds_daily_limit:
+            error_message = f"SMS daily limit of {service.sms_daily_limit} would be exceeded if job {job_id} is sent. Job size: {job.notification_count} Total SMS sent today + job size: {total_post_send} Over by: {total_post_send - service.sms_daily_limit}"
+    else:
+        total_post_send = fetch_todays_email_count(service.id) + job.notification_count
+        total_sent_this_fiscal = fetch_notification_status_totals_for_service_by_fiscal_year(
+            service.id, get_fiscal_year(datetime.utcnow()), notification_type=EMAIL_TYPE
+        )
+        send_exceeds_annual_limit = (total_post_send + total_sent_this_fiscal) > service.email_annual_limit
+        send_exceeds_daily_limit = total_post_send > service.message_limit
+
+        if send_exceeds_annual_limit and current_app.config["FF_ANNUAL_LIMIT"]:
+            error_message = f"Email annual limit of {service.email_annual_limit} would be exceeded if job {job_id} is sent. Job size: {job.notification_count} Total email sent this fiscal + job size: {total_post_send + total_sent_this_fiscal} Over limit by: {total_post_send + total_sent_this_fiscal - service.email_annual_limit}"
+        elif send_exceeds_daily_limit:
+            error_message = f"Email daily limit of {service.email_annual_limit} would be exceeded if job {job_id} is sent. Job size: {job.notification_count} Total email sent today + job size: {total_post_send + total_sent_this_fiscal} Over limit by: {total_post_send + total_sent_this_fiscal - service.email_annual_limit}"
 
-    if total_sent + job.notification_count > service.message_limit:
+    if error_message:
         job.job_status = JOB_STATUS_SENDING_LIMITS_EXCEEDED
         job.processing_finished = datetime.utcnow()
         dao_update_job(job)
-        current_app.logger.info(
-            "Job {} size {} error. Sending limits {} exceeded".format(job_id, job.notification_count, service.message_limit)
-        )
+        current_app.logger.info(error_message)
         return True
     return False
 

app/config.py

@@ -585,8 +585,8 @@ class Config(object):
         "job": "{}-dvla-file-per-job".format(os.getenv("NOTIFY_ENVIRONMENT", "development")),
         "notification": "{}-dvla-letter-api-files".format(os.getenv("NOTIFY_ENVIRONMENT", "development")),
     }
-    SERVICE_ANNUAL_EMAIL_LIMIT = env.int("SERVICE_ANNUAL_EMAIL_LIMIT", 10_000_000)
-    SERVICE_ANNUAL_SMS_LIMIT = env.int("SERVICE_ANNUAL_SMS_LIMIT", 25_000)
+    SERVICE_ANNUAL_EMAIL_LIMIT = env.int("SERVICE_ANNUAL_EMAIL_LIMIT", 20_000_000)
+    SERVICE_ANNUAL_SMS_LIMIT = env.int("SERVICE_ANNUAL_SMS_LIMIT", 100_000)
 
     FREE_SMS_TIER_FRAGMENT_COUNT = 250000
 

app/dao/fact_notification_status_dao.py

@@ -32,6 +32,7 @@
     User,
 )
 from app.utils import (
+    get_fiscal_dates,
     get_local_timezone_midnight_in_utc,
     get_local_timezone_month_from_utc_column,
 )
@@ -916,3 +917,23 @@ def fetch_quarter_data(start_date, end_date, service_ids):
         .group_by(FactNotificationStatus.service_id, FactNotificationStatus.notification_type)
     )
     return query.all()
+
+
+def fetch_notification_status_totals_for_service_by_fiscal_year(service_id, fiscal_year, notification_type=None):
+    start_date, end_date = get_fiscal_dates(year=fiscal_year)
+
+    filters = [
+        FactNotificationStatus.service_id == (service_id),
+        FactNotificationStatus.bst_date >= start_date,
+        FactNotificationStatus.bst_date <= end_date,
+    ]
+
+    if notification_type:
+        filters.append(FactNotificationStatus.notification_type == notification_type)
+
+    query = (
+        db.session.query(func.sum(FactNotificationStatus.notification_count).label("notification_count"))
+        .filter(*filters)
+        .scalar()
+    )
+    return query or 0

app/job/rest.py

@@ -33,7 +33,9 @@
 )
 from app.notifications.process_notifications import simulated_recipient
 from app.notifications.validators import (
+    check_email_annual_limit,
     check_email_daily_limit,
+    check_sms_annual_limit,
     check_sms_daily_limit,
     increment_email_daily_count_send_warnings_if_needed,
     increment_sms_daily_count_send_warnings_if_needed,
@@ -183,6 +185,7 @@ def create_job(service_id):
         is_test_notification = len(recipient_csv) == numberOfSimulated
 
         if not is_test_notification:
+            check_sms_annual_limit(service, len(recipient_csv))
             check_sms_daily_limit(service, len(recipient_csv))
             increment_sms_daily_count_send_warnings_if_needed(service, len(recipient_csv))
 
@@ -195,6 +198,7 @@ def create_job(service_id):
             )
             notification_count = len(recipient_csv)
 
+        check_email_annual_limit(service, notification_count)
         check_email_daily_limit(service, notification_count)
 
         scheduled_for = datetime.fromisoformat(data.get("scheduled_for")) if data.get("scheduled_for") else None

app/models.py

@@ -65,8 +65,8 @@
 DELIVERY_STATUS_CALLBACK_TYPE = "delivery_status"
 COMPLAINT_CALLBACK_TYPE = "complaint"
 SERVICE_CALLBACK_TYPES = [DELIVERY_STATUS_CALLBACK_TYPE, COMPLAINT_CALLBACK_TYPE]
-DEFAULT_SMS_ANNUAL_LIMIT = 25000
-DEFAULT_EMAIL_ANNUAL_LIMIT = 10000000
+DEFAULT_SMS_ANNUAL_LIMIT = 100000
+DEFAULT_EMAIL_ANNUAL_LIMIT = 20000000
 
 sms_sending_vehicles = db.Enum(*[vehicle.value for vehicle in SmsSendingVehicles], name="sms_sending_vehicles")
 

app/notifications/rest.py

@@ -22,8 +22,10 @@
     simulated_recipient,
 )
 from app.notifications.validators import (
+    check_email_annual_limit,
     check_email_daily_limit,
     check_rate_limiting,
+    check_sms_annual_limit,
     check_template_is_active,
     check_template_is_for_notification_type,
     service_has_permission,
@@ -113,6 +115,7 @@ def send_notification(notification_type: NotificationType):
 
     simulated = simulated_recipient(notification_form["to"], notification_type)
     if not simulated != api_user.key_type == KEY_TYPE_TEST and notification_type == EMAIL_TYPE:
+        check_email_annual_limit(authenticated_service, 1)
         check_email_daily_limit(authenticated_service, 1)
 
     check_template_is_for_notification_type(notification_type, template.template_type)
@@ -129,6 +132,8 @@ def send_notification(notification_type: NotificationType):
 
     if notification_type == SMS_TYPE:
         _service_can_send_internationally(authenticated_service, notification_form["to"])
+        if not simulated and api_user.key_type != KEY_TYPE_TEST:
+            check_sms_annual_limit(authenticated_service, 1)
     # Do not persist or send notification to the queue if it is a simulated recipient
 
     notification_model = persist_notification(