Merge branch 'main' of github.com:cds-snc/notification-api into pytho…

…n-3.12-second-try

.github/workflows/docker.yaml

@@ -97,15 +97,15 @@ jobs:
     - name: Update images in staging
       run: |
         DOCKER_TAG=${GITHUB_SHA::7}
-        kubectl set image deployment.apps/api api=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-beat celery-beat=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-sms celery-sms=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-primary celery-primary=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-scalable celery-scalable=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-sms-send-primary celery-sms-send-primary=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-sms-send-scalable celery-sms-send-scalable=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-email-send-primary celery-email-send-primary=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-email-send-scalable celery-email-send-scalable=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config        
+        kubectl set image deployment.apps/notify-api notify-api=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/notify-celery-beat notify-celery-beat=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/notify-celery-sms notify-celery-sms=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/notify-celery-primary notify-celery-primary=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/notify-celery-scalable notify-celery-scalable=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/notify-celery-sms-send-primary notify-celery-sms-send-primary=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/notify-celery-sms-send-scalable notify-celery-sms-send-scalable=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/notify-celery-email-send-primary notify-celery-email-send-primary=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/notify-celery-email-send-scalable notify-celery-email-send-scalable=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config        
 
 
     - name: my-app-install token

app/celery/tasks.py

@@ -39,6 +39,9 @@
     put_batch_saving_bulk_processed,
 )
 from app.config import Config, Priorities, QueueNames
+from app.dao.fact_notification_status_dao import (
+    fetch_notification_status_totals_for_service_by_fiscal_year,
+)
 from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
 from app.dao.jobs_dao import dao_get_in_progress_jobs, dao_get_job_by_id, dao_update_job
 from app.dao.notifications_dao import (
@@ -52,11 +55,9 @@
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_inbound_api_dao import get_service_inbound_api_for_service
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
-from app.dao.services_dao import (
-    dao_fetch_service_by_id,
-    fetch_todays_total_message_count,
-)
+from app.dao.services_dao import dao_fetch_service_by_id
 from app.dao.templates_dao import dao_get_template_by_id
+from app.email_limit_utils import fetch_todays_email_count
 from app.encryption import SignedNotification
 from app.exceptions import DVLAException
 from app.models import (
@@ -81,8 +82,9 @@
     persist_notifications,
     send_notification_to_queue,
 )
+from app.sms_fragment_utils import fetch_todays_requested_sms_count
 from app.types import VerifiedNotification
-from app.utils import get_csv_max_rows, get_delivery_queue_for_template
+from app.utils import get_csv_max_rows, get_delivery_queue_for_template, get_fiscal_year
 from app.v2.errors import (
     LiveServiceTooManyRequestsError,
     LiveServiceTooManySMSRequestsError,
@@ -117,9 +119,6 @@ def process_job(job_id):
         current_app.logger.warning("Job {} has been cancelled, service {} is inactive".format(job_id, service.id))
         return
 
-    if __sending_limits_for_job_exceeded(service, job, job_id):
-        return
-
     job.job_status = JOB_STATUS_IN_PROGRESS
     job.processing_started = start
     dao_update_job(job)
@@ -205,15 +204,38 @@ def process_rows(rows: List, template: Template, job: Job, service: Service):
 
 
 def __sending_limits_for_job_exceeded(service, job: Job, job_id):
-    total_sent = fetch_todays_total_message_count(service.id)
+    error_message = None
+
+    if job.template.template_type == SMS_TYPE:
+        total_post_send = fetch_todays_requested_sms_count(service.id) + job.notification_count
+        total_sent_this_fiscal = fetch_notification_status_totals_for_service_by_fiscal_year(
+            service.id, get_fiscal_year(datetime.utcnow()), notification_type=SMS_TYPE
+        )
+        send_exceeds_annual_limit = (total_post_send + total_sent_this_fiscal) > service.sms_annual_limit
+        send_exceeds_daily_limit = total_post_send > service.sms_daily_limit
 
-    if total_sent + job.notification_count > service.message_limit:
+        if send_exceeds_annual_limit and current_app.config["FF_ANNUAL_LIMIT"]:
+            error_message = f"SMS annual limit of {service.sms_annual_limit} would be exceeded if job {job_id} is sent. Job size: {job.notification_count} Total SMS sent this fiscal + job size: {total_post_send + total_sent_this_fiscal} Over by: {total_post_send + total_sent_this_fiscal - service.sms_annual_limit}"
+        elif send_exceeds_daily_limit:
+            error_message = f"SMS daily limit of {service.sms_daily_limit} would be exceeded if job {job_id} is sent. Job size: {job.notification_count} Total SMS sent today + job size: {total_post_send} Over by: {total_post_send - service.sms_daily_limit}"
+    else:
+        total_post_send = fetch_todays_email_count(service.id) + job.notification_count
+        total_sent_this_fiscal = fetch_notification_status_totals_for_service_by_fiscal_year(
+            service.id, get_fiscal_year(datetime.utcnow()), notification_type=EMAIL_TYPE
+        )
+        send_exceeds_annual_limit = (total_post_send + total_sent_this_fiscal) > service.email_annual_limit
+        send_exceeds_daily_limit = total_post_send > service.message_limit
+
+        if send_exceeds_annual_limit and current_app.config["FF_ANNUAL_LIMIT"]:
+            error_message = f"Email annual limit of {service.email_annual_limit} would be exceeded if job {job_id} is sent. Job size: {job.notification_count} Total email sent this fiscal + job size: {total_post_send + total_sent_this_fiscal} Over limit by: {total_post_send + total_sent_this_fiscal - service.email_annual_limit}"
+        elif send_exceeds_daily_limit:
+            error_message = f"Email daily limit of {service.email_annual_limit} would be exceeded if job {job_id} is sent. Job size: {job.notification_count} Total email sent today + job size: {total_post_send + total_sent_this_fiscal} Over limit by: {total_post_send + total_sent_this_fiscal - service.email_annual_limit}"
+
+    if error_message:
         job.job_status = JOB_STATUS_SENDING_LIMITS_EXCEEDED
         job.processing_finished = datetime.utcnow()
         dao_update_job(job)
-        current_app.logger.info(
-            "Job {} size {} error. Sending limits {} exceeded".format(job_id, job.notification_count, service.message_limit)
-        )
+        current_app.logger.info(error_message)
         return True
     return False
 

app/config.py

@@ -585,8 +585,8 @@ class Config(object):
         "job": "{}-dvla-file-per-job".format(os.getenv("NOTIFY_ENVIRONMENT", "development")),
         "notification": "{}-dvla-letter-api-files".format(os.getenv("NOTIFY_ENVIRONMENT", "development")),
     }
-    SERVICE_ANNUAL_EMAIL_LIMIT = env.int("SERVICE_ANNUAL_EMAIL_LIMIT", 10_000_000)
-    SERVICE_ANNUAL_SMS_LIMIT = env.int("SERVICE_ANNUAL_SMS_LIMIT", 25_000)
+    SERVICE_ANNUAL_EMAIL_LIMIT = env.int("SERVICE_ANNUAL_EMAIL_LIMIT", 20_000_000)
+    SERVICE_ANNUAL_SMS_LIMIT = env.int("SERVICE_ANNUAL_SMS_LIMIT", 100_000)
 
     FREE_SMS_TIER_FRAGMENT_COUNT = 250000
 

app/job/rest.py

@@ -33,7 +33,9 @@
 )
 from app.notifications.process_notifications import simulated_recipient
 from app.notifications.validators import (
+    check_email_annual_limit,
     check_email_daily_limit,
+    check_sms_annual_limit,
     check_sms_daily_limit,
     increment_email_daily_count_send_warnings_if_needed,
     increment_sms_daily_count_send_warnings_if_needed,
@@ -183,6 +185,7 @@ def create_job(service_id):
         is_test_notification = len(recipient_csv) == numberOfSimulated
 
         if not is_test_notification:
+            check_sms_annual_limit(service, len(recipient_csv))
             check_sms_daily_limit(service, len(recipient_csv))
             increment_sms_daily_count_send_warnings_if_needed(service, len(recipient_csv))
 
@@ -195,6 +198,7 @@ def create_job(service_id):
             )
             notification_count = len(recipient_csv)
 
+        check_email_annual_limit(service, notification_count)
         check_email_daily_limit(service, notification_count)
 
         scheduled_for = datetime.fromisoformat(data.get("scheduled_for")) if data.get("scheduled_for") else None

app/models.py

@@ -65,8 +65,8 @@
 DELIVERY_STATUS_CALLBACK_TYPE = "delivery_status"
 COMPLAINT_CALLBACK_TYPE = "complaint"
 SERVICE_CALLBACK_TYPES = [DELIVERY_STATUS_CALLBACK_TYPE, COMPLAINT_CALLBACK_TYPE]
-DEFAULT_SMS_ANNUAL_LIMIT = 25000
-DEFAULT_EMAIL_ANNUAL_LIMIT = 10000000
+DEFAULT_SMS_ANNUAL_LIMIT = 100000
+DEFAULT_EMAIL_ANNUAL_LIMIT = 20000000
 
 sms_sending_vehicles = db.Enum(*[vehicle.value for vehicle in SmsSendingVehicles], name="sms_sending_vehicles")
 

app/notifications/rest.py

@@ -22,8 +22,10 @@
     simulated_recipient,
 )
 from app.notifications.validators import (
+    check_email_annual_limit,
     check_email_daily_limit,
     check_rate_limiting,
+    check_sms_annual_limit,
     check_template_is_active,
     check_template_is_for_notification_type,
     service_has_permission,
@@ -113,6 +115,7 @@ def send_notification(notification_type: NotificationType):
 
     simulated = simulated_recipient(notification_form["to"], notification_type)
     if not simulated != api_user.key_type == KEY_TYPE_TEST and notification_type == EMAIL_TYPE:
+        check_email_annual_limit(authenticated_service, 1)
         check_email_daily_limit(authenticated_service, 1)
 
     check_template_is_for_notification_type(notification_type, template.template_type)
@@ -129,6 +132,8 @@ def send_notification(notification_type: NotificationType):
 
     if notification_type == SMS_TYPE:
         _service_can_send_internationally(authenticated_service, notification_form["to"])
+        if not simulated and api_user.key_type != KEY_TYPE_TEST:
+            check_sms_annual_limit(authenticated_service, 1)
     # Do not persist or send notification to the queue if it is a simulated recipient
 
     notification_model = persist_notification(

app/notifications/validators.py

@@ -198,8 +198,8 @@ def check_email_annual_limit(service: Service, requested_emails=0):
 
     if not send_exceeds_annual_limit:
         # Will this send put the service right at their limit?
-        if send_reaches_annual_limit and not annual_limit_client.check_has_over_limit_been_sent(service.id, SMS_TYPE):
-            annual_limit_client.set_over_sms_limit(service.id)
+        if send_reaches_annual_limit and not annual_limit_client.check_has_over_limit_been_sent(service.id, EMAIL_TYPE):
+            annual_limit_client.set_over_email_limit(service.id)
             current_app.logger.info(
                 f"Service {service.id} reached their annual email limit of {service.email_annual_limit} when sending {requested_emails} messages. Sending reached annual limit email."
             )
@@ -218,7 +218,7 @@ def check_email_annual_limit(service: Service, requested_emails=0):
         return
 
     current_app.logger.info(
-        f"Service {service.id} is exceeding their annual email limit [total sent this fiscal: {int(emails_sent_today + emails_sent_this_fiscal)} limit: {service.email_annual_limit}, attempted send: {requested_emails}"
+        f"{'Trial service' if service.restricted else 'Service'} {service.id} is exceeding their annual email limit [total sent this fiscal: {int(emails_sent_today + emails_sent_this_fiscal)} limit: {service.email_annual_limit}, attempted send: {requested_emails}"
     )
     if service.restricted:
         raise TrialServiceRequestExceedsEmailAnnualLimitError(service.email_annual_limit)
@@ -257,16 +257,16 @@ def check_sms_annual_limit(service: Service, requested_sms=0):
             current_app.logger.info(
                 f"Service {service.id} reached their annual SMS limit of {service.sms_annual_limit} messages. Sending reached annual limit email."
             )
-            send_annual_limit_reached_email(service, "email", current_fiscal_year + 1)
+            send_annual_limit_reached_email(service, "sms", current_fiscal_year + 1)
 
         # Will this send put annual usage within 80% of the limit?
-        if is_near_annual_limit and not annual_limit_client.check_has_warning_been_sent(service.id, EMAIL_TYPE):
-            annual_limit_client.set_nearing_email_limit(service.id)
+        if is_near_annual_limit and not annual_limit_client.check_has_warning_been_sent(service.id, SMS_TYPE):
+            annual_limit_client.set_nearing_sms_limit(service.id)
             current_app.logger.info(
                 f"Service {service.id} reached 80% of their annual SMS limit of {service.sms_annual_limit} messages. Sending annual limit usage warning email."
             )
             send_near_annual_limit_warning_email(
-                service, "email", int(sms_sent_today + sms_sent_this_fiscal), current_fiscal_year + 1
+                service, "sms", int(sms_sent_today + sms_sent_this_fiscal), current_fiscal_year + 1
             )
 
         return
@@ -497,8 +497,9 @@ def send_annual_limit_reached_email(service: Service, notification_type: Notific
         service_id=service.id,
         template_id=current_app.config["REACHED_ANNUAL_LIMIT_TEMPLATE_ID"],
         personalisation={
-            "message_type_en": notification_type,
-            "message_type_fr": "Courriel" if notification_type == EMAIL_TYPE else "SMS",
+            "service_name": service.name,
+            "message_type_en": "emails" if notification_type == EMAIL_TYPE else "text messages",
+            "message_type_fr": "courriels" if notification_type == EMAIL_TYPE else "messages texte",
             "fiscal_end": fiscal_end,
             "hyperlink_to_page_en": f"{current_app.config['ADMIN_BASE_URL']}/services/{service.id}/monthly",
             "hyperlink_to_page_fr": f"{current_app.config['ADMIN_BASE_URL']}/services/{service.id}/monthly?lang=fr",
@@ -512,15 +513,17 @@ def send_near_annual_limit_warning_email(service: Service, notification_type: No
     if notification_type == EMAIL_TYPE:
         message_limit_fr = "{:,}".format(service.email_annual_limit).replace(",", " ")
         message_limit_en = service.email_annual_limit
-        message_type_fr = "Courriel"
-        remaining_en = service.email_annual_limit - count_en
-        remaining_fr = "{:,}".format(remaining_en).replace(",", " ")
+        message_type_fr = "courriels"
+        message_type_en = "emails"
+        remaining_en = "{:,}".format(service.email_annual_limit - count_en)
+        remaining_fr = "{:,}".format(service.email_annual_limit - count_en).replace(",", " ")
     else:
         message_limit_fr = "{:,}".format(service.sms_annual_limit).replace(",", " ")
         message_limit_en = service.sms_annual_limit
-        message_type_fr = "sms"
-        remaining_en = service.sms_annual_limit - count_en
-        remaining_fr = "{:,}".format(remaining_en).replace(",", " ")
+        message_type_en = "text messages"
+        message_type_fr = "messages texte"
+        remaining_en = "{:,}".format(service.sms_annual_limit - count_en)
+        remaining_fr = "{:,}".format(service.sms_annual_limit - count_en).replace(",", " ")
 
     send_notification_to_service_users(
         service_id=service.id,
@@ -533,7 +536,7 @@ def send_near_annual_limit_warning_email(service: Service, notification_type: No
             "count_fr": count_fr,
             "message_limit_en": message_limit_en,
             "message_limit_fr": message_limit_fr,
-            "message_type_en": notification_type,
+            "message_type_en": message_type_en,
             "message_type_fr": message_type_fr,
             "remaining_en": remaining_en,
             "remaining_fr": remaining_fr,
@@ -544,24 +547,6 @@ def send_near_annual_limit_warning_email(service: Service, notification_type: No
     )
 
 
-def send_annual_limit_updated_email(service: Service, notification_type: NotificationType, fiscal_end: int):
-    send_notification_to_service_users(
-        service_id=service.id,
-        template_id=current_app.config["ANNUAL_LIMIT_UPDATED_TEMPLATE_ID"],
-        personalisation={
-            "message_type_en": notification_type,
-            "message_type_fr": "Courriel" if notification_type == EMAIL_TYPE else "SMS",
-            "message_limit_en": service.email_annual_limit if notification_type == EMAIL_TYPE else service.sms_annual_limit,
-            "message_limit_fr": "{:,}".format(
-                service.email_annual_limit if notification_type == EMAIL_TYPE else service.sms_annual_limit
-            ).replace(",", " "),
-            "hyperlink_to_page_en": f"{current_app.config['ADMIN_BASE_URL']}/services/{service.id}/monthly",
-            "hyperlink_to_page_fr": f"{current_app.config['ADMIN_BASE_URL']}/services/{service.id}/monthly?lang=fr",
-        },
-        include_user_fields=["name"],
-    )
-
-
 def check_template_is_for_notification_type(notification_type: NotificationType, template_type: TemplateType):
     if notification_type != template_type:
         message = "{0} template is not suitable for {1} notification".format(template_type, notification_type)