Merge branch 'main' into chore/python-3.12

app/dao/fact_notification_status_dao.py

@@ -32,6 +32,7 @@
     User,
 )
 from app.utils import (
+    get_fiscal_dates,
     get_local_timezone_midnight_in_utc,
     get_local_timezone_month_from_utc_column,
 )
@@ -916,3 +917,23 @@ def fetch_quarter_data(start_date, end_date, service_ids):
         .group_by(FactNotificationStatus.service_id, FactNotificationStatus.notification_type)
     )
     return query.all()
+
+
+def fetch_notification_status_totals_for_service_by_fiscal_year(service_id, fiscal_year, notification_type=None):
+    start_date, end_date = get_fiscal_dates(year=fiscal_year)
+
+    filters = [
+        FactNotificationStatus.service_id == (service_id),
+        FactNotificationStatus.bst_date >= start_date,
+        FactNotificationStatus.bst_date <= end_date,
+    ]
+
+    if notification_type:
+        filters.append(FactNotificationStatus.notification_type == notification_type)
+
+    query = (
+        db.session.query(func.sum(FactNotificationStatus.notification_count).label("notification_count"))
+        .filter(*filters)
+        .scalar()
+    )
+    return query or 0

app/notifications/validators.py

@@ -14,6 +14,7 @@
     over_sms_daily_limit_cache_key,
     rate_limit_cache_key,
 )
+from notifications_utils.decorators import requires_feature
 from notifications_utils.recipients import (
     get_international_phone_info,
     validate_and_format_email_address,
@@ -22,8 +23,11 @@
 from notifications_utils.statsd_decorators import statsd_catch
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import redis_store
+from app import annual_limit_client, redis_store
 from app.dao import services_dao, templates_dao
+from app.dao.fact_notification_status_dao import (
+    fetch_notification_status_totals_for_service_by_fiscal_year,
+)
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_letter_contact_dao import dao_get_letter_contact_by_id
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
@@ -53,22 +57,28 @@
 )
 from app.utils import (
     get_document_url,
+    get_fiscal_year,
     get_limit_reset_time_et,
     get_public_notify_type_text,
     is_blank,
 )
 from app.v2.errors import (
     BadRequestError,
+    LiveServiceRequestExceedsEmailAnnualLimitError,
+    LiveServiceRequestExceedsSMSAnnualLimitError,
     LiveServiceTooManyEmailRequestsError,
     LiveServiceTooManyRequestsError,
     LiveServiceTooManySMSRequestsError,
     RateLimitError,
+    TrialServiceRequestExceedsEmailAnnualLimitError,
+    TrialServiceRequestExceedsSMSAnnualLimitError,
     TrialServiceTooManyEmailRequestsError,
     TrialServiceTooManyRequestsError,
     TrialServiceTooManySMSRequestsError,
 )
 
 NEAR_DAILY_LIMIT_PERCENTAGE = 80 / 100
+NEAR_ANNUAL_LIMIT_PERCENTAGE = 80 / 100
 
 
 def check_service_over_api_rate_limit_and_update_rate(service: Service, api_key: ApiKey):
@@ -162,6 +172,115 @@ def check_email_daily_limit(service: Service, requested_email=0):
         raise LiveServiceTooManyEmailRequestsError(service.message_limit)
 
 
+# TODO: FF_ANNUAL_LIMIT removal
+@requires_feature("FF_ANNUAL_LIMIT")
+@statsd_catch(
+    namespace="validators",
+    counter_name="rate_limit.trial_service_annual_email",
+    exception=TrialServiceRequestExceedsEmailAnnualLimitError,
+)
+@statsd_catch(
+    namespace="validators",
+    counter_name="rate_limit.live_service_annual_email",
+    exception=LiveServiceRequestExceedsEmailAnnualLimitError,
+)
+def check_email_annual_limit(service: Service, requested_emails=0):
+    current_fiscal_year = get_fiscal_year(datetime.utcnow())
+    emails_sent_today = fetch_todays_email_count(service.id)
+    emails_sent_this_fiscal = fetch_notification_status_totals_for_service_by_fiscal_year(
+        service.id, current_fiscal_year, notification_type=EMAIL_TYPE
+    )
+    send_exceeds_annual_limit = (emails_sent_today + emails_sent_this_fiscal + requested_emails) > service.email_annual_limit
+    send_reaches_annual_limit = (emails_sent_today + emails_sent_this_fiscal + requested_emails) == service.email_annual_limit
+    is_near_annual_limit = (emails_sent_today + emails_sent_this_fiscal + requested_emails) > (
+        service.email_annual_limit * NEAR_ANNUAL_LIMIT_PERCENTAGE
+    )
+
+    if not send_exceeds_annual_limit:
+        # Will this send put the service right at their limit?
+        if send_reaches_annual_limit and not annual_limit_client.check_has_over_limit_been_sent(service.id, SMS_TYPE):
+            annual_limit_client.set_over_sms_limit(service.id)
+            current_app.logger.info(
+                f"Service {service.id} reached their annual email limit of {service.email_annual_limit} when sending {requested_emails} messages. Sending reached annual limit email."
+            )
+            send_annual_limit_reached_email(service, "email", current_fiscal_year + 1)
+
+        # Will this send put annual usage within 80% of the limit?
+        if is_near_annual_limit and not annual_limit_client.check_has_warning_been_sent(service.id, EMAIL_TYPE):
+            annual_limit_client.set_nearing_email_limit(service.id)
+            current_app.logger.info(
+                f"Service {service.id} reached 80% of their annual email limit of {service.email_annual_limit} messages. Sending annual limit usage warning email."
+            )
+            send_near_annual_limit_warning_email(
+                service, "email", int(emails_sent_today + emails_sent_this_fiscal), current_fiscal_year + 1
+            )
+
+        return
+
+    current_app.logger.info(
+        f"Service {service.id} is exceeding their annual email limit [total sent this fiscal: {int(emails_sent_today + emails_sent_this_fiscal)} limit: {service.email_annual_limit}, attempted send: {requested_emails}"
+    )
+    if service.restricted:
+        raise TrialServiceRequestExceedsEmailAnnualLimitError(service.email_annual_limit)
+    else:
+        raise LiveServiceRequestExceedsEmailAnnualLimitError(service.email_annual_limit)
+
+
+# TODO: FF_ANNUAL_LIMIT removal
+@requires_feature("FF_ANNUAL_LIMIT")
+@statsd_catch(
+    namespace="validators",
+    counter_name="rate_limit.trial_service_annual_sms",
+    exception=TrialServiceRequestExceedsSMSAnnualLimitError,
+)
+@statsd_catch(
+    namespace="validators",
+    counter_name="rate_limit.live_service_annual_sms",
+    exception=LiveServiceRequestExceedsSMSAnnualLimitError,
+)
+def check_sms_annual_limit(service: Service, requested_sms=0):
+    current_fiscal_year = get_fiscal_year(datetime.utcnow())
+    sms_sent_today = fetch_todays_requested_sms_count(service.id)
+    sms_sent_this_fiscal = fetch_notification_status_totals_for_service_by_fiscal_year(
+        service.id, current_fiscal_year, notification_type=SMS_TYPE
+    )
+    send_exceeds_annual_limit = (sms_sent_today + sms_sent_this_fiscal + requested_sms) > service.sms_annual_limit
+    send_reaches_annual_limit = (sms_sent_today + sms_sent_this_fiscal + requested_sms) == service.sms_annual_limit
+    is_near_annual_limit = (sms_sent_today + sms_sent_this_fiscal + requested_sms) >= (
+        service.sms_annual_limit * NEAR_ANNUAL_LIMIT_PERCENTAGE
+    )
+
+    if not send_exceeds_annual_limit:
+        # Will this send put the service right at their limit?
+        if send_reaches_annual_limit and not annual_limit_client.check_has_over_limit_been_sent(service.id, SMS_TYPE):
+            annual_limit_client.set_over_sms_limit(service.id)
+            current_app.logger.info(
+                f"Service {service.id} reached their annual SMS limit of {service.sms_annual_limit} messages. Sending reached annual limit email."
+            )
+            send_annual_limit_reached_email(service, "email", current_fiscal_year + 1)
+
+        # Will this send put annual usage within 80% of the limit?
+        if is_near_annual_limit and not annual_limit_client.check_has_warning_been_sent(service.id, EMAIL_TYPE):
+            annual_limit_client.set_nearing_email_limit(service.id)
+            current_app.logger.info(
+                f"Service {service.id} reached 80% of their annual SMS limit of {service.sms_annual_limit} messages. Sending annual limit usage warning email."
+            )
+            send_near_annual_limit_warning_email(
+                service, "email", int(sms_sent_today + sms_sent_this_fiscal), current_fiscal_year + 1
+            )
+
+        return
+
+    current_app.logger.info(
+        f"Service {service.id} is exceeding their annual SMS limit [total sent this fiscal: {int(sms_sent_today + sms_sent_this_fiscal)} limit: {service.sms_annual_limit}, attempted send: {requested_sms}"
+    )
+
+    if service.restricted:
+        raise TrialServiceRequestExceedsSMSAnnualLimitError(service.sms_annual_limit)
+    else:
+        raise LiveServiceRequestExceedsSMSAnnualLimitError(service.sms_annual_limit)
+
+
 def send_warning_email_limit_emails_if_needed(service: Service) -> None:
     """
     Function that decides if we should send email warnings about nearing or reaching the email daily limit.
@@ -373,6 +492,76 @@ def send_email_limit_reached_email(service: Service):
     )
 
 
+def send_annual_limit_reached_email(service: Service, notification_type: NotificationType, fiscal_end: int):
+    send_notification_to_service_users(
+        service_id=service.id,
+        template_id=current_app.config["REACHED_ANNUAL_LIMIT_TEMPLATE_ID"],
+        personalisation={
+            "message_type_en": notification_type,
+            "message_type_fr": "Courriel" if notification_type == EMAIL_TYPE else "SMS",
+            "fiscal_end": fiscal_end,
+            "hyperlink_to_page_en": f"{current_app.config['ADMIN_BASE_URL']}/services/{service.id}/monthly",
+            "hyperlink_to_page_fr": f"{current_app.config['ADMIN_BASE_URL']}/services/{service.id}/monthly?lang=fr",
+        },
+        include_user_fields=["name"],
+    )
+
+
+def send_near_annual_limit_warning_email(service: Service, notification_type: NotificationType, count_en: int, fiscal_end: int):
+    count_fr = "{:,}".format(count_en).replace(",", " ")
+    if notification_type == EMAIL_TYPE:
+        message_limit_fr = "{:,}".format(service.email_annual_limit).replace(",", " ")
+        message_limit_en = service.email_annual_limit
+        message_type_fr = "Courriel"
+        remaining_en = service.email_annual_limit - count_en
+        remaining_fr = "{:,}".format(remaining_en).replace(",", " ")
+    else:
+        message_limit_fr = "{:,}".format(service.sms_annual_limit).replace(",", " ")
+        message_limit_en = service.sms_annual_limit
+        message_type_fr = "sms"
+        remaining_en = service.sms_annual_limit - count_en
+        remaining_fr = "{:,}".format(remaining_en).replace(",", " ")
+
+    send_notification_to_service_users(
+        service_id=service.id,
+        template_id=current_app.config["NEAR_ANNUAL_LIMIT_TEMPLATE_ID"],
+        personalisation={
+            "message_type": notification_type,
+            "fiscal_end": fiscal_end,
+            "service_name": service.name,
+            "count_en": count_en,
+            "count_fr": count_fr,
+            "message_limit_en": message_limit_en,
+            "message_limit_fr": message_limit_fr,
+            "message_type_en": notification_type,
+            "message_type_fr": message_type_fr,
+            "remaining_en": remaining_en,
+            "remaining_fr": remaining_fr,
+            "hyperlink_to_page_en": f"{current_app.config['ADMIN_BASE_URL']}/services/{service.id}/monthly",
+            "hyperlink_to_page_fr": f"{current_app.config['ADMIN_BASE_URL']}/services/{service.id}/monthly?lang=fr",
+        },
+        include_user_fields=["name"],
+    )
+
+
+def send_annual_limit_updated_email(service: Service, notification_type: NotificationType, fiscal_end: int):
+    send_notification_to_service_users(
+        service_id=service.id,
+        template_id=current_app.config["ANNUAL_LIMIT_UPDATED_TEMPLATE_ID"],
+        personalisation={
+            "message_type_en": notification_type,
+            "message_type_fr": "Courriel" if notification_type == EMAIL_TYPE else "SMS",
+            "message_limit_en": service.email_annual_limit if notification_type == EMAIL_TYPE else service.sms_annual_limit,
+            "message_limit_fr": "{:,}".format(
+                service.email_annual_limit if notification_type == EMAIL_TYPE else service.sms_annual_limit
+            ).replace(",", " "),
+            "hyperlink_to_page_en": f"{current_app.config['ADMIN_BASE_URL']}/services/{service.id}/monthly",
+            "hyperlink_to_page_fr": f"{current_app.config['ADMIN_BASE_URL']}/services/{service.id}/monthly?lang=fr",
+        },
+        include_user_fields=["name"],
+    )
+
+
 def check_template_is_for_notification_type(notification_type: NotificationType, template_type: TemplateType):
     if notification_type != template_type:
         message = "{0} template is not suitable for {1} notification".format(template_type, notification_type)

app/utils.py

@@ -256,3 +256,62 @@ def prepare_notification_counts_for_seeding(notification_counts: list) -> dict:
         for _, notification_type, status, count in notification_counts
         if status in DELIVERED_STATUSES or status in FAILURE_STATUSES
     }
+
+
+def get_fiscal_year(current_date=None):
+    """
+    Determine the fiscal year for a given date.
+
+    Args:
+        current_date (datetime.date, optional): The date to determine the fiscal year for.
+                                                Defaults to today's date.
+
+    Returns:
+        int: The fiscal year (starting year).
+    """
+    if current_date is None:
+        current_date = datetime.today()
+
+    # Fiscal year starts on April 1st
+    fiscal_year_start_month = 4
+    if current_date.month >= fiscal_year_start_month:
+        return current_date.year
+    else:
+        return current_date.year - 1
+
+
+def get_fiscal_dates(current_date=None, year=None):
+    """
+    Determine the start and end dates of the fiscal year for a given date or year.
+    If no parameters are passed into the method, the fiscal year for the current date will be determined.
+
+    Args:
+        current_date (datetime.date, optional): The date to determine the fiscal year for.
+        year (int, optional): The year to determine the fiscal year for.
+
+    Returns:
+        tuple: A tuple containing the start and end dates of the fiscal year (datetime.date).
+    """
+    if current_date and year:
+        raise ValueError("Only one of current_date or year should be provided.")
+
+    if not current_date and not year:
+        current_date = datetime.today()
+
+    # Fiscal year starts on April 1st
+    fiscal_year_start_month = 4
+    fiscal_year_start_day = 1
+
+    if current_date:
+        if current_date.month >= fiscal_year_start_month:
+            fiscal_year_start = datetime(current_date.year, fiscal_year_start_month, fiscal_year_start_day)
+            fiscal_year_end = datetime(current_date.year + 1, fiscal_year_start_month - 1, 31)  # March 31 of the next year
+        else:
+            fiscal_year_start = datetime(current_date.year - 1, fiscal_year_start_month, fiscal_year_start_day)
+            fiscal_year_end = datetime(current_date.year, fiscal_year_start_month - 1, 31)  # March 31 of the current year
+
+    if year:
+        fiscal_year_start = datetime(year, fiscal_year_start_month, fiscal_year_start_day)
+        fiscal_year_end = datetime(year + 1, fiscal_year_start_month - 1, 31)
+
+    return fiscal_year_start, fiscal_year_end

app/v2/errors.py

@@ -46,6 +46,22 @@ def __init__(self, sending_limit):
         self.message = self.messsage_template.format(sending_limit)
 
 
+class RequestExceedsEmailAnnualLimitError(InvalidRequest):
+    status_code = 429
+    message_template = "Exceeded annual email sending limit of {} messages"
+
+    def __init__(self, annual_limit):
+        self.message = self.message_template.format(annual_limit)
+
+
+class RequestExceedsSMSAnnualLimitError(InvalidRequest):
+    status_code = 429
+    message_template = "Exceeded annual SMS sending limit of {} messages"
+
+    def __init__(self, annual_limit):
+        self.message = self.message_template.format(annual_limit)
+
+
 class LiveServiceTooManyRequestsError(TooManyRequestsError):
     pass
 
@@ -58,6 +74,22 @@ class LiveServiceTooManyEmailRequestsError(TooManyEmailRequestsError):
     pass
 
 
+class LiveServiceRequestExceedsEmailAnnualLimitError(RequestExceedsEmailAnnualLimitError):
+    pass
+
+
+class LiveServiceRequestExceedsSMSAnnualLimitError(RequestExceedsSMSAnnualLimitError):
+    pass
+
+
+class TrialServiceRequestExceedsEmailAnnualLimitError(RequestExceedsEmailAnnualLimitError):
+    pass
+
+
+class TrialServiceRequestExceedsSMSAnnualLimitError(RequestExceedsSMSAnnualLimitError):
+    pass
+
+
 class TrialServiceTooManyRequestsError(TooManyRequestsError):
     pass
 

scripts/run_celery_delivery.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+# Runs celery with only the delivery-receipts queue.
+
+echo "Start celery, concurrency: ${CELERY_CONCURRENCY-4}"
+
+celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency="${CELERY_CONCURRENCY-4}" -Q delivery-receipts