Backend: Task to institute a Daily email limit (#1937)

* Validation for deciding whether to send email limit emails

* Add db migrations

* update notification utils

* update waffles

* Makefile updates

* try

* tests for validators

* Add check for redis in jobs and fix tests

* Add feature flag where daily message check is called in tasks

* Add code for validators and post notifications

* try

* fix

* poetry lock update

.github/workflows/test.yaml

@@ -67,7 +67,7 @@ jobs:
       run: |
         cp -f .env.example .env
     - name: Checks for new endpoints against AWS WAF rules
-      uses: cds-snc/notification-utils/.github/actions/[email protected].3
+      uses: cds-snc/notification-utils/.github/actions/[email protected].4
       with:
         app-loc: '/github/workspace'
         app-libs: '/github/workspace/env/site-packages'

Makefile

@@ -69,3 +69,7 @@ run-celery-beat: ## Run the celery beat
 .PHONY: run-celery-purge
 run-celery-purge: ## Purge the celery queues
 	./scripts/run_celery_purge.sh
+
+.PHONY: run-db
+run-db: ## psql to access dev database
+	psql postgres://postgres:chummy@db:5432/notification_api

app/celery/tasks.py

@@ -306,7 +306,8 @@ def save_smss(self, service_id: Optional[str], signed_notifications: List[Signed
     current_app.logger.info(f"Sending following sms notifications to AWS: {notification_id_queue.keys()}")
     for notification_obj in saved_notifications:
         try:
-            check_service_over_daily_message_limit(notification_obj.key_type, service)
+            if not current_app.config["FF_EMAIL_DAILY_LIMIT"]:
+                check_service_over_daily_message_limit(notification_obj.key_type, service)
             queue = notification_id_queue.get(notification_obj.id) or template.queue_to_use()  # type: ignore
             send_notification_to_queue(
                 notification_obj,
@@ -433,7 +434,8 @@ def try_to_send_notifications_to_queue(notification_id_queue, service, saved_not
     research_mode = service.research_mode  # type: ignore
     for notification_obj in saved_notifications:
         try:
-            check_service_over_daily_message_limit(notification_obj.key_type, service)
+            if not current_app.config["FF_EMAIL_DAILY_LIMIT"]:
+                check_service_over_daily_message_limit(notification_obj.key_type, service)
             queue = notification_id_queue.get(notification_obj.id) or template.queue_to_use()  # type: ignore
             send_notification_to_queue(
                 notification_obj,

app/config.py

@@ -270,6 +270,9 @@ class Config(object):
     REACHED_DAILY_SMS_LIMIT_TEMPLATE_ID = "a646e614-c527-4f94-a955-ed7185d577f4"
     DAILY_SMS_LIMIT_UPDATED_TEMPLATE_ID = "6ec12dd0-680a-4073-8d58-91d17cc8442f"
     CONTACT_FORM_DIRECT_EMAIL_TEMPLATE_ID = "b04beb4a-8408-4280-9a5c-6a046b6f7704"
+    NEAR_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "9aa60ad7-2d7f-46f0-8cbe-2bac3d4d77d8"
+    REACHED_DAILY_EMAIL_LIMIT_TEMPLATE_ID = "ee036547-e51b-49f1-862b-10ea982cfceb"
+    DAILY_EMAIL_LIMIT_UPDATED_TEMPLATE_ID = "97dade64-ea8d-460f-8a34-900b74ee5eb0"
 
     # Allowed service IDs able to send HTML through their templates.
     ALLOW_HTML_SERVICE_IDS: List[str] = [id.strip() for id in os.getenv("ALLOW_HTML_SERVICE_IDS", "").split(",")]
@@ -525,6 +528,9 @@ class Config(object):
     # Timestamp in epoch milliseconds to seed the bounce rate. We will seed data for (24, the below config) included.
     FF_BOUNCE_RATE_SEED_EPOCH_MS = os.getenv("FF_BOUNCE_RATE_SEED_EPOCH_MS", False)
 
+    # Feature flags for email_daily_limit
+    FF_EMAIL_DAILY_LIMIT = env.bool("FF_EMAIL_DAILY_LIMIT", False)
+
     @classmethod
     def get_sensitive_config(cls) -> list[str]:
         "List of config keys that contain sensitive information"
@@ -621,6 +627,7 @@ class Test(Development):
 
     TEMPLATE_PREVIEW_API_HOST = "http://localhost:9999"
     FF_BOUNCE_RATE_BACKEND = True
+    FF_EMAIL_DAILY_LIMIT = False
 
 
 class Production(Config):

app/dao/services_dao.py

@@ -454,7 +454,7 @@ def fetch_service_email_limit(service_id: uuid.UUID) -> int:
 def fetch_todays_total_email_count(service_id: uuid.UUID) -> int:
     midnight = get_midnight(datetime.now(tz=pytz.utc))
     result = (
-        db.session.query(func.count(Notification).label("total_email_notifications"))
+        db.session.query(func.count(Notification.id).label("total_email_notifications"))
         .filter(
             Notification.service_id == service_id,
             Notification.key_type != KEY_TYPE_TEST,

app/email_limit_utils.py

@@ -23,7 +23,6 @@ def fetch_todays_email_count(service_id: UUID) -> int:
 def increment_todays_email_count(service_id: UUID, increment_by: int) -> None:
     if not current_app.config["REDIS_ENABLED"]:
         return
-
     fetch_todays_email_count(service_id)  # to make sure it's set in redis
     cache_key = email_daily_count_cache_key(service_id)
     redis_store.incrby(cache_key, increment_by)

app/job/rest.py

@@ -22,14 +22,15 @@
 from app.dao.templates_dao import dao_get_template_by_id
 from app.errors import InvalidRequest, register_errors
 from app.models import (
+    EMAIL_TYPE,
     JOB_STATUS_CANCELLED,
     JOB_STATUS_PENDING,
     JOB_STATUS_SCHEDULED,
-    LETTER_TYPE,
     SMS_TYPE,
 )
 from app.notifications.process_notifications import simulated_recipient
 from app.notifications.validators import (
+    check_email_limit_increment_redis_send_warnings_if_needed,
     check_sms_limit_increment_redis_send_warnings_if_needed,
 )
 from app.schemas import (
@@ -144,14 +145,15 @@ def create_job(service_id):
     data["template"] = data.pop("template_id")
     template = dao_get_template_by_id(data["template"])
 
+    job = get_job_from_s3(service_id, data["id"])
+    recipient_csv = RecipientCSV(
+        job,
+        template_type=template.template_type,
+        placeholders=template._as_utils_template().placeholders,
+        template=Template(template.__dict__),
+    )
+
     if template.template_type == SMS_TYPE:
-        job = get_job_from_s3(service_id, data["id"])
-        recipient_csv = RecipientCSV(
-            job,
-            template_type=template.template_type,
-            placeholders=template._as_utils_template().placeholders,
-            template=Template(template.__dict__),
-        )
         # calculate the number of simulated recipients
         numberOfSimulated = sum(
             simulated_recipient(i["phone_number"].data, template.template_type) for i in list(recipient_csv.get_rows())
@@ -167,8 +169,8 @@ def create_job(service_id):
         if not is_test_notification:
             check_sms_limit_increment_redis_send_warnings_if_needed(service, recipient_csv.sms_fragment_count)
 
-    if template.template_type == LETTER_TYPE and service.restricted:
-        raise InvalidRequest("Create letter job is not allowed for service in trial mode ", 403)
+    if template.template_type == EMAIL_TYPE:
+        check_email_limit_increment_redis_send_warnings_if_needed(service, len(list(recipient_csv.get_rows())))
 
     if data.get("valid") != "True":
         raise InvalidRequest("File is not valid, can't create job", 400)

app/notifications/validators.py

@@ -7,8 +7,10 @@
 from notifications_utils.clients.redis import (
     daily_limit_cache_key,
     near_daily_limit_cache_key,
+    near_email_daily_limit_cache_key,
     near_sms_daily_limit_cache_key,
     over_daily_limit_cache_key,
+    over_email_daily_limit_cache_key,
     over_sms_daily_limit_cache_key,
     rate_limit_cache_key,
 )
@@ -25,6 +27,7 @@
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_letter_contact_dao import dao_get_letter_contact_by_id
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
+from app.email_limit_utils import fetch_todays_email_count, increment_todays_email_count
 from app.models import (
     EMAIL_TYPE,
     INTERNATIONAL_SMS_TYPE,
@@ -51,9 +54,11 @@
 from app.utils import get_document_url, get_public_notify_type_text, is_blank
 from app.v2.errors import (
     BadRequestError,
+    LiveServiceTooManyEmailRequestsError,
     LiveServiceTooManyRequestsError,
     LiveServiceTooManySMSRequestsError,
     RateLimitError,
+    TrialServiceTooManyEmailRequestsError,
     TrialServiceTooManyRequestsError,
     TrialServiceTooManySMSRequestsError,
 )
@@ -125,6 +130,58 @@ def check_sms_daily_limit(service: Service, requested_sms=0):
         raise LiveServiceTooManySMSRequestsError(service.sms_daily_limit)
 
 
+@statsd_catch(
+    namespace="validators",
+    counter_name="rate_limit.trial_service_daily_email",
+    exception=TrialServiceTooManyEmailRequestsError,
+)
+@statsd_catch(
+    namespace="validators",
+    counter_name="rate_limit.live_service_daily_email",
+    exception=LiveServiceTooManyEmailRequestsError,
+)
+def check_email_daily_limit(service: Service, requested_email=0):
+    emails_sent_today = fetch_todays_email_count(service.id)
+    bool_over_email_daily_limit = (emails_sent_today + requested_email) > service.message_limit
+
+    # Send a warning when reaching the daily email limit
+    if not bool_over_email_daily_limit:
+        return
+
+    current_app.logger.info(
+        f"service {service.id} is exceeding their daily email limit [total sent today: {int(emails_sent_today)} limit: {service.message_limit}, attempted send: {requested_email}"
+    )
+    if service.restricted:
+        raise TrialServiceTooManyEmailRequestsError(service.message_limit)
+    else:
+        raise LiveServiceTooManyEmailRequestsError(service.message_limit)
+
+
+def send_warning_email_limit_emails_if_needed(service: Service) -> None:
+    """
+    Function that decides if we should send email warnings about nearing or reaching the email daily limit.
+    """
+    todays_current_email_count = fetch_todays_email_count(service.id)
+    bool_nearing_email_daily_limit = todays_current_email_count >= NEAR_DAILY_LIMIT_PERCENTAGE * service.message_limit
+    bool_at_or_over_email_daily_limit = todays_current_email_count >= service.message_limit
+    current_time = datetime.utcnow().isoformat()
+    cache_expiration = int(time_until_end_of_day().total_seconds())
+
+    # Send a warning when reaching 80% of the daily limit
+    if bool_nearing_email_daily_limit:
+        cache_key = near_email_daily_limit_cache_key(service.id)
+        if not redis_store.get(cache_key):
+            send_near_email_limit_email(service)
+            redis_store.set(cache_key, current_time, ex=cache_expiration)
+
+    # Send a warning when reaching the daily message limit
+    if bool_at_or_over_email_daily_limit:
+        cache_key = over_email_daily_limit_cache_key(service.id)
+        if not redis_store.get(cache_key):
+            send_email_limit_reached_email(service)
+            redis_store.set(cache_key, current_time, ex=cache_expiration)
+
+
 def send_warning_sms_limit_emails_if_needed(service: Service):
     todays_requested_sms = fetch_todays_requested_sms_count(service.id)
     nearing_sms_daily_limit = todays_requested_sms >= NEAR_DAILY_LIMIT_PERCENTAGE * service.sms_daily_limit
@@ -167,9 +224,19 @@ def check_sms_limit_increment_redis_send_warnings_if_needed(service: Service, re
     send_warning_sms_limit_emails_if_needed(service)
 
 
+def check_email_limit_increment_redis_send_warnings_if_needed(service: Service, requested_email=0) -> None:
+    if not current_app.config["FF_EMAIL_DAILY_LIMIT"]:
+        return
+
+    check_email_daily_limit(service, requested_email)
+    increment_todays_email_count(service.id, requested_email)
+    send_warning_email_limit_emails_if_needed(service)
+
+
 def check_rate_limiting(service: Service, api_key: ApiKey):
     check_service_over_api_rate_limit_and_update_rate(service, api_key)
-    check_service_over_daily_message_limit(api_key.key_type, service)
+    if not current_app.config["FF_EMAIL_DAILY_LIMIT"]:
+        check_service_over_daily_message_limit(api_key.key_type, service)
 
 
 def warn_about_daily_message_limit(service: Service, messages_sent):
@@ -242,6 +309,25 @@ def send_near_sms_limit_email(service: Service):
     current_app.logger.info(f"service {service.id} is approaching its daily sms limit of {service.sms_daily_limit}")
 
 
+def send_near_email_limit_email(service: Service) -> None:
+    """
+    Send an email to service users when nearing the daily email limit.
+
+    """
+    send_notification_to_service_users(
+        service_id=service.id,
+        template_id=current_app.config["NEAR_DAILY_EMAIL_LIMIT_TEMPLATE_ID"],
+        personalisation={
+            "service_name": service.name,
+            "contact_url": f"{current_app.config['ADMIN_BASE_URL']}/contact",
+            "message_limit_en": "{:,}".format(service.message_limit),
+            "message_limit_fr": "{:,}".format(service.message_limit).replace(",", " "),
+        },
+        include_user_fields=["name"],
+    )
+    current_app.logger.info(f"service {service.id} is approaching its daily email limit of {service.message_limit}")
+
+
 def send_sms_limit_reached_email(service: Service):
     send_notification_to_service_users(
         service_id=service.id,
@@ -258,6 +344,20 @@ def send_sms_limit_reached_email(service: Service):
     )
 
 
+def send_email_limit_reached_email(service: Service):
+    send_notification_to_service_users(
+        service_id=service.id,
+        template_id=current_app.config["REACHED_DAILY_EMAIL_LIMIT_TEMPLATE_ID"],
+        personalisation={
+            "service_name": service.name,
+            "contact_url": f"{current_app.config['ADMIN_BASE_URL']}/contact",
+            "message_limit_en": "{:,}".format(service.message_limit),
+            "message_limit_fr": "{:,}".format(service.message_limit).replace(",", " "),
+        },
+        include_user_fields=["name"],
+    )
+
+
 def check_template_is_for_notification_type(notification_type: NotificationType, template_type: TemplateType):
     if notification_type != template_type:
         message = "{0} template is not suitable for {1} notification".format(template_type, notification_type)

app/service/rest.py

@@ -329,7 +329,9 @@ def update_service(service_id):
 def _warn_service_users_about_message_limit_changed(service_id, data):
     send_notification_to_service_users(
         service_id=service_id,
-        template_id=current_app.config["DAILY_LIMIT_UPDATED_TEMPLATE_ID"],
+        template_id=current_app.config["DAILY_EMAIL_LIMIT_UPDATED_TEMPLATE_ID"]
+        if current_app.config["FF_EMAIL_DAILY_LIMIT"]
+        else current_app.config["DAILY_LIMIT_UPDATED_TEMPLATE_ID"],
         personalisation={
             "service_name": data["name"],
             "message_limit_en": "{:,}".format(data["message_limit"]),

app/service/send_notification.py

@@ -33,6 +33,7 @@
     simulated_recipient,
 )
 from app.notifications.validators import (
+    check_email_limit_increment_redis_send_warnings_if_needed,
     check_service_has_permission,
     check_service_over_daily_message_limit,
     check_sms_limit_increment_redis_send_warnings_if_needed,
@@ -63,11 +64,14 @@ def send_one_off_notification(service_id, post_data):
 
     _, template_with_content = validate_template(template.id, personalisation, service, template.template_type)
 
-    check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
+    if not current_app.config["FF_EMAIL_DAILY_LIMIT"]:
+        check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
     if template.template_type == SMS_TYPE:
         is_test_notification = simulated_recipient(post_data["to"], template.template_type)
         if not is_test_notification:
             check_sms_limit_increment_redis_send_warnings_if_needed(service, template_with_content.fragment_count)
+    elif template.template_type == EMAIL_TYPE and current_app.config["FF_EMAIL_DAILY_LIMIT"]:
+        check_email_limit_increment_redis_send_warnings_if_needed(service, 1)  # 1 email
 
     validate_and_format_recipient(
         send_to=post_data["to"],

app/v2/errors.py

@@ -38,6 +38,14 @@ def __init__(self, sending_limit):
         self.message = self.message_template.format(sending_limit)
 
 
+class TooManyEmailRequestsError(InvalidRequest):
+    status_code = 429
+    messsage_template = "Exceeded email daily sending limit of {} messages"
+
+    def __init__(self, sending_limit):
+        self.message = self.messsage_template.format(sending_limit)
+
+
 class LiveServiceTooManyRequestsError(TooManyRequestsError):
     pass
 
@@ -46,6 +54,10 @@ class LiveServiceTooManySMSRequestsError(TooManySMSRequestsError):
     pass
 
 
+class LiveServiceTooManyEmailRequestsError(TooManyEmailRequestsError):
+    pass
+
+
 class TrialServiceTooManyRequestsError(TooManyRequestsError):
     pass
 
@@ -54,6 +66,10 @@ class TrialServiceTooManySMSRequestsError(TooManySMSRequestsError):
     pass
 
 
+class TrialServiceTooManyEmailRequestsError(TooManyEmailRequestsError):
+    pass
+
+
 class RateLimitError(InvalidRequest):
     status_code = 429
     message_template = "Exceeded rate limit for key type {} of {} requests per {} seconds"

app/v2/notifications/post_notifications.py

@@ -72,6 +72,7 @@
     transform_notification,
 )
 from app.notifications.validators import (
+    check_email_limit_increment_redis_send_warnings_if_needed,
     check_rate_limiting,
     check_service_can_schedule_notification,
     check_service_email_reply_to_id,
@@ -213,6 +214,9 @@ def post_bulk():
             )
             raise BadRequestError(message=message)
 
+    if template.template_type == EMAIL_TYPE and api_user.key_type != KEY_TYPE_TEST:
+        check_email_limit_increment_redis_send_warnings_if_needed(authenticated_service, len(list(recipient_csv.get_rows())))
+
     if template.template_type == SMS_TYPE:
         # calculate the number of simulated recipients
         numberOfSimulated = sum(
@@ -274,6 +278,9 @@ def post_notification(notification_type: NotificationType):
         notification_type,
     )
 
+    if template.template_type == EMAIL_TYPE and api_user.key_type != KEY_TYPE_TEST:
+        check_email_limit_increment_redis_send_warnings_if_needed(authenticated_service, 1)  # 1 email
+
     if template.template_type == SMS_TYPE:
         is_test_notification = api_user.key_type == KEY_TYPE_TEST or simulated_recipient(form["phone_number"], notification_type)
         if not is_test_notification: