From 431772c95c22d242376c4d1c4a2f90cd9802c778 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Tue, 26 Nov 2024 18:30:15 +0000
Subject: [PATCH 01/28] feat: validate annual limit

---
 app/main/views/send.py                        |  43 +++++-
 app/notify_client/annual_limit_client.py      | 134 ++++++++++++++++++
 .../template_statistics_api_client.py         |  46 ++++++
 .../check/too-many-email-messages.html        |   8 +-
 app/templates/views/check/column-errors.html  |   6 +-
 pyproject.toml                                |   2 +-
 6 files changed, 227 insertions(+), 12 deletions(-)
 create mode 100644 app/notify_client/annual_limit_client.py

diff --git a/app/main/views/send.py b/app/main/views/send.py
index 2a5720e38b..bc7ce03cfb 100644
--- a/app/main/views/send.py
+++ b/app/main/views/send.py
@@ -1,5 +1,6 @@
 import itertools
 import json
+from datetime import datetime, timezone
 from string import ascii_uppercase
 from zipfile import BadZipFile
 
@@ -53,6 +54,7 @@
 )
 from app.main.views.dashboard import aggregate_notifications_stats
 from app.models.user import Users
+from app.notify_client.annual_limit_client import annual_limit_client
 from app.s3_client.s3_csv_client import (
     copy_bulk_send_file_to_uploads,
     list_bulk_send_uploads,
@@ -649,8 +651,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
 
     sms_fragments_sent_today = daily_sms_fragment_count(service_id)
     emails_sent_today = daily_email_count(service_id)
-    remaining_sms_message_fragments = current_service.sms_daily_limit - sms_fragments_sent_today
-    remaining_email_messages = current_service.message_limit - emails_sent_today
+    remaining_sms_message_fragments_today = current_service.sms_daily_limit - sms_fragments_sent_today
+    remaining_email_messages_today = current_service.message_limit - emails_sent_today
 
     contents = s3download(service_id, upload_id)
 
@@ -659,9 +661,27 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
     email_reply_to = None
     sms_sender = None
     recipients_remaining_messages = (
-        remaining_email_messages if db_template["template_type"] == "email" else remaining_sms_message_fragments
+        remaining_email_messages_today if db_template["template_type"] == "email" else remaining_sms_message_fragments_today
     )
 
+
+    # Override the remaining messages counts with the remaining annual counts, if the latter are lower
+    over_annual_limit = False
+    if current_app.config["FF_ANNUAL_LIMIT"]:
+        stats_ytd = annual_limit_client.get_all_notification_counts_ytd(service_id, datetime.now(timezone.utc).year)
+        remaining_sms_this_year = current_service.sms_annual_limit - stats_ytd["sms"]["requested"]
+        remaining_email_this_year = current_service.email_annual_limit - stats_ytd["email"]["requested"]
+
+        # if this is an email send
+        if db_template["template_type"] == "email":
+            if remaining_email_this_year < remaining_email_messages_today:
+                recipients_remaining_messages = remaining_email_this_year
+                over_annual_limit = True
+        else:
+            if remaining_sms_this_year < remaining_sms_message_fragments_today:
+                recipients_remaining_messages = remaining_sms_this_year
+                over_annual_limit = True
+
     if db_template["template_type"] == "email":
         email_reply_to = get_email_reply_to_address_from_session()
     elif db_template["template_type"] == "sms":
@@ -743,8 +763,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
         original_file_name=request.args.get("original_file_name", ""),
         upload_id=upload_id,
         form=CsvUploadForm(),
-        remaining_messages=remaining_email_messages,
-        remaining_sms_message_fragments=remaining_sms_message_fragments,
+        remaining_messages=remaining_email_messages_today,
+        remaining_sms_message_fragments=remaining_sms_message_fragments_today,
         sms_parts_to_send=sms_parts_to_send,
         is_sms_parts_estimated=is_sms_parts_estimated,
         choose_time_form=choose_time_form,
@@ -764,6 +784,7 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
             db_template["version"],
             request.args.get("original_file_name", ""),
         ),
+        over_annual_limit=over_annual_limit
     )
 
 
@@ -783,7 +804,15 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
     data["original_file_name"] = SanitiseASCII.encode(data.get("original_file_name", ""))
     data["sms_parts_requested"] = data["stats_daily"]["sms"]["requested"]
     data["sms_parts_remaining"] = current_service.sms_daily_limit - daily_sms_fragment_count(service_id)
-    data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
+
+    if current_app.config["FF_ANNUAL_LIMIT"]:
+        # Show annual limit validation over the daily one (even if both are true)
+        if data["over_annual_limit"]:
+            data["send_exceeds_annual_limit"] = True
+        else:
+            data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
+    else:
+        data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
 
     if (
         data["recipients"].too_many_rows
@@ -801,7 +830,7 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
     if data["errors"] or data["trying_to_send_letters_in_trial_mode"]:
         return render_template("views/check/column-errors.html", **data)
 
-    if data["send_exceeds_daily_limit"]:
+    if data["send_exceeds_daily_limit"] or data["send_exceeds_annual_limit"]:
         return render_template("views/check/column-errors.html", **data)
 
     metadata_kwargs = {
diff --git a/app/notify_client/annual_limit_client.py b/app/notify_client/annual_limit_client.py
new file mode 100644
index 0000000000..f47411051b
--- /dev/null
+++ b/app/notify_client/annual_limit_client.py
@@ -0,0 +1,134 @@
+from notifications_utils.clients.redis.annual_limit import RedisAnnualLimit
+from notifications_utils.clients.redis.redis_client import RedisClient
+
+from app import service_api_client, template_statistics_client
+from app.utils import DELIVERED_STATUSES, FAILURE_STATUSES
+
+
+class AnnualLimitWrapper:
+    def __init__(self, client):
+        self.client = client
+
+
+    def get_all_notification_counts_for_today(self, service_id):
+        """
+        Get total number of notifications by type for the current service
+
+        Return value:
+        {
+            'sms': {'requested': int, 'delivered': int, 'failed': int}
+            'email': {'requested': int, 'delivered': int, 'failed': int}
+        }
+
+        """
+        if self.client.was_seeded_today(service_id):
+            # annual limits client returns this data structure:
+            # {
+            #   sms_delivered: int,
+            #   email_delivered: int,
+            #   sms_failed: int,
+            #   email_failed: int
+            # }
+            stats = self.client.get_all_notification_counts(service_id)
+
+            return {
+                "sms": {
+                    "requested": stats["sms_delivered"] + stats["sms_failed"],
+                    "delivered": stats["sms_delivered"],
+                    "failed": stats["sms_failed"]
+                },
+                "email": {
+                    "requested": stats["email_delivered"] + stats["email_failed"],
+                    "delivered": stats["email_delivered"],
+                    "failed": stats["email_failed"]
+                }
+            }
+        else:
+            stats = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1)
+            transformed_stats = _aggregate_notifications_stats(stats)
+
+            return transformed_stats
+
+    def get_all_notification_counts_for_year(self, service_id, year):
+        """
+        Get total number of notifications by type for the current service for the current year
+
+        Return value:
+        {
+            'sms': {'requested': int, 'delivered': int, 'failed': int}
+            'email': {'requested': int, 'delivered': int, 'failed': int}
+        }
+
+        """
+        stats_today = self.get_all_notification_counts_for_today(service_id)
+        stats_this_year = service_api_client.get_monthly_notification_stats(service_id, year)["data"]
+        stats_this_year = _aggregate_stats_from_service_api(stats_this_year)
+        # aggregate stats_today and stats_this_year
+        for template_type in ["sms", "email"]:
+            for status in ["requested", "delivered", "failed"]:
+                stats_this_year[template_type][status] += stats_today[template_type][status]
+
+        return stats_this_year
+
+    def get_all_notification_counts_ytd(self, service_id, year):
+        """
+        Get total number of notifications by type for the current service for the year to date
+
+        Return value:
+        {
+            'sms': {'requested': int, 'delivered': int, 'failed': int}
+            'email': {'requested': int, 'delivered': int, 'failed': int}
+        }
+
+        """
+        stats_today = self.get_all_notification_counts_for_today(service_id)
+        stats_this_year = self.get_all_notification_counts_for_year(service_id, year)
+
+        # aggregate stats_today and stats_this_year
+        for template_type in ["sms", "email"]:
+            for status in ["requested", "delivered", "failed"]:
+                stats_this_year[template_type][status] += stats_today[template_type][status]
+
+        return stats_this_year
+
+# TODO: consolidate this function and other functions that transform the results of template_statistics_client calls
+def _aggregate_notifications_stats(template_statistics):
+    template_statistics = _filter_out_cancelled_stats(template_statistics)
+    notifications = {
+        template_type: {status: 0 for status in ("requested", "delivered", "failed")} for template_type in ["sms", "email"]
+    }
+    for stat in template_statistics:
+        notifications[stat["template_type"]]["requested"] += stat["count"]
+        if stat["status"] in DELIVERED_STATUSES:
+            notifications[stat["template_type"]]["delivered"] += stat["count"]
+        elif stat["status"] in FAILURE_STATUSES:
+            notifications[stat["template_type"]]["failed"] += stat["count"]
+
+    return notifications
+
+def _filter_out_cancelled_stats(template_statistics):
+    return [s for s in template_statistics if s["status"] != "cancelled"]
+
+def _aggregate_stats_from_service_api(stats):
+    """Aggregate monthly notification stats excluding cancelled"""
+    total_stats = {"sms": {}, "email": {}}
+
+    for month_data in stats.values():
+        for msg_type in ["sms", "email"]:
+            if msg_type in month_data:
+                for status, count in month_data[msg_type].items():
+                    if status != "cancelled":
+                        if status not in total_stats[msg_type]:
+                            total_stats[msg_type][status] = 0
+                        total_stats[msg_type][status] += count
+
+    return {
+        msg_type: {
+            "requested": sum(counts.values()),
+            "delivered": sum(counts.get(status, 0) for status in DELIVERED_STATUSES),
+            "failed": sum(counts.get(status, 0) for status in FAILURE_STATUSES)
+        }
+        for msg_type, counts in total_stats.items()
+    }
+
+annual_limit_client = AnnualLimitWrapper(RedisAnnualLimit(RedisClient()))
diff --git a/app/notify_client/template_statistics_api_client.py b/app/notify_client/template_statistics_api_client.py
index 132f8938fa..429bdcd313 100644
--- a/app/notify_client/template_statistics_api_client.py
+++ b/app/notify_client/template_statistics_api_client.py
@@ -1,4 +1,7 @@
+from itertools import groupby
+
 from app.notify_client import NotifyAdminAPIClient
+from app.utils import DELIVERED_STATUSES, FAILURE_STATUSES
 
 
 class TemplateStatisticsApiClient(NotifyAdminAPIClient):
@@ -16,4 +19,47 @@ def get_template_statistics_for_template(self, service_id, template_id):
         return self.get(url="/service/{}/template-statistics/{}".format(service_id, template_id))["data"]
 
 
+
 template_statistics_client = TemplateStatisticsApiClient()
+
+class TemplateStatistics:
+    def __init__(self, stats):
+        self.stats = stats
+
+    def as_aggregates(self):
+        template_statistics = self._filter_out_cancelled_stats(self.stats)
+        notifications = {
+            template_type: {status: 0 for status in ("requested", "delivered", "failed")} for template_type in ["sms", "email"]
+        }
+        for stat in template_statistics:
+            notifications[stat["template_type"]]["requested"] += stat["count"]
+            if stat["status"] in DELIVERED_STATUSES:
+                notifications[stat["template_type"]]["delivered"] += stat["count"]
+            elif stat["status"] in FAILURE_STATUSES:
+                notifications[stat["template_type"]]["failed"] += stat["count"]
+
+        return notifications
+
+    def as_template_usage(self, sort_key="count"):
+        template_statistics = self._filter_out_cancelled_stats(self.stats)
+        templates = []
+        for k, v in groupby(
+            sorted(template_statistics, key=lambda x: x["template_id"]),
+            key=lambda x: x["template_id"],
+        ):
+            template_stats = list(v)
+
+            templates.append(
+                {
+                    "template_id": k,
+                    "template_name": template_stats[0]["template_name"],
+                    "template_type": template_stats[0]["template_type"],
+                    "is_precompiled_letter": template_stats[0]["is_precompiled_letter"],
+                    "count": sum(s["count"] for s in template_stats),
+                }
+            )
+
+        return sorted(templates, key=lambda x: x[sort_key], reverse=True)
+
+    def _filter_out_cancelled_stats(self, template_statistics):
+        return [s for s in template_statistics if s["status"] != "cancelled"]
diff --git a/app/templates/partials/check/too-many-email-messages.html b/app/templates/partials/check/too-many-email-messages.html
index 987846c472..67c2b6b4cd 100644
--- a/app/templates/partials/check/too-many-email-messages.html
+++ b/app/templates/partials/check/too-many-email-messages.html
@@ -4,6 +4,12 @@
   {%- if current_service.trial_mode %}
     {{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
   {% else %}
-    {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
+    {% if send_exceeds_annual_limit %}
+      {{ _("To request an annual limit above {} emails, {}").format(current_service.email_annual_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
+    {% endif %}
+    {% if send_exceeds_daily_limit %}
+      {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
+    {% endif %}
+    
   {%- endif -%}
 </p>
\ No newline at end of file
diff --git a/app/templates/views/check/column-errors.html b/app/templates/views/check/column-errors.html
index 32f2ae9e8e..aca0d8950b 100644
--- a/app/templates/views/check/column-errors.html
+++ b/app/templates/views/check/column-errors.html
@@ -10,10 +10,10 @@
 {% set prefix_txt = _('a column called') %}
 {% set prefix_plural_txt = _('columns called') %}
 
-{% if send_exceeds_daily_limit and (sms_parts_remaining <= 0) %}
-  {% set page_title = _('These messages exceed your daily limit') %}
-{% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %}
+{% if send_exceeds_daily_limit %}
   {% set page_title = _('These messages exceed your daily limit') %}
+{% elif send_exceeds_annual_limit %}
+  {% set page_title = _('These messages exceed the annual limit') %}
 {% else %}
   {% set page_title = _('Check there’s a column for each variable') %}
 {% endif %}
diff --git a/pyproject.toml b/pyproject.toml
index 65718753e0..dd16bc94af 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -246,7 +246,7 @@ select = [
     "I001",
     "I002"
 ]
-ignore = ["E203", "E501", "E402"]
+ignore = ["E203", "E501", "E402", "F401"]
 
 # Provide line length leeway for docstrings
 [tool.ruff.lint.pycodestyle]

From 76faa15dcb7bb9cf75aded3aee5982af5a5cd38b Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Wed, 27 Nov 2024 20:22:51 +0000
Subject: [PATCH 02/28] fix: modify wrapper to work with older redis keys

---
 app/notify_client/annual_limit_client.py | 72 ++++++++----------------
 1 file changed, 23 insertions(+), 49 deletions(-)

diff --git a/app/notify_client/annual_limit_client.py b/app/notify_client/annual_limit_client.py
index f47411051b..5d9881ecde 100644
--- a/app/notify_client/annual_limit_client.py
+++ b/app/notify_client/annual_limit_client.py
@@ -1,7 +1,12 @@
+
+from notifications_utils.clients.redis import (
+    email_daily_count_cache_key,
+    sms_daily_count_cache_key,
+)
 from notifications_utils.clients.redis.annual_limit import RedisAnnualLimit
 from notifications_utils.clients.redis.redis_client import RedisClient
 
-from app import service_api_client, template_statistics_client
+from app import redis_client, service_api_client, template_statistics_client
 from app.utils import DELIVERED_STATUSES, FAILURE_STATUSES
 
 
@@ -11,38 +16,16 @@ def __init__(self, client):
 
 
     def get_all_notification_counts_for_today(self, service_id):
-        """
-        Get total number of notifications by type for the current service
-
-        Return value:
-        {
-            'sms': {'requested': int, 'delivered': int, 'failed': int}
-            'email': {'requested': int, 'delivered': int, 'failed': int}
-        }
-
-        """
-        if self.client.was_seeded_today(service_id):
-            # annual limits client returns this data structure:
-            # {
-            #   sms_delivered: int,
-            #   email_delivered: int,
-            #   sms_failed: int,
-            #   email_failed: int
-            # }
-            stats = self.client.get_all_notification_counts(service_id)
+        # try to get today's stats from redis
+        todays_sms = redis_client.get(sms_daily_count_cache_key(service_id))
+        todays_email = redis_client.get(email_daily_count_cache_key(service_id))
 
+        if todays_sms is not None and todays_email is not None:
             return {
-                "sms": {
-                    "requested": stats["sms_delivered"] + stats["sms_failed"],
-                    "delivered": stats["sms_delivered"],
-                    "failed": stats["sms_failed"]
-                },
-                "email": {
-                    "requested": stats["email_delivered"] + stats["email_failed"],
-                    "delivered": stats["email_delivered"],
-                    "failed": stats["email_failed"]
-                }
+                'sms': todays_sms,
+                'email': todays_email
             }
+        # fallback to the API if the stats are not in redis
         else:
             stats = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1)
             transformed_stats = _aggregate_notifications_stats(stats)
@@ -55,8 +38,8 @@ def get_all_notification_counts_for_year(self, service_id, year):
 
         Return value:
         {
-            'sms': {'requested': int, 'delivered': int, 'failed': int}
-            'email': {'requested': int, 'delivered': int, 'failed': int}
+            'sms': int,
+            'email': int
         }
 
         """
@@ -65,8 +48,7 @@ def get_all_notification_counts_for_year(self, service_id, year):
         stats_this_year = _aggregate_stats_from_service_api(stats_this_year)
         # aggregate stats_today and stats_this_year
         for template_type in ["sms", "email"]:
-            for status in ["requested", "delivered", "failed"]:
-                stats_this_year[template_type][status] += stats_today[template_type][status]
+            stats_this_year[template_type] += stats_today[template_type]
 
         return stats_this_year
 
@@ -76,8 +58,8 @@ def get_all_notification_counts_ytd(self, service_id, year):
 
         Return value:
         {
-            'sms': {'requested': int, 'delivered': int, 'failed': int}
-            'email': {'requested': int, 'delivered': int, 'failed': int}
+            'sms': int,
+            'email': int
         }
 
         """
@@ -86,8 +68,7 @@ def get_all_notification_counts_ytd(self, service_id, year):
 
         # aggregate stats_today and stats_this_year
         for template_type in ["sms", "email"]:
-            for status in ["requested", "delivered", "failed"]:
-                stats_this_year[template_type][status] += stats_today[template_type][status]
+            stats_this_year[template_type] += stats_today[template_type]
 
         return stats_this_year
 
@@ -95,14 +76,11 @@ def get_all_notification_counts_ytd(self, service_id, year):
 def _aggregate_notifications_stats(template_statistics):
     template_statistics = _filter_out_cancelled_stats(template_statistics)
     notifications = {
-        template_type: {status: 0 for status in ("requested", "delivered", "failed")} for template_type in ["sms", "email"]
+        'sms': 0,
+        'email': 0
     }
     for stat in template_statistics:
-        notifications[stat["template_type"]]["requested"] += stat["count"]
-        if stat["status"] in DELIVERED_STATUSES:
-            notifications[stat["template_type"]]["delivered"] += stat["count"]
-        elif stat["status"] in FAILURE_STATUSES:
-            notifications[stat["template_type"]]["failed"] += stat["count"]
+        notifications[stat["template_type"]] += stat["count"]
 
     return notifications
 
@@ -123,11 +101,7 @@ def _aggregate_stats_from_service_api(stats):
                         total_stats[msg_type][status] += count
 
     return {
-        msg_type: {
-            "requested": sum(counts.values()),
-            "delivered": sum(counts.get(status, 0) for status in DELIVERED_STATUSES),
-            "failed": sum(counts.get(status, 0) for status in FAILURE_STATUSES)
-        }
+        msg_type: sum(counts.values())
         for msg_type, counts in total_stats.items()
     }
 

From 7e0bd02d55be25bb07b64b60080edb052f74c97c Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Wed, 27 Nov 2024 20:22:58 +0000
Subject: [PATCH 03/28] test: add tests!

---
 .../notify_client/test_annual_limit_client.py | 111 ++++++++++++++++++
 1 file changed, 111 insertions(+)
 create mode 100644 tests/app/notify_client/test_annual_limit_client.py

diff --git a/tests/app/notify_client/test_annual_limit_client.py b/tests/app/notify_client/test_annual_limit_client.py
new file mode 100644
index 0000000000..b4605ed7a7
--- /dev/null
+++ b/tests/app/notify_client/test_annual_limit_client.py
@@ -0,0 +1,111 @@
+from unittest.mock import Mock, patch
+
+import pytest
+
+from app.notify_client.annual_limit_client import AnnualLimitWrapper
+
+
+@pytest.fixture
+def mock_redis():
+    with patch('app.notify_client.annual_limit_client.redis_client') as mock:
+        yield mock
+
+@pytest.fixture
+def mock_template_stats():
+    with patch('app.notify_client.annual_limit_client.template_statistics_client') as mock:
+        yield mock
+
+@pytest.fixture
+def mock_service_api():
+    with patch('app.notify_client.annual_limit_client.service_api_client') as mock:
+        yield mock
+
+class TestAnnualLimitWrapper:
+    def test_get_all_notification_counts_for_today_redis_has_data(self, mock_redis):
+        # Setup
+        mock_redis.get.side_effect = [5, 10]  # sms, email
+        wrapper = AnnualLimitWrapper(Mock())
+
+        # Execute
+        result = wrapper.get_all_notification_counts_for_today('service-123')
+
+        # Assert
+        assert result == {'sms': 5, 'email': 10}
+        assert mock_redis.get.call_count == 2
+
+    @pytest.mark.parametrize('redis_side_effect, expected_result', [
+        ([None, None], {'sms': 10, 'email': 10}),
+        ([None, 10], {'sms': 10, 'email': 10}),  # Falls back to API if either is None
+        ([10, None], {'sms': 10, 'email': 10}),  # Falls back to API if either is None
+    ])
+    def test_get_all_notification_counts_for_today_redis_missing_data(self, mock_redis, mock_template_stats, redis_side_effect, expected_result):
+        # Setup
+        mock_redis.get.side_effect = redis_side_effect
+        mock_template_stats.get_template_statistics_for_service.return_value = [
+            {'template_id': 'a1', 'template_type': 'sms', 'count': 3, 'status': 'delivered'},
+            {'template_id': 'a2', 'template_type': 'email', 'count': 7, 'status': 'temporary-failure'},
+            {'template_id': 'a3', 'template_type': 'email', 'count': 3, 'status': 'delivered'},
+            {'template_id': 'a4', 'template_type': 'sms', 'count': 7, 'status': 'delivered'},
+        ]
+
+        wrapper = AnnualLimitWrapper(Mock())
+
+        # Execute
+        result = wrapper.get_all_notification_counts_for_today('service-123')
+
+        # Assert
+        assert result == {'sms': 10, 'email': 10}
+        mock_template_stats.get_template_statistics_for_service.assert_called_once()
+
+    def test_get_all_notification_counts_for_year(self, mock_service_api):
+        # Setup
+        mock_service_api.get_monthly_notification_stats.return_value = {
+            "data": {
+                "2024-01": {
+                    'sms': {'sent': 1, 'temporary-failure:': 22 },
+                    'email': {'delivered': 1, 'permanent-failure': 1, 'sending': 12, 'technical-failure': 1}
+                },
+                "2024-02": {
+                    'sms': {'sent': 1},
+                    'email': {'delivered': 1}
+                }
+            }
+        }
+        wrapper = AnnualLimitWrapper(Mock())
+
+        with patch.object(wrapper, 'get_all_notification_counts_for_today') as mock_today:
+            mock_today.return_value = {
+                'sms': 5,
+                'email': 5
+            }
+
+            # Execute
+            result = wrapper.get_all_notification_counts_for_year('service-123', 2024)
+
+            # Assert
+            assert result['sms'] == 29  # 1 + 22 + 1 + 5
+            assert result['email']== 21  # 1 + 1 + 12 + 1 + 1 + 5
+
+    def test_get_all_notification_counts_ytd(self):
+        # Setup
+        wrapper = AnnualLimitWrapper(Mock())
+
+        with patch.object(wrapper, 'get_all_notification_counts_for_today') as mock_today, \
+             patch.object(wrapper, 'get_all_notification_counts_for_year') as mock_year:
+
+            mock_today.return_value = {
+                'sms': 2,
+                'email': 4
+            }
+            mock_year.return_value = {
+                'sms': 10,
+                'email': 20
+            }
+
+            # Execute
+            result = wrapper.get_all_notification_counts_ytd('service-123', 2024)
+
+            # Assert
+            assert result['sms'] == 12
+            assert result['email'] == 24
+

From e949bf1fcbdd5355c8104961dd9c0751591c7896 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 28 Nov 2024 12:59:36 +0000
Subject: [PATCH 04/28] chore: `utcnow` is deprecated, use `now` instead

---
 app/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/__init__.py b/app/__init__.py
index 0025ac021d..6e42f80bc9 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -226,7 +226,7 @@ def get_locale():
     application.jinja_env.globals["show_tou_prompt"] = show_tou_prompt
     application.jinja_env.globals["parse_ua"] = parse
     application.jinja_env.globals["events_key"] = EVENTS_KEY
-    application.jinja_env.globals["now"] = datetime.utcnow
+    application.jinja_env.globals["now"] = datetime.now
 
     # Initialize Salesforce Account list
     if application.config["FF_SALESFORCE_CONTACT"]:

From 7170035092526bdfaa3b906ed5a91eeceee24c23 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 28 Nov 2024 13:00:07 +0000
Subject: [PATCH 05/28] feat(annual limits): show error if user is over annual
 limits

---
 app/main/views/send.py | 42 +++++++++++++++++-------------------------
 1 file changed, 17 insertions(+), 25 deletions(-)

diff --git a/app/main/views/send.py b/app/main/views/send.py
index bc7ce03cfb..8778e53d60 100644
--- a/app/main/views/send.py
+++ b/app/main/views/send.py
@@ -54,7 +54,7 @@
 )
 from app.main.views.dashboard import aggregate_notifications_stats
 from app.models.user import Users
-from app.notify_client.annual_limit_client import annual_limit_client
+from app.notify_client.notification_counts_client import notification_counts_client
 from app.s3_client.s3_csv_client import (
     copy_bulk_send_file_to_uploads,
     list_bulk_send_uploads,
@@ -664,24 +664,6 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
         remaining_email_messages_today if db_template["template_type"] == "email" else remaining_sms_message_fragments_today
     )
 
-
-    # Override the remaining messages counts with the remaining annual counts, if the latter are lower
-    over_annual_limit = False
-    if current_app.config["FF_ANNUAL_LIMIT"]:
-        stats_ytd = annual_limit_client.get_all_notification_counts_ytd(service_id, datetime.now(timezone.utc).year)
-        remaining_sms_this_year = current_service.sms_annual_limit - stats_ytd["sms"]["requested"]
-        remaining_email_this_year = current_service.email_annual_limit - stats_ytd["email"]["requested"]
-
-        # if this is an email send
-        if db_template["template_type"] == "email":
-            if remaining_email_this_year < remaining_email_messages_today:
-                recipients_remaining_messages = remaining_email_this_year
-                over_annual_limit = True
-        else:
-            if remaining_sms_this_year < remaining_sms_message_fragments_today:
-                recipients_remaining_messages = remaining_sms_this_year
-                over_annual_limit = True
-
     if db_template["template_type"] == "email":
         email_reply_to = get_email_reply_to_address_from_session()
     elif db_template["template_type"] == "sms":
@@ -784,7 +766,6 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
             db_template["version"],
             request.args.get("original_file_name", ""),
         ),
-        over_annual_limit=over_annual_limit
     )
 
 
@@ -806,13 +787,24 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
     data["sms_parts_remaining"] = current_service.sms_daily_limit - daily_sms_fragment_count(service_id)
 
     if current_app.config["FF_ANNUAL_LIMIT"]:
+        # Override the remaining messages counts with the remaining annual counts, if the latter are lower
+        stats_ytd = notification_counts_client.get_all_notification_counts_for_year(service_id, datetime.now(timezone.utc).year)
+        remaining_sms_this_year = current_service.sms_annual_limit - stats_ytd["sms"]
+        remaining_email_this_year = current_service.email_annual_limit - stats_ytd["email"]
+
         # Show annual limit validation over the daily one (even if both are true)
-        if data["over_annual_limit"]:
-            data["send_exceeds_annual_limit"] = True
+        if data["template"].template_type == "email":
+            if remaining_email_this_year < data["count_of_recipients"]:
+                data["recipients_remaining_messages"] = remaining_email_this_year
+                data["send_exceeds_annual_limit"] = True
+            else:
+                 data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
         else:
-            data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
-    else:
-        data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
+            if remaining_sms_this_year < data["count_of_recipients"]:
+                data["recipients_remaining_messages"] = remaining_sms_this_year
+                data["send_exceeds_annual_limit"] = True
+            else:
+                 data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
 
     if (
         data["recipients"].too_many_rows

From 6529f2d466eeef2bc285b71fd0a183fc111a6756 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 28 Nov 2024 13:01:03 +0000
Subject: [PATCH 06/28] fix: move daily limit message into the appropriate
 template

---
 .../partials/check/too-many-email-messages.html       |  6 ++++++
 app/templates/views/check/column-errors.html          | 11 +----------
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/app/templates/partials/check/too-many-email-messages.html b/app/templates/partials/check/too-many-email-messages.html
index 67c2b6b4cd..0617d5d0b2 100644
--- a/app/templates/partials/check/too-many-email-messages.html
+++ b/app/templates/partials/check/too-many-email-messages.html
@@ -9,6 +9,12 @@
     {% endif %}
     {% if send_exceeds_daily_limit %}
       {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
+
+      <h2 class="heading-medium">{{ _('You cannot send all these email messages today') }}</h2>
+      <p>
+        {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang],
+        content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
+      </p>
     {% endif %}
     
   {%- endif -%}
diff --git a/app/templates/views/check/column-errors.html b/app/templates/views/check/column-errors.html
index aca0d8950b..26cb381b05 100644
--- a/app/templates/views/check/column-errors.html
+++ b/app/templates/views/check/column-errors.html
@@ -171,17 +171,8 @@ <h2 class="heading-medium">{{ _('You cannot send all these text messages today')
       {% elif recipients.more_rows_than_can_send %}
           {% call banner_wrapper(type='dangerous') %}
             {% include "partials/check/too-many-email-messages.html" %}
-          {% endcall %}
-          <h2 class="heading-medium">{{ _('You cannot send all these email messages today') }}</h2>
-          <p>
-            {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang],
-            content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
-          </p>
-            
-          
+          {% endcall %}           
       {% endif %}
-
-    
   </div>
 
   {% if not send_exceeds_daily_limit %}

From 6c814a34d8ef5d2bf077a12c23b8d1128b5a3474 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 28 Nov 2024 13:01:35 +0000
Subject: [PATCH 07/28] feat: add a client to get daily/yearly sent stats using
 caching where possible

---
 .../notification_counts_client.py             |  76 ++++++++++++
 .../notify_client/test_annual_limit_client.py | 111 ------------------
 .../test_notification_counts_client.py        |  89 ++++++++++++++
 3 files changed, 165 insertions(+), 111 deletions(-)
 create mode 100644 app/notify_client/notification_counts_client.py
 delete mode 100644 tests/app/notify_client/test_annual_limit_client.py
 create mode 100644 tests/app/notify_client/test_notification_counts_client.py

diff --git a/app/notify_client/notification_counts_client.py b/app/notify_client/notification_counts_client.py
new file mode 100644
index 0000000000..ae3bc3a68f
--- /dev/null
+++ b/app/notify_client/notification_counts_client.py
@@ -0,0 +1,76 @@
+from notifications_utils.clients.redis import (
+    email_daily_count_cache_key,
+    sms_daily_count_cache_key,
+)
+
+from app import redis_client, service_api_client, template_statistics_client
+from app.utils import DELIVERED_STATUSES, FAILURE_STATUSES
+
+
+class NotificationCounts:
+    def get_all_notification_counts_for_today(self, service_id):
+        # try to get today's stats from redis
+        todays_sms = redis_client.get(sms_daily_count_cache_key(service_id))
+        todays_email = redis_client.get(email_daily_count_cache_key(service_id))
+
+        if todays_sms is not None and todays_email is not None:
+            return {"sms": todays_sms, "email": todays_email}
+        # fallback to the API if the stats are not in redis
+        else:
+            stats = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1)
+            transformed_stats = _aggregate_notifications_stats(stats)
+
+            return transformed_stats
+
+    def get_all_notification_counts_for_year(self, service_id, year):
+        """
+        Get total number of notifications by type for the current service for the current year
+
+        Return value:
+        {
+            'sms': int,
+            'email': int
+        }
+
+        """
+        stats_today = self.get_all_notification_counts_for_today(service_id)
+        stats_this_year = service_api_client.get_monthly_notification_stats(service_id, year)["data"]
+        stats_this_year = _aggregate_stats_from_service_api(stats_this_year)
+        # aggregate stats_today and stats_this_year
+        for template_type in ["sms", "email"]:
+            stats_this_year[template_type] += stats_today[template_type]
+
+        return stats_this_year
+
+
+# TODO: consolidate this function and other functions that transform the results of template_statistics_client calls
+def _aggregate_notifications_stats(template_statistics):
+    template_statistics = _filter_out_cancelled_stats(template_statistics)
+    notifications = {"sms": 0, "email": 0}
+    for stat in template_statistics:
+        notifications[stat["template_type"]] += stat["count"]
+
+    return notifications
+
+
+def _filter_out_cancelled_stats(template_statistics):
+    return [s for s in template_statistics if s["status"] != "cancelled"]
+
+
+def _aggregate_stats_from_service_api(stats):
+    """Aggregate monthly notification stats excluding cancelled"""
+    total_stats = {"sms": {}, "email": {}}
+
+    for month_data in stats.values():
+        for msg_type in ["sms", "email"]:
+            if msg_type in month_data:
+                for status, count in month_data[msg_type].items():
+                    if status != "cancelled":
+                        if status not in total_stats[msg_type]:
+                            total_stats[msg_type][status] = 0
+                        total_stats[msg_type][status] += count
+
+    return {msg_type: sum(counts.values()) for msg_type, counts in total_stats.items()}
+
+
+notification_counts_client = NotificationCounts()
diff --git a/tests/app/notify_client/test_annual_limit_client.py b/tests/app/notify_client/test_annual_limit_client.py
deleted file mode 100644
index b4605ed7a7..0000000000
--- a/tests/app/notify_client/test_annual_limit_client.py
+++ /dev/null
@@ -1,111 +0,0 @@
-from unittest.mock import Mock, patch
-
-import pytest
-
-from app.notify_client.annual_limit_client import AnnualLimitWrapper
-
-
-@pytest.fixture
-def mock_redis():
-    with patch('app.notify_client.annual_limit_client.redis_client') as mock:
-        yield mock
-
-@pytest.fixture
-def mock_template_stats():
-    with patch('app.notify_client.annual_limit_client.template_statistics_client') as mock:
-        yield mock
-
-@pytest.fixture
-def mock_service_api():
-    with patch('app.notify_client.annual_limit_client.service_api_client') as mock:
-        yield mock
-
-class TestAnnualLimitWrapper:
-    def test_get_all_notification_counts_for_today_redis_has_data(self, mock_redis):
-        # Setup
-        mock_redis.get.side_effect = [5, 10]  # sms, email
-        wrapper = AnnualLimitWrapper(Mock())
-
-        # Execute
-        result = wrapper.get_all_notification_counts_for_today('service-123')
-
-        # Assert
-        assert result == {'sms': 5, 'email': 10}
-        assert mock_redis.get.call_count == 2
-
-    @pytest.mark.parametrize('redis_side_effect, expected_result', [
-        ([None, None], {'sms': 10, 'email': 10}),
-        ([None, 10], {'sms': 10, 'email': 10}),  # Falls back to API if either is None
-        ([10, None], {'sms': 10, 'email': 10}),  # Falls back to API if either is None
-    ])
-    def test_get_all_notification_counts_for_today_redis_missing_data(self, mock_redis, mock_template_stats, redis_side_effect, expected_result):
-        # Setup
-        mock_redis.get.side_effect = redis_side_effect
-        mock_template_stats.get_template_statistics_for_service.return_value = [
-            {'template_id': 'a1', 'template_type': 'sms', 'count': 3, 'status': 'delivered'},
-            {'template_id': 'a2', 'template_type': 'email', 'count': 7, 'status': 'temporary-failure'},
-            {'template_id': 'a3', 'template_type': 'email', 'count': 3, 'status': 'delivered'},
-            {'template_id': 'a4', 'template_type': 'sms', 'count': 7, 'status': 'delivered'},
-        ]
-
-        wrapper = AnnualLimitWrapper(Mock())
-
-        # Execute
-        result = wrapper.get_all_notification_counts_for_today('service-123')
-
-        # Assert
-        assert result == {'sms': 10, 'email': 10}
-        mock_template_stats.get_template_statistics_for_service.assert_called_once()
-
-    def test_get_all_notification_counts_for_year(self, mock_service_api):
-        # Setup
-        mock_service_api.get_monthly_notification_stats.return_value = {
-            "data": {
-                "2024-01": {
-                    'sms': {'sent': 1, 'temporary-failure:': 22 },
-                    'email': {'delivered': 1, 'permanent-failure': 1, 'sending': 12, 'technical-failure': 1}
-                },
-                "2024-02": {
-                    'sms': {'sent': 1},
-                    'email': {'delivered': 1}
-                }
-            }
-        }
-        wrapper = AnnualLimitWrapper(Mock())
-
-        with patch.object(wrapper, 'get_all_notification_counts_for_today') as mock_today:
-            mock_today.return_value = {
-                'sms': 5,
-                'email': 5
-            }
-
-            # Execute
-            result = wrapper.get_all_notification_counts_for_year('service-123', 2024)
-
-            # Assert
-            assert result['sms'] == 29  # 1 + 22 + 1 + 5
-            assert result['email']== 21  # 1 + 1 + 12 + 1 + 1 + 5
-
-    def test_get_all_notification_counts_ytd(self):
-        # Setup
-        wrapper = AnnualLimitWrapper(Mock())
-
-        with patch.object(wrapper, 'get_all_notification_counts_for_today') as mock_today, \
-             patch.object(wrapper, 'get_all_notification_counts_for_year') as mock_year:
-
-            mock_today.return_value = {
-                'sms': 2,
-                'email': 4
-            }
-            mock_year.return_value = {
-                'sms': 10,
-                'email': 20
-            }
-
-            # Execute
-            result = wrapper.get_all_notification_counts_ytd('service-123', 2024)
-
-            # Assert
-            assert result['sms'] == 12
-            assert result['email'] == 24
-
diff --git a/tests/app/notify_client/test_notification_counts_client.py b/tests/app/notify_client/test_notification_counts_client.py
new file mode 100644
index 0000000000..0faf35082f
--- /dev/null
+++ b/tests/app/notify_client/test_notification_counts_client.py
@@ -0,0 +1,89 @@
+from unittest.mock import Mock, patch
+
+import pytest
+
+from app.notify_client.notification_counts_client import NotificationCounts
+
+
+@pytest.fixture
+def mock_redis():
+    with patch("app.notify_client.notification_counts_client.redis_client") as mock:
+        yield mock
+
+
+@pytest.fixture
+def mock_template_stats():
+    with patch("app.notify_client.notification_counts_client.template_statistics_client") as mock:
+        yield mock
+
+
+@pytest.fixture
+def mock_service_api():
+    with patch("app.notify_client.notification_counts_client.service_api_client") as mock:
+        yield mock
+
+
+class TestNotificationCounts:
+    def test_get_all_notification_counts_for_today_redis_has_data(self, mock_redis):
+        # Setup
+        mock_redis.get.side_effect = [5, 10]  # sms, email
+        wrapper = NotificationCounts()
+
+        # Execute
+        result = wrapper.get_all_notification_counts_for_today("service-123")
+
+        # Assert
+        assert result == {"sms": 5, "email": 10}
+        assert mock_redis.get.call_count == 2
+
+    @pytest.mark.parametrize(
+        "redis_side_effect, expected_result",
+        [
+            ([None, None], {"sms": 10, "email": 10}),
+            ([None, 10], {"sms": 10, "email": 10}),  # Falls back to API if either is None
+            ([10, None], {"sms": 10, "email": 10}),  # Falls back to API if either is None
+        ],
+    )
+    def test_get_all_notification_counts_for_today_redis_missing_data(
+        self, mock_redis, mock_template_stats, redis_side_effect, expected_result
+    ):
+        # Setup
+        mock_redis.get.side_effect = redis_side_effect
+        mock_template_stats.get_template_statistics_for_service.return_value = [
+            {"template_id": "a1", "template_type": "sms", "count": 3, "status": "delivered"},
+            {"template_id": "a2", "template_type": "email", "count": 7, "status": "temporary-failure"},
+            {"template_id": "a3", "template_type": "email", "count": 3, "status": "delivered"},
+            {"template_id": "a4", "template_type": "sms", "count": 7, "status": "delivered"},
+        ]
+
+        wrapper = NotificationCounts()
+
+        # Execute
+        result = wrapper.get_all_notification_counts_for_today("service-123")
+
+        # Assert
+        assert result == {"sms": 10, "email": 10}
+        mock_template_stats.get_template_statistics_for_service.assert_called_once()
+
+    def test_get_all_notification_counts_for_year(self, mock_service_api):
+        # Setup
+        mock_service_api.get_monthly_notification_stats.return_value = {
+            "data": {
+                "2024-01": {
+                    "sms": {"sent": 1, "temporary-failure:": 22},
+                    "email": {"delivered": 1, "permanent-failure": 1, "sending": 12, "technical-failure": 1},
+                },
+                "2024-02": {"sms": {"sent": 1}, "email": {"delivered": 1}},
+            }
+        }
+        wrapper = NotificationCounts()
+
+        with patch.object(wrapper, "get_all_notification_counts_for_today") as mock_today:
+            mock_today.return_value = {"sms": 5, "email": 5}
+
+            # Execute
+            result = wrapper.get_all_notification_counts_for_year("service-123", 2024)
+
+            # Assert
+            assert result["sms"] == 29  # 1 + 22 + 1 + 5
+            assert result["email"] == 21  # 1 + 1 + 12 + 1 + 1 + 5

From a890ce1a2d36a963f95896f4f57f2dbfe6f43a0c Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 28 Nov 2024 13:17:18 +0000
Subject: [PATCH 08/28] add translations

---
 app/notify_client/annual_limit_client.py      | 108 ------------------
 .../check/too-many-email-messages.html        |   8 +-
 app/translations/csv/fr.csv                   |   6 +-
 3 files changed, 12 insertions(+), 110 deletions(-)
 delete mode 100644 app/notify_client/annual_limit_client.py

diff --git a/app/notify_client/annual_limit_client.py b/app/notify_client/annual_limit_client.py
deleted file mode 100644
index 5d9881ecde..0000000000
--- a/app/notify_client/annual_limit_client.py
+++ /dev/null
@@ -1,108 +0,0 @@
-
-from notifications_utils.clients.redis import (
-    email_daily_count_cache_key,
-    sms_daily_count_cache_key,
-)
-from notifications_utils.clients.redis.annual_limit import RedisAnnualLimit
-from notifications_utils.clients.redis.redis_client import RedisClient
-
-from app import redis_client, service_api_client, template_statistics_client
-from app.utils import DELIVERED_STATUSES, FAILURE_STATUSES
-
-
-class AnnualLimitWrapper:
-    def __init__(self, client):
-        self.client = client
-
-
-    def get_all_notification_counts_for_today(self, service_id):
-        # try to get today's stats from redis
-        todays_sms = redis_client.get(sms_daily_count_cache_key(service_id))
-        todays_email = redis_client.get(email_daily_count_cache_key(service_id))
-
-        if todays_sms is not None and todays_email is not None:
-            return {
-                'sms': todays_sms,
-                'email': todays_email
-            }
-        # fallback to the API if the stats are not in redis
-        else:
-            stats = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1)
-            transformed_stats = _aggregate_notifications_stats(stats)
-
-            return transformed_stats
-
-    def get_all_notification_counts_for_year(self, service_id, year):
-        """
-        Get total number of notifications by type for the current service for the current year
-
-        Return value:
-        {
-            'sms': int,
-            'email': int
-        }
-
-        """
-        stats_today = self.get_all_notification_counts_for_today(service_id)
-        stats_this_year = service_api_client.get_monthly_notification_stats(service_id, year)["data"]
-        stats_this_year = _aggregate_stats_from_service_api(stats_this_year)
-        # aggregate stats_today and stats_this_year
-        for template_type in ["sms", "email"]:
-            stats_this_year[template_type] += stats_today[template_type]
-
-        return stats_this_year
-
-    def get_all_notification_counts_ytd(self, service_id, year):
-        """
-        Get total number of notifications by type for the current service for the year to date
-
-        Return value:
-        {
-            'sms': int,
-            'email': int
-        }
-
-        """
-        stats_today = self.get_all_notification_counts_for_today(service_id)
-        stats_this_year = self.get_all_notification_counts_for_year(service_id, year)
-
-        # aggregate stats_today and stats_this_year
-        for template_type in ["sms", "email"]:
-            stats_this_year[template_type] += stats_today[template_type]
-
-        return stats_this_year
-
-# TODO: consolidate this function and other functions that transform the results of template_statistics_client calls
-def _aggregate_notifications_stats(template_statistics):
-    template_statistics = _filter_out_cancelled_stats(template_statistics)
-    notifications = {
-        'sms': 0,
-        'email': 0
-    }
-    for stat in template_statistics:
-        notifications[stat["template_type"]] += stat["count"]
-
-    return notifications
-
-def _filter_out_cancelled_stats(template_statistics):
-    return [s for s in template_statistics if s["status"] != "cancelled"]
-
-def _aggregate_stats_from_service_api(stats):
-    """Aggregate monthly notification stats excluding cancelled"""
-    total_stats = {"sms": {}, "email": {}}
-
-    for month_data in stats.values():
-        for msg_type in ["sms", "email"]:
-            if msg_type in month_data:
-                for status, count in month_data[msg_type].items():
-                    if status != "cancelled":
-                        if status not in total_stats[msg_type]:
-                            total_stats[msg_type][status] = 0
-                        total_stats[msg_type][status] += count
-
-    return {
-        msg_type: sum(counts.values())
-        for msg_type, counts in total_stats.items()
-    }
-
-annual_limit_client = AnnualLimitWrapper(RedisAnnualLimit(RedisClient()))
diff --git a/app/templates/partials/check/too-many-email-messages.html b/app/templates/partials/check/too-many-email-messages.html
index 0617d5d0b2..ff7dd02953 100644
--- a/app/templates/partials/check/too-many-email-messages.html
+++ b/app/templates/partials/check/too-many-email-messages.html
@@ -5,7 +5,13 @@
     {{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
   {% else %}
     {% if send_exceeds_annual_limit %}
-      {{ _("To request an annual limit above {} emails, {}").format(current_service.email_annual_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
+
+      <p>{{ _('<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets'.format(current_service.name, recipients_remaining_messages)) }}</p>
+      <p>
+        {{ _('To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. '.format(recipients_remaining_messages)) }}
+        {{ _('To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.'.format(now().year)) }}
+      </p>
+      
     {% endif %}
     {% if send_exceeds_daily_limit %}
       {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
diff --git a/app/translations/csv/fr.csv b/app/translations/csv/fr.csv
index 57aead4f21..fe2c41b635 100644
--- a/app/translations/csv/fr.csv
+++ b/app/translations/csv/fr.csv
@@ -2027,4 +2027,8 @@
 "Annual usage","Utilisation annuelle"
 "resets at 7pm Eastern Time","Réinitialisation à 19 h, heure de l’Est"
 "Visit usage report","Consulter le rapport d’utilisation"
-"Month by month totals","Totaux mensuels"
\ No newline at end of file
+"Month by month totals","Totaux mensuels"
+"<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets","FR: <strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets"
+"To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. ","FR: To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. "
+"To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.","FR: To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way"
+"These messages exceed the annual limit","FR: These messages exceed the annual limit"
\ No newline at end of file

From 5e61b91e5d3ce3e8392e152953fd6d3d6bc26aac Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 28 Nov 2024 13:18:59 +0000
Subject: [PATCH 09/28] chore: formatting

---
 app/main/views/send.py                              | 4 ++--
 app/notify_client/template_statistics_api_client.py | 2 +-
 gunicorn_config.py                                  | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/app/main/views/send.py b/app/main/views/send.py
index 8778e53d60..271d8870f2 100644
--- a/app/main/views/send.py
+++ b/app/main/views/send.py
@@ -798,13 +798,13 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
                 data["recipients_remaining_messages"] = remaining_email_this_year
                 data["send_exceeds_annual_limit"] = True
             else:
-                 data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
+                data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
         else:
             if remaining_sms_this_year < data["count_of_recipients"]:
                 data["recipients_remaining_messages"] = remaining_sms_this_year
                 data["send_exceeds_annual_limit"] = True
             else:
-                 data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
+                data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
 
     if (
         data["recipients"].too_many_rows
diff --git a/app/notify_client/template_statistics_api_client.py b/app/notify_client/template_statistics_api_client.py
index 429bdcd313..fb6acbc138 100644
--- a/app/notify_client/template_statistics_api_client.py
+++ b/app/notify_client/template_statistics_api_client.py
@@ -19,9 +19,9 @@ def get_template_statistics_for_template(self, service_id, template_id):
         return self.get(url="/service/{}/template-statistics/{}".format(service_id, template_id))["data"]
 
 
-
 template_statistics_client = TemplateStatisticsApiClient()
 
+
 class TemplateStatistics:
     def __init__(self, stats):
         self.stats = stats
diff --git a/gunicorn_config.py b/gunicorn_config.py
index 63325ca386..cf6e5e7112 100644
--- a/gunicorn_config.py
+++ b/gunicorn_config.py
@@ -51,6 +51,7 @@
 # Start timer for total running time
 start_time = time.time()
 
+
 def on_starting(server):
     server.log.info("Starting Notifications Admin")
 

From e1fe31f03ea63121c4467b8ade3af79452c6d631 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 28 Nov 2024 14:03:52 +0000
Subject: [PATCH 10/28] fix send to work without FF too

---
 app/main/views/send.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/main/views/send.py b/app/main/views/send.py
index 271d8870f2..706f208327 100644
--- a/app/main/views/send.py
+++ b/app/main/views/send.py
@@ -797,14 +797,14 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
             if remaining_email_this_year < data["count_of_recipients"]:
                 data["recipients_remaining_messages"] = remaining_email_this_year
                 data["send_exceeds_annual_limit"] = True
-            else:
-                data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
         else:
             if remaining_sms_this_year < data["count_of_recipients"]:
                 data["recipients_remaining_messages"] = remaining_sms_this_year
                 data["send_exceeds_annual_limit"] = True
-            else:
-                data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
+
+        data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
+    else:
+         data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
 
     if (
         data["recipients"].too_many_rows

From 10ffa2740bb952b0b3b07019c1c0cf20da92ef4e Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 28 Nov 2024 17:24:30 +0000
Subject: [PATCH 11/28] =?UTF-8?q?feat:=20fix=20sending;=20write=20tests;?=
 =?UTF-8?q?=20=F0=9F=99=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 app/main/views/send.py                        |   7 +-
 .../check/too-many-email-messages.html        |   7 +-
 .../check/too-many-sms-message-parts.html     |   2 +-
 app/templates/views/check/column-errors.html  |  10 +-
 tests/app/main/views/test_send.py             | 215 +++++++++++++++++-
 tests/conftest.py                             |  28 ++-
 6 files changed, 255 insertions(+), 14 deletions(-)

diff --git a/app/main/views/send.py b/app/main/views/send.py
index 706f208327..ab6412caf1 100644
--- a/app/main/views/send.py
+++ b/app/main/views/send.py
@@ -793,6 +793,8 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
         remaining_email_this_year = current_service.email_annual_limit - stats_ytd["email"]
 
         # Show annual limit validation over the daily one (even if both are true)
+        data["send_exceeds_annual_limit"] = False
+        data["send_exceeds_daily_limit"] = False
         if data["template"].template_type == "email":
             if remaining_email_this_year < data["count_of_recipients"]:
                 data["recipients_remaining_messages"] = remaining_email_this_year
@@ -801,10 +803,11 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
             if remaining_sms_this_year < data["count_of_recipients"]:
                 data["recipients_remaining_messages"] = remaining_sms_this_year
                 data["send_exceeds_annual_limit"] = True
+            else:
+                data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
 
-        data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
     else:
-         data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
+        data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
 
     if (
         data["recipients"].too_many_rows
diff --git a/app/templates/partials/check/too-many-email-messages.html b/app/templates/partials/check/too-many-email-messages.html
index ff7dd02953..ed75a1ab29 100644
--- a/app/templates/partials/check/too-many-email-messages.html
+++ b/app/templates/partials/check/too-many-email-messages.html
@@ -6,18 +6,17 @@
   {% else %}
     {% if send_exceeds_annual_limit %}
 
-      <p>{{ _('<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets'.format(current_service.name, recipients_remaining_messages)) }}</p>
+      <p data-testid="exceeds-annual">{{ _('<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets'.format(current_service.name, recipients_remaining_messages)) }}</p>
       <p>
         {{ _('To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. '.format(recipients_remaining_messages)) }}
         {{ _('To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.'.format(now().year)) }}
       </p>
       
-    {% endif %}
-    {% if send_exceeds_daily_limit %}
+    {% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %}
       {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
 
       <h2 class="heading-medium">{{ _('You cannot send all these email messages today') }}</h2>
-      <p>
+      <p data-testid="exceeds-daily">
         {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang],
         content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
       </p>
diff --git a/app/templates/partials/check/too-many-sms-message-parts.html b/app/templates/partials/check/too-many-sms-message-parts.html
index 1000fe1ac5..8ceee20762 100644
--- a/app/templates/partials/check/too-many-sms-message-parts.html
+++ b/app/templates/partials/check/too-many-sms-message-parts.html
@@ -1,6 +1,6 @@
 {% from "components/links.html" import content_link %}
 
-<p>
+<p data-testid="exceeds-daily">
   {%- if current_service.trial_mode %}
     {{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
   {% else %}
diff --git a/app/templates/views/check/column-errors.html b/app/templates/views/check/column-errors.html
index 26cb381b05..b60c6457ff 100644
--- a/app/templates/views/check/column-errors.html
+++ b/app/templates/views/check/column-errors.html
@@ -10,10 +10,12 @@
 {% set prefix_txt = _('a column called') %}
 {% set prefix_plural_txt = _('columns called') %}
 
-{% if send_exceeds_daily_limit %}
-  {% set page_title = _('These messages exceed your daily limit') %}
-{% elif send_exceeds_annual_limit %}
+{% if send_exceeds_annual_limit %}
   {% set page_title = _('These messages exceed the annual limit') %}
+{% elif send_exceeds_daily_limit and (sms_parts_remaining <= 0) %}
+  {% set page_title = _('These messages exceed your daily limit') %}
+{% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %}
+  {% set page_title = _('These messages exceed your daily limit') %}
 {% else %}
   {% set page_title = _('Check there’s a column for each variable') %}
 {% endif %}
@@ -168,7 +170,7 @@ <h2 class="heading-medium">{{ _('You cannot send all these text messages today')
         {% call banner_wrapper(type='dangerous') %}
           {% include "partials/check/too-many-email-messages.html" %}
         {% endcall %}
-      {% elif recipients.more_rows_than_can_send %}
+      {% elif recipients.more_rows_than_can_send or  send_exceeds_annual_limit %}
           {% call banner_wrapper(type='dangerous') %}
             {% include "partials/check/too-many-email-messages.html" %}
           {% endcall %}           
diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py
index 78df4b10cc..bb431810ee 100644
--- a/tests/app/main/views/test_send.py
+++ b/tests/app/main/views/test_send.py
@@ -6,6 +6,7 @@
 from io import BytesIO
 from itertools import repeat
 from os import path
+from unittest.mock import patch
 from uuid import uuid4
 from zipfile import BadZipFile
 
@@ -41,6 +42,7 @@
     mock_get_service_letter_template,
     mock_get_service_template,
     normalize_spaces,
+    set_config,
 )
 
 template_types = ["email", "sms"]
@@ -2543,6 +2545,7 @@ def test_check_messages_shows_too_many_sms_messages_errors(
     mock_get_jobs,
     mock_s3_download,
     mock_s3_set_metadata,
+    mock_notification_counts_client,
     fake_uuid,
     num_requested,
     expected_msg,
@@ -2560,6 +2563,10 @@ def test_check_messages_shows_too_many_sms_messages_errors(
         },
     )
 
+    mock_notification_counts_client.get_all_notification_counts_for_year.return_value = {
+        "sms": 0,
+        "email": 0,
+    }
     with client_request.session_transaction() as session:
         session["file_uploads"] = {
             fake_uuid: {
@@ -2584,6 +2591,30 @@ def test_check_messages_shows_too_many_sms_messages_errors(
     assert details == expected_msg
 
 
+@pytest.fixture
+def mock_notification_counts_client():
+    with patch("app.main.views.send.notification_counts_client") as mock:
+        yield mock
+
+
+@pytest.fixture
+def mock_daily_sms_fragment_count():
+    with patch("app.main.views.send.daily_sms_fragment_count") as mock:
+        yield mock
+
+
+@pytest.fixture
+def mock_daily_email_count():
+    with patch("app.main.views.send.daily_email_count") as mock:
+        yield mock
+
+
+@pytest.fixture
+def mock_get_service_template_annual_limits():
+    with patch("app.service_api_client.get_service_template") as mock:
+        yield mock
+
+
 @pytest.mark.parametrize(
     "num_requested,expected_msg",
     [
@@ -2601,6 +2632,7 @@ def test_check_messages_shows_too_many_email_messages_errors(
     mock_get_template_statistics,
     mock_get_job_doesnt_exist,
     mock_get_jobs,
+    mock_notification_counts_client,
     fake_uuid,
     num_requested,
     expected_msg,
@@ -2617,7 +2649,10 @@ def test_check_messages_shows_too_many_email_messages_errors(
             "email": {"requested": num_requested, "delivered": 0, "failed": 0},
         },
     )
-
+    mock_notification_counts_client.get_all_notification_counts_for_year.return_value = {
+        "sms": 0,
+        "email": 0,
+    }
     with client_request.session_transaction() as session:
         session["file_uploads"] = {
             fake_uuid: {
@@ -3401,3 +3436,181 @@ class Object(object):
     multiple_choise_options = [x.text.strip() for x in options]
 
     assert multiple_choise_options == expected_filenames
+
+
+class TestAnnualLimitsSend:
+    @pytest.mark.parametrize(
+        "num_being_sent, num_sent_today, num_sent_this_year, expect_to_see_annual_limit_msg, expect_to_see_daily_limit_msg",
+        [
+            # annual limit for mock_get_live_service is 10,000email/10,000sms
+            # daily limit for mock_get_live_service is 1,000email/1,000sms
+            # 1000 have already been sent today, trying to send 100 more [over both limits]
+            (100, 1000, 10000, True, False),
+            # No sent yet today or this year, trying to send 1001 [over both limits]
+            (10001, 0, 0, True, False),
+            # 600 have already been sent this year, trying to send 500 more [over annual limit but not daily]
+            (500, 0, 9600, True, False),
+            # No sent yet today or this year, trying to send 1001 [over daily limit but not annual]
+            (1001, 0, 0, False, True),
+            # No sent yet today or this year, trying to send 100 [over neither limit]
+            (100, 0, 0, False, False),
+        ],
+        ids=[
+            "email_over_both_limits",
+            "email_over_both_limits2",
+            "email_over_annual_but_not_daily",
+            "email_over_daily_but_not_annual",
+            "email_over_neither",
+        ],
+    )
+    def test_email_send_fails_approrpiately_when_over_limits(
+        self,
+        mocker,
+        client_request,
+        mock_get_live_service,  # set email_annual_limit and sms_annual_limit to 1000
+        mock_get_users_by_service,
+        mock_get_service_email_template_without_placeholders,
+        mock_get_template_statistics,
+        mock_get_job_doesnt_exist,
+        mock_get_jobs,
+        mock_s3_set_metadata,
+        mock_notification_counts_client,
+        mock_daily_sms_fragment_count,
+        mock_daily_email_count,
+        fake_uuid,
+        num_being_sent,
+        num_sent_today,
+        num_sent_this_year,
+        expect_to_see_annual_limit_msg,
+        expect_to_see_daily_limit_msg,
+    ):
+        mocker.patch(
+            "app.main.views.send.s3download",
+            return_value=",\n".join(["email address"] + ([mock_get_users_by_service(None)[0]["email_address"]] * num_being_sent)),
+        )
+
+        # mock that `num_sent_this_year` have already been sent this year
+        mock_notification_counts_client.get_all_notification_counts_for_year.return_value = {
+            "sms": 1000,  # not used in test but needs a value
+            "email": num_sent_this_year,
+        }
+
+        # mock that we've already sent `emails_sent_today` emails today
+        mock_daily_email_count.return_value = num_sent_today
+        mock_daily_sms_fragment_count.return_value = 900  # not used in test but needs a value
+
+        with client_request.session_transaction() as session:
+            session["file_uploads"] = {
+                fake_uuid: {
+                    "template_id": fake_uuid,
+                    "notification_count": 1,
+                    "valid": True,
+                }
+            }
+
+        page = client_request.get(
+            "main.check_messages",
+            service_id=SERVICE_ONE_ID,
+            template_id=fake_uuid,
+            upload_id=fake_uuid,
+            original_file_name="valid.csv",
+            _test_page_title=False,
+        )
+
+        if expect_to_see_annual_limit_msg:
+            assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None
+        else:
+            assert page.find(attrs={"data-testid": "exceeds-annual"}) is None
+
+        if expect_to_see_daily_limit_msg:
+            assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None
+        else:
+            assert page.find(attrs={"data-testid": "exceeds-daily"}) is None
+
+    @pytest.mark.parametrize(
+        "num_being_sent, num_sent_today, num_sent_this_year, expect_to_see_annual_limit_msg, expect_to_see_daily_limit_msg",
+        [
+            # annual limit for mock_get_live_service is 10,000email/10,000sms
+            # daily limit for mock_get_live_service is 1,000email/1,000sms
+            # 1000 have already been sent today, trying to send 100 more [over both limits]
+            (100, 1000, 10000, True, False),
+            # No sent yet today or this year, trying to send 1001 [over both limits]
+            (10001, 0, 0, True, False),
+            # 600 have already been sent this year, trying to send 500 more [over annual limit but not daily]
+            (500, 0, 9600, True, False),
+            # No sent yet today or this year, trying to send 1001 [over daily limit but not annual]
+            (1001, 0, 0, False, True),
+            # No sent yet today or this year, trying to send 100 [over neither limit]
+            (100, 0, 0, False, False),
+        ],
+        ids=[
+            "sms_over_both_limits",
+            "sms_over_both_limits2",
+            "sms_over_annual_but_not_daily",
+            "sms_over_daily_but_not_annual",
+            "sms_over_neither",
+        ],
+    )
+    def test_sms_send_fails_approrpiately_when_over_limits(
+        self,
+        mocker,
+        client_request,
+        mock_get_live_service,  # set email_annual_limit and sms_annual_limit to 1000
+        mock_get_users_by_service,
+        mock_get_service_sms_template_without_placeholders,
+        mock_get_template_statistics,
+        mock_get_job_doesnt_exist,
+        mock_get_jobs,
+        mock_s3_set_metadata,
+        mock_notification_counts_client,
+        mock_daily_sms_fragment_count,
+        mock_daily_email_count,
+        fake_uuid,
+        num_being_sent,
+        num_sent_today,
+        num_sent_this_year,
+        expect_to_see_annual_limit_msg,
+        expect_to_see_daily_limit_msg,
+    ):
+        mocker.patch(
+            "app.main.views.send.s3download",
+            return_value=",\n".join(["phone number"] + ([mock_get_users_by_service(None)[0]["mobile_number"]] * num_being_sent)),
+        )
+
+        # mock that `num_sent_this_year` have already been sent this year
+        mock_notification_counts_client.get_all_notification_counts_for_year.return_value = {
+            "sms": num_sent_this_year,
+            "email": 1000,  # not used in test but needs a value
+        }
+
+        # mock that we've already sent `emails_sent_today` emails today
+        mock_daily_email_count.return_value = 900  # not used in test but needs a value
+        mock_daily_sms_fragment_count.return_value = num_sent_today
+
+        with client_request.session_transaction() as session:
+            session["file_uploads"] = {
+                fake_uuid: {
+                    "template_id": fake_uuid,
+                    "notification_count": 1,
+                    "valid": True,
+                }
+            }
+
+        page = client_request.get(
+            "main.check_messages",
+            service_id=SERVICE_ONE_ID,
+            template_id=fake_uuid,
+            upload_id=fake_uuid,
+            original_file_name="valid.csv",
+            _test_page_title=False,
+        )
+
+        if expect_to_see_annual_limit_msg:
+            assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None
+        else:
+            assert page.find(attrs={"data-testid": "exceeds-annual"}) is None
+
+        if expect_to_see_daily_limit_msg:
+            assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None
+        else:
+            assert page.find(attrs={"data-testid": "exceeds-daily"}) is None
diff --git a/tests/conftest.py b/tests/conftest.py
index 94ff0ca5be..21ed1f782b 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -550,7 +550,14 @@ def fake_uuid():
 @pytest.fixture(scope="function")
 def mock_get_service(mocker, api_user_active):
     def _get(service_id):
-        service = service_json(service_id, users=[api_user_active["id"]], message_limit=50, sms_daily_limit=20)
+        service = service_json(
+            service_id,
+            users=[api_user_active["id"]],
+            message_limit=50,
+            sms_daily_limit=20,
+            email_annual_limit=1000,
+            sms_annual_limit=1000,
+        )
         return {"data": service}
 
     return mocker.patch("app.service_api_client.get_service", side_effect=_get)
@@ -675,7 +682,9 @@ def mock_service_email_from_is_unique(mocker):
 @pytest.fixture(scope="function")
 def mock_get_live_service(mocker, api_user_active):
     def _get(service_id):
-        service = service_json(service_id, users=[api_user_active["id"]], restricted=False)
+        service = service_json(
+            service_id, users=[api_user_active["id"]], restricted=False, sms_annual_limit=10000, email_annual_limit=10000
+        )
         return {"data": service}
 
     return mocker.patch("app.service_api_client.get_service", side_effect=_get)
@@ -971,6 +980,21 @@ def _get(service_id, template_id, version=None):
     return mocker.patch("app.service_api_client.get_service_template", side_effect=_get)
 
 
+@pytest.fixture(scope="function")
+def mock_get_service_sms_template_without_placeholders(mocker):
+    def _get(service_id, template_id, version=None):
+        template = template_json(
+            service_id,
+            template_id,
+            "Two week reminder",
+            "sms",
+            "Yo.",
+        )
+        return {"data": template}
+
+    return mocker.patch("app.service_api_client.get_service_template", side_effect=_get)
+
+
 @pytest.fixture(scope="function")
 def mock_get_service_letter_template(mocker, content=None, subject=None, postage="second"):
     def _get(service_id, template_id, version=None, postage=postage):

From 7f1aeb686ab97355c92234edeff936386687dfdd Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 28 Nov 2024 18:31:04 +0000
Subject: [PATCH 12/28] fix: only check for `send_exceeds_annual_limit` when FF
 is on

---
 app/main/views/send.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/main/views/send.py b/app/main/views/send.py
index ab6412caf1..e96c3f4f1d 100644
--- a/app/main/views/send.py
+++ b/app/main/views/send.py
@@ -825,9 +825,13 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
     if data["errors"] or data["trying_to_send_letters_in_trial_mode"]:
         return render_template("views/check/column-errors.html", **data)
 
-    if data["send_exceeds_daily_limit"] or data["send_exceeds_annual_limit"]:
+    if data["send_exceeds_daily_limit"]:
         return render_template("views/check/column-errors.html", **data)
 
+    if current_app.config["FF_ANNUAL_LIMIT"]:
+        if data["send_exceeds_annual_limit"]:
+            return render_template("views/check/column-errors.html", **data)
+
     metadata_kwargs = {
         "notification_count": data["count_of_recipients"],
         "template_id": str(template_id),

From cde386261373a105f19b5b2a136f1b0d6356c8bc Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 28 Nov 2024 19:32:02 +0000
Subject: [PATCH 13/28] fix(tests): only run the new tests with the FF on (they
 cant pass with it off!)

---
 tests/app/main/views/test_send.py | 152 ++++++++++++++++--------------
 1 file changed, 80 insertions(+), 72 deletions(-)

diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py
index bb431810ee..c1350bdb46 100644
--- a/tests/app/main/views/test_send.py
+++ b/tests/app/main/views/test_send.py
@@ -3483,49 +3483,53 @@ def test_email_send_fails_approrpiately_when_over_limits(
         num_sent_this_year,
         expect_to_see_annual_limit_msg,
         expect_to_see_daily_limit_msg,
+        app_,
     ):
-        mocker.patch(
-            "app.main.views.send.s3download",
-            return_value=",\n".join(["email address"] + ([mock_get_users_by_service(None)[0]["email_address"]] * num_being_sent)),
-        )
-
-        # mock that `num_sent_this_year` have already been sent this year
-        mock_notification_counts_client.get_all_notification_counts_for_year.return_value = {
-            "sms": 1000,  # not used in test but needs a value
-            "email": num_sent_this_year,
-        }
+        with set_config(app_, "FF_ANNUAL_LIMIT", True):
+            mocker.patch(
+                "app.main.views.send.s3download",
+                return_value=",\n".join(
+                    ["email address"] + ([mock_get_users_by_service(None)[0]["email_address"]] * num_being_sent)
+                ),
+            )
 
-        # mock that we've already sent `emails_sent_today` emails today
-        mock_daily_email_count.return_value = num_sent_today
-        mock_daily_sms_fragment_count.return_value = 900  # not used in test but needs a value
+            # mock that `num_sent_this_year` have already been sent this year
+            mock_notification_counts_client.get_all_notification_counts_for_year.return_value = {
+                "sms": 1000,  # not used in test but needs a value
+                "email": num_sent_this_year,
+            }
 
-        with client_request.session_transaction() as session:
-            session["file_uploads"] = {
-                fake_uuid: {
-                    "template_id": fake_uuid,
-                    "notification_count": 1,
-                    "valid": True,
+            # mock that we've already sent `emails_sent_today` emails today
+            mock_daily_email_count.return_value = num_sent_today
+            mock_daily_sms_fragment_count.return_value = 900  # not used in test but needs a value
+
+            with client_request.session_transaction() as session:
+                session["file_uploads"] = {
+                    fake_uuid: {
+                        "template_id": fake_uuid,
+                        "notification_count": 1,
+                        "valid": True,
+                    }
                 }
-            }
 
-        page = client_request.get(
-            "main.check_messages",
-            service_id=SERVICE_ONE_ID,
-            template_id=fake_uuid,
-            upload_id=fake_uuid,
-            original_file_name="valid.csv",
-            _test_page_title=False,
-        )
+            page = client_request.get(
+                "main.check_messages",
+                service_id=SERVICE_ONE_ID,
+                template_id=fake_uuid,
+                upload_id=fake_uuid,
+                original_file_name="valid.csv",
+                _test_page_title=False,
+            )
 
-        if expect_to_see_annual_limit_msg:
-            assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None
-        else:
-            assert page.find(attrs={"data-testid": "exceeds-annual"}) is None
+            if expect_to_see_annual_limit_msg:
+                assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None
+            else:
+                assert page.find(attrs={"data-testid": "exceeds-annual"}) is None
 
-        if expect_to_see_daily_limit_msg:
-            assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None
-        else:
-            assert page.find(attrs={"data-testid": "exceeds-daily"}) is None
+            if expect_to_see_daily_limit_msg:
+                assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None
+            else:
+                assert page.find(attrs={"data-testid": "exceeds-daily"}) is None
 
     @pytest.mark.parametrize(
         "num_being_sent, num_sent_today, num_sent_this_year, expect_to_see_annual_limit_msg, expect_to_see_daily_limit_msg",
@@ -3571,46 +3575,50 @@ def test_sms_send_fails_approrpiately_when_over_limits(
         num_sent_this_year,
         expect_to_see_annual_limit_msg,
         expect_to_see_daily_limit_msg,
+        app_,
     ):
-        mocker.patch(
-            "app.main.views.send.s3download",
-            return_value=",\n".join(["phone number"] + ([mock_get_users_by_service(None)[0]["mobile_number"]] * num_being_sent)),
-        )
-
-        # mock that `num_sent_this_year` have already been sent this year
-        mock_notification_counts_client.get_all_notification_counts_for_year.return_value = {
-            "sms": num_sent_this_year,
-            "email": 1000,  # not used in test but needs a value
-        }
+        with set_config(app_, "FF_ANNUAL_LIMIT", True):  # REMOVE LINE WHEN FF REMOVED
+            mocker.patch(
+                "app.main.views.send.s3download",
+                return_value=",\n".join(
+                    ["phone number"] + ([mock_get_users_by_service(None)[0]["mobile_number"]] * num_being_sent)
+                ),
+            )
 
-        # mock that we've already sent `emails_sent_today` emails today
-        mock_daily_email_count.return_value = 900  # not used in test but needs a value
-        mock_daily_sms_fragment_count.return_value = num_sent_today
+            # mock that `num_sent_this_year` have already been sent this year
+            mock_notification_counts_client.get_all_notification_counts_for_year.return_value = {
+                "sms": num_sent_this_year,
+                "email": 1000,  # not used in test but needs a value
+            }
 
-        with client_request.session_transaction() as session:
-            session["file_uploads"] = {
-                fake_uuid: {
-                    "template_id": fake_uuid,
-                    "notification_count": 1,
-                    "valid": True,
+            # mock that we've already sent `emails_sent_today` emails today
+            mock_daily_email_count.return_value = 900  # not used in test but needs a value
+            mock_daily_sms_fragment_count.return_value = num_sent_today
+
+            with client_request.session_transaction() as session:
+                session["file_uploads"] = {
+                    fake_uuid: {
+                        "template_id": fake_uuid,
+                        "notification_count": 1,
+                        "valid": True,
+                    }
                 }
-            }
 
-        page = client_request.get(
-            "main.check_messages",
-            service_id=SERVICE_ONE_ID,
-            template_id=fake_uuid,
-            upload_id=fake_uuid,
-            original_file_name="valid.csv",
-            _test_page_title=False,
-        )
+            page = client_request.get(
+                "main.check_messages",
+                service_id=SERVICE_ONE_ID,
+                template_id=fake_uuid,
+                upload_id=fake_uuid,
+                original_file_name="valid.csv",
+                _test_page_title=False,
+            )
 
-        if expect_to_see_annual_limit_msg:
-            assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None
-        else:
-            assert page.find(attrs={"data-testid": "exceeds-annual"}) is None
+            if expect_to_see_annual_limit_msg:
+                assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None
+            else:
+                assert page.find(attrs={"data-testid": "exceeds-annual"}) is None
 
-        if expect_to_see_daily_limit_msg:
-            assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None
-        else:
-            assert page.find(attrs={"data-testid": "exceeds-daily"}) is None
+            if expect_to_see_daily_limit_msg:
+                assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None
+            else:
+                assert page.find(attrs={"data-testid": "exceeds-daily"}) is None

From aa5b18effc5b4ef79c540db42265d80788738f20 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 28 Nov 2024 19:34:05 +0000
Subject: [PATCH 14/28] chore: remove unused imports

---
 app/notify_client/notification_counts_client.py            | 1 -
 tests/app/notify_client/test_notification_counts_client.py | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/notify_client/notification_counts_client.py b/app/notify_client/notification_counts_client.py
index ae3bc3a68f..ae6e1a4930 100644
--- a/app/notify_client/notification_counts_client.py
+++ b/app/notify_client/notification_counts_client.py
@@ -4,7 +4,6 @@
 )
 
 from app import redis_client, service_api_client, template_statistics_client
-from app.utils import DELIVERED_STATUSES, FAILURE_STATUSES
 
 
 class NotificationCounts:
diff --git a/tests/app/notify_client/test_notification_counts_client.py b/tests/app/notify_client/test_notification_counts_client.py
index 0faf35082f..fa7183aea8 100644
--- a/tests/app/notify_client/test_notification_counts_client.py
+++ b/tests/app/notify_client/test_notification_counts_client.py
@@ -1,4 +1,4 @@
-from unittest.mock import Mock, patch
+from unittest.mock import patch
 
 import pytest
 

From e7ad1debbc1578adf7ce9a88046242702498c295 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Tue, 3 Dec 2024 13:58:40 +0000
Subject: [PATCH 15/28] fix: move secondary message outside banner

---
 app/templates/partials/check/too-many-email-messages.html | 7 +++++--
 app/templates/views/check/column-errors.html              | 4 +---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/app/templates/partials/check/too-many-email-messages.html b/app/templates/partials/check/too-many-email-messages.html
index ed75a1ab29..4ead4e464b 100644
--- a/app/templates/partials/check/too-many-email-messages.html
+++ b/app/templates/partials/check/too-many-email-messages.html
@@ -1,3 +1,4 @@
+{% from "components/banner.html" import banner_wrapper %}
 {% from "components/links.html" import content_link %}
 
 <p>
@@ -13,8 +14,10 @@
       </p>
       
     {% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %}
-      {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
-
+      {% call banner_wrapper(type='dangerous') %}
+        {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
+      {% endcall %}
+      
       <h2 class="heading-medium">{{ _('You cannot send all these email messages today') }}</h2>
       <p data-testid="exceeds-daily">
         {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang],
diff --git a/app/templates/views/check/column-errors.html b/app/templates/views/check/column-errors.html
index b60c6457ff..852d30f0bd 100644
--- a/app/templates/views/check/column-errors.html
+++ b/app/templates/views/check/column-errors.html
@@ -171,9 +171,7 @@ <h2 class="heading-medium">{{ _('You cannot send all these text messages today')
           {% include "partials/check/too-many-email-messages.html" %}
         {% endcall %}
       {% elif recipients.more_rows_than_can_send or  send_exceeds_annual_limit %}
-          {% call banner_wrapper(type='dangerous') %}
-            {% include "partials/check/too-many-email-messages.html" %}
-          {% endcall %}           
+          {% include "partials/check/too-many-email-messages.html" %}
       {% endif %}
   </div>
 

From aa849892736a2702aa1ff8028709f760034bbe1f Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Tue, 3 Dec 2024 14:02:48 +0000
Subject: [PATCH 16/28] fix: undo ruff change made by accident

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 8c37921bf2..662a8a46ce 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -247,7 +247,7 @@ select = [
     "I001",
     "I002"
 ]
-ignore = ["E203", "E501", "E402", "F401"]
+ignore = ["E203", "E501", "E402"]
 
 # Provide line length leeway for docstrings
 [tool.ruff.lint.pycodestyle]

From bb34be9f26aea5611e1ac253b8126eede75ca1a7 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Wed, 4 Dec 2024 16:32:42 +0000
Subject: [PATCH 17/28] feat(error message): split annual into its own

---
 app/main/views/send.py                        |  6 +++++
 .../check/too-many-email-messages-annual.html | 13 +++++++++++
 .../check/too-many-email-messages.html        | 22 +------------------
 app/templates/views/check/column-errors.html  | 13 ++++++++---
 app/templates/views/notifications/check.html  |  6 +++++
 5 files changed, 36 insertions(+), 24 deletions(-)
 create mode 100644 app/templates/partials/check/too-many-email-messages-annual.html

diff --git a/app/main/views/send.py b/app/main/views/send.py
index e96c3f4f1d..f7984c827f 100644
--- a/app/main/views/send.py
+++ b/app/main/views/send.py
@@ -809,6 +809,8 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
     else:
         data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
 
+    data["send_exceeds_annual_limit"] = False
+
     if (
         data["recipients"].too_many_rows
         or not data["count_of_recipients"]
@@ -1113,6 +1115,10 @@ def get_template_error_dict(exception):
         error = "too-many-sms-messages"
     elif "Content for template has a character count greater than the limit of" in exception.message:
         error = "message-too-long"
+    elif "Exceeded annual email sending limit" in exception.message:
+        error = "too-many-email-annual"
+    elif "Exceeded annual SMS sending limit" in exception.message:
+        error = "too-many-sms-annual"
     else:
         raise exception
 
diff --git a/app/templates/partials/check/too-many-email-messages-annual.html b/app/templates/partials/check/too-many-email-messages-annual.html
new file mode 100644
index 0000000000..de30481845
--- /dev/null
+++ b/app/templates/partials/check/too-many-email-messages-annual.html
@@ -0,0 +1,13 @@
+{% from "components/links.html" import content_link %}
+
+<p>
+  {%- if current_service.trial_mode %}
+    {{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
+  {% else %}
+    <p data-testid="exceeds-annual">{{ _('<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets'.format(current_service.name, recipients_remaining_messages)) }}</p>
+    <p>
+      {{ _('To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. '.format(recipients_remaining_messages)) }}
+      {{ _('To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.'.format(now().year)) }}
+    </p>
+  {%- endif -%}
+</p>
\ No newline at end of file
diff --git a/app/templates/partials/check/too-many-email-messages.html b/app/templates/partials/check/too-many-email-messages.html
index 4ead4e464b..987846c472 100644
--- a/app/templates/partials/check/too-many-email-messages.html
+++ b/app/templates/partials/check/too-many-email-messages.html
@@ -1,29 +1,9 @@
-{% from "components/banner.html" import banner_wrapper %}
 {% from "components/links.html" import content_link %}
 
 <p>
   {%- if current_service.trial_mode %}
     {{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
   {% else %}
-    {% if send_exceeds_annual_limit %}
-
-      <p data-testid="exceeds-annual">{{ _('<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets'.format(current_service.name, recipients_remaining_messages)) }}</p>
-      <p>
-        {{ _('To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. '.format(recipients_remaining_messages)) }}
-        {{ _('To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.'.format(now().year)) }}
-      </p>
-      
-    {% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %}
-      {% call banner_wrapper(type='dangerous') %}
-        {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
-      {% endcall %}
-      
-      <h2 class="heading-medium">{{ _('You cannot send all these email messages today') }}</h2>
-      <p data-testid="exceeds-daily">
-        {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang],
-        content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
-      </p>
-    {% endif %}
-    
+    {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }}
   {%- endif -%}
 </p>
\ No newline at end of file
diff --git a/app/templates/views/check/column-errors.html b/app/templates/views/check/column-errors.html
index 852d30f0bd..0b9cecaafe 100644
--- a/app/templates/views/check/column-errors.html
+++ b/app/templates/views/check/column-errors.html
@@ -166,12 +166,19 @@ <h2 class="heading-medium">{{ _('You cannot send all these text messages today')
             {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang],
             content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
           </p>
-      {% elif recipients.more_rows_than_can_send and false %}
+      {% elif send_exceeds_annual_limit %}
         {% call banner_wrapper(type='dangerous') %}
-          {% include "partials/check/too-many-email-messages.html" %}
+          {% include "partials/check/too-many-email-messages-annual.html" %}
         {% endcall %}
-      {% elif recipients.more_rows_than_can_send or  send_exceeds_annual_limit %}
+      {% elif recipients.more_rows_than_can_send %}
+        {% call banner_wrapper(type='dangerous') %}
           {% include "partials/check/too-many-email-messages.html" %}
+        {% endcall %}
+        <h2 class="heading-medium">{{ _('You cannot send all these email messages today') }}</h2>
+        <p>
+          {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang],
+          content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
+        </p>
       {% endif %}
   </div>
 
diff --git a/app/templates/views/notifications/check.html b/app/templates/views/notifications/check.html
index 17c668b0c8..089e19a951 100644
--- a/app/templates/views/notifications/check.html
+++ b/app/templates/views/notifications/check.html
@@ -66,6 +66,12 @@ <h2 class="heading-medium">{{_('You cannot send this email message today') }}</h
         content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
       </p>
     </div>
+  {% elif error == 'too-many-email-messages-annual' %}
+    {{ page_header(_('This message exceeds your annual limit'), back_link=back_link) }}
+    <div class="mb-12 clear-both contain-floats">
+      {% call banner_wrapper(type='dangerous') %}
+        {% include "partials/check/too-many-email-messages-annual.html" %}
+      {% endcall %}
   {% elif error == 'message-too-long' %}
     {# the only row_errors we can get when sending one off messages is that the message is too long #}
     {{ govuk_back_link(back_link) }}

From b308d2d5ca1ec3a092ac835712b44246439fc23f Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Wed, 4 Dec 2024 18:32:36 +0000
Subject: [PATCH 18/28] feat(send): refactor limit checker to use
 `notification_counts_client` class when sending

---
 app/main/views/send.py                        | 26 +++----
 .../notification_counts_client.py             | 71 +++++++++++++++++++
 2 files changed, 80 insertions(+), 17 deletions(-)

diff --git a/app/main/views/send.py b/app/main/views/send.py
index f7984c827f..53acc9ba63 100644
--- a/app/main/views/send.py
+++ b/app/main/views/send.py
@@ -1,6 +1,5 @@
 import itertools
 import json
-from datetime import datetime, timezone
 from string import ascii_uppercase
 from zipfile import BadZipFile
 
@@ -787,30 +786,23 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
     data["sms_parts_remaining"] = current_service.sms_daily_limit - daily_sms_fragment_count(service_id)
 
     if current_app.config["FF_ANNUAL_LIMIT"]:
-        # Override the remaining messages counts with the remaining annual counts, if the latter are lower
-        stats_ytd = notification_counts_client.get_all_notification_counts_for_year(service_id, datetime.now(timezone.utc).year)
-        remaining_sms_this_year = current_service.sms_annual_limit - stats_ytd["sms"]
-        remaining_email_this_year = current_service.email_annual_limit - stats_ytd["email"]
-
-        # Show annual limit validation over the daily one (even if both are true)
         data["send_exceeds_annual_limit"] = False
         data["send_exceeds_daily_limit"] = False
-        if data["template"].template_type == "email":
-            if remaining_email_this_year < data["count_of_recipients"]:
-                data["recipients_remaining_messages"] = remaining_email_this_year
-                data["send_exceeds_annual_limit"] = True
+        # determine the remaining sends for daily + annual
+        limit_stats = notification_counts_client.get_limit_stats(current_service)
+        remaining_annual = limit_stats[data["template"].template_type]["annual"]["remaining"]
+
+        if remaining_annual < data["count_of_recipients"]:
+            data["recipients_remaining_messages"] = remaining_annual
+            data["send_exceeds_annual_limit"] = True
         else:
-            if remaining_sms_this_year < data["count_of_recipients"]:
-                data["recipients_remaining_messages"] = remaining_sms_this_year
-                data["send_exceeds_annual_limit"] = True
-            else:
+            # if they arent over their limit, and its sms, check if they are over their daily limit
+            if data["template"].template_type == "sms":
                 data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
 
     else:
         data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"]
 
-    data["send_exceeds_annual_limit"] = False
-
     if (
         data["recipients"].too_many_rows
         or not data["count_of_recipients"]
diff --git a/app/notify_client/notification_counts_client.py b/app/notify_client/notification_counts_client.py
index ae6e1a4930..cd1c6b7781 100644
--- a/app/notify_client/notification_counts_client.py
+++ b/app/notify_client/notification_counts_client.py
@@ -1,9 +1,12 @@
+from datetime import datetime
+
 from notifications_utils.clients.redis import (
     email_daily_count_cache_key,
     sms_daily_count_cache_key,
 )
 
 from app import redis_client, service_api_client, template_statistics_client
+from app.models.service import Service
 
 
 class NotificationCounts:
@@ -41,6 +44,74 @@ def get_all_notification_counts_for_year(self, service_id, year):
 
         return stats_this_year
 
+    def get_limit_stats(self, service: Service):
+        """
+        Get the limit stats for the current service, by notification type, including:
+         - how many notifications were sent today and this year
+         - the monthy and daily limits
+         - the number of notifications remaining today and this year
+        Returns:
+            dict: A dictionary containing the limit stats for email and SMS notifications. The structure is as follows:
+                {
+                    "email": {
+                        "annual": {
+                            "limit": int,  # The annual limit for email notifications
+                            "sent": int,   # The number of email notifications sent this year
+                            "remaining": int,  # The number of email notifications remaining this year
+                        },
+                        "daily": {
+                            "limit": int,  # The daily limit for email notifications
+                            "sent": int,   # The number of email notifications sent today
+                            "remaining": int,  # The number of email notifications remaining today
+                        },
+                    },
+                    "sms": {
+                        "annual": {
+                            "limit": int,  # The annual limit for SMS notifications
+                            "sent": int,   # The number of SMS notifications sent this year
+                            "remaining": int,  # The number of SMS notifications remaining this year
+                        },
+                        "daily": {
+                            "limit": int,  # The daily limit for SMS notifications
+                            "sent": int,   # The number of SMS notifications sent today
+                            "remaining": int,  # The number of SMS notifications remaining today
+                        },
+                    }
+                }
+        """
+
+        sent_today = self.get_all_notification_counts_for_today(service.id)
+        sent_thisyear = self.get_all_notification_counts_for_year(service.id, datetime.now().year)
+
+        limit_stats = {
+            "email": {
+                "annual": {
+                    "limit": service.email_annual_limit,
+                    "sent": sent_thisyear["email"],
+                    "remaining": service.email_annual_limit - sent_thisyear["email"],
+                },
+                "daily": {
+                    "limit": service.message_limit,
+                    "sent": sent_today["email"],
+                    "remaining": service.message_limit - sent_today["email"],
+                },
+            },
+            "sms": {
+                "annual": {
+                    "limit": service.sms_annual_limit,
+                    "sent": sent_thisyear["sms"],
+                    "remaining": service.sms_annual_limit - sent_thisyear["sms"],
+                },
+                "daily": {
+                    "limit": service.sms_daily_limit,
+                    "sent": sent_today["sms"],
+                    "remaining": service.sms_daily_limit - sent_today["sms"],
+                },
+            },
+        }
+
+        return limit_stats
+
 
 # TODO: consolidate this function and other functions that transform the results of template_statistics_client calls
 def _aggregate_notifications_stats(template_statistics):

From d84ba18f92106ab0b944f757795c2c21383d4e85 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Wed, 4 Dec 2024 18:33:25 +0000
Subject: [PATCH 19/28] feat(remaining messages summary): add heading mode; fix
 missing translations

---
 .../components/remaining-messages-summary.html       | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/templates/components/remaining-messages-summary.html b/app/templates/components/remaining-messages-summary.html
index 42175d47fd..b685e03aff 100644
--- a/app/templates/components/remaining-messages-summary.html
+++ b/app/templates/components/remaining-messages-summary.html
@@ -1,4 +1,4 @@
-{% macro remaining_messages_summary(dailyLimit, dailyUsed, yearlyLimit, yearlyUsed, notification_type, textOnly=None) %}
+{% macro remaining_messages_summary(dailyLimit, dailyUsed, yearlyLimit, yearlyUsed, notification_type, headingMode=False, textOnly=None) %}
   <!-- Validate textOnly param -->
   {% set textOnly_allowed_values = ['text', 'emoji'] %}
   {% if textOnly not in textOnly_allowed_values %}
@@ -95,14 +95,14 @@
     </div>
   {% endif %}
 
-  {% if sections[0].skip %}
+  {% if not headingMode and sections[0].skip %}
     <p class="mt-4 pl-10 py-4 border-l-4 border-gray-300" data-testid="yearly-sending-paused">
-      Sending paused until annual limit resets
+      {{ _('Sending paused until annual limit resets') }}
     </p>
-  {% elif sections[0].remaining == "0" %}
+  {% elif not headingMode and sections[0].remaining == "0" %}
     <p class="mt-4 pl-10 py-4 border-l-4 border-gray-300" data-testid="daily-sending-paused">
-      Sending paused until 7pm ET. You can schedule more messages to send later.
+      {{ _('Sending paused until 7pm ET. You can schedule more messages to send later.') }}      
     </p>
   {% endif %}
   
-{% endmacro %}
+{% endmacro %}
\ No newline at end of file

From 8a5f916574230924240d6a91a6406210de215c27 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Wed, 4 Dec 2024 18:33:56 +0000
Subject: [PATCH 20/28] feat(ready to send): show RMS componont on page,
 prevent users from moving forward if service is at either limit

---
 app/main/views/templates.py | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/app/main/views/templates.py b/app/main/views/templates.py
index 6c7f8a9dce..d099e2cff1 100644
--- a/app/main/views/templates.py
+++ b/app/main/views/templates.py
@@ -57,6 +57,7 @@
     TemplateList,
     TemplateLists,
 )
+from app.notify_client.notification_counts_client import notification_counts_client
 from app.template_previews import TemplatePreview, get_page_count_for_letter
 from app.utils import (
     email_or_sms_not_enabled,
@@ -134,6 +135,26 @@ def view_template(service_id, template_id):
 
     user_has_template_permission = current_user.has_template_folder_permission(template_folder)
 
+    # get the limit stats for the current service
+    limit_stats = notification_counts_client.get_limit_stats(current_service)
+
+    # transform the stats into a format that can be used in the template
+    notification_type = template["template_type"]
+    dailyLimit = limit_stats[notification_type]["daily"]["limit"]
+    dailyUsed = limit_stats[notification_type]["daily"]["sent"]
+    dailyRemaining = limit_stats[notification_type]["daily"]["remaining"]
+    yearlyLimit = limit_stats[notification_type]["annual"]["limit"]
+    yearlyUsed = limit_stats[notification_type]["annual"]["sent"]
+    yearlyRemaining = limit_stats[notification_type]["annual"]["remaining"]
+
+    # determine ready to send heading
+    if yearlyRemaining == 0:
+        heading = _("Sending paused until annual limit resets")
+    elif dailyRemaining == 0:
+        heading = _("Sending paused until 7pm ET. You can schedule more messages to send later.")
+    else:
+        heading = _("Ready to send?")
+
     if should_skip_template_page(template["template_type"]):
         return redirect(url_for(".send_one_off", service_id=service_id, template_id=template_id))
 
@@ -142,6 +163,14 @@ def view_template(service_id, template_id):
         template=get_email_preview_template(template, template_id, service_id),
         template_postage=template["postage"],
         user_has_template_permission=user_has_template_permission,
+        dailyLimit=dailyLimit,
+        dailyUsed=dailyUsed,
+        yearlyLimit=yearlyLimit,
+        yearlyUsed=yearlyUsed,
+        notification_type=notification_type,
+        dailyRemaining=dailyRemaining,
+        yearlyRemaining=yearlyRemaining,
+        heading=heading,
     )
 
 

From 2d5e949c8cb30062c9227c2f5fb49d8de723a352 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Wed, 4 Dec 2024 18:34:16 +0000
Subject: [PATCH 21/28] feat(ready to send): update view template

---
 app/templates/views/templates/_template.html | 33 +++++++++++++-------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/app/templates/views/templates/_template.html b/app/templates/views/templates/_template.html
index 86c19a127e..038db7fb62 100644
--- a/app/templates/views/templates/_template.html
+++ b/app/templates/views/templates/_template.html
@@ -1,4 +1,13 @@
 {% from 'components/message-count-label.html' import message_count_label %}
+{% from 'components/remaining-messages-summary.html' import remaining_messages_summary with context %}
+
+
+<!-- dailyLimit: {{ dailyLimit }} <br />
+dailyUsed: {{ dailyUsed }} <br />
+yearlyLimit: {{ yearlyLimit }} <br />
+yearlyUsed: {{ yearlyUsed }} <br />
+dailyRemaining : {{ dailyRemaining }} <br />
+yearlyRemaining : {{ yearlyRemaining }} <br /> -->
 
 <div>
   {% if template._template.archived %}
@@ -13,20 +22,22 @@
     </p>
   {% else %}
     {% if current_user.has_permissions('send_messages', restrict_admin_usage=True) %}
-    <h2 class="heading-medium">{{ _('Ready to send?') }}</h2>
+      <h2 class="heading-medium">{{ heading }}</h2>
+
+      {{ remaining_messages_summary(dailyLimit, dailyUsed, yearlyLimit, yearlyUsed, notification_type, yearlyRemaining == 0 or dailyRemaining == 0) }}
 
-    <div class="mb-12">
-      <a href="{{ url_for('.add_recipients', service_id=current_service.id, template_id=template.id) }}" class="button" data-testid="add-recipients">
-        {{ _('Yes, add recipients') }}
-      </a>
-      <a href="{{ url_for('.send_test', service_id=current_service.id, template_id=template.id) }}" class="button button-secondary">{{ _('No, send yourself this message') }}</a>
-    </div>
+      {% if yearlyRemaining > 0 and dailyRemaining > 0 %}
+        <div class="mb-12 mt-12">
+          <a href="{{ url_for('.add_recipients', service_id=current_service.id, template_id=template.id) }}" class="button" data-testid="add-recipients">
+            {{ _('Yes, add recipients') }}
+          </a>
+          <a href="{{ url_for('.send_test', service_id=current_service.id, template_id=template.id) }}" class="button button-secondary">{{ _('No, send yourself this message') }}</a>
+        </div>
+      {% endif %}
     {% endif %}
   {% endif %}
 </div>
 
-<div class="w-full template-container">
+<div class="w-full template-container mt-10">
   {{ template|string|translate_preview_template }}
-</div>
-
-
+</div>
\ No newline at end of file

From 80e3cd31600bccb595d79ac41afab4bce917ee17 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Wed, 4 Dec 2024 18:34:50 +0000
Subject: [PATCH 22/28] feat(annual error mesage email): refactor page a bit,
 add case for remaining = 0

---
 .../check/too-many-email-messages-annual.html   | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/app/templates/partials/check/too-many-email-messages-annual.html b/app/templates/partials/check/too-many-email-messages-annual.html
index de30481845..f9b3abd6f6 100644
--- a/app/templates/partials/check/too-many-email-messages-annual.html
+++ b/app/templates/partials/check/too-many-email-messages-annual.html
@@ -1,13 +1,18 @@
 {% from "components/links.html" import content_link %}
 
-<p>
+<p data-testid="exceeds-annual">
   {%- if current_service.trial_mode %}
     {{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
   {% else %}
-    <p data-testid="exceeds-annual">{{ _('<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets'.format(current_service.name, recipients_remaining_messages)) }}</p>
-    <p>
-      {{ _('To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. '.format(recipients_remaining_messages)) }}
-      {{ _('To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.'.format(now().year)) }}
-    </p>
+    {% if recipients_remaining_messages > 0 %}
+      <p>{{ _('<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets'.format(current_service.name, recipients_remaining_messages)) }}</p>
+      <p>
+        {{ _('To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. '.format(recipients_remaining_messages)) }}
+        {{ _('To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.'.format(now().year)) }}
+      </p>
+    {% else %}
+      <p>{{ _('<strong>{}</strong> cannot send any more email messages until <strong>April 1, {}</strong>'.format(current_service.name, now().year)) }}</p>
+      <p>{{ _('For more information, visit the <a href={}>usage report for {}</a>.'.format(url_for('.monthly', service_id=current_service.id), current_service.name)) }}</p>
+    {% endif %}
   {%- endif -%}
 </p>
\ No newline at end of file

From f0ed140eefe7a4a9c9c4ace07d62ce229bebe4ed Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 5 Dec 2024 15:04:05 +0000
Subject: [PATCH 23/28] fix(notifications_count_client): cast bytes to int (why
 are they stored as bytes?)

---
 app/notify_client/notification_counts_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/notify_client/notification_counts_client.py b/app/notify_client/notification_counts_client.py
index cd1c6b7781..0d04291450 100644
--- a/app/notify_client/notification_counts_client.py
+++ b/app/notify_client/notification_counts_client.py
@@ -12,8 +12,8 @@
 class NotificationCounts:
     def get_all_notification_counts_for_today(self, service_id):
         # try to get today's stats from redis
-        todays_sms = redis_client.get(sms_daily_count_cache_key(service_id))
-        todays_email = redis_client.get(email_daily_count_cache_key(service_id))
+        todays_sms = int(redis_client.get(sms_daily_count_cache_key(service_id)))
+        todays_email = int(redis_client.get(email_daily_count_cache_key(service_id)))
 
         if todays_sms is not None and todays_email is not None:
             return {"sms": todays_sms, "email": todays_email}

From 091e2ee512666fd26840c7906dd0d36758803ae9 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 5 Dec 2024 15:06:31 +0000
Subject: [PATCH 24/28] fix: refactor error message views to make them
 re-usable

---
 .../partials/check/too-many-email-messages.html        |  2 +-
 ...sages-annual.html => too-many-messages-annual.html} | 10 ++++++++--
 app/templates/views/check/column-errors.html           |  2 +-
 app/templates/views/notifications/check.html           |  7 ++++---
 4 files changed, 14 insertions(+), 7 deletions(-)
 rename app/templates/partials/check/{too-many-email-messages-annual.html => too-many-messages-annual.html} (70%)

diff --git a/app/templates/partials/check/too-many-email-messages.html b/app/templates/partials/check/too-many-email-messages.html
index 987846c472..9f66acd9c3 100644
--- a/app/templates/partials/check/too-many-email-messages.html
+++ b/app/templates/partials/check/too-many-email-messages.html
@@ -1,6 +1,6 @@
 {% from "components/links.html" import content_link %}
 
-<p>
+<p data-testid="exceeds-daily">
   {%- if current_service.trial_mode %}
     {{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
   {% else %}
diff --git a/app/templates/partials/check/too-many-email-messages-annual.html b/app/templates/partials/check/too-many-messages-annual.html
similarity index 70%
rename from app/templates/partials/check/too-many-email-messages-annual.html
rename to app/templates/partials/check/too-many-messages-annual.html
index f9b3abd6f6..86e4cb3ae9 100644
--- a/app/templates/partials/check/too-many-email-messages-annual.html
+++ b/app/templates/partials/check/too-many-messages-annual.html
@@ -1,17 +1,23 @@
 {% from "components/links.html" import content_link %}
 
+{% if template.template_type == 'email' %}
+  {% set units = _('email messages') %}
+{% else %}
+  {% set units = _('text messages') %}
+{% endif %}
+
 <p data-testid="exceeds-annual">
   {%- if current_service.trial_mode %}
     {{ _("Your service is in trial mode. To send more messages, <a href='{}'>request to go live</a>").format(url_for('main.request_to_go_live', service_id=current_service.id)) }}
   {% else %}
     {% if recipients_remaining_messages > 0 %}
-      <p>{{ _('<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets'.format(current_service.name, recipients_remaining_messages)) }}</p>
+      <p>{{ _('<strong>{}</strong> can only send <strong>{}</strong> more {} until annual limit resets'.format(current_service.name, recipients_remaining_messages, units)) }}</p>
       <p>
         {{ _('To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. '.format(recipients_remaining_messages)) }}
         {{ _('To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.'.format(now().year)) }}
       </p>
     {% else %}
-      <p>{{ _('<strong>{}</strong> cannot send any more email messages until <strong>April 1, {}</strong>'.format(current_service.name, now().year)) }}</p>
+      <p>{{ _('<strong>{}</strong> cannot send any more {} until <strong>April 1, {}</strong>'.format(current_service.name, units, now().year)) }}</p>
       <p>{{ _('For more information, visit the <a href={}>usage report for {}</a>.'.format(url_for('.monthly', service_id=current_service.id), current_service.name)) }}</p>
     {% endif %}
   {%- endif -%}
diff --git a/app/templates/views/check/column-errors.html b/app/templates/views/check/column-errors.html
index 0b9cecaafe..1e46c28333 100644
--- a/app/templates/views/check/column-errors.html
+++ b/app/templates/views/check/column-errors.html
@@ -168,7 +168,7 @@ <h2 class="heading-medium">{{ _('You cannot send all these text messages today')
           </p>
       {% elif send_exceeds_annual_limit %}
         {% call banner_wrapper(type='dangerous') %}
-          {% include "partials/check/too-many-email-messages-annual.html" %}
+          {% include "partials/check/too-many-messages-annual.html" %}
         {% endcall %}
       {% elif recipients.more_rows_than_can_send %}
         {% call banner_wrapper(type='dangerous') %}
diff --git a/app/templates/views/notifications/check.html b/app/templates/views/notifications/check.html
index 089e19a951..0161a74f17 100644
--- a/app/templates/views/notifications/check.html
+++ b/app/templates/views/notifications/check.html
@@ -66,11 +66,12 @@ <h2 class="heading-medium">{{_('You cannot send this email message today') }}</h
         content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}
       </p>
     </div>
-  {% elif error == 'too-many-email-messages-annual' %}
-    {{ page_header(_('This message exceeds your annual limit'), back_link=back_link) }}
+  {% elif error == 'too-many-email-annual' or error == 'too-many-sms-annual' %}
+    {{ page_header(_('These messages exceed the annual limit'), back_link=back_link) }}
     <div class="mb-12 clear-both contain-floats">
       {% call banner_wrapper(type='dangerous') %}
-        {% include "partials/check/too-many-email-messages-annual.html" %}
+        {% set recipients_remaining_messages = 0 %}
+        {% include "partials/check/too-many-messages-annual.html" %}
       {% endcall %}
   {% elif error == 'message-too-long' %}
     {# the only row_errors we can get when sending one off messages is that the message is too long #}

From 87f465e9c22c79243a4abf86b772fc8486d6dc87 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 5 Dec 2024 15:06:46 +0000
Subject: [PATCH 25/28] chore: translations

---
 app/translations/csv/fr.csv | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/app/translations/csv/fr.csv b/app/translations/csv/fr.csv
index fe2c41b635..d6af104602 100644
--- a/app/translations/csv/fr.csv
+++ b/app/translations/csv/fr.csv
@@ -2028,7 +2028,13 @@
 "resets at 7pm Eastern Time","Réinitialisation à 19 h, heure de l’Est"
 "Visit usage report","Consulter le rapport d’utilisation"
 "Month by month totals","Totaux mensuels"
-"<strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets","FR: <strong>{}</strong> can only send <strong>{}</strong> more email messages until annual limit resets"
+"<strong>{}</strong> can only send <strong>{}</strong> more {} until annual limit resets","FR: <strong>{}</strong> can only send <strong>{}</strong> more {} until annual limit resets"
 "To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. ","FR: To send some of these messages now, edit the spreadsheet to <strong>{}</strong> recipients maximum. "
 "To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way.","FR: To send to recipients you removed, wait until <strong>April 1, {}</strong> or contact them some other way"
-"These messages exceed the annual limit","FR: These messages exceed the annual limit"
\ No newline at end of file
+"These messages exceed the annual limit","FR: These messages exceed the annual limit"
+"<strong>{}</strong> cannot send any more {} until <strong>April 1, {}</strong>","FR: <strong>{}</strong> cannot send any more {} until <strong>April 1, {}</strong>"
+"For more information, visit the <a href={}>usage report for {}</a>.","FR: For more information, visit the <a href={}>usage report for {}</a>."
+"This message exceeds your annual limit", "FR: This message exceeds your annual limit"
+"email messages","courriels"
+"Sending paused until annual limit resets","FR: Sending paused until annual limit resets"
+"Sending paused until 7pm ET. You can schedule more messages to send later.","FR: Sending paused until 7pm ET. You can schedule more messages to send later."
\ No newline at end of file

From 31780b4b498946f6c1aa231d76baa3b378b5803e Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 5 Dec 2024 15:07:15 +0000
Subject: [PATCH 26/28] tests: add tests for new send code, template code, and
 changes to notification_counts_client

---
 tests/app/main/views/test_send.py             | 129 ++++++++++++++++--
 tests/app/main/views/test_templates.py        |  69 +++++++++-
 .../test_notification_counts_client.py        | 112 ++++++++++++++-
 3 files changed, 297 insertions(+), 13 deletions(-)

diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py
index c1350bdb46..5967046fd4 100644
--- a/tests/app/main/views/test_send.py
+++ b/tests/app/main/views/test_send.py
@@ -3493,10 +3493,21 @@ def test_email_send_fails_approrpiately_when_over_limits(
                 ),
             )
 
-            # mock that `num_sent_this_year` have already been sent this year
-            mock_notification_counts_client.get_all_notification_counts_for_year.return_value = {
-                "sms": 1000,  # not used in test but needs a value
-                "email": num_sent_this_year,
+            mock_notification_counts_client.get_limit_stats.return_value = {
+                "email": {
+                    "annual": {
+                        "limit": 1,  # doesn't matter for our test
+                        "sent": 1,  # doesn't matter for our test
+                        "remaining": 10000
+                        - num_sent_this_year
+                        - num_sent_today,  # The number of email notifications remaining this year
+                    },
+                    "daily": {
+                        "limit": 1,  # doesn't matter for our test
+                        "sent": 1,  # doesn't matter for our test
+                        "remaining": 1000 - num_sent_today,  # The number of email notifications remaining today
+                    },
+                }
             }
 
             # mock that we've already sent `emails_sent_today` emails today
@@ -3584,14 +3595,23 @@ def test_sms_send_fails_approrpiately_when_over_limits(
                     ["phone number"] + ([mock_get_users_by_service(None)[0]["mobile_number"]] * num_being_sent)
                 ),
             )
-
-            # mock that `num_sent_this_year` have already been sent this year
-            mock_notification_counts_client.get_all_notification_counts_for_year.return_value = {
-                "sms": num_sent_this_year,
-                "email": 1000,  # not used in test but needs a value
+            mock_notification_counts_client.get_limit_stats.return_value = {
+                "sms": {
+                    "annual": {
+                        "limit": 1,  # doesn't matter for our test
+                        "sent": 1,  # doesn't matter for our test
+                        "remaining": 10000
+                        - num_sent_this_year
+                        - num_sent_today,  # The number of email notifications remaining this year
+                    },
+                    "daily": {
+                        "limit": 1,  # doesn't matter for our test
+                        "sent": 1,  # doesn't matter for our test
+                        "remaining": 1000 - num_sent_today,  # The number of email notifications remaining today
+                    },
+                }
             }
-
-            # mock that we've already sent `emails_sent_today` emails today
+            # mock that we've already sent `num_sent_today` emails today
             mock_daily_email_count.return_value = 900  # not used in test but needs a value
             mock_daily_sms_fragment_count.return_value = num_sent_today
 
@@ -3622,3 +3642,90 @@ def test_sms_send_fails_approrpiately_when_over_limits(
                 assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None
             else:
                 assert page.find(attrs={"data-testid": "exceeds-daily"}) is None
+
+    @pytest.mark.parametrize(
+        "num_to_send, remaining_daily, remaining_annual, error_shown",
+        [
+            (2, 2, 2, "none"),
+            (5, 5, 4, "annual"),
+            (5, 4, 5, "daily"),
+            (5, 4, 4, "annual"),
+        ],
+    )
+    def test_correct_error_displayed(
+        self,
+        mocker,
+        client_request,
+        mock_get_live_service,  # set email_annual_limit and sms_annual_limit to 1000
+        mock_get_users_by_service,
+        mock_get_service_email_template_without_placeholders,
+        mock_get_template_statistics,
+        mock_get_job_doesnt_exist,
+        mock_get_jobs,
+        mock_s3_set_metadata,
+        mock_daily_email_count,
+        mock_notification_counts_client,
+        fake_uuid,
+        num_to_send,
+        remaining_daily,
+        remaining_annual,
+        error_shown,
+        app_,
+    ):
+        with set_config(app_, "FF_ANNUAL_LIMIT", True):  # REMOVE LINE WHEN FF REMOVED
+            # mock that `num_sent_this_year` have already been sent this year
+            mock_notification_counts_client.get_limit_stats.return_value = {
+                "email": {
+                    "annual": {
+                        "limit": 1,  # doesn't matter for our test
+                        "sent": 1,  # doesn't matter for our test
+                        "remaining": remaining_annual,  # The number of email notifications remaining this year
+                    },
+                    "daily": {
+                        "limit": 1,  # doesn't matter for our test
+                        "sent": 1,  # doesn't matter for our test
+                        "remaining": remaining_daily,  # The number of email notifications remaining today
+                    },
+                }
+            }
+
+            # only change this value when we're expecting an error
+            if error_shown != "none":
+                mock_daily_email_count.return_value = 1000 - (
+                    num_to_send - 1
+                )  # svc limit is 1000 - exceeding the daily limit is calculated based off of this
+            else:
+                mock_daily_email_count.return_value = 0  # none sent
+
+            mocker.patch(
+                "app.main.views.send.s3download",
+                return_value=",\n".join(
+                    ["email address"] + ([mock_get_users_by_service(None)[0]["email_address"]] * num_to_send)
+                ),
+            )
+            with client_request.session_transaction() as session:
+                session["file_uploads"] = {
+                    fake_uuid: {
+                        "template_id": fake_uuid,
+                        "notification_count": 1,
+                        "valid": True,
+                    }
+                }
+            page = client_request.get(
+                "main.check_messages",
+                service_id=SERVICE_ONE_ID,
+                template_id=fake_uuid,
+                upload_id=fake_uuid,
+                original_file_name="valid.csv",
+                _test_page_title=False,
+            )
+
+            if error_shown == "annual":
+                assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None
+                assert page.find(attrs={"data-testid": "exceeds-daily"}) is None
+            elif error_shown == "daily":
+                assert page.find(attrs={"data-testid": "exceeds-annual"}) is None
+                assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None
+            elif error_shown == "none":
+                assert page.find(attrs={"data-testid": "exceeds-annual"}) is None
+                assert page.find(attrs={"data-testid": "exceeds-daily"}) is None
diff --git a/tests/app/main/views/test_templates.py b/tests/app/main/views/test_templates.py
index b48234af1f..54df21e58f 100644
--- a/tests/app/main/views/test_templates.py
+++ b/tests/app/main/views/test_templates.py
@@ -1,6 +1,6 @@
 from datetime import datetime
 from functools import partial
-from unittest.mock import ANY, MagicMock, Mock
+from unittest.mock import ANY, MagicMock, Mock, patch
 
 import pytest
 from flask import url_for
@@ -54,6 +54,12 @@
 DEFAULT_PROCESS_TYPE = TemplateProcessTypes.BULK.value
 
 
+@pytest.fixture
+def mock_notification_counts_client():
+    with patch("app.main.views.templates.notification_counts_client") as mock:
+        yield mock
+
+
 class TestRedisPreviewUtilities:
     def test_set_get(self, fake_uuid, mocker):
         mock_redis_obj = MockRedis()
@@ -2742,3 +2748,64 @@ def test_should_hide_category_name_from_template_list_if_marked_hidden(
     # assert that "HIDDEN_CATEGORY" is not found anywhere in the page using beautifulsoup
     assert "HIDDEN_CATEGORY" not in page.text
     assert not page.find(text="HIDDEN_CATEGORY")
+
+
+class TestAnnualLimits:
+    @pytest.mark.parametrize(
+        "remaining_daily, remaining_annual, buttons_shown",
+        [
+            (10, 100, True),  # Within both limits
+            (0, 100, False),  # Exceeds daily limit
+            (10, 0, False),  # Exceeds annual limit
+            (0, 0, False),  # Exceeds both limits
+            (1, 1, True),  # Exactly at both limits
+        ],
+    )
+    def test_should_hide_send_buttons_when_appropriate(
+        self,
+        client_request,
+        mock_get_service_template,
+        mock_get_template_folders,
+        mock_notification_counts_client,
+        fake_uuid,
+        remaining_daily,
+        remaining_annual,
+        buttons_shown,
+    ):
+        mock_notification_counts_client.get_limit_stats.return_value = {
+            "email": {
+                "annual": {
+                    "limit": 1,  # doesn't matter for our test
+                    "sent": 1,  # doesn't matter for our test
+                    "remaining": remaining_annual,  # The number of email notifications remaining this year
+                },
+                "daily": {
+                    "limit": 1,  # doesn't matter for our test
+                    "sent": 1,  # doesn't matter for our test
+                    "remaining": remaining_daily,  # The number of email notifications remaining today
+                },
+            },
+            "sms": {
+                "annual": {
+                    "limit": 1,  # doesn't matter for our test
+                    "sent": 1,  # doesn't matter for our test
+                    "remaining": remaining_annual,  # The number of email notifications remaining this year
+                },
+                "daily": {
+                    "limit": 1,  # doesn't matter for our test
+                    "sent": 1,  # doesn't matter for our test
+                    "remaining": remaining_daily,  # The number of email notifications remaining today
+                },
+            },
+        }
+
+        page = client_request.get(
+            ".view_template",
+            service_id=SERVICE_ONE_ID,
+            template_id=fake_uuid,
+            _test_page_title=False,
+        )
+        if buttons_shown:
+            assert page.find(attrs={"data-testid": "send-buttons"}) is not None
+        else:
+            assert page.find(attrs={"data-testid": "send-buttons"}) is None
diff --git a/tests/app/notify_client/test_notification_counts_client.py b/tests/app/notify_client/test_notification_counts_client.py
index fa7183aea8..342aaa2917 100644
--- a/tests/app/notify_client/test_notification_counts_client.py
+++ b/tests/app/notify_client/test_notification_counts_client.py
@@ -1,4 +1,5 @@
-from unittest.mock import patch
+from datetime import datetime
+from unittest.mock import Mock, patch
 
 import pytest
 
@@ -23,6 +24,12 @@ def mock_service_api():
         yield mock
 
 
+@pytest.fixture
+def mock_get_all_notification_counts_for_today():
+    with patch("app.notify_client.notification_counts_client.get_all_notification_counts_for_today") as mock:
+        yield mock
+
+
 class TestNotificationCounts:
     def test_get_all_notification_counts_for_today_redis_has_data(self, mock_redis):
         # Setup
@@ -87,3 +94,106 @@ def test_get_all_notification_counts_for_year(self, mock_service_api):
             # Assert
             assert result["sms"] == 29  # 1 + 22 + 1 + 5
             assert result["email"] == 21  # 1 + 1 + 12 + 1 + 1 + 5
+
+    def test_get_limit_stats(self, mocker):
+        # Setup
+        mock_service = Mock(id="service-1", email_annual_limit=1000, sms_annual_limit=500, message_limit=100, sms_daily_limit=50)
+
+        mock_notification_client = NotificationCounts()
+
+        # Mock the dependency methods
+
+        mocker.patch.object(
+            mock_notification_client, "get_all_notification_counts_for_today", return_value={"email": 20, "sms": 10}
+        )
+        mocker.patch.object(
+            mock_notification_client, "get_all_notification_counts_for_year", return_value={"email": 200, "sms": 100}
+        )
+
+        # Execute
+        result = mock_notification_client.get_limit_stats(mock_service)
+
+        # Assert
+        assert result == {
+            "email": {
+                "annual": {
+                    "limit": 1000,
+                    "sent": 200,
+                    "remaining": 800,
+                },
+                "daily": {
+                    "limit": 100,
+                    "sent": 20,
+                    "remaining": 80,
+                },
+            },
+            "sms": {
+                "annual": {
+                    "limit": 500,
+                    "sent": 100,
+                    "remaining": 400,
+                },
+                "daily": {
+                    "limit": 50,
+                    "sent": 10,
+                    "remaining": 40,
+                },
+            },
+        }
+
+    @pytest.mark.parametrize(
+        "today_counts,year_counts,expected_remaining",
+        [
+            (
+                {"email": 0, "sms": 0},
+                {"email": 0, "sms": 0},
+                {"email": {"annual": 1000, "daily": 100}, "sms": {"annual": 500, "daily": 50}},
+            ),
+            (
+                {"email": 100, "sms": 50},
+                {"email": 1000, "sms": 500},
+                {"email": {"annual": 0, "daily": 0}, "sms": {"annual": 0, "daily": 0}},
+            ),
+            (
+                {"email": 50, "sms": 25},
+                {"email": 500, "sms": 250},
+                {"email": {"annual": 500, "daily": 50}, "sms": {"annual": 250, "daily": 25}},
+            ),
+        ],
+    )
+    def test_get_limit_stats_remaining_calculations(self, mocker, today_counts, year_counts, expected_remaining):
+        # Setup
+        mock_service = Mock(id="service-1", email_annual_limit=1000, sms_annual_limit=500, message_limit=100, sms_daily_limit=50)
+
+        mock_notification_client = NotificationCounts()
+
+        mocker.patch.object(mock_notification_client, "get_all_notification_counts_for_today", return_value=today_counts)
+        mocker.patch.object(mock_notification_client, "get_all_notification_counts_for_year", return_value=year_counts)
+
+        # Execute
+        result = mock_notification_client.get_limit_stats(mock_service)
+
+        # Assert remaining counts
+        assert result["email"]["annual"]["remaining"] == expected_remaining["email"]["annual"]
+        assert result["email"]["daily"]["remaining"] == expected_remaining["email"]["daily"]
+        assert result["sms"]["annual"]["remaining"] == expected_remaining["sms"]["annual"]
+        assert result["sms"]["daily"]["remaining"] == expected_remaining["sms"]["daily"]
+
+    def test_get_limit_stats_dependencies_called(self, mocker):
+        # Setup
+        mock_service = Mock(id="service-1", email_annual_limit=1000, sms_annual_limit=500, message_limit=100, sms_daily_limit=50)
+        mock_notification_client = NotificationCounts()
+
+        mock_today = mocker.patch.object(
+            mock_notification_client, "get_all_notification_counts_for_today", return_value={"email": 0, "sms": 0}
+        )
+        mock_year = mocker.patch.object(
+            mock_notification_client, "get_all_notification_counts_for_year", return_value={"email": 0, "sms": 0}
+        )
+
+        # Execute
+        mock_notification_client.get_limit_stats(mock_service)
+
+        # Assert dependencies called
+        mock_today.assert_called_once_with(mock_service.id)
+        mock_year.assert_called_once_with(mock_service.id, datetime.now().year)

From e3b67c121a547b2d2b90c034fd8369a9dba55ddb Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 5 Dec 2024 15:07:57 +0000
Subject: [PATCH 27/28] chore: fix translation

---
 app/translations/csv/fr.csv | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/translations/csv/fr.csv b/app/translations/csv/fr.csv
index d6af104602..be9346b1c1 100644
--- a/app/translations/csv/fr.csv
+++ b/app/translations/csv/fr.csv
@@ -2034,7 +2034,7 @@
 "These messages exceed the annual limit","FR: These messages exceed the annual limit"
 "<strong>{}</strong> cannot send any more {} until <strong>April 1, {}</strong>","FR: <strong>{}</strong> cannot send any more {} until <strong>April 1, {}</strong>"
 "For more information, visit the <a href={}>usage report for {}</a>.","FR: For more information, visit the <a href={}>usage report for {}</a>."
-"This message exceeds your annual limit", "FR: This message exceeds your annual limit"
+"This message exceeds your annual limit","FR: This message exceeds your annual limit"
 "email messages","courriels"
 "Sending paused until annual limit resets","FR: Sending paused until annual limit resets"
 "Sending paused until 7pm ET. You can schedule more messages to send later.","FR: Sending paused until 7pm ET. You can schedule more messages to send later."
\ No newline at end of file

From 0aeecf77f812488552d5ee9d5f90287501263b45 Mon Sep 17 00:00:00 2001
From: Andrew Leith <andrew.leith@cds-snc.ca>
Date: Thu, 5 Dec 2024 15:34:48 +0000
Subject: [PATCH 28/28] chore: remove dupe translation due to merge

---
 app/translations/csv/fr.csv | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/translations/csv/fr.csv b/app/translations/csv/fr.csv
index 2a4de35570..6b2b5dc2cb 100644
--- a/app/translations/csv/fr.csv
+++ b/app/translations/csv/fr.csv
@@ -2038,4 +2038,3 @@
 "email messages","courriels"
 "Sending paused until annual limit resets","FR: Sending paused until annual limit resets"
 "Sending paused until 7pm ET. You can schedule more messages to send later.","FR: Sending paused until 7pm ET. You can schedule more messages to send later."
-"Month by month totals","Totaux mensuels"