From f53833222a70e603010f05ab5842d5fa609a4681 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 25 Nov 2024 10:04:20 -0400 Subject: [PATCH 01/20] Revert 1989: Add annual limits to dashboard and usage reports (#2000) * annual-limits-error-msg * chore: fix poetry.lock --- app/config.py | 4 +- app/extensions.py | 3 - app/main/views/dashboard.py | 98 +------ app/notify_client/service_api_client.py | 27 +- .../views/dashboard/_totals_annual.html | 25 -- .../views/dashboard/_totals_daily.html | 26 +- app/templates/views/dashboard/dashboard.html | 5 +- app/templates/views/dashboard/monthly.html | 53 +--- app/translations/csv/fr.csv | 9 - poetry.lock | 8 +- pyproject.toml | 2 +- tests/app/main/views/test_dashboard.py | 253 +++--------------- tests/conftest.py | 33 --- 13 files changed, 60 insertions(+), 486 deletions(-) delete mode 100644 app/templates/views/dashboard/_totals_annual.html diff --git a/app/config.py b/app/config.py index 9b37a55798..0d2d7ed1f1 100644 --- a/app/config.py +++ b/app/config.py @@ -80,7 +80,7 @@ class Config(object): # FEATURE FLAGS FF_SALESFORCE_CONTACT = env.bool("FF_SALESFORCE_CONTACT", True) FF_RTL = env.bool("FF_RTL", True) - FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", False) + FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", True) FREE_YEARLY_EMAIL_LIMIT = env.int("FREE_YEARLY_EMAIL_LIMIT", 20_000_000) FREE_YEARLY_SMS_LIMIT = env.int("FREE_YEARLY_SMS_LIMIT", 100_000) @@ -215,7 +215,7 @@ class Test(Development): NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef" FF_RTL = True - FF_ANNUAL_LIMIT = True + FF_ANNUAL_LIMIT = False class ProductionFF(Config): diff --git a/app/extensions.py b/app/extensions.py index 51bebe6e38..8421b1e560 100644 --- a/app/extensions.py +++ b/app/extensions.py @@ -1,6 +1,5 @@ from flask_caching import Cache from notifications_utils.clients.antivirus.antivirus_client import AntivirusClient -from notifications_utils.clients.redis.annual_limit import RedisAnnualLimit from notifications_utils.clients.redis.bounce_rate import RedisBounceRate from notifications_utils.clients.redis.redis_client import RedisClient from notifications_utils.clients.statsd.statsd_client import StatsdClient @@ -11,6 +10,4 @@ zendesk_client = ZendeskClient() redis_client = RedisClient() bounce_rate_client = RedisBounceRate(redis_client) -annual_limit_client = RedisAnnualLimit(redis_client) - cache = Cache(config={"CACHE_TYPE": "simple"}) # TODO: pull config out to config.py later diff --git a/app/main/views/dashboard.py b/app/main/views/dashboard.py index 5a3414a260..f1bd214bfe 100644 --- a/app/main/views/dashboard.py +++ b/app/main/views/dashboard.py @@ -24,7 +24,7 @@ service_api_client, template_statistics_client, ) -from app.extensions import annual_limit_client, bounce_rate_client +from app.extensions import bounce_rate_client from app.main import main from app.models.enum.bounce_rate_status import BounceRateStatus from app.models.enum.notification_statuses import NotificationStatuses @@ -229,83 +229,16 @@ def usage(service_id): @main.route("/services//monthly") @user_has_permissions("view_activity") def monthly(service_id): - def combine_daily_to_annual(daily, annual, mode): - if mode == "redis": - # the redis client omits properties if there are no counts yet, so account for this here\ - daily_redis = { - field: daily.get(field, 0) for field in ["sms_delivered", "sms_failed", "email_delivered", "email_failed"] - } - annual["sms"] += daily_redis["sms_delivered"] + daily_redis["sms_failed"] - annual["email"] += daily_redis["email_delivered"] + daily_redis["email_failed"] - elif mode == "db": - annual["sms"] += daily["sms"]["requested"] - annual["email"] += daily["email"]["requested"] - - return annual - - def combine_daily_to_monthly(daily, monthly, mode): - if mode == "redis": - # the redis client omits properties if there are no counts yet, so account for this here\ - daily_redis = { - field: daily.get(field, 0) for field in ["sms_delivered", "sms_failed", "email_delivered", "email_failed"] - } - - monthly[0]["sms_counts"]["failed"] += daily_redis["sms_failed"] - monthly[0]["sms_counts"]["requested"] += daily_redis["sms_failed"] + daily_redis["sms_delivered"] - monthly[0]["email_counts"]["failed"] += daily_redis["email_failed"] - monthly[0]["email_counts"]["requested"] += daily_redis["email_failed"] + daily_redis["email_delivered"] - elif mode == "db": - monthly[0]["sms_counts"]["failed"] += daily["sms"]["failed"] - monthly[0]["sms_counts"]["requested"] += daily["sms"]["requested"] - monthly[0]["email_counts"]["failed"] += daily["email"]["failed"] - monthly[0]["email_counts"]["requested"] += daily["email"]["requested"] - - return monthly - - def aggregate_by_type(notification_data): - counts = {"sms": 0, "email": 0, "letter": 0} - for month_data in notification_data["data"].values(): - for message_type, message_counts in month_data.items(): - if isinstance(message_counts, dict): - counts[message_type] += sum(message_counts.values()) - - # return the result - return counts - year, current_financial_year = requested_and_current_financial_year(request) - monthly_data = service_api_client.get_monthly_notification_stats(service_id, year) - annual_data = aggregate_by_type(monthly_data) - - todays_data = annual_limit_client.get_all_notification_counts(current_service.id) - - # if redis is empty, query the db - if todays_data is None: - todays_data = service_api_client.get_service_statistics(service_id, limit_days=1, today_only=False) - annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "db") - - months = (format_monthly_stats_to_list(monthly_data["data"]),) - monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "db") - else: - # aggregate daily + annual - current_app.logger.info("todays data" + str(todays_data)) - annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "redis") - - months = (format_monthly_stats_to_list(monthly_data["data"]),) - monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "redis") - - # add today's data to monthly data - return render_template( "views/dashboard/monthly.html", - months=monthly_data_aggregate, + months=format_monthly_stats_to_list(service_api_client.get_monthly_notification_stats(service_id, year)["data"]), years=get_tuples_of_financial_years( partial_url=partial(url_for, ".monthly", service_id=service_id), start=current_financial_year - 2, end=current_financial_year, ), - annual_data=annual_data_aggregate, selected_year=year, - current_financial_year=current_financial_year, ) @@ -351,21 +284,6 @@ def aggregate_notifications_stats(template_statistics): def get_dashboard_partials(service_id): - def aggregate_by_type(data, daily_data): - counts = {"sms": 0, "email": 0, "letter": 0} - # flatten out this structure to match the above - for month_data in data["data"].values(): - for message_type, message_counts in month_data.items(): - if isinstance(message_counts, dict): - counts[message_type] += sum(message_counts.values()) - - # add todays data to the annual data - counts = { - "sms": counts["sms"] + daily_data["sms"]["requested"], - "email": counts["email"] + daily_data["email"]["requested"], - } - return counts - all_statistics_weekly = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=7) template_statistics_weekly = aggregate_template_usage(all_statistics_weekly) @@ -382,10 +300,6 @@ def aggregate_by_type(data, daily_data): dashboard_totals_weekly = (get_dashboard_totals(stats_weekly),) bounce_rate_data = get_bounce_rate_data_from_redis(service_id) - # get annual data from fact table (all data this year except today) - annual_data = service_api_client.get_monthly_notification_stats(service_id, year=get_current_financial_year()) - annual_data = aggregate_by_type(annual_data, dashboard_totals_daily[0]) - return { "upcoming": render_template("views/dashboard/_upcoming.html", scheduled_jobs=scheduled_jobs), "daily_totals": render_template( @@ -394,13 +308,6 @@ def aggregate_by_type(data, daily_data): statistics=dashboard_totals_daily[0], column_width=column_width, ), - "annual_totals": render_template( - "views/dashboard/_totals_annual.html", - service_id=service_id, - statistics=dashboard_totals_daily[0], - statistics_annual=annual_data, - column_width=column_width, - ), "weekly_totals": render_template( "views/dashboard/_totals.html", service_id=service_id, @@ -422,7 +329,6 @@ def aggregate_by_type(data, daily_data): def _get_daily_stats(service_id): - # TODO: get from redis, else fallback to template_statistics_client.get_template_statistics_for_service all_statistics_daily = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1) stats_daily = aggregate_notifications_stats(all_statistics_daily) dashboard_totals_daily = (get_dashboard_totals(stats_daily),) diff --git a/app/notify_client/service_api_client.py b/app/notify_client/service_api_client.py index b1274337c7..6054c374d2 100644 --- a/app/notify_client/service_api_client.py +++ b/app/notify_client/service_api_client.py @@ -1,5 +1,5 @@ import json -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from flask import current_app from flask_login import current_user @@ -9,12 +9,6 @@ from app.notify_client import NotifyAdminAPIClient, _attach_current_user, cache -def _seconds_until_midnight(): - now = datetime.now(timezone.utc) - midnight = datetime.combine(now + timedelta(days=1), datetime.min.time()) - return int((midnight - now).total_seconds()) - - class ServiceAPIClient(NotifyAdminAPIClient): @cache.delete("user-{user_id}") def create_service( @@ -383,15 +377,8 @@ def is_service_email_from_unique(self, service_id, email_from): def get_service_history(self, service_id): return self.get("/service/{0}/history".format(service_id)) - # TODO: cache this once the backend is updated to exlude data from the current day - # @flask_cache.memoize(timeout=_seconds_until_midnight()) def get_monthly_notification_stats(self, service_id, year): - return self.get( - url="/service/{}/notifications/monthly?year={}".format( - service_id, - year, - ) - ) + return self.get(url="/service/{}/notifications/monthly?year={}".format(service_id, year)) def get_safelist(self, service_id): return self.get(url="/service/{}/safelist".format(service_id)) @@ -635,15 +622,5 @@ def _use_case_data_name(self, service_id): def _tos_key_name(self, service_id): return f"tos-accepted-{service_id}" - def aggregate_by_type(self, notification_data): - counts = {"sms": 0, "email": 0, "letter": 0} - for month_data in notification_data["data"].values(): - for message_type, message_counts in month_data.items(): - if isinstance(message_counts, dict): - counts[message_type] += sum(message_counts.values()) - - # return the result - return counts - service_api_client = ServiceAPIClient() diff --git a/app/templates/views/dashboard/_totals_annual.html b/app/templates/views/dashboard/_totals_annual.html deleted file mode 100644 index 4985ec80be..0000000000 --- a/app/templates/views/dashboard/_totals_annual.html +++ /dev/null @@ -1,25 +0,0 @@ -{% from "components/big-number.html" import big_number %} -{% from "components/message-count-label.html" import message_count_label %} -{% from 'components/remaining-messages.html' import remaining_messages %} -{% from "components/show-more.html" import show_more %} - -
-

- {{ _('Annual usage') }} -
- - {% set current_year = current_year or (now().year if now().month < 4 else now().year + 1) %} - {{ _('resets on April 1, ') ~ current_year }} - -

-
-
- {{ remaining_messages(header=_('emails'), total=current_service.email_annual_limit, used=statistics_annual['email'], muted=true) }} -
-
- {{ remaining_messages(header=_('text messages'), total=current_service.sms_annual_limit, used=statistics_annual['sms'], muted=true) }} -
-
- {{ show_more(url_for('.monthly', service_id=current_service.id), _('Visit usage report')) }} -
- diff --git a/app/templates/views/dashboard/_totals_daily.html b/app/templates/views/dashboard/_totals_daily.html index 98195f1a5d..8b9d9e0dcb 100644 --- a/app/templates/views/dashboard/_totals_daily.html +++ b/app/templates/views/dashboard/_totals_daily.html @@ -1,29 +1,8 @@ {% from "components/big-number.html" import big_number %} {% from "components/message-count-label.html" import message_count_label %} {% from 'components/remaining-messages.html' import remaining_messages %} -{% from "components/show-more.html" import show_more %} -{% if config["FF_ANNUAL_LIMIT"] %} -
-

- {{ _('Daily usage') }} -
- - {{ _('resets at 7pm Eastern Time') }} - -

-
-
- {{ remaining_messages(header=_('emails'), total=current_service.message_limit, used=statistics['email']['requested'], muted=true) }} -
-
- {{ remaining_messages(header=_('text messages'), total=current_service.sms_daily_limit, used=statistics['sms']['requested'], muted=true) }} -
-
- {{ show_more(url_for('main.contact'), _('Request a daily limit increase')) }} -
-{% else %} -
+

{{ _('Daily usage') }}
@@ -39,5 +18,4 @@

{{ remaining_messages(header=_('text messages'), total=current_service.sms_daily_limit, used=statistics['sms']['requested']) }}

- -{% endif %} + diff --git a/app/templates/views/dashboard/dashboard.html b/app/templates/views/dashboard/dashboard.html index 4e005d86bb..110a94f241 100644 --- a/app/templates/views/dashboard/dashboard.html +++ b/app/templates/views/dashboard/dashboard.html @@ -26,10 +26,7 @@

{{ _("Scheduled sends") }}

{{ ajax_block(partials, updates_url, 'weekly_totals', interval=5) }} {{ ajax_block(partials, updates_url, 'daily_totals', interval=5) }} - {% if config["FF_ANNUAL_LIMIT"] %} - {{ ajax_block(partials, updates_url, 'annual_totals', interval=5) }} - {% endif %} - +
{% if partials['has_template_statistics'] %} diff --git a/app/templates/views/dashboard/monthly.html b/app/templates/views/dashboard/monthly.html index 5fe77f7253..7aacbb685a 100644 --- a/app/templates/views/dashboard/monthly.html +++ b/app/templates/views/dashboard/monthly.html @@ -1,23 +1,22 @@ -{% from "components/big-number.html" import big_number_with_status, big_number, big_number_simple %} +{% from "components/big-number.html" import big_number_with_status, big_number %} {% from "components/pill.html" import pill %} {% from "components/table.html" import list_table, field, hidden_field_heading, right_aligned_field_heading, row_heading %} {% from "components/message-count-label.html" import message_count_label %} -{% from 'components/remaining-messages.html' import remaining_messages %} {% extends "admin_template.html" %} {% block service_page_title %} - {{ _('Usage report') }} + {{ _('Messages sent,') }} {{ selected_year }} {{ _('to') }} {{ selected_year + 1 }} {{ _('fiscal year') }} {% endblock %} {% block maincolumn_content %}

- {{ _('Usage report') }} + {{ _('Messages sent') }}

-
+
{{ pill( items=years, current_value=selected_year, @@ -26,50 +25,6 @@

) }}

- {% if config["FF_ANNUAL_LIMIT"] %} -

- {% if selected_year == current_financial_year %} - {{ _('Annual limit overview') }} - {% else %} - {{ _('Annual overview') }} - {% endif %} -
- - {{ _('Fiscal year begins April 1, ') ~ selected_year ~ _(' and ends March 31, ') ~ (selected_year + 1) }} - -

-
- {% if selected_year == current_financial_year %} -
- {{ remaining_messages(header=_('emails'), total=current_service.email_annual_limit, used=annual_data['email']) }} -
-
- {{ remaining_messages(header=_('text messages'), total=current_service.sms_annual_limit, used=annual_data['sms']) }} -
- {% else %} -
- {{ big_number_simple( - annual_data['email'], - _('emails'), - - ) - }} -
-
- {{ big_number_simple( - annual_data['sms'], - _('text messages'), - - ) - }} -
- {% endif %} -
-

- {{ _('Month by month totals') }} -

- {% endif %} - {% if months %} {% set spend_txt = _('Total spend') %} {% set heading_1 = _('Month') %} diff --git a/app/translations/csv/fr.csv b/app/translations/csv/fr.csv index 0f5e7e0d26..23ba5ac52d 100644 --- a/app/translations/csv/fr.csv +++ b/app/translations/csv/fr.csv @@ -2004,12 +2004,3 @@ "Annual text message limit","(FR) Limite maximale de messages texte par exercice financier" "Annual email message limit","(FR) Limite maximale de messages électroniques par exercice financier" "Annual email limit","(FR) Limite maximale de courriels par exercice financier" -" and ends March 31, "," et se termine le 31 mars " -"Annual limit overview","Aperçu de la limite annuelle" -"Usage report","Rapport d’utilisation" -"Fiscal year begins April 1, ","Réinitialisation le 1er avril " -"resets on April 1, ","Réinitialisation le 1er avril " -"Annual usage","Utilisation annuelle" -"resets at 7pm Eastern Time","Réinitialisation à 19 h, heure de l’Est" -"Visit usage report","Consulter le rapport d’utilisation" -"Month by month totals","Totaux mensuels" \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 005883ce93..80bf69fe26 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1552,7 +1552,7 @@ requests = ">=2.0.0" [[package]] name = "notifications-utils" -version = "52.3.9" +version = "52.3.5" description = "Shared python code for Notification - Provides logging utils etc." optional = false python-versions = "~3.10.9" @@ -1588,8 +1588,8 @@ werkzeug = "3.0.4" [package.source] type = "git" url = "https://github.com/cds-snc/notifier-utils.git" -reference = "52.3.9" -resolved_reference = "b344e5a74c79a8fa8ca4f722691850ac0d277959" +reference = "52.3.5" +resolved_reference = "953ee170b4c47465bef047f1060d17a7702edeeb" [[package]] name = "openpyxl" @@ -2758,4 +2758,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "~3.10.9" -content-hash = "443df8a67497588c1801bfac747fde95ecaffda93675b6f038906750e891316b" +content-hash = "587e62c5c8f700ef83c6a39d3faca6dde4d10c92ee627c771ecdf538b405a77f" diff --git a/pyproject.toml b/pyproject.toml index e44d4352e3..65718753e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ unidecode = "^1.3.8" # PaaS awscli-cwlogs = "^1.4.6" -notifications-utils = { git = "https://github.com/cds-snc/notifier-utils.git", tag = "52.3.9" } +notifications-utils = { git = "https://github.com/cds-snc/notifier-utils.git", tag = "52.3.5" } # Pinned dependencies diff --git a/tests/app/main/views/test_dashboard.py b/tests/app/main/views/test_dashboard.py index a00980bc6c..b87baa165b 100644 --- a/tests/app/main/views/test_dashboard.py +++ b/tests/app/main/views/test_dashboard.py @@ -1,9 +1,7 @@ import copy import re -from unittest.mock import ANY import pytest -from bs4 import BeautifulSoup from flask import url_for from freezegun import freeze_time @@ -23,7 +21,6 @@ create_active_caseworking_user, create_active_user_view_permissions, normalize_spaces, - set_config, ) stub_template_stats = [ @@ -140,7 +137,6 @@ def test_task_shortcuts_are_visible_based_on_permissions( mock_get_service_templates, mock_get_jobs, mock_get_template_statistics, - mock_get_service_statistics, permissions: list, text_in_page: list, text_not_in_page: list, @@ -174,7 +170,6 @@ def test_survey_widget_presence( mock_get_service_templates, mock_get_jobs, mock_get_template_statistics, - mock_get_service_statistics, mocker, admin_url, is_widget_present, @@ -198,7 +193,6 @@ def test_sending_link_has_query_param( mock_get_service_templates, mock_get_jobs, mock_get_template_statistics, - mock_get_service_statistics, ): active_user_with_permissions["permissions"][SERVICE_ONE_ID] = ["view_activity", "send_messages"] client_request.login(active_user_with_permissions) @@ -215,7 +209,6 @@ def test_no_sending_link_if_no_templates( client_request: ClientRequest, mock_get_service_templates_when_no_templates_exist, mock_get_template_statistics, - mock_get_service_statistics, mock_get_jobs, ): page = client_request.get("main.service_dashboard", service_id=SERVICE_ONE_ID) @@ -312,7 +305,11 @@ def test_should_show_monthly_breakdown_of_template_usage( def test_anyone_can_see_monthly_breakdown( - client, api_user_active, service_one, mocker, mock_get_monthly_notification_stats, mock_get_service_statistics + client, + api_user_active, + service_one, + mocker, + mock_get_monthly_notification_stats, ): validate_route_permission_with_client( mocker, @@ -327,14 +324,16 @@ def test_anyone_can_see_monthly_breakdown( def test_monthly_shows_letters_in_breakdown( - client_request, service_one, mock_get_monthly_notification_stats, mock_get_service_statistics + client_request, + service_one, + mock_get_monthly_notification_stats, ): page = client_request.get("main.monthly", service_id=service_one["id"]) columns = page.select(".table-field-left-aligned .big-number-label") - assert normalize_spaces(columns[2].text) == "emails" - assert normalize_spaces(columns[3].text) == "text messages" + assert normalize_spaces(columns[0].text) == "emails" + assert normalize_spaces(columns[1].text) == "text messages" @pytest.mark.parametrize( @@ -346,7 +345,10 @@ def test_monthly_shows_letters_in_breakdown( ) @freeze_time("2015-01-01 15:15:15.000000") def test_stats_pages_show_last_3_years( - client_request, endpoint, mock_get_monthly_notification_stats, mock_get_monthly_template_usage, mock_get_service_statistics + client_request, + endpoint, + mock_get_monthly_notification_stats, + mock_get_monthly_template_usage, ): page = client_request.get( endpoint, @@ -359,7 +361,9 @@ def test_stats_pages_show_last_3_years( def test_monthly_has_equal_length_tables( - client_request, service_one, mock_get_monthly_notification_stats, mock_get_service_statistics + client_request, + service_one, + mock_get_monthly_notification_stats, ): page = client_request.get("main.monthly", service_id=service_one["id"]) @@ -397,6 +401,31 @@ def test_should_show_upcoming_jobs_on_dashboard( assert table_rows[1].find_all("td")[0].text.strip() == "Scheduled to send to 30 recipients" +@pytest.mark.parametrize( + "permissions, column_name, expected_column_count", + [ + (["email", "sms"], ".w-1\\/2", 6), + (["email", "sms"], ".w-1\\/2", 6), + ], +) +def test_correct_columns_display_on_dashboard_v15( + client_request: ClientRequest, + mock_get_service_templates, + mock_get_template_statistics, + mock_get_service_statistics, + mock_get_jobs, + service_one, + permissions, + expected_column_count, + column_name, + app_, +): + service_one["permissions"] = permissions + + page = client_request.get("main.service_dashboard", service_id=service_one["id"]) + assert len(page.select(column_name)) == expected_column_count + + def test_daily_usage_section_shown( client_request, mocker, @@ -1395,201 +1424,3 @@ def test_dashboard_daily_limits( ) == 2 ) - - -class TestAnnualLimits: - def test_daily_usage_uses_muted_component( - self, - logged_in_client, - mocker, - mock_get_service_templates_when_no_templates_exist, - mock_get_jobs, - mock_get_service_statistics, - mock_get_usage, - app_, - ): - with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED - mocker.patch( - "app.template_statistics_client.get_template_statistics_for_service", - return_value=copy.deepcopy(stub_template_stats), - ) - - url = url_for("main.service_dashboard", service_id=SERVICE_ONE_ID) - response = logged_in_client.get(url) - page = BeautifulSoup(response.data.decode("utf-8"), "html.parser") - - # ensure both email + sms widgets are muted - assert len(page.select("[data-testid='daily-usage'] .remaining-messages.muted")) == 2 - - def test_annual_usage_uses_muted_component( - self, - logged_in_client, - mocker, - mock_get_service_templates_when_no_templates_exist, - mock_get_jobs, - mock_get_service_statistics, - mock_get_usage, - app_, - ): - with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED - mocker.patch( - "app.template_statistics_client.get_template_statistics_for_service", - return_value=copy.deepcopy(stub_template_stats), - ) - - url = url_for("main.service_dashboard", service_id=SERVICE_ONE_ID) - response = logged_in_client.get(url) - page = BeautifulSoup(response.data.decode("utf-8"), "html.parser") - - # ensure both email + sms widgets are muted - assert len(page.select("[data-testid='annual-usage'] .remaining-messages.muted")) == 2 - - @freeze_time("2024-11-25 12:12:12") - @pytest.mark.parametrize( - "redis_daily_data, monthly_data, expected_data", - [ - ( - {"sms_delivered": 100, "email_delivered": 50, "sms_failed": 1000, "email_failed": 500}, - { - "data": { - "2024-04": {"sms": {}, "email": {}, "letter": {}}, - "2024-05": {"sms": {}, "email": {}, "letter": {}}, - "2024-06": {"sms": {}, "email": {}, "letter": {}}, - "2024-07": {"sms": {}, "email": {}, "letter": {}}, - "2024-08": {"sms": {}, "email": {}, "letter": {}}, - "2024-09": {"sms": {}, "email": {}, "letter": {}}, - "2024-10": { - "sms": {"delivered": 5, "permanent-failure": 50, "sending": 5, "technical-failure": 100}, - "email": {"delivered": 10, "permanent-failure": 110, "sending": 50, "technical-failure": 50}, - "letter": {}, - }, - "2024-11": { - "sms": {"delivered": 5, "permanent-failure": 50, "sending": 5, "technical-failure": 100}, - "email": {"delivered": 10, "permanent-failure": 110, "sending": 50, "technical-failure": 50}, - "letter": {}, - }, - } - }, - {"email": 990, "letter": 0, "sms": 1420}, - ), - ( - {"sms_delivered": 6, "email_delivered": 6, "sms_failed": 6, "email_failed": 6}, - { - "data": { - "2024-10": { - "sms": {"delivered": 6, "permanent-failure": 6, "sending": 6, "technical-failure": 6}, - "email": {"delivered": 6, "permanent-failure": 6, "sending": 6, "technical-failure": 6}, - "letter": {}, - }, - } - }, - {"email": 36, "letter": 0, "sms": 36}, - ), - ], - ) - def test_usage_report_aggregates_calculated_properly_with_redis( - self, - logged_in_client, - mocker, - mock_get_service_templates_when_no_templates_exist, - mock_get_jobs, - mock_get_service_statistics, - mock_get_usage, - app_, - redis_daily_data, - monthly_data, - expected_data, - ): - with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED - # mock annual_limit_client.get_all_notification_counts - mocker.patch( - "app.main.views.dashboard.annual_limit_client.get_all_notification_counts", - return_value=redis_daily_data, - ) - - mocker.patch( - "app.service_api_client.get_monthly_notification_stats", - return_value=copy.deepcopy(monthly_data), - ) - - mock_render_template = mocker.patch("app.main.views.dashboard.render_template") - - url = url_for("main.monthly", service_id=SERVICE_ONE_ID) - logged_in_client.get(url) - - mock_render_template.assert_called_with( - ANY, months=ANY, years=ANY, annual_data=expected_data, selected_year=ANY, current_financial_year=ANY - ) - - @freeze_time("2024-11-25 12:12:12") - @pytest.mark.parametrize( - "daily_data, monthly_data, expected_data", - [ - ( - { - "sms": {"requested": 100, "delivered": 50, "failed": 50}, - "email": {"requested": 100, "delivered": 50, "failed": 50}, - "letter": {"requested": 0, "delivered": 0, "failed": 0}, - }, - { - "data": { - "2024-04": {"sms": {}, "email": {}, "letter": {}}, - "2024-05": {"sms": {}, "email": {}, "letter": {}}, - "2024-06": {"sms": {}, "email": {}, "letter": {}}, - "2024-07": {"sms": {}, "email": {}, "letter": {}}, - "2024-08": {"sms": {}, "email": {}, "letter": {}}, - "2024-09": {"sms": {}, "email": {}, "letter": {}}, - "2024-10": { - "sms": {"delivered": 5, "permanent-failure": 50, "sending": 5, "technical-failure": 100}, - "email": {"delivered": 10, "permanent-failure": 110, "sending": 50, "technical-failure": 50}, - "letter": {}, - }, - "2024-11": { - "sms": {"delivered": 5, "permanent-failure": 50, "sending": 5, "technical-failure": 100}, - "email": {"delivered": 10, "permanent-failure": 110, "sending": 50, "technical-failure": 50}, - "letter": {}, - }, - } - }, - {"email": 540, "letter": 0, "sms": 420}, - ) - ], - ) - def test_usage_report_aggregates_calculated_properly_without_redis( - self, - logged_in_client, - mocker, - mock_get_service_templates_when_no_templates_exist, - mock_get_jobs, - mock_get_service_statistics, - mock_get_usage, - app_, - daily_data, - monthly_data, - expected_data, - ): - with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED - # mock annual_limit_client.get_all_notification_counts - mocker.patch( - "app.main.views.dashboard.annual_limit_client.get_all_notification_counts", - return_value=None, - ) - - mocker.patch( - "app.service_api_client.get_service_statistics", - return_value=copy.deepcopy(daily_data), - ) - - mocker.patch( - "app.service_api_client.get_monthly_notification_stats", - return_value=copy.deepcopy(monthly_data), - ) - - mock_render_template = mocker.patch("app.main.views.dashboard.render_template") - - url = url_for("main.monthly", service_id=SERVICE_ONE_ID) - logged_in_client.get(url) - - mock_render_template.assert_called_with( - ANY, months=ANY, years=ANY, annual_data=expected_data, selected_year=ANY, current_financial_year=ANY - ) diff --git a/tests/conftest.py b/tests/conftest.py index 94ff0ca5be..a0c964cb90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -578,42 +578,9 @@ def _get(service_id, today_only, limit_days=None): "letter": {"requested": 0, "delivered": 0, "failed": 0}, } - # mock these stats at the same time - def _get_monthly_stats(service_id, year): - return { - "data": { - "2024-04": {"sms": {}, "email": {}, "letter": {}}, - "2024-05": {"sms": {}, "email": {}, "letter": {}}, - "2024-06": {"sms": {}, "email": {}, "letter": {}}, - "2024-07": {"sms": {}, "email": {}, "letter": {}}, - "2024-08": {"sms": {}, "email": {}, "letter": {}}, - "2024-09": {"sms": {}, "email": {}, "letter": {}}, - "2024-10": {"sms": {}, "email": {}, "letter": {}}, - "2024-11": { - "sms": {"sent": 1}, - "email": {"delivered": 1, "permanent-failure": 1, "sending": 3, "technical-failure": 1}, - "letter": {}, - }, - } - } - - mocker.patch("app.service_api_client.get_monthly_notification_stats", side_effect=_get_monthly_stats) - return mocker.patch("app.service_api_client.get_service_statistics", side_effect=_get) -@pytest.fixture(scope="function") -def mock_get_annual_statistics(mocker, api_user_active): - def _get(service_id, year): - return { - "email": 100, - "sms": 200, - "letter": 300, - } - - return mocker.patch("app.service_api_client.get_monthly_notification_stats", side_effect=_get) - - @pytest.fixture(scope="function") def mock_get_detailed_services(mocker, fake_uuid): service_one = service_json( From 57487374b34579c16545374b8cc54f40602b7c66 Mon Sep 17 00:00:00 2001 From: Philippe Caron Date: Mon, 25 Nov 2024 13:03:35 -0500 Subject: [PATCH 02/20] fix various translations (#1997) * fix various translations * fix: update translation strings to include space * fix: remove line break from text * fix: update `make test-translations` to look for translations inside of `.html` files and not just `.py` files * chore: add missing translation * update settings page title * remove redundant translations * add missing translation * csv lint --------- Co-authored-by: Andrew Leith --- .../components/message-count-label.html | 4 +- .../callbacks/delivery-status-callback.html | 4 +- .../received-text-messages-callback.html | 4 +- app/templates/views/check/column-errors.html | 6 ++- .../views/email-branding/branding-goc.html | 3 +- .../views/email-branding/branding-pool.html | 3 +- .../branding-request-submitted.html | 2 +- app/templates/views/notifications/check.html | 6 ++- .../views/service-settings/email_from.html | 3 +- .../set-free-sms-allowance.html | 2 +- app/translations/csv/fr.csv | 38 +++++++++++++++---- babel.cfg | 1 + 12 files changed, 53 insertions(+), 23 deletions(-) diff --git a/app/templates/components/message-count-label.html b/app/templates/components/message-count-label.html index 81b6764baf..444f58b1ab 100644 --- a/app/templates/components/message-count-label.html +++ b/app/templates/components/message-count-label.html @@ -104,9 +104,9 @@ {%- if session["userlang"] == "fr" -%} {%- if count <= 1 -%} - {{ _('addresse courriel problématique') }} + addresse courriel problématique {%- else -%} - {{ _('addresses courriel problématiques') }} + addresses courriel problématiques {%- endif %} {{" "}} {%- endif %} diff --git a/app/templates/views/api/callbacks/delivery-status-callback.html b/app/templates/views/api/callbacks/delivery-status-callback.html index 92c5b0e118..e561c16236 100644 --- a/app/templates/views/api/callbacks/delivery-status-callback.html +++ b/app/templates/views/api/callbacks/delivery-status-callback.html @@ -41,13 +41,13 @@

autocomplete='new-password' ) }} {% set test_response_txt = _('Test response time') if has_callback_config else None %} - {% set test_response_value = _('test_response_time') if has_callback_config else None %} + {% set test_response_value = 'test_response_time' if has_callback_config else None %} {% set display_footer = is_deleting if is_deleting else False %} {% set delete_link = url_for('.delete_delivery_status_callback', service_id=current_service.id) if has_callback_config else None%} {% if not display_footer %} {{ sticky_page_footer_two_submit_buttons_and_delete_link( button1_text=_('Save'), - button1_value=_('save'), + button1_value='save', button2_text=test_response_txt, button2_value=test_response_value, delete_link=delete_link, diff --git a/app/templates/views/api/callbacks/received-text-messages-callback.html b/app/templates/views/api/callbacks/received-text-messages-callback.html index 7056ffa71d..e43e2b55e7 100644 --- a/app/templates/views/api/callbacks/received-text-messages-callback.html +++ b/app/templates/views/api/callbacks/received-text-messages-callback.html @@ -36,14 +36,14 @@

autocomplete='new-password' ) }} {% set test_response_txt = _('Test response time') if has_callback_config else None %} - {% set test_response_value = _('test_response_time') if has_callback_config else None %} + {% set test_response_value = 'test_response_time' if has_callback_config else None %} {% set display_footer = is_deleting if is_deleting else False %} {% set delete_link = url_for('.delete_received_text_messages_callback', service_id=current_service.id) if has_callback_config else None%} {% set delete_link_text = _('Delete') if has_callback_config else None %} {% if not display_footer %} {{ sticky_page_footer_two_submit_buttons_and_delete_link( button1_text=_('Save'), - button1_value=_('save'), + button1_value='save', button2_text=test_response_txt, button2_value=test_response_value, delete_link=delete_link, diff --git a/app/templates/views/check/column-errors.html b/app/templates/views/check/column-errors.html index c3ef2200f8..32f2ae9e8e 100644 --- a/app/templates/views/check/column-errors.html +++ b/app/templates/views/check/column-errors.html @@ -161,7 +161,8 @@

- {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], content_link(_("your current local time"), 'https://nrc.canada.ca/en/web-clock/', is_external_link=true))}} + {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], + content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}

{% elif recipients.more_rows_than_can_send and false %} {% call banner_wrapper(type='dangerous') %} @@ -173,7 +174,8 @@

{{ _('You cannot send all these text messages today') {% endcall %}

{{ _('You cannot send all these email messages today') }}

- {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], content_link(_("your current local time"), 'https://nrc.canada.ca/en/web-clock/', is_external_link=true))}} + {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], + content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}

diff --git a/app/templates/views/email-branding/branding-goc.html b/app/templates/views/email-branding/branding-goc.html index d9fc13c351..e059b08f82 100644 --- a/app/templates/views/email-branding/branding-goc.html +++ b/app/templates/views/email-branding/branding-goc.html @@ -24,7 +24,8 @@

{{ _('Change the default langu {% call form_wrapper() %} {{ radios(form.goc_branding, hide_legend=True, testid="goc_branding") }} - {{ _('{}').format(url_for('main.review_branding_pool', service_id=current_service.id), _('Select alternate logo')) }} + {{ _('Select alternate logo') }}
{{ page_footer( diff --git a/app/templates/views/email-branding/branding-pool.html b/app/templates/views/email-branding/branding-pool.html index daa3366124..1d293017c7 100644 --- a/app/templates/views/email-branding/branding-pool.html +++ b/app/templates/views/email-branding/branding-pool.html @@ -29,7 +29,8 @@
- {{ _('{}').format(url_for('main.create_branding_request', service_id=current_service.id), _('Request a new logo')) }} + {{ + _('Request a new logo') }}
{{ page_footer(_('Preview'), testid="preview") }} diff --git a/app/templates/views/email-branding/branding-request-submitted.html b/app/templates/views/email-branding/branding-request-submitted.html index 968dad1359..70ab504f0c 100644 --- a/app/templates/views/email-branding/branding-request-submitted.html +++ b/app/templates/views/email-branding/branding-request-submitted.html @@ -31,7 +31,7 @@ {{ task_shortcut( description=_("Explore other settings"), link_url=url_for('main.service_settings', service_id=current_service.id), - link_text=_("Go to your Settings"), + link_text=_("Go to your settings"), icon="arrow-right" )}}
diff --git a/app/templates/views/notifications/check.html b/app/templates/views/notifications/check.html index 044eb29281..17c668b0c8 100644 --- a/app/templates/views/notifications/check.html +++ b/app/templates/views/notifications/check.html @@ -43,7 +43,8 @@ {% endcall %}

{{_('You cannot send this text message today') }}

- {{ _("You can try sending this message after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], content_link(_("your current local time"), 'https://nrc.canada.ca/en/web-clock/', is_external_link=true))}} + {{ _("You can try sending this message after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], + content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}

{% elif error == 'too-many-messages' %} @@ -61,7 +62,8 @@

{{_('You cannot send this text message today') }}

{{_('You cannot send this email message today') }}

- {{ _("You can try sending this message after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], content_link(_("your current local time"), 'https://nrc.canada.ca/en/web-clock/', is_external_link=true))}} + {{ _("You can try sending this message after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], + content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}

{% elif error == 'message-too-long' %} diff --git a/app/templates/views/service-settings/email_from.html b/app/templates/views/service-settings/email_from.html index 321b6ee3f0..b9b9851efc 100644 --- a/app/templates/views/service-settings/email_from.html +++ b/app/templates/views/service-settings/email_from.html @@ -23,8 +23,7 @@ {% call form_wrapper() %} {% set save_txt = _('Save') %} - {% set hint_txt = _('Maximum 64 characters with no spaces. Characters can include letters, numbers, dots, dashes, and - underscores.') %} + {% set hint_txt = _('Maximum 64 characters with no spaces. Characters can include letters, numbers, dots, dashes, and underscores.') %} {{ textbox(form.email_from, hint=hint_txt) }}
{% call confirmation_preview() %} diff --git a/app/templates/views/service-settings/set-free-sms-allowance.html b/app/templates/views/service-settings/set-free-sms-allowance.html index c410b58d10..d06a4ccd00 100644 --- a/app/templates/views/service-settings/set-free-sms-allowance.html +++ b/app/templates/views/service-settings/set-free-sms-allowance.html @@ -12,7 +12,7 @@ {% call form_wrapper() %} {{ page_header( - _('Free text messages per year'), + _('Free text messages per fiscal year'), back_link=url_for('.service_settings', service_id=current_service.id) ) }} {{ textbox(form.free_sms_allowance) }} diff --git a/app/translations/csv/fr.csv b/app/translations/csv/fr.csv index 23ba5ac52d..1485bc4300 100644 --- a/app/translations/csv/fr.csv +++ b/app/translations/csv/fr.csv @@ -1,6 +1,6 @@ "source","target" "\!/\!/ HELLO DEVS, READ THIS FOR DOCUMENTATION \!/\!/","Read more about how translations work in scripts/generate_en_translations.py" -":"," :" +": "," : " "English","Anglais" "French","Français" "Your account and language","Votre compte et langue" @@ -56,6 +56,7 @@ "Who runs this service?","Qui assure la gestion du service?" "What’s your key called?","Quel est le nom de votre clé?" "Numbers of text messages per fiscal year","Nombre de messages texte par exercice financier" +"Free text messages per fiscal year","Messages texte gratuits par exercice financier" "Enter password","Entrez votre mot de passe" "Invalid password","Mot de passe non valide" "Template name","Nom du gabarit" @@ -127,6 +128,7 @@ "Text message","Message texte" "Choose a folder","Choisissez un dossier" "Create template","Créer un gabarit" +"Create category","Créer une catégorie" "Will you send the message by email or text?","Enverrez-vous le message par courriel ou par texte?" "What’s their name?","Quel est son nom?" "What’s their email address?","Quelle est son adresse courriel?" @@ -653,6 +655,7 @@ "Count in list of live services","Compte dans la liste de services activés" "Organisation","Organisation" "Free text message allowance","Nombre de messages texte gratuits" +"Free text messages per year","Nombre de messages texte gratuits par année" "text messages per fiscal year","Allocation de messages texte par exercice financier" "Letter branding","Image de marque de la lettre" "Data retention","Rétention des données" @@ -1080,6 +1083,7 @@ "Daily text message limit","Limite quotidienne de message texte" "Last edited","Dernière modification : " "See previous versions","Voir les versions précédentes" +"Version {}","Version {}" "Delete this template","Supprimer ce gabarit" "Redact personalised variable content after sending","Masquer le contenu variable personnalisé après l’envoi" "Personalised variable content redacted after sending","Le contenu variable personnalisé masqué après l’envoi" @@ -1426,6 +1430,7 @@ "Filter by year","Filtrer par année" "Filter by status","Filtrer par état de livraison" "Filter by template type","Filtrer par type de gabarit" +"Filter by template type and category","Filtrer par type et catégorie de gabarit" "Top of page","Haut de page" "Your service is in trial mode. Trial mode limits your service to sending notifications to yourself and to your team members within GC Notify.","Votre service est en mode d’essai. Le mode d’essai limite l’envoi de notifications à vous-même et aux autres membres de votre équipe sur Notification GC." "Complete the following steps to go live and send notifications to more people.","Complétez les étapes suivantes pour activer votre service et envoyer des messages à d’autres personnes." @@ -1637,6 +1642,8 @@ "Back to template {}","Retour au gabarit {}" "Previewing template {}","Aperçu du gabarit {}" "You need a new password","Vous devez créer un nouveau mot de passe" +"You need to create a new password","Vous devez créer un nouveau mot de passe" +"GC Notify needs you to create a new password for this account.","Notification GC vous demande de créer un nouveau mot de passe pour ce compte." "As a security precaution, all users of GC Notify must change their password.","Par mesure de sécurité, tou·te·s les utilisateur·rice·s de Notification GC doivent changer leur mot de passe." "Check your email. If you did not receive the link,","Nous vous avons envoyé un courriel contenant un lien de réinitialisation. Si vous ne l’avez pas reçu," "contact support.","veuillez contacter notre équipe de soutien." @@ -1650,7 +1657,7 @@ "The link in the email we sent you has expired","Le lien que nous vous avons envoyé par courriel a expiré." "Check your email. We sent you a password reset link.","Vérifiez votre courriel. Nous vous avons envoyé un lien pour réinitialiser votre mot de passe." "Why we are asking you to create a new password","Pourquoi nous vous demandons de créer un nouveau mot de passe." -"GC Notify has expired all user passwords out of an abundance of caution following the discovery of a potential security risk on March 29, 2022.", Notification GC a expiré le mot de passe de tous les utilisateurs par excès de prudence, suivant la découverte d’une faille potentielle le 29 mars 2022." +"GC Notify has expired all user passwords out of an abundance of caution following the discovery of a potential security risk on March 29, 2022.","Notification GC a expiré le mot de passe de tous les utilisateurs par excès de prudence, suivant la découverte d’une faille potentielle le 29 mars 2022." "If you don't receive a password reset link in your inbox,","Si vous ne recevez pas de courriel pour réinitialiser votre mot de passe," "please contact our support team.","contactez notre équipe de soutien technique." "A password that is hard to guess contains:","Un mot de passe difficile à deviner contient les caractéristiques suivantes:" @@ -1705,7 +1712,8 @@ "This message exceeds your daily email limit","Ce message dépasse votre limite quotidienne d'envoi de courriels" "You’ve sent too many text messages.","Vous avez envoyé trop de messages texte." "You can send more text messages after {} Eastern Time. To raise your daily limit, {contact_us}.","Vous pourrez envoyer d’autres messages texte après {} heures, heure de l’Est. Pour augmenter votre limite d’envoi quotidienne, {contact_us}." -"You can try sending these messages after {} Eastern Time. Check your current local time.","Vous pourrez envoyer ces messages après {} heures, heure de l’Est. Comparez les heures officielles au Canada." +"You can try sending these messages after {} Eastern Time. Check {}.","Vous pourrez envoyer ces messages après {} heures, heure de l’Est. Comparez {}." +"https://nrc.canada.ca/en/web-clock/","https://nrc.canada.ca/fr/horloge-web/" "Attachment has virus","La pièce jointe contient un virus" "Review reports","Examiner les rapports" "Enter name of your group","Saisissez le nom de votre groupe" @@ -1716,8 +1724,9 @@ "For example: Treasury Board of Canada Secretariat","Par exemple : Secrétariat du Conseil du Trésor du Canada" "For example: Canadian Digital Service","Par exemple : Service numérique canadien" "Not on list?","Vous ne trouvez pas?" +"Not on the list? Add your organization","Vous ne trouvez pas? Ajoutez votre organisation" "Choose name from drop-down menu","Choisissez un nom dans le menu déroulant" -"Tech issue","Problème technique", +"Tech issue","Problème technique" "Content or inbox issue","Problème de contenu ou de boîte de réception" "In transit","Envoi en cours" "Exceeds Protected A","Niveau supérieur à Protégé A" @@ -1766,8 +1775,7 @@ "Annual maximum
(April 1 to March 31)","Maximum par exercice financier" "To request a daily limit above {} emails, {}","Si vous désirez obtenir une limite quotidienne supérieure à {} courriels, veuillez {}" "To request a daily limit above {} text messages, {}","Si vous désirez obtenir une limite quotidienne supérieure à {} messages texte, veuillez {}" -"You can try sending these messages after {} Eastern Time. Check your {}.","Vous pourrez envoyer ces messages après {} heures, heure de l’Est. Comparez {}" -"You can try sending this message after {} Eastern Time. Check your {}.","Vous pourrez envoyer ce message après {} heures, heure de l’Est. Comparez {}" +"You can try sending this message after {} Eastern Time. Check {}.","Vous pourrez envoyer ce message après {} heures, heure de l’Est. Comparez {}." "your current local time","les heures officielles au Canada" "You cannot send this email message today","Vous ne pouvez pas envoyer ce courriel aujourd’hui." "You cannot send this text message today","Vous ne pouvez pas envoyer ce message texte aujourd’hui." @@ -1776,12 +1784,16 @@ "of","de" "Sent since 7 pm Eastern Time","Envoyé depuis 19 h, heure de l'Est" "You are nearing the daily {} limit","Vous approchez de la limite quotidienne de {}" +"Below limit: ","Vous êtes en dessous de votre limite : " +"Near limit: ","Vous approchez de votre limite : " +"At limit: ","Vous avez atteint votre limite : " "Daily usage","Utilisation quotidienne" "Message limits reset each night at 7pm Eastern Time","Les limites d’envoi sont réinitialisées chaque soir à 19 h, heure de l’Est" "Maximum 612 characters. Some messages may be too long due to custom content.","612 caractères au maximum. Certains messages peuvent être trop longs en raison de leur contenu personnalisé." "Too many characters","Limite de caractère atteinte" "New features","Nouvelles fonctionnalités" "Your","Votre" +"your","votre" "You are browsing templates. Create and copy template or add new folder.","Vous explorez les gabarits. Créer et copier un gabarit ou créer un nouveau dossier." "Move templates to a new or existing folder","Déplacer les gabarits dans un dossier" "You are selecting templates. Move templates into a new or existing folder.","Vous sélectionnez les gabarits. Déplacer les gabarits dans un dossier." @@ -1918,7 +1930,7 @@ "Read and agree to the terms of use","Lisez et acceptez les conditions d’utilisation" "Read and agree to continue","Lisez et acceptez les conditions d’utilisation" "Agree follows terms of use","Accepter suite aux conditions d'utilisation" -"Priority","Envoi prioritaire", +"Priority","Envoi prioritaire" "Bulk","Envoi de masse" "Text message priority","Niveau de priorité des messages texte" "Hide category","Visibilité de la catégorie" @@ -2004,3 +2016,15 @@ "Annual text message limit","(FR) Limite maximale de messages texte par exercice financier" "Annual email message limit","(FR) Limite maximale de messages électroniques par exercice financier" "Annual email limit","(FR) Limite maximale de courriels par exercice financier" +"Test response time","Tester le temps de réponse" +"No records found.","Aucun enregistrement trouvé." +" and ends March 31, "," et se termine le 31 mars " +"Annual limit overview","Aperçu de la limite annuelle" +"Annual overview","Aperçu annuel" +"Usage report","Rapport d’utilisation" +"Fiscal year begins April 1, ","Réinitialisation le 1er avril " +"resets on April 1, ","Réinitialisation le 1er avril " +"Annual usage","Utilisation annuelle" +"resets at 7pm Eastern Time","Réinitialisation à 19 h, heure de l’Est" +"Visit usage report","Consulter le rapport d’utilisation" +"Month by month totals","Totaux mensuels" diff --git a/babel.cfg b/babel.cfg index 19d23b74a1..2c649d5cee 100644 --- a/babel.cfg +++ b/babel.cfg @@ -1 +1,2 @@ [python: app/**.py] +[jinja2: app/templates/**.html] \ No newline at end of file From b7859811d8b57673e800cb87fa5cd816365a1f89 Mon Sep 17 00:00:00 2001 From: Andrew Date: Tue, 26 Nov 2024 14:32:48 -0400 Subject: [PATCH 03/20] Feat/add annual limits to dash and usage 2 (#2001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add client to fetch and cache annual usage for a given service * feat(dashboard): fetch and use annual send counts on the dashboard * chore: undo new client class * feat(service_api_client): get annual data excluding today and cache it for use in the dashboard * chore: formatting (why is this still happening?!?!) * feat(dashboard): display annual data on dashboard (cached); show aggregates at top of usage report page * chore: translations * test(dashboard): add a mock for annual stats whenever dashboard is tested * chore: formatting * chore: remove duplicate translation * chore: default `FF_ANNUAL_LIMITS` to true for staging config * fix: only show new ui related to annual limits if `FF_ANNUAL_LIMIT` is true * chore: update utils * chore: use annual_limit_client to get cached stats for today * feat: cache the result of get_monthly_notification_stats * chore: translation * feat(dashboard): get daily data from redis where possible and aggregate with monthly sources * test: add testids * chore(service_api_client): cache call * test: add tests for dashboard and usage report changes * chore: regen poetry lock * fix: align with data structure in annual_limits client * chore: bump utils * chore: add TODO around caching for when API is updated * chore: fix failing tests * chore: formtting * chore: update poetry.lock * chore: fix tests * debug: add logging stmt to see whats going wrong in staging * chore: fix loggin stmt * chore: log properly? maybe? 😱 * fix: add some missing node checks to dail limits redis structure to ensure code does error out * fix: mistakenly trying to use redis data in "db" mode, run formatting * fix: remove "notifications" node on redis data, as it isnt there after all * tests: update data format to align with redis annual_limit client * fix: use explicit timezone * Default the FF to OFF if it isnt in the ENV vars * chore: formatting * fix: avoid uninitialized var --- app/config.py | 4 +- app/extensions.py | 3 + app/main/views/dashboard.py | 105 +++++++- app/notify_client/service_api_client.py | 27 +- .../views/dashboard/_totals_annual.html | 25 ++ .../views/dashboard/_totals_daily.html | 26 +- app/templates/views/dashboard/dashboard.html | 5 +- app/templates/views/dashboard/monthly.html | 53 +++- app/translations/csv/fr.csv | 2 +- poetry.lock | 8 +- pyproject.toml | 2 +- tests/app/main/views/test_dashboard.py | 253 +++++++++++++++--- tests/conftest.py | 33 +++ 13 files changed, 485 insertions(+), 61 deletions(-) create mode 100644 app/templates/views/dashboard/_totals_annual.html diff --git a/app/config.py b/app/config.py index 0d2d7ed1f1..9b37a55798 100644 --- a/app/config.py +++ b/app/config.py @@ -80,7 +80,7 @@ class Config(object): # FEATURE FLAGS FF_SALESFORCE_CONTACT = env.bool("FF_SALESFORCE_CONTACT", True) FF_RTL = env.bool("FF_RTL", True) - FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", True) + FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", False) FREE_YEARLY_EMAIL_LIMIT = env.int("FREE_YEARLY_EMAIL_LIMIT", 20_000_000) FREE_YEARLY_SMS_LIMIT = env.int("FREE_YEARLY_SMS_LIMIT", 100_000) @@ -215,7 +215,7 @@ class Test(Development): NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef" FF_RTL = True - FF_ANNUAL_LIMIT = False + FF_ANNUAL_LIMIT = True class ProductionFF(Config): diff --git a/app/extensions.py b/app/extensions.py index 8421b1e560..51bebe6e38 100644 --- a/app/extensions.py +++ b/app/extensions.py @@ -1,5 +1,6 @@ from flask_caching import Cache from notifications_utils.clients.antivirus.antivirus_client import AntivirusClient +from notifications_utils.clients.redis.annual_limit import RedisAnnualLimit from notifications_utils.clients.redis.bounce_rate import RedisBounceRate from notifications_utils.clients.redis.redis_client import RedisClient from notifications_utils.clients.statsd.statsd_client import StatsdClient @@ -10,4 +11,6 @@ zendesk_client = ZendeskClient() redis_client = RedisClient() bounce_rate_client = RedisBounceRate(redis_client) +annual_limit_client = RedisAnnualLimit(redis_client) + cache = Cache(config={"CACHE_TYPE": "simple"}) # TODO: pull config out to config.py later diff --git a/app/main/views/dashboard.py b/app/main/views/dashboard.py index f1bd214bfe..54c141a82d 100644 --- a/app/main/views/dashboard.py +++ b/app/main/views/dashboard.py @@ -24,7 +24,7 @@ service_api_client, template_statistics_client, ) -from app.extensions import bounce_rate_client +from app.extensions import annual_limit_client, bounce_rate_client from app.main import main from app.models.enum.bounce_rate_status import BounceRateStatus from app.models.enum.notification_statuses import NotificationStatuses @@ -229,16 +229,90 @@ def usage(service_id): @main.route("/services//monthly") @user_has_permissions("view_activity") def monthly(service_id): + def combine_daily_to_annual(daily, annual, mode): + if mode == "redis": + # the redis client omits properties if there are no counts yet, so account for this here\ + daily_redis = { + field: daily.get(field, 0) for field in ["sms_delivered", "sms_failed", "email_delivered", "email_failed"] + } + annual["sms"] += daily_redis["sms_delivered"] + daily_redis["sms_failed"] + annual["email"] += daily_redis["email_delivered"] + daily_redis["email_failed"] + elif mode == "db": + annual["sms"] += daily["sms"]["requested"] + annual["email"] += daily["email"]["requested"] + + return annual + + def combine_daily_to_monthly(daily, monthly, mode): + if mode == "redis": + # the redis client omits properties if there are no counts yet, so account for this here\ + daily_redis = { + field: daily.get(field, 0) for field in ["sms_delivered", "sms_failed", "email_delivered", "email_failed"] + } + + monthly[0]["sms_counts"]["failed"] += daily_redis["sms_failed"] + monthly[0]["sms_counts"]["requested"] += daily_redis["sms_failed"] + daily_redis["sms_delivered"] + monthly[0]["email_counts"]["failed"] += daily_redis["email_failed"] + monthly[0]["email_counts"]["requested"] += daily_redis["email_failed"] + daily_redis["email_delivered"] + elif mode == "db": + monthly[0]["sms_counts"]["failed"] += daily["sms"]["failed"] + monthly[0]["sms_counts"]["requested"] += daily["sms"]["requested"] + monthly[0]["email_counts"]["failed"] += daily["email"]["failed"] + monthly[0]["email_counts"]["requested"] += daily["email"]["requested"] + + return monthly + + def aggregate_by_type(notification_data): + counts = {"sms": 0, "email": 0, "letter": 0} + for month_data in notification_data["data"].values(): + for message_type, message_counts in month_data.items(): + if isinstance(message_counts, dict): + counts[message_type] += sum(message_counts.values()) + + # return the result + return counts + year, current_financial_year = requested_and_current_financial_year(request) + + # if FF_ANNUAL is on + if current_app.config["FF_ANNUAL_LIMIT"]: + monthly_data = service_api_client.get_monthly_notification_stats(service_id, year) + annual_data = aggregate_by_type(monthly_data) + + todays_data = annual_limit_client.get_all_notification_counts(current_service.id) + + # if redis is empty, query the db + if todays_data is None: + todays_data = service_api_client.get_service_statistics(service_id, limit_days=1, today_only=False) + annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "db") + + months = (format_monthly_stats_to_list(monthly_data["data"]),) + monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "db") + else: + # aggregate daily + annual + current_app.logger.info("todays data" + str(todays_data)) + annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "redis") + + months = (format_monthly_stats_to_list(monthly_data["data"]),) + monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "redis") + else: + monthly_data_aggregate = ( + format_monthly_stats_to_list(service_api_client.get_monthly_notification_stats(service_id, year)["data"]), + ) + monthly_data_aggregate = monthly_data_aggregate[0] + annual_data_aggregate = None + return render_template( "views/dashboard/monthly.html", - months=format_monthly_stats_to_list(service_api_client.get_monthly_notification_stats(service_id, year)["data"]), + months=monthly_data_aggregate, years=get_tuples_of_financial_years( partial_url=partial(url_for, ".monthly", service_id=service_id), start=current_financial_year - 2, end=current_financial_year, ), + annual_data=annual_data_aggregate, selected_year=year, + current_financial_year=current_financial_year, ) @@ -284,6 +358,21 @@ def aggregate_notifications_stats(template_statistics): def get_dashboard_partials(service_id): + def aggregate_by_type(data, daily_data): + counts = {"sms": 0, "email": 0, "letter": 0} + # flatten out this structure to match the above + for month_data in data["data"].values(): + for message_type, message_counts in month_data.items(): + if isinstance(message_counts, dict): + counts[message_type] += sum(message_counts.values()) + + # add todays data to the annual data + counts = { + "sms": counts["sms"] + daily_data["sms"]["requested"], + "email": counts["email"] + daily_data["email"]["requested"], + } + return counts + all_statistics_weekly = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=7) template_statistics_weekly = aggregate_template_usage(all_statistics_weekly) @@ -300,6 +389,10 @@ def get_dashboard_partials(service_id): dashboard_totals_weekly = (get_dashboard_totals(stats_weekly),) bounce_rate_data = get_bounce_rate_data_from_redis(service_id) + # get annual data from fact table (all data this year except today) + annual_data = service_api_client.get_monthly_notification_stats(service_id, year=get_current_financial_year()) + annual_data = aggregate_by_type(annual_data, dashboard_totals_daily[0]) + return { "upcoming": render_template("views/dashboard/_upcoming.html", scheduled_jobs=scheduled_jobs), "daily_totals": render_template( @@ -308,6 +401,13 @@ def get_dashboard_partials(service_id): statistics=dashboard_totals_daily[0], column_width=column_width, ), + "annual_totals": render_template( + "views/dashboard/_totals_annual.html", + service_id=service_id, + statistics=dashboard_totals_daily[0], + statistics_annual=annual_data, + column_width=column_width, + ), "weekly_totals": render_template( "views/dashboard/_totals.html", service_id=service_id, @@ -329,6 +429,7 @@ def get_dashboard_partials(service_id): def _get_daily_stats(service_id): + # TODO: get from redis, else fallback to template_statistics_client.get_template_statistics_for_service all_statistics_daily = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1) stats_daily = aggregate_notifications_stats(all_statistics_daily) dashboard_totals_daily = (get_dashboard_totals(stats_daily),) diff --git a/app/notify_client/service_api_client.py b/app/notify_client/service_api_client.py index 6054c374d2..b1274337c7 100644 --- a/app/notify_client/service_api_client.py +++ b/app/notify_client/service_api_client.py @@ -1,5 +1,5 @@ import json -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from flask import current_app from flask_login import current_user @@ -9,6 +9,12 @@ from app.notify_client import NotifyAdminAPIClient, _attach_current_user, cache +def _seconds_until_midnight(): + now = datetime.now(timezone.utc) + midnight = datetime.combine(now + timedelta(days=1), datetime.min.time()) + return int((midnight - now).total_seconds()) + + class ServiceAPIClient(NotifyAdminAPIClient): @cache.delete("user-{user_id}") def create_service( @@ -377,8 +383,15 @@ def is_service_email_from_unique(self, service_id, email_from): def get_service_history(self, service_id): return self.get("/service/{0}/history".format(service_id)) + # TODO: cache this once the backend is updated to exlude data from the current day + # @flask_cache.memoize(timeout=_seconds_until_midnight()) def get_monthly_notification_stats(self, service_id, year): - return self.get(url="/service/{}/notifications/monthly?year={}".format(service_id, year)) + return self.get( + url="/service/{}/notifications/monthly?year={}".format( + service_id, + year, + ) + ) def get_safelist(self, service_id): return self.get(url="/service/{}/safelist".format(service_id)) @@ -622,5 +635,15 @@ def _use_case_data_name(self, service_id): def _tos_key_name(self, service_id): return f"tos-accepted-{service_id}" + def aggregate_by_type(self, notification_data): + counts = {"sms": 0, "email": 0, "letter": 0} + for month_data in notification_data["data"].values(): + for message_type, message_counts in month_data.items(): + if isinstance(message_counts, dict): + counts[message_type] += sum(message_counts.values()) + + # return the result + return counts + service_api_client = ServiceAPIClient() diff --git a/app/templates/views/dashboard/_totals_annual.html b/app/templates/views/dashboard/_totals_annual.html new file mode 100644 index 0000000000..4985ec80be --- /dev/null +++ b/app/templates/views/dashboard/_totals_annual.html @@ -0,0 +1,25 @@ +{% from "components/big-number.html" import big_number %} +{% from "components/message-count-label.html" import message_count_label %} +{% from 'components/remaining-messages.html' import remaining_messages %} +{% from "components/show-more.html" import show_more %} + +
+

+ {{ _('Annual usage') }} +
+ + {% set current_year = current_year or (now().year if now().month < 4 else now().year + 1) %} + {{ _('resets on April 1, ') ~ current_year }} + +

+
+
+ {{ remaining_messages(header=_('emails'), total=current_service.email_annual_limit, used=statistics_annual['email'], muted=true) }} +
+
+ {{ remaining_messages(header=_('text messages'), total=current_service.sms_annual_limit, used=statistics_annual['sms'], muted=true) }} +
+
+ {{ show_more(url_for('.monthly', service_id=current_service.id), _('Visit usage report')) }} +
+ diff --git a/app/templates/views/dashboard/_totals_daily.html b/app/templates/views/dashboard/_totals_daily.html index 8b9d9e0dcb..98195f1a5d 100644 --- a/app/templates/views/dashboard/_totals_daily.html +++ b/app/templates/views/dashboard/_totals_daily.html @@ -1,8 +1,29 @@ {% from "components/big-number.html" import big_number %} {% from "components/message-count-label.html" import message_count_label %} {% from 'components/remaining-messages.html' import remaining_messages %} +{% from "components/show-more.html" import show_more %} -
+{% if config["FF_ANNUAL_LIMIT"] %} +
+

+ {{ _('Daily usage') }} +
+ + {{ _('resets at 7pm Eastern Time') }} + +

+
+
+ {{ remaining_messages(header=_('emails'), total=current_service.message_limit, used=statistics['email']['requested'], muted=true) }} +
+
+ {{ remaining_messages(header=_('text messages'), total=current_service.sms_daily_limit, used=statistics['sms']['requested'], muted=true) }} +
+
+ {{ show_more(url_for('main.contact'), _('Request a daily limit increase')) }} +
+{% else %} +

{{ _('Daily usage') }}
@@ -18,4 +39,5 @@

{{ remaining_messages(header=_('text messages'), total=current_service.sms_daily_limit, used=statistics['sms']['requested']) }}

-
+ +{% endif %} diff --git a/app/templates/views/dashboard/dashboard.html b/app/templates/views/dashboard/dashboard.html index 110a94f241..4e005d86bb 100644 --- a/app/templates/views/dashboard/dashboard.html +++ b/app/templates/views/dashboard/dashboard.html @@ -26,7 +26,10 @@

{{ _("Scheduled sends") }}

{{ ajax_block(partials, updates_url, 'weekly_totals', interval=5) }} {{ ajax_block(partials, updates_url, 'daily_totals', interval=5) }} - + {% if config["FF_ANNUAL_LIMIT"] %} + {{ ajax_block(partials, updates_url, 'annual_totals', interval=5) }} + {% endif %} +
{% if partials['has_template_statistics'] %} diff --git a/app/templates/views/dashboard/monthly.html b/app/templates/views/dashboard/monthly.html index 7aacbb685a..5fe77f7253 100644 --- a/app/templates/views/dashboard/monthly.html +++ b/app/templates/views/dashboard/monthly.html @@ -1,22 +1,23 @@ -{% from "components/big-number.html" import big_number_with_status, big_number %} +{% from "components/big-number.html" import big_number_with_status, big_number, big_number_simple %} {% from "components/pill.html" import pill %} {% from "components/table.html" import list_table, field, hidden_field_heading, right_aligned_field_heading, row_heading %} {% from "components/message-count-label.html" import message_count_label %} +{% from 'components/remaining-messages.html' import remaining_messages %} {% extends "admin_template.html" %} {% block service_page_title %} - {{ _('Messages sent,') }} + {{ _('Usage report') }} {{ selected_year }} {{ _('to') }} {{ selected_year + 1 }} {{ _('fiscal year') }} {% endblock %} {% block maincolumn_content %}

- {{ _('Messages sent') }} + {{ _('Usage report') }}

-
+
{{ pill( items=years, current_value=selected_year, @@ -25,6 +26,50 @@

) }}

+ {% if config["FF_ANNUAL_LIMIT"] %} +

+ {% if selected_year == current_financial_year %} + {{ _('Annual limit overview') }} + {% else %} + {{ _('Annual overview') }} + {% endif %} +
+ + {{ _('Fiscal year begins April 1, ') ~ selected_year ~ _(' and ends March 31, ') ~ (selected_year + 1) }} + +

+
+ {% if selected_year == current_financial_year %} +
+ {{ remaining_messages(header=_('emails'), total=current_service.email_annual_limit, used=annual_data['email']) }} +
+
+ {{ remaining_messages(header=_('text messages'), total=current_service.sms_annual_limit, used=annual_data['sms']) }} +
+ {% else %} +
+ {{ big_number_simple( + annual_data['email'], + _('emails'), + + ) + }} +
+
+ {{ big_number_simple( + annual_data['sms'], + _('text messages'), + + ) + }} +
+ {% endif %} +
+

+ {{ _('Month by month totals') }} +

+ {% endif %} + {% if months %} {% set spend_txt = _('Total spend') %} {% set heading_1 = _('Month') %} diff --git a/app/translations/csv/fr.csv b/app/translations/csv/fr.csv index 1485bc4300..57aead4f21 100644 --- a/app/translations/csv/fr.csv +++ b/app/translations/csv/fr.csv @@ -2027,4 +2027,4 @@ "Annual usage","Utilisation annuelle" "resets at 7pm Eastern Time","Réinitialisation à 19 h, heure de l’Est" "Visit usage report","Consulter le rapport d’utilisation" -"Month by month totals","Totaux mensuels" +"Month by month totals","Totaux mensuels" \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 80bf69fe26..005883ce93 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1552,7 +1552,7 @@ requests = ">=2.0.0" [[package]] name = "notifications-utils" -version = "52.3.5" +version = "52.3.9" description = "Shared python code for Notification - Provides logging utils etc." optional = false python-versions = "~3.10.9" @@ -1588,8 +1588,8 @@ werkzeug = "3.0.4" [package.source] type = "git" url = "https://github.com/cds-snc/notifier-utils.git" -reference = "52.3.5" -resolved_reference = "953ee170b4c47465bef047f1060d17a7702edeeb" +reference = "52.3.9" +resolved_reference = "b344e5a74c79a8fa8ca4f722691850ac0d277959" [[package]] name = "openpyxl" @@ -2758,4 +2758,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "~3.10.9" -content-hash = "587e62c5c8f700ef83c6a39d3faca6dde4d10c92ee627c771ecdf538b405a77f" +content-hash = "443df8a67497588c1801bfac747fde95ecaffda93675b6f038906750e891316b" diff --git a/pyproject.toml b/pyproject.toml index 65718753e0..e44d4352e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ unidecode = "^1.3.8" # PaaS awscli-cwlogs = "^1.4.6" -notifications-utils = { git = "https://github.com/cds-snc/notifier-utils.git", tag = "52.3.5" } +notifications-utils = { git = "https://github.com/cds-snc/notifier-utils.git", tag = "52.3.9" } # Pinned dependencies diff --git a/tests/app/main/views/test_dashboard.py b/tests/app/main/views/test_dashboard.py index b87baa165b..a00980bc6c 100644 --- a/tests/app/main/views/test_dashboard.py +++ b/tests/app/main/views/test_dashboard.py @@ -1,7 +1,9 @@ import copy import re +from unittest.mock import ANY import pytest +from bs4 import BeautifulSoup from flask import url_for from freezegun import freeze_time @@ -21,6 +23,7 @@ create_active_caseworking_user, create_active_user_view_permissions, normalize_spaces, + set_config, ) stub_template_stats = [ @@ -137,6 +140,7 @@ def test_task_shortcuts_are_visible_based_on_permissions( mock_get_service_templates, mock_get_jobs, mock_get_template_statistics, + mock_get_service_statistics, permissions: list, text_in_page: list, text_not_in_page: list, @@ -170,6 +174,7 @@ def test_survey_widget_presence( mock_get_service_templates, mock_get_jobs, mock_get_template_statistics, + mock_get_service_statistics, mocker, admin_url, is_widget_present, @@ -193,6 +198,7 @@ def test_sending_link_has_query_param( mock_get_service_templates, mock_get_jobs, mock_get_template_statistics, + mock_get_service_statistics, ): active_user_with_permissions["permissions"][SERVICE_ONE_ID] = ["view_activity", "send_messages"] client_request.login(active_user_with_permissions) @@ -209,6 +215,7 @@ def test_no_sending_link_if_no_templates( client_request: ClientRequest, mock_get_service_templates_when_no_templates_exist, mock_get_template_statistics, + mock_get_service_statistics, mock_get_jobs, ): page = client_request.get("main.service_dashboard", service_id=SERVICE_ONE_ID) @@ -305,11 +312,7 @@ def test_should_show_monthly_breakdown_of_template_usage( def test_anyone_can_see_monthly_breakdown( - client, - api_user_active, - service_one, - mocker, - mock_get_monthly_notification_stats, + client, api_user_active, service_one, mocker, mock_get_monthly_notification_stats, mock_get_service_statistics ): validate_route_permission_with_client( mocker, @@ -324,16 +327,14 @@ def test_anyone_can_see_monthly_breakdown( def test_monthly_shows_letters_in_breakdown( - client_request, - service_one, - mock_get_monthly_notification_stats, + client_request, service_one, mock_get_monthly_notification_stats, mock_get_service_statistics ): page = client_request.get("main.monthly", service_id=service_one["id"]) columns = page.select(".table-field-left-aligned .big-number-label") - assert normalize_spaces(columns[0].text) == "emails" - assert normalize_spaces(columns[1].text) == "text messages" + assert normalize_spaces(columns[2].text) == "emails" + assert normalize_spaces(columns[3].text) == "text messages" @pytest.mark.parametrize( @@ -345,10 +346,7 @@ def test_monthly_shows_letters_in_breakdown( ) @freeze_time("2015-01-01 15:15:15.000000") def test_stats_pages_show_last_3_years( - client_request, - endpoint, - mock_get_monthly_notification_stats, - mock_get_monthly_template_usage, + client_request, endpoint, mock_get_monthly_notification_stats, mock_get_monthly_template_usage, mock_get_service_statistics ): page = client_request.get( endpoint, @@ -361,9 +359,7 @@ def test_stats_pages_show_last_3_years( def test_monthly_has_equal_length_tables( - client_request, - service_one, - mock_get_monthly_notification_stats, + client_request, service_one, mock_get_monthly_notification_stats, mock_get_service_statistics ): page = client_request.get("main.monthly", service_id=service_one["id"]) @@ -401,31 +397,6 @@ def test_should_show_upcoming_jobs_on_dashboard( assert table_rows[1].find_all("td")[0].text.strip() == "Scheduled to send to 30 recipients" -@pytest.mark.parametrize( - "permissions, column_name, expected_column_count", - [ - (["email", "sms"], ".w-1\\/2", 6), - (["email", "sms"], ".w-1\\/2", 6), - ], -) -def test_correct_columns_display_on_dashboard_v15( - client_request: ClientRequest, - mock_get_service_templates, - mock_get_template_statistics, - mock_get_service_statistics, - mock_get_jobs, - service_one, - permissions, - expected_column_count, - column_name, - app_, -): - service_one["permissions"] = permissions - - page = client_request.get("main.service_dashboard", service_id=service_one["id"]) - assert len(page.select(column_name)) == expected_column_count - - def test_daily_usage_section_shown( client_request, mocker, @@ -1424,3 +1395,201 @@ def test_dashboard_daily_limits( ) == 2 ) + + +class TestAnnualLimits: + def test_daily_usage_uses_muted_component( + self, + logged_in_client, + mocker, + mock_get_service_templates_when_no_templates_exist, + mock_get_jobs, + mock_get_service_statistics, + mock_get_usage, + app_, + ): + with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED + mocker.patch( + "app.template_statistics_client.get_template_statistics_for_service", + return_value=copy.deepcopy(stub_template_stats), + ) + + url = url_for("main.service_dashboard", service_id=SERVICE_ONE_ID) + response = logged_in_client.get(url) + page = BeautifulSoup(response.data.decode("utf-8"), "html.parser") + + # ensure both email + sms widgets are muted + assert len(page.select("[data-testid='daily-usage'] .remaining-messages.muted")) == 2 + + def test_annual_usage_uses_muted_component( + self, + logged_in_client, + mocker, + mock_get_service_templates_when_no_templates_exist, + mock_get_jobs, + mock_get_service_statistics, + mock_get_usage, + app_, + ): + with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED + mocker.patch( + "app.template_statistics_client.get_template_statistics_for_service", + return_value=copy.deepcopy(stub_template_stats), + ) + + url = url_for("main.service_dashboard", service_id=SERVICE_ONE_ID) + response = logged_in_client.get(url) + page = BeautifulSoup(response.data.decode("utf-8"), "html.parser") + + # ensure both email + sms widgets are muted + assert len(page.select("[data-testid='annual-usage'] .remaining-messages.muted")) == 2 + + @freeze_time("2024-11-25 12:12:12") + @pytest.mark.parametrize( + "redis_daily_data, monthly_data, expected_data", + [ + ( + {"sms_delivered": 100, "email_delivered": 50, "sms_failed": 1000, "email_failed": 500}, + { + "data": { + "2024-04": {"sms": {}, "email": {}, "letter": {}}, + "2024-05": {"sms": {}, "email": {}, "letter": {}}, + "2024-06": {"sms": {}, "email": {}, "letter": {}}, + "2024-07": {"sms": {}, "email": {}, "letter": {}}, + "2024-08": {"sms": {}, "email": {}, "letter": {}}, + "2024-09": {"sms": {}, "email": {}, "letter": {}}, + "2024-10": { + "sms": {"delivered": 5, "permanent-failure": 50, "sending": 5, "technical-failure": 100}, + "email": {"delivered": 10, "permanent-failure": 110, "sending": 50, "technical-failure": 50}, + "letter": {}, + }, + "2024-11": { + "sms": {"delivered": 5, "permanent-failure": 50, "sending": 5, "technical-failure": 100}, + "email": {"delivered": 10, "permanent-failure": 110, "sending": 50, "technical-failure": 50}, + "letter": {}, + }, + } + }, + {"email": 990, "letter": 0, "sms": 1420}, + ), + ( + {"sms_delivered": 6, "email_delivered": 6, "sms_failed": 6, "email_failed": 6}, + { + "data": { + "2024-10": { + "sms": {"delivered": 6, "permanent-failure": 6, "sending": 6, "technical-failure": 6}, + "email": {"delivered": 6, "permanent-failure": 6, "sending": 6, "technical-failure": 6}, + "letter": {}, + }, + } + }, + {"email": 36, "letter": 0, "sms": 36}, + ), + ], + ) + def test_usage_report_aggregates_calculated_properly_with_redis( + self, + logged_in_client, + mocker, + mock_get_service_templates_when_no_templates_exist, + mock_get_jobs, + mock_get_service_statistics, + mock_get_usage, + app_, + redis_daily_data, + monthly_data, + expected_data, + ): + with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED + # mock annual_limit_client.get_all_notification_counts + mocker.patch( + "app.main.views.dashboard.annual_limit_client.get_all_notification_counts", + return_value=redis_daily_data, + ) + + mocker.patch( + "app.service_api_client.get_monthly_notification_stats", + return_value=copy.deepcopy(monthly_data), + ) + + mock_render_template = mocker.patch("app.main.views.dashboard.render_template") + + url = url_for("main.monthly", service_id=SERVICE_ONE_ID) + logged_in_client.get(url) + + mock_render_template.assert_called_with( + ANY, months=ANY, years=ANY, annual_data=expected_data, selected_year=ANY, current_financial_year=ANY + ) + + @freeze_time("2024-11-25 12:12:12") + @pytest.mark.parametrize( + "daily_data, monthly_data, expected_data", + [ + ( + { + "sms": {"requested": 100, "delivered": 50, "failed": 50}, + "email": {"requested": 100, "delivered": 50, "failed": 50}, + "letter": {"requested": 0, "delivered": 0, "failed": 0}, + }, + { + "data": { + "2024-04": {"sms": {}, "email": {}, "letter": {}}, + "2024-05": {"sms": {}, "email": {}, "letter": {}}, + "2024-06": {"sms": {}, "email": {}, "letter": {}}, + "2024-07": {"sms": {}, "email": {}, "letter": {}}, + "2024-08": {"sms": {}, "email": {}, "letter": {}}, + "2024-09": {"sms": {}, "email": {}, "letter": {}}, + "2024-10": { + "sms": {"delivered": 5, "permanent-failure": 50, "sending": 5, "technical-failure": 100}, + "email": {"delivered": 10, "permanent-failure": 110, "sending": 50, "technical-failure": 50}, + "letter": {}, + }, + "2024-11": { + "sms": {"delivered": 5, "permanent-failure": 50, "sending": 5, "technical-failure": 100}, + "email": {"delivered": 10, "permanent-failure": 110, "sending": 50, "technical-failure": 50}, + "letter": {}, + }, + } + }, + {"email": 540, "letter": 0, "sms": 420}, + ) + ], + ) + def test_usage_report_aggregates_calculated_properly_without_redis( + self, + logged_in_client, + mocker, + mock_get_service_templates_when_no_templates_exist, + mock_get_jobs, + mock_get_service_statistics, + mock_get_usage, + app_, + daily_data, + monthly_data, + expected_data, + ): + with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED + # mock annual_limit_client.get_all_notification_counts + mocker.patch( + "app.main.views.dashboard.annual_limit_client.get_all_notification_counts", + return_value=None, + ) + + mocker.patch( + "app.service_api_client.get_service_statistics", + return_value=copy.deepcopy(daily_data), + ) + + mocker.patch( + "app.service_api_client.get_monthly_notification_stats", + return_value=copy.deepcopy(monthly_data), + ) + + mock_render_template = mocker.patch("app.main.views.dashboard.render_template") + + url = url_for("main.monthly", service_id=SERVICE_ONE_ID) + logged_in_client.get(url) + + mock_render_template.assert_called_with( + ANY, months=ANY, years=ANY, annual_data=expected_data, selected_year=ANY, current_financial_year=ANY + ) diff --git a/tests/conftest.py b/tests/conftest.py index a0c964cb90..94ff0ca5be 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -578,9 +578,42 @@ def _get(service_id, today_only, limit_days=None): "letter": {"requested": 0, "delivered": 0, "failed": 0}, } + # mock these stats at the same time + def _get_monthly_stats(service_id, year): + return { + "data": { + "2024-04": {"sms": {}, "email": {}, "letter": {}}, + "2024-05": {"sms": {}, "email": {}, "letter": {}}, + "2024-06": {"sms": {}, "email": {}, "letter": {}}, + "2024-07": {"sms": {}, "email": {}, "letter": {}}, + "2024-08": {"sms": {}, "email": {}, "letter": {}}, + "2024-09": {"sms": {}, "email": {}, "letter": {}}, + "2024-10": {"sms": {}, "email": {}, "letter": {}}, + "2024-11": { + "sms": {"sent": 1}, + "email": {"delivered": 1, "permanent-failure": 1, "sending": 3, "technical-failure": 1}, + "letter": {}, + }, + } + } + + mocker.patch("app.service_api_client.get_monthly_notification_stats", side_effect=_get_monthly_stats) + return mocker.patch("app.service_api_client.get_service_statistics", side_effect=_get) +@pytest.fixture(scope="function") +def mock_get_annual_statistics(mocker, api_user_active): + def _get(service_id, year): + return { + "email": 100, + "sms": 200, + "letter": 300, + } + + return mocker.patch("app.service_api_client.get_monthly_notification_stats", side_effect=_get) + + @pytest.fixture(scope="function") def mock_get_detailed_services(mocker, fake_uuid): service_one = service_json( From 1d9da00b037d19b8c79871f4a1c47b68cecf609a Mon Sep 17 00:00:00 2001 From: Ben Larabie Date: Wed, 27 Nov 2024 09:00:00 -0500 Subject: [PATCH 04/20] Allowing manual runs on cypress tests (#2004) --- .github/workflows/cypress-staging.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cypress-staging.yaml b/.github/workflows/cypress-staging.yaml index 361009c393..9ecc28d1a2 100644 --- a/.github/workflows/cypress-staging.yaml +++ b/.github/workflows/cypress-staging.yaml @@ -3,6 +3,7 @@ name: Cypress staging a11y tests on: schedule: - cron: 0 */3 * * * + workflow_dispatch: defaults: run: shell: bash From e4f6a101680aca10d7243320196921ca90512a3f Mon Sep 17 00:00:00 2001 From: Ben Larabie Date: Wed, 27 Nov 2024 09:33:32 -0500 Subject: [PATCH 05/20] github secrets (#2003) --- .github/workflows/docker.yaml | 6 ++---- .github/workflows/test-admin-delete-unused.yaml | 4 ++-- .github/workflows/test-admin-deploy.yaml | 8 ++++---- .github/workflows/test-admin-remove.yaml | 4 ++-- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 1031535048..6c1871f36d 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -8,7 +8,6 @@ env: AWS_REGION: ca-central-1 DOCKER_ORG: public.ecr.aws/v6b8u5o6 DOCKER_SLUG: public.ecr.aws/v6b8u5o6/notify-admin - WORKFLOW_PAT: ${{ secrets.WORKFLOW_GITHUB_PAT }} OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} permissions: @@ -57,8 +56,8 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} aws-region: ca-central-1 - name: Install OpenVPN @@ -87,7 +86,6 @@ jobs: uses: "kota65535/github-openvpn-connect-action@cd2ed8a90cc7b060dc4e001143e811b5f7ea0af5" with: config_file: /var/tmp/staging.ovpn - client_key: ${{ secrets.STAGING_OVPN_CLIENT_KEY }} echo_config: false - name: Configure kubeconfig diff --git a/.github/workflows/test-admin-delete-unused.yaml b/.github/workflows/test-admin-delete-unused.yaml index 5beaa3611b..620526f00f 100644 --- a/.github/workflows/test-admin-delete-unused.yaml +++ b/.github/workflows/test-admin-delete-unused.yaml @@ -19,8 +19,8 @@ jobs: id: aws-creds uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} aws-region: ca-central-1 - name: Delete old PR review environments diff --git a/.github/workflows/test-admin-deploy.yaml b/.github/workflows/test-admin-deploy.yaml index c216e28f30..4428177b02 100644 --- a/.github/workflows/test-admin-deploy.yaml +++ b/.github/workflows/test-admin-deploy.yaml @@ -32,8 +32,8 @@ jobs: id: aws-creds uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} aws-region: ca-central-1 - name: Login to ECR @@ -79,8 +79,8 @@ jobs: id: aws-creds uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} aws-region: ca-central-1 - name: Create/Update lambda function diff --git a/.github/workflows/test-admin-remove.yaml b/.github/workflows/test-admin-remove.yaml index 5c3f0f2b28..c7381b3b0e 100644 --- a/.github/workflows/test-admin-remove.yaml +++ b/.github/workflows/test-admin-remove.yaml @@ -26,8 +26,8 @@ jobs: id: aws-creds uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} aws-region: ca-central-1 - name: Delete lambda function resources From 9b682f1f873c421c7617d213cbfe4346a0b533e6 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 27 Nov 2024 11:03:47 -0400 Subject: [PATCH 06/20] fix: remove the hardcoded FF instances from config (#2005) --- app/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/config.py b/app/config.py index 9b37a55798..99dab44c5b 100644 --- a/app/config.py +++ b/app/config.py @@ -188,7 +188,6 @@ class Development(Config): SESSION_PROTECTION = None SYSTEM_STATUS_URL = "https://localhost:3000" NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef" - FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", True) class Test(Development): @@ -252,7 +251,6 @@ class Production(Config): NOTIFY_LOG_LEVEL = "INFO" SYSTEM_STATUS_URL = "https://status.notification.canada.ca" NO_BRANDING_ID = "760c802a-7762-4f71-b19e-f93c66c92f1a" - FF_ANNUAL_LIMIT = False class Staging(Production): @@ -260,7 +258,6 @@ class Staging(Production): NOTIFY_LOG_LEVEL = "INFO" SYSTEM_STATUS_URL = "https://status.staging.notification.cdssandbox.xyz" NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef" - FF_ANNUAL_LIMIT = True class Scratch(Production): From 8ffc6490c3e50d58a56824628e6ae2647b7d9036 Mon Sep 17 00:00:00 2001 From: William B <7444334+whabanks@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:55:08 -0400 Subject: [PATCH 07/20] Add pre-commit hook for ruff (#1990) * Add pre-commit hook for ruff * Update lock file * Scope Ruff format-on-save to python files only * Disable format on save by default --- .devcontainer/devcontainer.json | 16 +++++++++------- .devcontainer/scripts/installations.sh | 3 +++ .pre-commit-config.yaml | 9 +++++++++ poetry.lock | 2 +- pyproject.toml | 1 + 5 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2f45fbc1f7..ce94cdc09a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -33,13 +33,15 @@ "vsliveshare.vsliveshare" ], "settings": { - "ruff.lint.run": "onSave", - "ruff.configurationPreference": "filesystemFirst", - "editor.defaultFormatter": "charliermarsh.ruff", - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.fixAll": "explicit" - } + "[python]": { + "ruff.lint.run": "onSave", + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": false, + "editor.codeActionsOnSave": { + "source.fixAll": "explicit" + } + }, + "ruff.configurationPreference": "filesystemFirst" } } }, diff --git a/.devcontainer/scripts/installations.sh b/.devcontainer/scripts/installations.sh index 7151480947..5a8ee3aa36 100644 --- a/.devcontainer/scripts/installations.sh +++ b/.devcontainer/scripts/installations.sh @@ -38,6 +38,9 @@ poetry completions zsh > ~/.zfunc/_poetry cd /workspace poetry install +# Install pre-commit hooks +poetry run pre-commit install + # Poe the Poet plugin tab completions touch ~/.zfunc/_poe poetry run poe _zsh_completion > ~/.zfunc/_poe diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..2f775686d5 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.5 + hooks: + # Run the linter + - id: ruff + args: [ --fix ] + # Run the formatter + - id: ruff-format \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 005883ce93..39aba7fe69 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2758,4 +2758,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "~3.10.9" -content-hash = "443df8a67497588c1801bfac747fde95ecaffda93675b6f038906750e891316b" +content-hash = "94c0fc0df7115b07d314521cd48aefc25a5bee81c7b0fa2477a86bb4db3ede81" diff --git a/pyproject.toml b/pyproject.toml index e44d4352e3..662a8a46ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ jinja2-cli = { version = "^0.8.2", extras = ["yaml"] } mypy = "1.11.2" monkeytype = "23.3.0" poethepoet = "^0.24.4" +pre-commit = "^3.7.1" ruff = "^0.6.9" # stubs libraries to keep mypy happy From d3887e5c6c77509f721bb9e55b64a86f07303d6c Mon Sep 17 00:00:00 2001 From: Andrew Date: Tue, 3 Dec 2024 11:24:53 -0400 Subject: [PATCH 08/20] feat: validate annual limit (#2002) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: validate annual limit * fix: modify wrapper to work with older redis keys * test: add tests! * chore: `utcnow` is deprecated, use `now` instead * feat(annual limits): show error if user is over annual limits * fix: move daily limit message into the appropriate template * feat: add a client to get daily/yearly sent stats using caching where possible * add translations * chore: formatting * fix send to work without FF too * feat: fix sending; write tests; 🙏 * fix: only check for `send_exceeds_annual_limit` when FF is on * fix(tests): only run the new tests with the FF on (they cant pass with it off!) * chore: remove unused imports * fix: move secondary message outside banner * fix: undo ruff change made by accident --------- Co-authored-by: William B <7444334+whabanks@users.noreply.github.com> --- app/__init__.py | 2 +- app/main/views/send.py | 40 +++- .../notification_counts_client.py | 75 ++++++ .../template_statistics_api_client.py | 46 ++++ .../check/too-many-email-messages.html | 22 +- .../check/too-many-sms-message-parts.html | 2 +- app/templates/views/check/column-errors.html | 19 +- app/translations/csv/fr.csv | 6 +- gunicorn_config.py | 1 + tests/app/main/views/test_send.py | 223 +++++++++++++++++- .../test_notification_counts_client.py | 89 +++++++ tests/conftest.py | 28 ++- 12 files changed, 526 insertions(+), 27 deletions(-) create mode 100644 app/notify_client/notification_counts_client.py create mode 100644 tests/app/notify_client/test_notification_counts_client.py diff --git a/app/__init__.py b/app/__init__.py index 0025ac021d..6e42f80bc9 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -226,7 +226,7 @@ def get_locale(): application.jinja_env.globals["show_tou_prompt"] = show_tou_prompt application.jinja_env.globals["parse_ua"] = parse application.jinja_env.globals["events_key"] = EVENTS_KEY - application.jinja_env.globals["now"] = datetime.utcnow + application.jinja_env.globals["now"] = datetime.now # Initialize Salesforce Account list if application.config["FF_SALESFORCE_CONTACT"]: diff --git a/app/main/views/send.py b/app/main/views/send.py index 2a5720e38b..e96c3f4f1d 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -1,5 +1,6 @@ import itertools import json +from datetime import datetime, timezone from string import ascii_uppercase from zipfile import BadZipFile @@ -53,6 +54,7 @@ ) from app.main.views.dashboard import aggregate_notifications_stats from app.models.user import Users +from app.notify_client.notification_counts_client import notification_counts_client from app.s3_client.s3_csv_client import ( copy_bulk_send_file_to_uploads, list_bulk_send_uploads, @@ -649,8 +651,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_ sms_fragments_sent_today = daily_sms_fragment_count(service_id) emails_sent_today = daily_email_count(service_id) - remaining_sms_message_fragments = current_service.sms_daily_limit - sms_fragments_sent_today - remaining_email_messages = current_service.message_limit - emails_sent_today + remaining_sms_message_fragments_today = current_service.sms_daily_limit - sms_fragments_sent_today + remaining_email_messages_today = current_service.message_limit - emails_sent_today contents = s3download(service_id, upload_id) @@ -659,7 +661,7 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_ email_reply_to = None sms_sender = None recipients_remaining_messages = ( - remaining_email_messages if db_template["template_type"] == "email" else remaining_sms_message_fragments + remaining_email_messages_today if db_template["template_type"] == "email" else remaining_sms_message_fragments_today ) if db_template["template_type"] == "email": @@ -743,8 +745,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_ original_file_name=request.args.get("original_file_name", ""), upload_id=upload_id, form=CsvUploadForm(), - remaining_messages=remaining_email_messages, - remaining_sms_message_fragments=remaining_sms_message_fragments, + remaining_messages=remaining_email_messages_today, + remaining_sms_message_fragments=remaining_sms_message_fragments_today, sms_parts_to_send=sms_parts_to_send, is_sms_parts_estimated=is_sms_parts_estimated, choose_time_form=choose_time_form, @@ -783,7 +785,29 @@ def check_messages(service_id, template_id, upload_id, row_index=2): data["original_file_name"] = SanitiseASCII.encode(data.get("original_file_name", "")) data["sms_parts_requested"] = data["stats_daily"]["sms"]["requested"] data["sms_parts_remaining"] = current_service.sms_daily_limit - daily_sms_fragment_count(service_id) - data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"] + + if current_app.config["FF_ANNUAL_LIMIT"]: + # Override the remaining messages counts with the remaining annual counts, if the latter are lower + stats_ytd = notification_counts_client.get_all_notification_counts_for_year(service_id, datetime.now(timezone.utc).year) + remaining_sms_this_year = current_service.sms_annual_limit - stats_ytd["sms"] + remaining_email_this_year = current_service.email_annual_limit - stats_ytd["email"] + + # Show annual limit validation over the daily one (even if both are true) + data["send_exceeds_annual_limit"] = False + data["send_exceeds_daily_limit"] = False + if data["template"].template_type == "email": + if remaining_email_this_year < data["count_of_recipients"]: + data["recipients_remaining_messages"] = remaining_email_this_year + data["send_exceeds_annual_limit"] = True + else: + if remaining_sms_this_year < data["count_of_recipients"]: + data["recipients_remaining_messages"] = remaining_sms_this_year + data["send_exceeds_annual_limit"] = True + else: + data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"] + + else: + data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"] if ( data["recipients"].too_many_rows @@ -804,6 +828,10 @@ def check_messages(service_id, template_id, upload_id, row_index=2): if data["send_exceeds_daily_limit"]: return render_template("views/check/column-errors.html", **data) + if current_app.config["FF_ANNUAL_LIMIT"]: + if data["send_exceeds_annual_limit"]: + return render_template("views/check/column-errors.html", **data) + metadata_kwargs = { "notification_count": data["count_of_recipients"], "template_id": str(template_id), diff --git a/app/notify_client/notification_counts_client.py b/app/notify_client/notification_counts_client.py new file mode 100644 index 0000000000..ae6e1a4930 --- /dev/null +++ b/app/notify_client/notification_counts_client.py @@ -0,0 +1,75 @@ +from notifications_utils.clients.redis import ( + email_daily_count_cache_key, + sms_daily_count_cache_key, +) + +from app import redis_client, service_api_client, template_statistics_client + + +class NotificationCounts: + def get_all_notification_counts_for_today(self, service_id): + # try to get today's stats from redis + todays_sms = redis_client.get(sms_daily_count_cache_key(service_id)) + todays_email = redis_client.get(email_daily_count_cache_key(service_id)) + + if todays_sms is not None and todays_email is not None: + return {"sms": todays_sms, "email": todays_email} + # fallback to the API if the stats are not in redis + else: + stats = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1) + transformed_stats = _aggregate_notifications_stats(stats) + + return transformed_stats + + def get_all_notification_counts_for_year(self, service_id, year): + """ + Get total number of notifications by type for the current service for the current year + + Return value: + { + 'sms': int, + 'email': int + } + + """ + stats_today = self.get_all_notification_counts_for_today(service_id) + stats_this_year = service_api_client.get_monthly_notification_stats(service_id, year)["data"] + stats_this_year = _aggregate_stats_from_service_api(stats_this_year) + # aggregate stats_today and stats_this_year + for template_type in ["sms", "email"]: + stats_this_year[template_type] += stats_today[template_type] + + return stats_this_year + + +# TODO: consolidate this function and other functions that transform the results of template_statistics_client calls +def _aggregate_notifications_stats(template_statistics): + template_statistics = _filter_out_cancelled_stats(template_statistics) + notifications = {"sms": 0, "email": 0} + for stat in template_statistics: + notifications[stat["template_type"]] += stat["count"] + + return notifications + + +def _filter_out_cancelled_stats(template_statistics): + return [s for s in template_statistics if s["status"] != "cancelled"] + + +def _aggregate_stats_from_service_api(stats): + """Aggregate monthly notification stats excluding cancelled""" + total_stats = {"sms": {}, "email": {}} + + for month_data in stats.values(): + for msg_type in ["sms", "email"]: + if msg_type in month_data: + for status, count in month_data[msg_type].items(): + if status != "cancelled": + if status not in total_stats[msg_type]: + total_stats[msg_type][status] = 0 + total_stats[msg_type][status] += count + + return {msg_type: sum(counts.values()) for msg_type, counts in total_stats.items()} + + +notification_counts_client = NotificationCounts() diff --git a/app/notify_client/template_statistics_api_client.py b/app/notify_client/template_statistics_api_client.py index 132f8938fa..fb6acbc138 100644 --- a/app/notify_client/template_statistics_api_client.py +++ b/app/notify_client/template_statistics_api_client.py @@ -1,4 +1,7 @@ +from itertools import groupby + from app.notify_client import NotifyAdminAPIClient +from app.utils import DELIVERED_STATUSES, FAILURE_STATUSES class TemplateStatisticsApiClient(NotifyAdminAPIClient): @@ -17,3 +20,46 @@ def get_template_statistics_for_template(self, service_id, template_id): template_statistics_client = TemplateStatisticsApiClient() + + +class TemplateStatistics: + def __init__(self, stats): + self.stats = stats + + def as_aggregates(self): + template_statistics = self._filter_out_cancelled_stats(self.stats) + notifications = { + template_type: {status: 0 for status in ("requested", "delivered", "failed")} for template_type in ["sms", "email"] + } + for stat in template_statistics: + notifications[stat["template_type"]]["requested"] += stat["count"] + if stat["status"] in DELIVERED_STATUSES: + notifications[stat["template_type"]]["delivered"] += stat["count"] + elif stat["status"] in FAILURE_STATUSES: + notifications[stat["template_type"]]["failed"] += stat["count"] + + return notifications + + def as_template_usage(self, sort_key="count"): + template_statistics = self._filter_out_cancelled_stats(self.stats) + templates = [] + for k, v in groupby( + sorted(template_statistics, key=lambda x: x["template_id"]), + key=lambda x: x["template_id"], + ): + template_stats = list(v) + + templates.append( + { + "template_id": k, + "template_name": template_stats[0]["template_name"], + "template_type": template_stats[0]["template_type"], + "is_precompiled_letter": template_stats[0]["is_precompiled_letter"], + "count": sum(s["count"] for s in template_stats), + } + ) + + return sorted(templates, key=lambda x: x[sort_key], reverse=True) + + def _filter_out_cancelled_stats(self, template_statistics): + return [s for s in template_statistics if s["status"] != "cancelled"] diff --git a/app/templates/partials/check/too-many-email-messages.html b/app/templates/partials/check/too-many-email-messages.html index 987846c472..4ead4e464b 100644 --- a/app/templates/partials/check/too-many-email-messages.html +++ b/app/templates/partials/check/too-many-email-messages.html @@ -1,9 +1,29 @@ +{% from "components/banner.html" import banner_wrapper %} {% from "components/links.html" import content_link %}

{%- if current_service.trial_mode %} {{ _("Your service is in trial mode. To send more messages, request to go live").format(url_for('main.request_to_go_live', service_id=current_service.id)) }} {% else %} - {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }} + {% if send_exceeds_annual_limit %} + +

{{ _('{} can only send {} more email messages until annual limit resets'.format(current_service.name, recipients_remaining_messages)) }}

+

+ {{ _('To send some of these messages now, edit the spreadsheet to {} recipients maximum. '.format(recipients_remaining_messages)) }} + {{ _('To send to recipients you removed, wait until April 1, {} or contact them some other way.'.format(now().year)) }} +

+ + {% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %} + {% call banner_wrapper(type='dangerous') %} + {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }} + {% endcall %} + +

{{ _('You cannot send all these email messages today') }}

+

+ {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], + content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}} +

+ {% endif %} + {%- endif -%}

\ No newline at end of file diff --git a/app/templates/partials/check/too-many-sms-message-parts.html b/app/templates/partials/check/too-many-sms-message-parts.html index 1000fe1ac5..8ceee20762 100644 --- a/app/templates/partials/check/too-many-sms-message-parts.html +++ b/app/templates/partials/check/too-many-sms-message-parts.html @@ -1,6 +1,6 @@ {% from "components/links.html" import content_link %} -

+

{%- if current_service.trial_mode %} {{ _("Your service is in trial mode. To send more messages, request to go live").format(url_for('main.request_to_go_live', service_id=current_service.id)) }} {% else %} diff --git a/app/templates/views/check/column-errors.html b/app/templates/views/check/column-errors.html index 32f2ae9e8e..852d30f0bd 100644 --- a/app/templates/views/check/column-errors.html +++ b/app/templates/views/check/column-errors.html @@ -10,7 +10,9 @@ {% set prefix_txt = _('a column called') %} {% set prefix_plural_txt = _('columns called') %} -{% if send_exceeds_daily_limit and (sms_parts_remaining <= 0) %} +{% if send_exceeds_annual_limit %} + {% set page_title = _('These messages exceed the annual limit') %} +{% elif send_exceeds_daily_limit and (sms_parts_remaining <= 0) %} {% set page_title = _('These messages exceed your daily limit') %} {% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %} {% set page_title = _('These messages exceed your daily limit') %} @@ -168,20 +170,9 @@

{{ _('You cannot send all these text messages today') {% call banner_wrapper(type='dangerous') %} {% include "partials/check/too-many-email-messages.html" %} {% endcall %} - {% elif recipients.more_rows_than_can_send %} - {% call banner_wrapper(type='dangerous') %} - {% include "partials/check/too-many-email-messages.html" %} - {% endcall %} -

{{ _('You cannot send all these email messages today') }}

-

- {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], - content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}} -

- - + {% elif recipients.more_rows_than_can_send or send_exceeds_annual_limit %} + {% include "partials/check/too-many-email-messages.html" %} {% endif %} - -
{% if not send_exceeds_daily_limit %} diff --git a/app/translations/csv/fr.csv b/app/translations/csv/fr.csv index 57aead4f21..fe2c41b635 100644 --- a/app/translations/csv/fr.csv +++ b/app/translations/csv/fr.csv @@ -2027,4 +2027,8 @@ "Annual usage","Utilisation annuelle" "resets at 7pm Eastern Time","Réinitialisation à 19 h, heure de l’Est" "Visit usage report","Consulter le rapport d’utilisation" -"Month by month totals","Totaux mensuels" \ No newline at end of file +"Month by month totals","Totaux mensuels" +"{} can only send {} more email messages until annual limit resets","FR: {} can only send {} more email messages until annual limit resets" +"To send some of these messages now, edit the spreadsheet to {} recipients maximum. ","FR: To send some of these messages now, edit the spreadsheet to {} recipients maximum. " +"To send to recipients you removed, wait until April 1, {} or contact them some other way.","FR: To send to recipients you removed, wait until April 1, {} or contact them some other way" +"These messages exceed the annual limit","FR: These messages exceed the annual limit" \ No newline at end of file diff --git a/gunicorn_config.py b/gunicorn_config.py index 63325ca386..cf6e5e7112 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -51,6 +51,7 @@ # Start timer for total running time start_time = time.time() + def on_starting(server): server.log.info("Starting Notifications Admin") diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 78df4b10cc..c1350bdb46 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -6,6 +6,7 @@ from io import BytesIO from itertools import repeat from os import path +from unittest.mock import patch from uuid import uuid4 from zipfile import BadZipFile @@ -41,6 +42,7 @@ mock_get_service_letter_template, mock_get_service_template, normalize_spaces, + set_config, ) template_types = ["email", "sms"] @@ -2543,6 +2545,7 @@ def test_check_messages_shows_too_many_sms_messages_errors( mock_get_jobs, mock_s3_download, mock_s3_set_metadata, + mock_notification_counts_client, fake_uuid, num_requested, expected_msg, @@ -2560,6 +2563,10 @@ def test_check_messages_shows_too_many_sms_messages_errors( }, ) + mock_notification_counts_client.get_all_notification_counts_for_year.return_value = { + "sms": 0, + "email": 0, + } with client_request.session_transaction() as session: session["file_uploads"] = { fake_uuid: { @@ -2584,6 +2591,30 @@ def test_check_messages_shows_too_many_sms_messages_errors( assert details == expected_msg +@pytest.fixture +def mock_notification_counts_client(): + with patch("app.main.views.send.notification_counts_client") as mock: + yield mock + + +@pytest.fixture +def mock_daily_sms_fragment_count(): + with patch("app.main.views.send.daily_sms_fragment_count") as mock: + yield mock + + +@pytest.fixture +def mock_daily_email_count(): + with patch("app.main.views.send.daily_email_count") as mock: + yield mock + + +@pytest.fixture +def mock_get_service_template_annual_limits(): + with patch("app.service_api_client.get_service_template") as mock: + yield mock + + @pytest.mark.parametrize( "num_requested,expected_msg", [ @@ -2601,6 +2632,7 @@ def test_check_messages_shows_too_many_email_messages_errors( mock_get_template_statistics, mock_get_job_doesnt_exist, mock_get_jobs, + mock_notification_counts_client, fake_uuid, num_requested, expected_msg, @@ -2617,7 +2649,10 @@ def test_check_messages_shows_too_many_email_messages_errors( "email": {"requested": num_requested, "delivered": 0, "failed": 0}, }, ) - + mock_notification_counts_client.get_all_notification_counts_for_year.return_value = { + "sms": 0, + "email": 0, + } with client_request.session_transaction() as session: session["file_uploads"] = { fake_uuid: { @@ -3401,3 +3436,189 @@ class Object(object): multiple_choise_options = [x.text.strip() for x in options] assert multiple_choise_options == expected_filenames + + +class TestAnnualLimitsSend: + @pytest.mark.parametrize( + "num_being_sent, num_sent_today, num_sent_this_year, expect_to_see_annual_limit_msg, expect_to_see_daily_limit_msg", + [ + # annual limit for mock_get_live_service is 10,000email/10,000sms + # daily limit for mock_get_live_service is 1,000email/1,000sms + # 1000 have already been sent today, trying to send 100 more [over both limits] + (100, 1000, 10000, True, False), + # No sent yet today or this year, trying to send 1001 [over both limits] + (10001, 0, 0, True, False), + # 600 have already been sent this year, trying to send 500 more [over annual limit but not daily] + (500, 0, 9600, True, False), + # No sent yet today or this year, trying to send 1001 [over daily limit but not annual] + (1001, 0, 0, False, True), + # No sent yet today or this year, trying to send 100 [over neither limit] + (100, 0, 0, False, False), + ], + ids=[ + "email_over_both_limits", + "email_over_both_limits2", + "email_over_annual_but_not_daily", + "email_over_daily_but_not_annual", + "email_over_neither", + ], + ) + def test_email_send_fails_approrpiately_when_over_limits( + self, + mocker, + client_request, + mock_get_live_service, # set email_annual_limit and sms_annual_limit to 1000 + mock_get_users_by_service, + mock_get_service_email_template_without_placeholders, + mock_get_template_statistics, + mock_get_job_doesnt_exist, + mock_get_jobs, + mock_s3_set_metadata, + mock_notification_counts_client, + mock_daily_sms_fragment_count, + mock_daily_email_count, + fake_uuid, + num_being_sent, + num_sent_today, + num_sent_this_year, + expect_to_see_annual_limit_msg, + expect_to_see_daily_limit_msg, + app_, + ): + with set_config(app_, "FF_ANNUAL_LIMIT", True): + mocker.patch( + "app.main.views.send.s3download", + return_value=",\n".join( + ["email address"] + ([mock_get_users_by_service(None)[0]["email_address"]] * num_being_sent) + ), + ) + + # mock that `num_sent_this_year` have already been sent this year + mock_notification_counts_client.get_all_notification_counts_for_year.return_value = { + "sms": 1000, # not used in test but needs a value + "email": num_sent_this_year, + } + + # mock that we've already sent `emails_sent_today` emails today + mock_daily_email_count.return_value = num_sent_today + mock_daily_sms_fragment_count.return_value = 900 # not used in test but needs a value + + with client_request.session_transaction() as session: + session["file_uploads"] = { + fake_uuid: { + "template_id": fake_uuid, + "notification_count": 1, + "valid": True, + } + } + + page = client_request.get( + "main.check_messages", + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + upload_id=fake_uuid, + original_file_name="valid.csv", + _test_page_title=False, + ) + + if expect_to_see_annual_limit_msg: + assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None + else: + assert page.find(attrs={"data-testid": "exceeds-annual"}) is None + + if expect_to_see_daily_limit_msg: + assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None + else: + assert page.find(attrs={"data-testid": "exceeds-daily"}) is None + + @pytest.mark.parametrize( + "num_being_sent, num_sent_today, num_sent_this_year, expect_to_see_annual_limit_msg, expect_to_see_daily_limit_msg", + [ + # annual limit for mock_get_live_service is 10,000email/10,000sms + # daily limit for mock_get_live_service is 1,000email/1,000sms + # 1000 have already been sent today, trying to send 100 more [over both limits] + (100, 1000, 10000, True, False), + # No sent yet today or this year, trying to send 1001 [over both limits] + (10001, 0, 0, True, False), + # 600 have already been sent this year, trying to send 500 more [over annual limit but not daily] + (500, 0, 9600, True, False), + # No sent yet today or this year, trying to send 1001 [over daily limit but not annual] + (1001, 0, 0, False, True), + # No sent yet today or this year, trying to send 100 [over neither limit] + (100, 0, 0, False, False), + ], + ids=[ + "sms_over_both_limits", + "sms_over_both_limits2", + "sms_over_annual_but_not_daily", + "sms_over_daily_but_not_annual", + "sms_over_neither", + ], + ) + def test_sms_send_fails_approrpiately_when_over_limits( + self, + mocker, + client_request, + mock_get_live_service, # set email_annual_limit and sms_annual_limit to 1000 + mock_get_users_by_service, + mock_get_service_sms_template_without_placeholders, + mock_get_template_statistics, + mock_get_job_doesnt_exist, + mock_get_jobs, + mock_s3_set_metadata, + mock_notification_counts_client, + mock_daily_sms_fragment_count, + mock_daily_email_count, + fake_uuid, + num_being_sent, + num_sent_today, + num_sent_this_year, + expect_to_see_annual_limit_msg, + expect_to_see_daily_limit_msg, + app_, + ): + with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED + mocker.patch( + "app.main.views.send.s3download", + return_value=",\n".join( + ["phone number"] + ([mock_get_users_by_service(None)[0]["mobile_number"]] * num_being_sent) + ), + ) + + # mock that `num_sent_this_year` have already been sent this year + mock_notification_counts_client.get_all_notification_counts_for_year.return_value = { + "sms": num_sent_this_year, + "email": 1000, # not used in test but needs a value + } + + # mock that we've already sent `emails_sent_today` emails today + mock_daily_email_count.return_value = 900 # not used in test but needs a value + mock_daily_sms_fragment_count.return_value = num_sent_today + + with client_request.session_transaction() as session: + session["file_uploads"] = { + fake_uuid: { + "template_id": fake_uuid, + "notification_count": 1, + "valid": True, + } + } + + page = client_request.get( + "main.check_messages", + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + upload_id=fake_uuid, + original_file_name="valid.csv", + _test_page_title=False, + ) + + if expect_to_see_annual_limit_msg: + assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None + else: + assert page.find(attrs={"data-testid": "exceeds-annual"}) is None + + if expect_to_see_daily_limit_msg: + assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None + else: + assert page.find(attrs={"data-testid": "exceeds-daily"}) is None diff --git a/tests/app/notify_client/test_notification_counts_client.py b/tests/app/notify_client/test_notification_counts_client.py new file mode 100644 index 0000000000..fa7183aea8 --- /dev/null +++ b/tests/app/notify_client/test_notification_counts_client.py @@ -0,0 +1,89 @@ +from unittest.mock import patch + +import pytest + +from app.notify_client.notification_counts_client import NotificationCounts + + +@pytest.fixture +def mock_redis(): + with patch("app.notify_client.notification_counts_client.redis_client") as mock: + yield mock + + +@pytest.fixture +def mock_template_stats(): + with patch("app.notify_client.notification_counts_client.template_statistics_client") as mock: + yield mock + + +@pytest.fixture +def mock_service_api(): + with patch("app.notify_client.notification_counts_client.service_api_client") as mock: + yield mock + + +class TestNotificationCounts: + def test_get_all_notification_counts_for_today_redis_has_data(self, mock_redis): + # Setup + mock_redis.get.side_effect = [5, 10] # sms, email + wrapper = NotificationCounts() + + # Execute + result = wrapper.get_all_notification_counts_for_today("service-123") + + # Assert + assert result == {"sms": 5, "email": 10} + assert mock_redis.get.call_count == 2 + + @pytest.mark.parametrize( + "redis_side_effect, expected_result", + [ + ([None, None], {"sms": 10, "email": 10}), + ([None, 10], {"sms": 10, "email": 10}), # Falls back to API if either is None + ([10, None], {"sms": 10, "email": 10}), # Falls back to API if either is None + ], + ) + def test_get_all_notification_counts_for_today_redis_missing_data( + self, mock_redis, mock_template_stats, redis_side_effect, expected_result + ): + # Setup + mock_redis.get.side_effect = redis_side_effect + mock_template_stats.get_template_statistics_for_service.return_value = [ + {"template_id": "a1", "template_type": "sms", "count": 3, "status": "delivered"}, + {"template_id": "a2", "template_type": "email", "count": 7, "status": "temporary-failure"}, + {"template_id": "a3", "template_type": "email", "count": 3, "status": "delivered"}, + {"template_id": "a4", "template_type": "sms", "count": 7, "status": "delivered"}, + ] + + wrapper = NotificationCounts() + + # Execute + result = wrapper.get_all_notification_counts_for_today("service-123") + + # Assert + assert result == {"sms": 10, "email": 10} + mock_template_stats.get_template_statistics_for_service.assert_called_once() + + def test_get_all_notification_counts_for_year(self, mock_service_api): + # Setup + mock_service_api.get_monthly_notification_stats.return_value = { + "data": { + "2024-01": { + "sms": {"sent": 1, "temporary-failure:": 22}, + "email": {"delivered": 1, "permanent-failure": 1, "sending": 12, "technical-failure": 1}, + }, + "2024-02": {"sms": {"sent": 1}, "email": {"delivered": 1}}, + } + } + wrapper = NotificationCounts() + + with patch.object(wrapper, "get_all_notification_counts_for_today") as mock_today: + mock_today.return_value = {"sms": 5, "email": 5} + + # Execute + result = wrapper.get_all_notification_counts_for_year("service-123", 2024) + + # Assert + assert result["sms"] == 29 # 1 + 22 + 1 + 5 + assert result["email"] == 21 # 1 + 1 + 12 + 1 + 1 + 5 diff --git a/tests/conftest.py b/tests/conftest.py index 94ff0ca5be..21ed1f782b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -550,7 +550,14 @@ def fake_uuid(): @pytest.fixture(scope="function") def mock_get_service(mocker, api_user_active): def _get(service_id): - service = service_json(service_id, users=[api_user_active["id"]], message_limit=50, sms_daily_limit=20) + service = service_json( + service_id, + users=[api_user_active["id"]], + message_limit=50, + sms_daily_limit=20, + email_annual_limit=1000, + sms_annual_limit=1000, + ) return {"data": service} return mocker.patch("app.service_api_client.get_service", side_effect=_get) @@ -675,7 +682,9 @@ def mock_service_email_from_is_unique(mocker): @pytest.fixture(scope="function") def mock_get_live_service(mocker, api_user_active): def _get(service_id): - service = service_json(service_id, users=[api_user_active["id"]], restricted=False) + service = service_json( + service_id, users=[api_user_active["id"]], restricted=False, sms_annual_limit=10000, email_annual_limit=10000 + ) return {"data": service} return mocker.patch("app.service_api_client.get_service", side_effect=_get) @@ -971,6 +980,21 @@ def _get(service_id, template_id, version=None): return mocker.patch("app.service_api_client.get_service_template", side_effect=_get) +@pytest.fixture(scope="function") +def mock_get_service_sms_template_without_placeholders(mocker): + def _get(service_id, template_id, version=None): + template = template_json( + service_id, + template_id, + "Two week reminder", + "sms", + "Yo.", + ) + return {"data": template} + + return mocker.patch("app.service_api_client.get_service_template", side_effect=_get) + + @pytest.fixture(scope="function") def mock_get_service_letter_template(mocker, content=None, subject=None, postage="second"): def _get(service_id, template_id, version=None, postage=postage): From 52dc3405a40224b49b4bd7fe159f639a9bc5c198 Mon Sep 17 00:00:00 2001 From: Ben Larabie Date: Tue, 3 Dec 2024 11:09:09 -0500 Subject: [PATCH 09/20] Admin workflow fix (#2006) * Admin workflow to use the correct secret for OP * commenting out sbom for now --- .github/workflows/docker.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 6c1871f36d..a6f8ceae91 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -8,7 +8,7 @@ env: AWS_REGION: ca-central-1 DOCKER_ORG: public.ecr.aws/v6b8u5o6 DOCKER_SLUG: public.ecr.aws/v6b8u5o6/notify-admin - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN_STAGING }} permissions: id-token: write # This is required for requesting the OIDC JWT @@ -108,15 +108,15 @@ jobs: env: TOKEN: ${{ steps.notify-pr-bot.outputs.token }} - - name: Docker generate SBOM - uses: cds-snc/security-tools/.github/actions/generate-sbom@34794baf2af592913bb5b51d8df4f8d0acc49b6f # v3.2.0 - env: - TRIVY_DB_REPOSITORY: ${{ vars.TRIVY_DB_REPOSITORY }} - with: - docker_image: "${{ env.DOCKER_SLUG }}:latest" - dockerfile_path: "ci/Dockerfile" - sbom_name: "notification-admin" - token: "${{ secrets.GITHUB_TOKEN }}" + #- name: Docker generate SBOM + # uses: cds-snc/security-tools/.github/actions/generate-sbom@34794baf2af592913bb5b51d8df4f8d0acc49b6f # v3.2.0 + # env: + # TRIVY_DB_REPOSITORY: ${{ vars.TRIVY_DB_REPOSITORY }} + # with: + # docker_image: "${{ env.DOCKER_SLUG }}:latest" + # dockerfile_path: "ci/Dockerfile" + # sbom_name: "notification-admin" + # token: "${{ secrets.GITHUB_TOKEN }}" - name: Notify Slack channel if this job failed if: ${{ failure() }} From 677f52f07eb528291da116a689fe9f3579535f9c Mon Sep 17 00:00:00 2001 From: Ben Larabie Date: Tue, 3 Dec 2024 11:41:42 -0500 Subject: [PATCH 10/20] adding docker sbom back (#2007) --- .github/workflows/docker.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index a6f8ceae91..71358535af 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -108,15 +108,15 @@ jobs: env: TOKEN: ${{ steps.notify-pr-bot.outputs.token }} - #- name: Docker generate SBOM - # uses: cds-snc/security-tools/.github/actions/generate-sbom@34794baf2af592913bb5b51d8df4f8d0acc49b6f # v3.2.0 - # env: - # TRIVY_DB_REPOSITORY: ${{ vars.TRIVY_DB_REPOSITORY }} - # with: - # docker_image: "${{ env.DOCKER_SLUG }}:latest" - # dockerfile_path: "ci/Dockerfile" - # sbom_name: "notification-admin" - # token: "${{ secrets.GITHUB_TOKEN }}" + - name: Docker generate SBOM + uses: cds-snc/security-tools/.github/actions/generate-sbom@34794baf2af592913bb5b51d8df4f8d0acc49b6f # v3.2.0 + env: + TRIVY_DB_REPOSITORY: ${{ vars.TRIVY_DB_REPOSITORY }} + with: + docker_image: "${{ env.DOCKER_SLUG }}:latest" + dockerfile_path: "ci/Dockerfile" + sbom_name: "notification-admin" + token: "${{ secrets.GITHUB_TOKEN }}" - name: Notify Slack channel if this job failed if: ${{ failure() }} From 10033ace21202ccd5217fc5998e67f3f2bcefebc Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 4 Dec 2024 10:41:20 -0400 Subject: [PATCH 11/20] Revert "feat: validate annual limit (#2002)" (#2009) This reverts commit d3887e5c6c77509f721bb9e55b64a86f07303d6c. --- app/__init__.py | 2 +- app/main/views/send.py | 40 +--- .../notification_counts_client.py | 75 ------ .../template_statistics_api_client.py | 46 ---- .../check/too-many-email-messages.html | 22 +- .../check/too-many-sms-message-parts.html | 2 +- app/templates/views/check/column-errors.html | 19 +- app/translations/csv/fr.csv | 6 +- gunicorn_config.py | 1 - tests/app/main/views/test_send.py | 223 +----------------- .../test_notification_counts_client.py | 89 ------- tests/conftest.py | 28 +-- 12 files changed, 27 insertions(+), 526 deletions(-) delete mode 100644 app/notify_client/notification_counts_client.py delete mode 100644 tests/app/notify_client/test_notification_counts_client.py diff --git a/app/__init__.py b/app/__init__.py index 6e42f80bc9..0025ac021d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -226,7 +226,7 @@ def get_locale(): application.jinja_env.globals["show_tou_prompt"] = show_tou_prompt application.jinja_env.globals["parse_ua"] = parse application.jinja_env.globals["events_key"] = EVENTS_KEY - application.jinja_env.globals["now"] = datetime.now + application.jinja_env.globals["now"] = datetime.utcnow # Initialize Salesforce Account list if application.config["FF_SALESFORCE_CONTACT"]: diff --git a/app/main/views/send.py b/app/main/views/send.py index e96c3f4f1d..2a5720e38b 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -1,6 +1,5 @@ import itertools import json -from datetime import datetime, timezone from string import ascii_uppercase from zipfile import BadZipFile @@ -54,7 +53,6 @@ ) from app.main.views.dashboard import aggregate_notifications_stats from app.models.user import Users -from app.notify_client.notification_counts_client import notification_counts_client from app.s3_client.s3_csv_client import ( copy_bulk_send_file_to_uploads, list_bulk_send_uploads, @@ -651,8 +649,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_ sms_fragments_sent_today = daily_sms_fragment_count(service_id) emails_sent_today = daily_email_count(service_id) - remaining_sms_message_fragments_today = current_service.sms_daily_limit - sms_fragments_sent_today - remaining_email_messages_today = current_service.message_limit - emails_sent_today + remaining_sms_message_fragments = current_service.sms_daily_limit - sms_fragments_sent_today + remaining_email_messages = current_service.message_limit - emails_sent_today contents = s3download(service_id, upload_id) @@ -661,7 +659,7 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_ email_reply_to = None sms_sender = None recipients_remaining_messages = ( - remaining_email_messages_today if db_template["template_type"] == "email" else remaining_sms_message_fragments_today + remaining_email_messages if db_template["template_type"] == "email" else remaining_sms_message_fragments ) if db_template["template_type"] == "email": @@ -745,8 +743,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_ original_file_name=request.args.get("original_file_name", ""), upload_id=upload_id, form=CsvUploadForm(), - remaining_messages=remaining_email_messages_today, - remaining_sms_message_fragments=remaining_sms_message_fragments_today, + remaining_messages=remaining_email_messages, + remaining_sms_message_fragments=remaining_sms_message_fragments, sms_parts_to_send=sms_parts_to_send, is_sms_parts_estimated=is_sms_parts_estimated, choose_time_form=choose_time_form, @@ -785,29 +783,7 @@ def check_messages(service_id, template_id, upload_id, row_index=2): data["original_file_name"] = SanitiseASCII.encode(data.get("original_file_name", "")) data["sms_parts_requested"] = data["stats_daily"]["sms"]["requested"] data["sms_parts_remaining"] = current_service.sms_daily_limit - daily_sms_fragment_count(service_id) - - if current_app.config["FF_ANNUAL_LIMIT"]: - # Override the remaining messages counts with the remaining annual counts, if the latter are lower - stats_ytd = notification_counts_client.get_all_notification_counts_for_year(service_id, datetime.now(timezone.utc).year) - remaining_sms_this_year = current_service.sms_annual_limit - stats_ytd["sms"] - remaining_email_this_year = current_service.email_annual_limit - stats_ytd["email"] - - # Show annual limit validation over the daily one (even if both are true) - data["send_exceeds_annual_limit"] = False - data["send_exceeds_daily_limit"] = False - if data["template"].template_type == "email": - if remaining_email_this_year < data["count_of_recipients"]: - data["recipients_remaining_messages"] = remaining_email_this_year - data["send_exceeds_annual_limit"] = True - else: - if remaining_sms_this_year < data["count_of_recipients"]: - data["recipients_remaining_messages"] = remaining_sms_this_year - data["send_exceeds_annual_limit"] = True - else: - data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"] - - else: - data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"] + data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"] if ( data["recipients"].too_many_rows @@ -828,10 +804,6 @@ def check_messages(service_id, template_id, upload_id, row_index=2): if data["send_exceeds_daily_limit"]: return render_template("views/check/column-errors.html", **data) - if current_app.config["FF_ANNUAL_LIMIT"]: - if data["send_exceeds_annual_limit"]: - return render_template("views/check/column-errors.html", **data) - metadata_kwargs = { "notification_count": data["count_of_recipients"], "template_id": str(template_id), diff --git a/app/notify_client/notification_counts_client.py b/app/notify_client/notification_counts_client.py deleted file mode 100644 index ae6e1a4930..0000000000 --- a/app/notify_client/notification_counts_client.py +++ /dev/null @@ -1,75 +0,0 @@ -from notifications_utils.clients.redis import ( - email_daily_count_cache_key, - sms_daily_count_cache_key, -) - -from app import redis_client, service_api_client, template_statistics_client - - -class NotificationCounts: - def get_all_notification_counts_for_today(self, service_id): - # try to get today's stats from redis - todays_sms = redis_client.get(sms_daily_count_cache_key(service_id)) - todays_email = redis_client.get(email_daily_count_cache_key(service_id)) - - if todays_sms is not None and todays_email is not None: - return {"sms": todays_sms, "email": todays_email} - # fallback to the API if the stats are not in redis - else: - stats = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1) - transformed_stats = _aggregate_notifications_stats(stats) - - return transformed_stats - - def get_all_notification_counts_for_year(self, service_id, year): - """ - Get total number of notifications by type for the current service for the current year - - Return value: - { - 'sms': int, - 'email': int - } - - """ - stats_today = self.get_all_notification_counts_for_today(service_id) - stats_this_year = service_api_client.get_monthly_notification_stats(service_id, year)["data"] - stats_this_year = _aggregate_stats_from_service_api(stats_this_year) - # aggregate stats_today and stats_this_year - for template_type in ["sms", "email"]: - stats_this_year[template_type] += stats_today[template_type] - - return stats_this_year - - -# TODO: consolidate this function and other functions that transform the results of template_statistics_client calls -def _aggregate_notifications_stats(template_statistics): - template_statistics = _filter_out_cancelled_stats(template_statistics) - notifications = {"sms": 0, "email": 0} - for stat in template_statistics: - notifications[stat["template_type"]] += stat["count"] - - return notifications - - -def _filter_out_cancelled_stats(template_statistics): - return [s for s in template_statistics if s["status"] != "cancelled"] - - -def _aggregate_stats_from_service_api(stats): - """Aggregate monthly notification stats excluding cancelled""" - total_stats = {"sms": {}, "email": {}} - - for month_data in stats.values(): - for msg_type in ["sms", "email"]: - if msg_type in month_data: - for status, count in month_data[msg_type].items(): - if status != "cancelled": - if status not in total_stats[msg_type]: - total_stats[msg_type][status] = 0 - total_stats[msg_type][status] += count - - return {msg_type: sum(counts.values()) for msg_type, counts in total_stats.items()} - - -notification_counts_client = NotificationCounts() diff --git a/app/notify_client/template_statistics_api_client.py b/app/notify_client/template_statistics_api_client.py index fb6acbc138..132f8938fa 100644 --- a/app/notify_client/template_statistics_api_client.py +++ b/app/notify_client/template_statistics_api_client.py @@ -1,7 +1,4 @@ -from itertools import groupby - from app.notify_client import NotifyAdminAPIClient -from app.utils import DELIVERED_STATUSES, FAILURE_STATUSES class TemplateStatisticsApiClient(NotifyAdminAPIClient): @@ -20,46 +17,3 @@ def get_template_statistics_for_template(self, service_id, template_id): template_statistics_client = TemplateStatisticsApiClient() - - -class TemplateStatistics: - def __init__(self, stats): - self.stats = stats - - def as_aggregates(self): - template_statistics = self._filter_out_cancelled_stats(self.stats) - notifications = { - template_type: {status: 0 for status in ("requested", "delivered", "failed")} for template_type in ["sms", "email"] - } - for stat in template_statistics: - notifications[stat["template_type"]]["requested"] += stat["count"] - if stat["status"] in DELIVERED_STATUSES: - notifications[stat["template_type"]]["delivered"] += stat["count"] - elif stat["status"] in FAILURE_STATUSES: - notifications[stat["template_type"]]["failed"] += stat["count"] - - return notifications - - def as_template_usage(self, sort_key="count"): - template_statistics = self._filter_out_cancelled_stats(self.stats) - templates = [] - for k, v in groupby( - sorted(template_statistics, key=lambda x: x["template_id"]), - key=lambda x: x["template_id"], - ): - template_stats = list(v) - - templates.append( - { - "template_id": k, - "template_name": template_stats[0]["template_name"], - "template_type": template_stats[0]["template_type"], - "is_precompiled_letter": template_stats[0]["is_precompiled_letter"], - "count": sum(s["count"] for s in template_stats), - } - ) - - return sorted(templates, key=lambda x: x[sort_key], reverse=True) - - def _filter_out_cancelled_stats(self, template_statistics): - return [s for s in template_statistics if s["status"] != "cancelled"] diff --git a/app/templates/partials/check/too-many-email-messages.html b/app/templates/partials/check/too-many-email-messages.html index 4ead4e464b..987846c472 100644 --- a/app/templates/partials/check/too-many-email-messages.html +++ b/app/templates/partials/check/too-many-email-messages.html @@ -1,29 +1,9 @@ -{% from "components/banner.html" import banner_wrapper %} {% from "components/links.html" import content_link %}

{%- if current_service.trial_mode %} {{ _("Your service is in trial mode. To send more messages, request to go live").format(url_for('main.request_to_go_live', service_id=current_service.id)) }} {% else %} - {% if send_exceeds_annual_limit %} - -

{{ _('{} can only send {} more email messages until annual limit resets'.format(current_service.name, recipients_remaining_messages)) }}

-

- {{ _('To send some of these messages now, edit the spreadsheet to {} recipients maximum. '.format(recipients_remaining_messages)) }} - {{ _('To send to recipients you removed, wait until April 1, {} or contact them some other way.'.format(now().year)) }} -

- - {% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %} - {% call banner_wrapper(type='dangerous') %} - {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }} - {% endcall %} - -

{{ _('You cannot send all these email messages today') }}

-

- {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], - content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}} -

- {% endif %} - + {{ _("To request a daily limit above {} emails, {}").format(current_service.message_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }} {%- endif -%}

\ No newline at end of file diff --git a/app/templates/partials/check/too-many-sms-message-parts.html b/app/templates/partials/check/too-many-sms-message-parts.html index 8ceee20762..1000fe1ac5 100644 --- a/app/templates/partials/check/too-many-sms-message-parts.html +++ b/app/templates/partials/check/too-many-sms-message-parts.html @@ -1,6 +1,6 @@ {% from "components/links.html" import content_link %} -

+

{%- if current_service.trial_mode %} {{ _("Your service is in trial mode. To send more messages, request to go live").format(url_for('main.request_to_go_live', service_id=current_service.id)) }} {% else %} diff --git a/app/templates/views/check/column-errors.html b/app/templates/views/check/column-errors.html index 852d30f0bd..32f2ae9e8e 100644 --- a/app/templates/views/check/column-errors.html +++ b/app/templates/views/check/column-errors.html @@ -10,9 +10,7 @@ {% set prefix_txt = _('a column called') %} {% set prefix_plural_txt = _('columns called') %} -{% if send_exceeds_annual_limit %} - {% set page_title = _('These messages exceed the annual limit') %} -{% elif send_exceeds_daily_limit and (sms_parts_remaining <= 0) %} +{% if send_exceeds_daily_limit and (sms_parts_remaining <= 0) %} {% set page_title = _('These messages exceed your daily limit') %} {% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %} {% set page_title = _('These messages exceed your daily limit') %} @@ -170,9 +168,20 @@

{{ _('You cannot send all these text messages today') {% call banner_wrapper(type='dangerous') %} {% include "partials/check/too-many-email-messages.html" %} {% endcall %} - {% elif recipients.more_rows_than_can_send or send_exceeds_annual_limit %} - {% include "partials/check/too-many-email-messages.html" %} + {% elif recipients.more_rows_than_can_send %} + {% call banner_wrapper(type='dangerous') %} + {% include "partials/check/too-many-email-messages.html" %} + {% endcall %} +

{{ _('You cannot send all these email messages today') }}

+

+ {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], + content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}} +

+ + {% endif %} + + {% if not send_exceeds_daily_limit %} diff --git a/app/translations/csv/fr.csv b/app/translations/csv/fr.csv index fe2c41b635..57aead4f21 100644 --- a/app/translations/csv/fr.csv +++ b/app/translations/csv/fr.csv @@ -2027,8 +2027,4 @@ "Annual usage","Utilisation annuelle" "resets at 7pm Eastern Time","Réinitialisation à 19 h, heure de l’Est" "Visit usage report","Consulter le rapport d’utilisation" -"Month by month totals","Totaux mensuels" -"{} can only send {} more email messages until annual limit resets","FR: {} can only send {} more email messages until annual limit resets" -"To send some of these messages now, edit the spreadsheet to {} recipients maximum. ","FR: To send some of these messages now, edit the spreadsheet to {} recipients maximum. " -"To send to recipients you removed, wait until April 1, {} or contact them some other way.","FR: To send to recipients you removed, wait until April 1, {} or contact them some other way" -"These messages exceed the annual limit","FR: These messages exceed the annual limit" \ No newline at end of file +"Month by month totals","Totaux mensuels" \ No newline at end of file diff --git a/gunicorn_config.py b/gunicorn_config.py index cf6e5e7112..63325ca386 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -51,7 +51,6 @@ # Start timer for total running time start_time = time.time() - def on_starting(server): server.log.info("Starting Notifications Admin") diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index c1350bdb46..78df4b10cc 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -6,7 +6,6 @@ from io import BytesIO from itertools import repeat from os import path -from unittest.mock import patch from uuid import uuid4 from zipfile import BadZipFile @@ -42,7 +41,6 @@ mock_get_service_letter_template, mock_get_service_template, normalize_spaces, - set_config, ) template_types = ["email", "sms"] @@ -2545,7 +2543,6 @@ def test_check_messages_shows_too_many_sms_messages_errors( mock_get_jobs, mock_s3_download, mock_s3_set_metadata, - mock_notification_counts_client, fake_uuid, num_requested, expected_msg, @@ -2563,10 +2560,6 @@ def test_check_messages_shows_too_many_sms_messages_errors( }, ) - mock_notification_counts_client.get_all_notification_counts_for_year.return_value = { - "sms": 0, - "email": 0, - } with client_request.session_transaction() as session: session["file_uploads"] = { fake_uuid: { @@ -2591,30 +2584,6 @@ def test_check_messages_shows_too_many_sms_messages_errors( assert details == expected_msg -@pytest.fixture -def mock_notification_counts_client(): - with patch("app.main.views.send.notification_counts_client") as mock: - yield mock - - -@pytest.fixture -def mock_daily_sms_fragment_count(): - with patch("app.main.views.send.daily_sms_fragment_count") as mock: - yield mock - - -@pytest.fixture -def mock_daily_email_count(): - with patch("app.main.views.send.daily_email_count") as mock: - yield mock - - -@pytest.fixture -def mock_get_service_template_annual_limits(): - with patch("app.service_api_client.get_service_template") as mock: - yield mock - - @pytest.mark.parametrize( "num_requested,expected_msg", [ @@ -2632,7 +2601,6 @@ def test_check_messages_shows_too_many_email_messages_errors( mock_get_template_statistics, mock_get_job_doesnt_exist, mock_get_jobs, - mock_notification_counts_client, fake_uuid, num_requested, expected_msg, @@ -2649,10 +2617,7 @@ def test_check_messages_shows_too_many_email_messages_errors( "email": {"requested": num_requested, "delivered": 0, "failed": 0}, }, ) - mock_notification_counts_client.get_all_notification_counts_for_year.return_value = { - "sms": 0, - "email": 0, - } + with client_request.session_transaction() as session: session["file_uploads"] = { fake_uuid: { @@ -3436,189 +3401,3 @@ class Object(object): multiple_choise_options = [x.text.strip() for x in options] assert multiple_choise_options == expected_filenames - - -class TestAnnualLimitsSend: - @pytest.mark.parametrize( - "num_being_sent, num_sent_today, num_sent_this_year, expect_to_see_annual_limit_msg, expect_to_see_daily_limit_msg", - [ - # annual limit for mock_get_live_service is 10,000email/10,000sms - # daily limit for mock_get_live_service is 1,000email/1,000sms - # 1000 have already been sent today, trying to send 100 more [over both limits] - (100, 1000, 10000, True, False), - # No sent yet today or this year, trying to send 1001 [over both limits] - (10001, 0, 0, True, False), - # 600 have already been sent this year, trying to send 500 more [over annual limit but not daily] - (500, 0, 9600, True, False), - # No sent yet today or this year, trying to send 1001 [over daily limit but not annual] - (1001, 0, 0, False, True), - # No sent yet today or this year, trying to send 100 [over neither limit] - (100, 0, 0, False, False), - ], - ids=[ - "email_over_both_limits", - "email_over_both_limits2", - "email_over_annual_but_not_daily", - "email_over_daily_but_not_annual", - "email_over_neither", - ], - ) - def test_email_send_fails_approrpiately_when_over_limits( - self, - mocker, - client_request, - mock_get_live_service, # set email_annual_limit and sms_annual_limit to 1000 - mock_get_users_by_service, - mock_get_service_email_template_without_placeholders, - mock_get_template_statistics, - mock_get_job_doesnt_exist, - mock_get_jobs, - mock_s3_set_metadata, - mock_notification_counts_client, - mock_daily_sms_fragment_count, - mock_daily_email_count, - fake_uuid, - num_being_sent, - num_sent_today, - num_sent_this_year, - expect_to_see_annual_limit_msg, - expect_to_see_daily_limit_msg, - app_, - ): - with set_config(app_, "FF_ANNUAL_LIMIT", True): - mocker.patch( - "app.main.views.send.s3download", - return_value=",\n".join( - ["email address"] + ([mock_get_users_by_service(None)[0]["email_address"]] * num_being_sent) - ), - ) - - # mock that `num_sent_this_year` have already been sent this year - mock_notification_counts_client.get_all_notification_counts_for_year.return_value = { - "sms": 1000, # not used in test but needs a value - "email": num_sent_this_year, - } - - # mock that we've already sent `emails_sent_today` emails today - mock_daily_email_count.return_value = num_sent_today - mock_daily_sms_fragment_count.return_value = 900 # not used in test but needs a value - - with client_request.session_transaction() as session: - session["file_uploads"] = { - fake_uuid: { - "template_id": fake_uuid, - "notification_count": 1, - "valid": True, - } - } - - page = client_request.get( - "main.check_messages", - service_id=SERVICE_ONE_ID, - template_id=fake_uuid, - upload_id=fake_uuid, - original_file_name="valid.csv", - _test_page_title=False, - ) - - if expect_to_see_annual_limit_msg: - assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None - else: - assert page.find(attrs={"data-testid": "exceeds-annual"}) is None - - if expect_to_see_daily_limit_msg: - assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None - else: - assert page.find(attrs={"data-testid": "exceeds-daily"}) is None - - @pytest.mark.parametrize( - "num_being_sent, num_sent_today, num_sent_this_year, expect_to_see_annual_limit_msg, expect_to_see_daily_limit_msg", - [ - # annual limit for mock_get_live_service is 10,000email/10,000sms - # daily limit for mock_get_live_service is 1,000email/1,000sms - # 1000 have already been sent today, trying to send 100 more [over both limits] - (100, 1000, 10000, True, False), - # No sent yet today or this year, trying to send 1001 [over both limits] - (10001, 0, 0, True, False), - # 600 have already been sent this year, trying to send 500 more [over annual limit but not daily] - (500, 0, 9600, True, False), - # No sent yet today or this year, trying to send 1001 [over daily limit but not annual] - (1001, 0, 0, False, True), - # No sent yet today or this year, trying to send 100 [over neither limit] - (100, 0, 0, False, False), - ], - ids=[ - "sms_over_both_limits", - "sms_over_both_limits2", - "sms_over_annual_but_not_daily", - "sms_over_daily_but_not_annual", - "sms_over_neither", - ], - ) - def test_sms_send_fails_approrpiately_when_over_limits( - self, - mocker, - client_request, - mock_get_live_service, # set email_annual_limit and sms_annual_limit to 1000 - mock_get_users_by_service, - mock_get_service_sms_template_without_placeholders, - mock_get_template_statistics, - mock_get_job_doesnt_exist, - mock_get_jobs, - mock_s3_set_metadata, - mock_notification_counts_client, - mock_daily_sms_fragment_count, - mock_daily_email_count, - fake_uuid, - num_being_sent, - num_sent_today, - num_sent_this_year, - expect_to_see_annual_limit_msg, - expect_to_see_daily_limit_msg, - app_, - ): - with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED - mocker.patch( - "app.main.views.send.s3download", - return_value=",\n".join( - ["phone number"] + ([mock_get_users_by_service(None)[0]["mobile_number"]] * num_being_sent) - ), - ) - - # mock that `num_sent_this_year` have already been sent this year - mock_notification_counts_client.get_all_notification_counts_for_year.return_value = { - "sms": num_sent_this_year, - "email": 1000, # not used in test but needs a value - } - - # mock that we've already sent `emails_sent_today` emails today - mock_daily_email_count.return_value = 900 # not used in test but needs a value - mock_daily_sms_fragment_count.return_value = num_sent_today - - with client_request.session_transaction() as session: - session["file_uploads"] = { - fake_uuid: { - "template_id": fake_uuid, - "notification_count": 1, - "valid": True, - } - } - - page = client_request.get( - "main.check_messages", - service_id=SERVICE_ONE_ID, - template_id=fake_uuid, - upload_id=fake_uuid, - original_file_name="valid.csv", - _test_page_title=False, - ) - - if expect_to_see_annual_limit_msg: - assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None - else: - assert page.find(attrs={"data-testid": "exceeds-annual"}) is None - - if expect_to_see_daily_limit_msg: - assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None - else: - assert page.find(attrs={"data-testid": "exceeds-daily"}) is None diff --git a/tests/app/notify_client/test_notification_counts_client.py b/tests/app/notify_client/test_notification_counts_client.py deleted file mode 100644 index fa7183aea8..0000000000 --- a/tests/app/notify_client/test_notification_counts_client.py +++ /dev/null @@ -1,89 +0,0 @@ -from unittest.mock import patch - -import pytest - -from app.notify_client.notification_counts_client import NotificationCounts - - -@pytest.fixture -def mock_redis(): - with patch("app.notify_client.notification_counts_client.redis_client") as mock: - yield mock - - -@pytest.fixture -def mock_template_stats(): - with patch("app.notify_client.notification_counts_client.template_statistics_client") as mock: - yield mock - - -@pytest.fixture -def mock_service_api(): - with patch("app.notify_client.notification_counts_client.service_api_client") as mock: - yield mock - - -class TestNotificationCounts: - def test_get_all_notification_counts_for_today_redis_has_data(self, mock_redis): - # Setup - mock_redis.get.side_effect = [5, 10] # sms, email - wrapper = NotificationCounts() - - # Execute - result = wrapper.get_all_notification_counts_for_today("service-123") - - # Assert - assert result == {"sms": 5, "email": 10} - assert mock_redis.get.call_count == 2 - - @pytest.mark.parametrize( - "redis_side_effect, expected_result", - [ - ([None, None], {"sms": 10, "email": 10}), - ([None, 10], {"sms": 10, "email": 10}), # Falls back to API if either is None - ([10, None], {"sms": 10, "email": 10}), # Falls back to API if either is None - ], - ) - def test_get_all_notification_counts_for_today_redis_missing_data( - self, mock_redis, mock_template_stats, redis_side_effect, expected_result - ): - # Setup - mock_redis.get.side_effect = redis_side_effect - mock_template_stats.get_template_statistics_for_service.return_value = [ - {"template_id": "a1", "template_type": "sms", "count": 3, "status": "delivered"}, - {"template_id": "a2", "template_type": "email", "count": 7, "status": "temporary-failure"}, - {"template_id": "a3", "template_type": "email", "count": 3, "status": "delivered"}, - {"template_id": "a4", "template_type": "sms", "count": 7, "status": "delivered"}, - ] - - wrapper = NotificationCounts() - - # Execute - result = wrapper.get_all_notification_counts_for_today("service-123") - - # Assert - assert result == {"sms": 10, "email": 10} - mock_template_stats.get_template_statistics_for_service.assert_called_once() - - def test_get_all_notification_counts_for_year(self, mock_service_api): - # Setup - mock_service_api.get_monthly_notification_stats.return_value = { - "data": { - "2024-01": { - "sms": {"sent": 1, "temporary-failure:": 22}, - "email": {"delivered": 1, "permanent-failure": 1, "sending": 12, "technical-failure": 1}, - }, - "2024-02": {"sms": {"sent": 1}, "email": {"delivered": 1}}, - } - } - wrapper = NotificationCounts() - - with patch.object(wrapper, "get_all_notification_counts_for_today") as mock_today: - mock_today.return_value = {"sms": 5, "email": 5} - - # Execute - result = wrapper.get_all_notification_counts_for_year("service-123", 2024) - - # Assert - assert result["sms"] == 29 # 1 + 22 + 1 + 5 - assert result["email"] == 21 # 1 + 1 + 12 + 1 + 1 + 5 diff --git a/tests/conftest.py b/tests/conftest.py index 21ed1f782b..94ff0ca5be 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -550,14 +550,7 @@ def fake_uuid(): @pytest.fixture(scope="function") def mock_get_service(mocker, api_user_active): def _get(service_id): - service = service_json( - service_id, - users=[api_user_active["id"]], - message_limit=50, - sms_daily_limit=20, - email_annual_limit=1000, - sms_annual_limit=1000, - ) + service = service_json(service_id, users=[api_user_active["id"]], message_limit=50, sms_daily_limit=20) return {"data": service} return mocker.patch("app.service_api_client.get_service", side_effect=_get) @@ -682,9 +675,7 @@ def mock_service_email_from_is_unique(mocker): @pytest.fixture(scope="function") def mock_get_live_service(mocker, api_user_active): def _get(service_id): - service = service_json( - service_id, users=[api_user_active["id"]], restricted=False, sms_annual_limit=10000, email_annual_limit=10000 - ) + service = service_json(service_id, users=[api_user_active["id"]], restricted=False) return {"data": service} return mocker.patch("app.service_api_client.get_service", side_effect=_get) @@ -980,21 +971,6 @@ def _get(service_id, template_id, version=None): return mocker.patch("app.service_api_client.get_service_template", side_effect=_get) -@pytest.fixture(scope="function") -def mock_get_service_sms_template_without_placeholders(mocker): - def _get(service_id, template_id, version=None): - template = template_json( - service_id, - template_id, - "Two week reminder", - "sms", - "Yo.", - ) - return {"data": template} - - return mocker.patch("app.service_api_client.get_service_template", side_effect=_get) - - @pytest.fixture(scope="function") def mock_get_service_letter_template(mocker, content=None, subject=None, postage="second"): def _get(service_id, template_id, version=None, postage=postage): From 992103fb2c36298e654f027c31963c6a3042a5e7 Mon Sep 17 00:00:00 2001 From: Steve Astels Date: Wed, 4 Dec 2024 12:54:04 -0500 Subject: [PATCH 12/20] Chore/python 3.12 upgrade (#1996) --- .devcontainer/Dockerfile | 2 +- .github/workflows/test.yaml | 2 +- .github/workflows/test_endpoints.yaml | 8 +- .github/workflows/test_prod_config.yaml | 2 +- Makefile | 4 + README.md | 34 ++-- ci/Dockerfile | 2 +- ci/Dockerfile.lambda | 4 +- ci/Dockerfile.test | 2 +- mypy.ini | 2 +- poetry.lock | 186 +++++++++--------- pyproject.toml | 6 +- tests/app/articles/test_fallback_cache.py | 2 +- tests/app/articles/test_pages.py | 2 +- tests/app/main/views/test_dashboard.py | 10 +- tests/app/main/views/test_notifications.py | 1 + .../app/salesforce/test_salesforce_account.py | 6 +- 17 files changed, 139 insertions(+), 136 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 2d1b24fa28..20279ddac1 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/vscode/devcontainers/python:1-3.10@sha256:c5b379b09a94ac1ccb437e000dd54c96164a8322d0c53d2bcb25f225e27924e6 +FROM mcr.microsoft.com/vscode/devcontainers/python:3.12 ENV POETRY_VERSION="1.7.1" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fbd6772dd8..f2e6f1ba01 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0 with: - python-version: '3.10' + python-version: '3.12' - uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v3.8.2 with: node-version: '16.x' diff --git a/.github/workflows/test_endpoints.yaml b/.github/workflows/test_endpoints.yaml index 419e570342..4fc17401fe 100644 --- a/.github/workflows/test_endpoints.yaml +++ b/.github/workflows/test_endpoints.yaml @@ -8,10 +8,10 @@ jobs: steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Set up Python 3.10 + - name: Set up Python 3.12 uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0 with: - python-version: '3.10' + python-version: '3.12' - name: Upgrade pip run: python -m pip install --upgrade pip @@ -45,7 +45,7 @@ jobs: working-directory: ${{ github.workspace }} shell: bash run: | - mkdir -p "${{ github.workspace }}/env/" && cp -fR $(poetry env list | poetry env info -p)/lib/python3.10/site-packages "${{ github.workspace }}/env/" + mkdir -p "${{ github.workspace }}/env/" && cp -fR $(poetry env list | poetry env info -p)/lib/python3.12/site-packages "${{ github.workspace }}/env/" - name: Install development .env file working-directory: ${{ github.workspace }} @@ -58,7 +58,7 @@ jobs: echo "PYTHONPATH=/github/workspace/env/site-packages:${{ env.PYTHONPATH}}" >> $GITHUB_ENV - name: Checks for new endpoints against AWS WAF rules - uses: cds-snc/notification-utils/.github/actions/waffles@52.2.2 + uses: cds-snc/notification-utils/.github/actions/waffles@53.0.1 with: app-loc: '/github/workspace' app-libs: '/github/workspace/env/site-packages' diff --git a/.github/workflows/test_prod_config.yaml b/.github/workflows/test_prod_config.yaml index d8c0fdc6dc..8e7ba34d00 100644 --- a/.github/workflows/test_prod_config.yaml +++ b/.github/workflows/test_prod_config.yaml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0 with: - python-version: '3.10' + python-version: '3.12' - uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v3.8.2 with: node-version: '16.x' diff --git a/Makefile b/Makefile index 8833fd26e4..71d608ebf6 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,10 @@ coverage: venv ## Create coverage report run-dev: poetry run flask run -p 6012 --host=localhost +.PHONY: run-gunicorn +run-gunicorn: + PORT=6012 poetry run gunicorn -c gunicorn_config.py application + .PHONY: format format: ruff check --select I --fix . diff --git a/README.md b/README.md index 77f29917a0..b679a8ff53 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ https://github.com/alphagov/notifications-admin ``` Languages needed -- Python 3.10 +- Python 3.12 - [Node](https://nodejs.org/) 10.15.3 or greater - [npm](https://www.npmjs.com/) 6.4.1 or greater @@ -58,15 +58,15 @@ On OS X: `brew install pyenv` -2. Install Python 3.10.8 or whatever is the latest +2. Install Python 3.12.7 or whatever is the latest -`pyenv install 3.10.8` +`pyenv install 3.12.7` -3. If you expect no conflicts, set `3.10.8` as you default +3. If you expect no conflicts, set `3.12.7` as you default -`pyenv global 3.10.8` +`pyenv global 3.12.7` -4. Ensure that version `3.10.8` is now the default by running +4. Ensure that version `3.12.7` is now the default by running `python --version` @@ -77,7 +77,7 @@ eval "$(pyenv init -)" ``` and open a new terminal. -If you are still not running Python 3.10.8 take a look here: https://github.com/pyenv/pyenv/issues/660 +If you are still not running Python 3.12.7 take a look here: https://github.com/pyenv/pyenv/issues/660 5. Install `poetry`: @@ -85,7 +85,7 @@ If you are still not running Python 3.10.8 take a look here: https://github.com/ 6. Restart your terminal and make your virtual environtment: -`mkvirtualenv -p ~/.pyenv/versions/3.10.8/bin/python notifications-admin` +`mkvirtualenv -p ~/.pyenv/versions/3.12.7/bin/python notifications-admin` 7. You can now return to your environment any time by entering @@ -278,7 +278,7 @@ https://github.com/alphagov/notifications-admin ``` Langages nécessaires -- Python 3.10 +- Python 3.12 - [Node](https://nodejs.org/) 10.15.3 ou supérieur - [npm](https://www.npmjs.com/) 6.4.1 ou plus ```shell @@ -302,15 +302,15 @@ Sur macOS : `brew install pyenv` -2. Installez Python 3.10.8 ou la dernière version +1. Installez Python 3.12.7 ou la dernière version -`pyenv install 3.10.8` +`pyenv install 3.12.7` -3. Si vous n'attendez aucun conflit, mettez `3.10.8` comme valeur par défaut +3. Si vous n'attendez aucun conflit, mettez `3.12.7` comme valeur par défaut -`pyenv global 3.10.8` +`pyenv global 3.12.7` -4. Assurez-vous que la version 3.10.8 est maintenant la version par défaut en exécutant +4. Assurez-vous que la version 3.12.7 est maintenant la version par défaut en exécutant `python --version` @@ -320,7 +320,7 @@ eval "$(pyenv init --path)" eval "$(pyenv init -)" ``` et ouvrez un nouveau terminal. -Si vous n’utilisez toujours pas Python 3.10.8, jetez un coup d’œil ici : https://github.com/pyenv/pyenv/issues/660 +Si vous n’utilisez toujours pas Python 3.12.7, jetez un coup d’œil ici : https://github.com/pyenv/pyenv/issues/660 5. Installez `virtualenv` : @@ -331,12 +331,12 @@ Si vous n’utilisez toujours pas Python 3.10.8, jetez un coup d’œil ici : ht ``` export WORKON_HOME=$HOME/.virtualenvs export PROJECT_HOME=$HOME/Devel -source ~/.pyenv/versions/3.10.8/bin/virtualenvwrapper.sh +source ~/.pyenv/versions/3.12.7/bin/virtualenvwrapper.sh ``` 7. Redémarrez votre terminal et créez votre environnement virtuel : -`mkvirtualenv -p ~/.pyenv/versions/3.10.8/bin/python notifications-admin` +`mkvirtualenv -p ~/.pyenv/versions/3.12.7/bin/python notifications-admin` 8. Vous pouvez maintenant retourner dans votre environnement à tout moment en entrant diff --git a/ci/Dockerfile b/ci/Dockerfile index 1520383fa5..f3addf1b18 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-alpine3.18@sha256:d5ee9613c89c9bd4c4112465d2136512ea8629bce6ff15fa27144f3cc16b5c6b +FROM python:3.12.7-alpine3.20 ENV PYTHONDONTWRITEBYTECODE 1 ENV POETRY_VERSION="1.7.1" diff --git a/ci/Dockerfile.lambda b/ci/Dockerfile.lambda index 9d96e1bc5c..d4f39b59b3 100644 --- a/ci/Dockerfile.lambda +++ b/ci/Dockerfile.lambda @@ -5,7 +5,7 @@ ARG POETRY_VERSION="1.7.1" ARG POETRY_VIRTUALENVS_CREATE="false" # Build image -FROM python:3.10-alpine3.18@sha256:d5ee9613c89c9bd4c4112465d2136512ea8629bce6ff15fa27144f3cc16b5c6b as base +FROM python:3.12-alpine3.20@sha256:5049c050bdc68575a10bcb1885baa0689b6c15152d8a56a7e399fb49f783bf98 as base ARG APP_DIR ARG APP_VENV @@ -68,7 +68,7 @@ RUN . ${APP_VENV}/bin/activate \ && make generate-version-file # Final image -FROM python:3.10-alpine3.18@sha256:d5ee9613c89c9bd4c4112465d2136512ea8629bce6ff15fa27144f3cc16b5c6b as lambda +FROM python:3.12-alpine3.20@sha256:5049c050bdc68575a10bcb1885baa0689b6c15152d8a56a7e399fb49f783bf98 as lambda ARG APP_DIR ARG APP_VENV diff --git a/ci/Dockerfile.test b/ci/Dockerfile.test index 418f87782c..4f59865132 100644 --- a/ci/Dockerfile.test +++ b/ci/Dockerfile.test @@ -1,4 +1,4 @@ -FROM python:3.10-alpine3.16@sha256:afe68972cc00883d70b3760ee0ffbb7375cf09706c122dda7063ffe64c5be21b +FROM python:3.12-alpine3.20@sha256:5049c050bdc68575a10bcb1885baa0689b6c15152d8a56a7e399fb49f783bf98 ENV PYTHONDONTWRITEBYTECODE 1 diff --git a/mypy.ini b/mypy.ini index 2d3d5c255f..05a2cf8485 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,5 +1,5 @@ [mypy] -python_version = 3.10 +python_version = 3.12 [mypy-pytest.*] ignore_missing_imports = True diff --git a/poetry.lock b/poetry.lock index 39aba7fe69..1acc726313 100644 --- a/poetry.lock +++ b/poetry.lock @@ -652,20 +652,6 @@ files = [ {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, ] -[[package]] -name = "exceptiongroup" -version = "1.2.2" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, - {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, -] - -[package.extras] -test = ["pytest (>=6)"] - [[package]] name = "execnet" version = "2.1.1" @@ -882,7 +868,7 @@ files = [ [package.dependencies] cffi = {version = ">=1.12.2", markers = "platform_python_implementation == \"CPython\" and sys_platform == \"win32\""} -greenlet = {version = ">=2.0.0", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.11\""} +greenlet = {version = ">=3.0rc3", markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\""} "zope.event" = "*" "zope.interface" = "*" @@ -895,79 +881,88 @@ test = ["cffi (>=1.12.2)", "coverage (>=5.0)", "dnspython (>=1.16.0,<2.0)", "idn [[package]] name = "greenlet" -version = "2.0.2" +version = "3.1.1" description = "Lightweight in-process concurrent programming" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" -files = [ - {file = "greenlet-2.0.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:bdfea8c661e80d3c1c99ad7c3ff74e6e87184895bbaca6ee8cc61209f8b9b85d"}, - {file = "greenlet-2.0.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:9d14b83fab60d5e8abe587d51c75b252bcc21683f24699ada8fb275d7712f5a9"}, - {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, - {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, - {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, - {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"}, - {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d75209eed723105f9596807495d58d10b3470fa6732dd6756595e89925ce2470"}, - {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3a51c9751078733d88e013587b108f1b7a1fb106d402fb390740f002b6f6551a"}, - {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, - {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, - {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, - {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, - {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:eff4eb9b7eb3e4d0cae3d28c283dc16d9bed6b193c2e1ace3ed86ce48ea8df19"}, - {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5454276c07d27a740c5892f4907c86327b632127dd9abec42ee62e12427ff7e3"}, - {file = "greenlet-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:7cafd1208fdbe93b67c7086876f061f660cfddc44f404279c1585bbf3cdc64c5"}, - {file = "greenlet-2.0.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:910841381caba4f744a44bf81bfd573c94e10b3045ee00de0cbf436fe50673a6"}, - {file = "greenlet-2.0.2-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:18a7f18b82b52ee85322d7a7874e676f34ab319b9f8cce5de06067384aa8ff43"}, - {file = "greenlet-2.0.2-cp35-cp35m-win32.whl", hash = "sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a"}, - {file = "greenlet-2.0.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4b58adb399c4d61d912c4c331984d60eb66565175cdf4a34792cd9600f21b394"}, - {file = "greenlet-2.0.2-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:703f18f3fda276b9a916f0934d2fb6d989bf0b4fb5a64825260eb9bfd52d78f0"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:32e5b64b148966d9cccc2c8d35a671409e45f195864560829f395a54226408d3"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dd11f291565a81d71dab10b7033395b7a3a5456e637cf997a6f33ebdf06f8db"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e0f72c9ddb8cd28532185f54cc1453f2c16fb417a08b53a855c4e6a418edd099"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd021c754b162c0fb55ad5d6b9d960db667faad0fa2ff25bb6e1301b0b6e6a75"}, - {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:3c9b12575734155d0c09d6c3e10dbd81665d5c18e1a7c6597df72fd05990c8cf"}, - {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b9ec052b06a0524f0e35bd8790686a1da006bd911dd1ef7d50b77bfbad74e292"}, - {file = "greenlet-2.0.2-cp36-cp36m-win32.whl", hash = "sha256:dbfcfc0218093a19c252ca8eb9aee3d29cfdcb586df21049b9d777fd32c14fd9"}, - {file = "greenlet-2.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:9f35ec95538f50292f6d8f2c9c9f8a3c6540bbfec21c9e5b4b751e0a7c20864f"}, - {file = "greenlet-2.0.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:d5508f0b173e6aa47273bdc0a0b5ba055b59662ba7c7ee5119528f466585526b"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9c59a2120b55788e800d82dfa99b9e156ff8f2227f07c5e3012a45a399620b7"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2780572ec463d44c1d3ae850239508dbeb9fed38e294c68d19a24d925d9223ca"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:937e9020b514ceedb9c830c55d5c9872abc90f4b5862f89c0887033ae33c6f73"}, - {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:36abbf031e1c0f79dd5d596bfaf8e921c41df2bdf54ee1eed921ce1f52999a86"}, - {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:18e98fb3de7dba1c0a852731c3070cf022d14f0d68b4c87a19cc1016f3bb8b33"}, - {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, - {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, - {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, - {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acd2162a36d3de67ee896c43effcd5ee3de247eb00354db411feb025aa319857"}, - {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0bf60faf0bc2468089bdc5edd10555bab6e85152191df713e2ab1fcc86382b5a"}, - {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, - {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, - {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, - {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"}, - {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be4ed120b52ae4d974aa40215fcdfde9194d63541c7ded40ee12eb4dda57b76b"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94c817e84245513926588caf1152e3b559ff794d505555211ca041f032abbb6b"}, - {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1a819eef4b0e0b96bb0d98d797bef17dc1b4a10e8d7446be32d1da33e095dbb8"}, - {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7efde645ca1cc441d6dc4b48c0f7101e8d86b54c8530141b09fd31cef5149ec9"}, - {file = "greenlet-2.0.2-cp39-cp39-win32.whl", hash = "sha256:ea9872c80c132f4663822dd2a08d404073a5a9b5ba6155bea72fb2a79d1093b5"}, - {file = "greenlet-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:db1a39669102a1d8d12b57de2bb7e2ec9066a6f2b3da35ae511ff93b01b5d564"}, - {file = "greenlet-2.0.2.tar.gz", hash = "sha256:e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0"}, +python-versions = ">=3.7" +files = [ + {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:36b89d13c49216cadb828db8dfa6ce86bbbc476a82d3a6c397f0efae0525bdd0"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94b6150a85e1b33b40b1464a3f9988dcc5251d6ed06842abff82e42632fac120"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93147c513fac16385d1036b7e5b102c7fbbdb163d556b791f0f11eada7ba65dc"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da7a9bff22ce038e19bf62c4dd1ec8391062878710ded0a845bcf47cc0200617"}, + {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b2795058c23988728eec1f36a4e5e4ebad22f8320c85f3587b539b9ac84128d7"}, + {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ed10eac5830befbdd0c32f83e8aa6288361597550ba669b04c48f0f9a2c843c6"}, + {file = "greenlet-3.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:77c386de38a60d1dfb8e55b8c1101d68c79dfdd25c7095d51fec2dd800892b80"}, + {file = "greenlet-3.1.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a"}, + {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511"}, + {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395"}, + {file = "greenlet-3.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39"}, + {file = "greenlet-3.1.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3a701fe5a9695b238503ce5bbe8218e03c3bcccf7e204e455e7462d770268aa"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2846930c65b47d70b9d178e89c7e1a69c95c1f68ea5aa0a58646b7a96df12441"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99cfaa2110534e2cf3ba31a7abcac9d328d1d9f1b95beede58294a60348fba36"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1443279c19fca463fc33e65ef2a935a5b09bb90f978beab37729e1c3c6c25fe9"}, + {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b7cede291382a78f7bb5f04a529cb18e068dd29e0fb27376074b6d0317bf4dd0"}, + {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:23f20bb60ae298d7d8656c6ec6db134bca379ecefadb0b19ce6f19d1f232a942"}, + {file = "greenlet-3.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:7124e16b4c55d417577c2077be379514321916d5790fa287c9ed6f23bd2ffd01"}, + {file = "greenlet-3.1.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:05175c27cb459dcfc05d026c4232f9de8913ed006d42713cb8a5137bd49375f1"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935e943ec47c4afab8965954bf49bfa639c05d4ccf9ef6e924188f762145c0ff"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:667a9706c970cb552ede35aee17339a18e8f2a87a51fba2ed39ceeeb1004798a"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8a678974d1f3aa55f6cc34dc480169d58f2e6d8958895d68845fa4ab566509e"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efc0f674aa41b92da8c49e0346318c6075d734994c3c4e4430b1c3f853e498e4"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e"}, + {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:275f72decf9932639c1c6dd1013a1bc266438eb32710016a1c742df5da6e60a1"}, + {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c4aab7f6381f38a4b42f269057aee279ab0fc7bf2e929e3d4abfae97b682a12c"}, + {file = "greenlet-3.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:b42703b1cf69f2aa1df7d1030b9d77d3e584a70755674d60e710f0af570f3761"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1695e76146579f8c06c1509c7ce4dfe0706f49c6831a817ac04eebb2fd02011"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7876452af029456b3f3549b696bb36a06db7c90747740c5302f74a9e9fa14b13"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ead44c85f8ab905852d3de8d86f6f8baf77109f9da589cb4fa142bd3b57b475"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8320f64b777d00dd7ccdade271eaf0cad6636343293a25074cc5566160e4de7b"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6510bf84a6b643dabba74d3049ead221257603a253d0a9873f55f6a59a65f822"}, + {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:04b013dc07c96f83134b1e99888e7a79979f1a247e2a9f59697fa14b5862ed01"}, + {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47da355d8687fd65240c364c90a31569a133b7b60de111c255ef5b606f2ae291"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98884ecf2ffb7d7fe6bd517e8eb99d31ff7855a840fa6d0d63cd07c037f6a981"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1d4aeb8891338e60d1ab6127af1fe45def5259def8094b9c7e34690c8858803"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db32b5348615a04b82240cc67983cb315309e88d444a288934ee6ceaebcad6cc"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dcc62f31eae24de7f8dce72134c8651c58000d3b1868e01392baea7c32c247de"}, + {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1d3755bcb2e02de341c55b4fca7a745a24a9e7212ac953f6b3a48d117d7257aa"}, + {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b8da394b34370874b4572676f36acabac172602abf054cbc4ac910219f3340af"}, + {file = "greenlet-3.1.1-cp37-cp37m-win32.whl", hash = "sha256:a0dfc6c143b519113354e780a50381508139b07d2177cb6ad6a08278ec655798"}, + {file = "greenlet-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:54558ea205654b50c438029505def3834e80f0869a70fb15b871c29b4575ddef"}, + {file = "greenlet-3.1.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:346bed03fe47414091be4ad44786d1bd8bef0c3fcad6ed3dee074a032ab408a9"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfc59d69fc48664bc693842bd57acfdd490acafda1ab52c7836e3fc75c90a111"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21e10da6ec19b457b82636209cbe2331ff4306b54d06fa04b7c138ba18c8a81"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37b9de5a96111fc15418819ab4c4432e4f3c2ede61e660b1e33971eba26ef9ba"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef9ea3f137e5711f0dbe5f9263e8c009b7069d8a1acea822bd5e9dae0ae49c8"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85f3ff71e2e60bd4b4932a043fbbe0f499e263c628390b285cb599154a3b03b1"}, + {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:95ffcf719966dd7c453f908e208e14cde192e09fde6c7186c8f1896ef778d8cd"}, + {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:03a088b9de532cbfe2ba2034b2b85e82df37874681e8c470d6fb2f8c04d7e4b7"}, + {file = "greenlet-3.1.1-cp38-cp38-win32.whl", hash = "sha256:8b8b36671f10ba80e159378df9c4f15c14098c4fd73a36b9ad715f057272fbef"}, + {file = "greenlet-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:7017b2be767b9d43cc31416aba48aab0d2309ee31b4dbf10a1d38fb7972bdf9d"}, + {file = "greenlet-3.1.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:396979749bd95f018296af156201d6211240e7a23090f50a8d5d18c370084dc3"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9d0ff5ad43e785350894d97e13633a66e2b50000e8a183a50a88d834752d42"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94ebba31df2aa506d7b14866fed00ac141a867e63143fe5bca82a8e503b36437"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aaad12ac0ff500f62cebed98d8789198ea0e6f233421059fa68a5aa7220145"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63e4844797b975b9af3a3fb8f7866ff08775f5426925e1e0bbcfe7932059a12c"}, + {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7939aa3ca7d2a1593596e7ac6d59391ff30281ef280d8632fa03d81f7c5f955e"}, + {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e"}, + {file = "greenlet-3.1.1-cp39-cp39-win32.whl", hash = "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c"}, + {file = "greenlet-3.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22"}, + {file = "greenlet-3.1.1.tar.gz", hash = "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467"}, ] [package.extras] -docs = ["Sphinx", "docutils (<0.18)"] +docs = ["Sphinx", "furo"] test = ["objgraph", "psutil"] [[package]] @@ -1463,7 +1458,6 @@ files = [ [package.dependencies] mypy-extensions = ">=1.0.0" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typing-extensions = ">=4.6.0" [package.extras] @@ -1552,10 +1546,10 @@ requests = ">=2.0.0" [[package]] name = "notifications-utils" -version = "52.3.9" +version = "53.0.1" description = "Shared python code for Notification - Provides logging utils etc." optional = false -python-versions = "~3.10.9" +python-versions = "~3.12.7" files = [] develop = false @@ -1580,7 +1574,7 @@ pypdf2 = "1.28.6" python-json-logger = "2.0.7" pytz = "2021.3" PyYAML = "6.0.1" -requests = "2.31.0" +requests = "2.32.2" smartypants = "2.0.1" statsd = "3.3.0" werkzeug = "3.0.4" @@ -1588,8 +1582,8 @@ werkzeug = "3.0.4" [package.source] type = "git" url = "https://github.com/cds-snc/notifier-utils.git" -reference = "52.3.9" -resolved_reference = "b344e5a74c79a8fa8ca4f722691850ac0d277959" +reference = "53.0.1" +resolved_reference = "8ee66e4ed598a694b778283573715d7d5b9eeb9c" [[package]] name = "openpyxl" @@ -1918,11 +1912,9 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] @@ -2142,13 +2134,13 @@ hiredis = ["hiredis (>=0.1.3)"] [[package]] name = "requests" -version = "2.31.0" +version = "2.32.2" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, + {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, ] [package.dependencies] @@ -2757,5 +2749,5 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" -python-versions = "~3.10.9" -content-hash = "94c0fc0df7115b07d314521cd48aefc25a5bee81c7b0fa2477a86bb4db3ede81" +python-versions = "~3.12.7" +content-hash = "981af1136bc73bbdab78ac127a508c97561d557aa8b3c521de03c3af14ff8c06" diff --git a/pyproject.toml b/pyproject.toml index 662a8a46ce..b2680bca2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ requires = ["poetry-core>=1.7.1"] build-backend = "poetry.core.masonry.api" [tool.poetry.dependencies] -python = "~3.10.9" +python = "~3.12.7" timeago = "1.0.16" Flask = "2.3.3" Flask-WTF = "1.2.1" @@ -43,13 +43,13 @@ user-agents = "2.2.0" WTForms = "3.1.2" email-validator = "1.3.1" Werkzeug = "3.0.4" -greenlet = "2.0.2" +greenlet = "3.1.1" mixpanel = "4.10.1" unidecode = "^1.3.8" # PaaS awscli-cwlogs = "^1.4.6" -notifications-utils = { git = "https://github.com/cds-snc/notifier-utils.git", tag = "52.3.9" } +notifications-utils = { git = "https://github.com/cds-snc/notifier-utils.git", tag = "53.0.1"} # Pinned dependencies diff --git a/tests/app/articles/test_fallback_cache.py b/tests/app/articles/test_fallback_cache.py index 5baeb9b649..5d91554175 100644 --- a/tests/app/articles/test_fallback_cache.py +++ b/tests/app/articles/test_fallback_cache.py @@ -71,5 +71,5 @@ def test_retrieve_existing_from_fallback_cache_on_http_error(app_, mocker): get_content(endpoint, {"slug": "mypage", "lang": "en"}, cacheable=True) assert mock_redis_method.get.called - assert mock_redis_method.get.called_with(cache_key) + mock_redis_method.get.assert_called_with(cache_key) assert mock_redis_method.get(cache_key) == json.dumps(response_json) diff --git a/tests/app/articles/test_pages.py b/tests/app/articles/test_pages.py index bc10fba10a..7a62dbb443 100644 --- a/tests/app/articles/test_pages.py +++ b/tests/app/articles/test_pages.py @@ -40,7 +40,7 @@ def test_get_page_by_slug_with_cache_retrieve_from_cache(app_, mocker): assert mock_redis_method.get.called assert mock_redis_method.get.call_count == 1 - assert mock_redis_method.get.called_with(cache_key) + mock_redis_method.get.assert_called_with(cache_key) assert mock_redis_method.get(cache_key) is not None assert not request_mock.called diff --git a/tests/app/main/views/test_dashboard.py b/tests/app/main/views/test_dashboard.py index a00980bc6c..8d5651cd01 100644 --- a/tests/app/main/views/test_dashboard.py +++ b/tests/app/main/views/test_dashboard.py @@ -314,6 +314,7 @@ def test_should_show_monthly_breakdown_of_template_usage( def test_anyone_can_see_monthly_breakdown( client, api_user_active, service_one, mocker, mock_get_monthly_notification_stats, mock_get_service_statistics ): + mocker.patch("app.main.views.dashboard.annual_limit_client.get_all_notification_counts", return_value={"data": service_one}) validate_route_permission_with_client( mocker, client, @@ -327,8 +328,9 @@ def test_anyone_can_see_monthly_breakdown( def test_monthly_shows_letters_in_breakdown( - client_request, service_one, mock_get_monthly_notification_stats, mock_get_service_statistics + client_request, service_one, mocker, mock_get_monthly_notification_stats, mock_get_service_statistics ): + mocker.patch("app.main.views.dashboard.annual_limit_client.get_all_notification_counts", return_value={"data": service_one}) page = client_request.get("main.monthly", service_id=service_one["id"]) columns = page.select(".table-field-left-aligned .big-number-label") @@ -346,8 +348,9 @@ def test_monthly_shows_letters_in_breakdown( ) @freeze_time("2015-01-01 15:15:15.000000") def test_stats_pages_show_last_3_years( - client_request, endpoint, mock_get_monthly_notification_stats, mock_get_monthly_template_usage, mock_get_service_statistics + client_request, endpoint, service_one, mocker, mock_get_monthly_notification_stats, mock_get_monthly_template_usage, mock_get_service_statistics ): + mocker.patch("app.main.views.dashboard.annual_limit_client.get_all_notification_counts", return_value={"data": service_one}) page = client_request.get( endpoint, service_id=SERVICE_ONE_ID, @@ -359,8 +362,9 @@ def test_stats_pages_show_last_3_years( def test_monthly_has_equal_length_tables( - client_request, service_one, mock_get_monthly_notification_stats, mock_get_service_statistics + client_request, service_one, mocker, mock_get_monthly_notification_stats, mock_get_service_statistics ): + mocker.patch("app.main.views.dashboard.annual_limit_client.get_all_notification_counts", return_value={"data": service_one}) page = client_request.get("main.monthly", service_id=service_one["id"]) assert page.select_one(".table-field-headings th")["style"] == "width: 33%" diff --git a/tests/app/main/views/test_notifications.py b/tests/app/main/views/test_notifications.py index a0b3eb690c..16110de5e2 100644 --- a/tests/app/main/views/test_notifications.py +++ b/tests/app/main/views/test_notifications.py @@ -814,6 +814,7 @@ def test_notification_page_has_expected_template_link_for_letter( assert link is None +@pytest.mark.skip(reason="feature not in use") def test_should_show_image_of_precompiled_letter_notification( logged_in_client, fake_uuid, diff --git a/tests/app/salesforce/test_salesforce_account.py b/tests/app/salesforce/test_salesforce_account.py index 5f2196ad9a..1348f14e49 100644 --- a/tests/app/salesforce/test_salesforce_account.py +++ b/tests/app/salesforce/test_salesforce_account.py @@ -1,3 +1,5 @@ +from unittest.mock import call + from app.salesforce import salesforce_account @@ -5,8 +7,8 @@ def test_get_accounts_requests_correct_url(mocker, app_): with app_.app_context(): mock_request = mocker.patch("app.salesforce.salesforce_account.requests.get") salesforce_account.get_accounts("www.test_url.ca", "secret_token", app_.logger) - assert mock_request.called_with("www.test_url.ca") - assert mock_request.called_with(headers={"Authorization": "token secret_token"}) + calls = [call("www.test_url.ca", headers={"Authorization": "token secret_token"})] + mock_request.assert_has_calls(calls) def test_get_accounts_sorts_alphabetically(mocker, app_): From 29f6354fbfcc20481d603c523be9f14695e12bd1 Mon Sep 17 00:00:00 2001 From: Jumana B Date: Wed, 4 Dec 2024 13:47:46 -0500 Subject: [PATCH 13/20] Change/annual limit (#2011) * Set annual limits to read from the DB * formatting --- app/templates/views/service-settings.html | 17 ++--------------- gunicorn_config.py | 1 + 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/app/templates/views/service-settings.html b/app/templates/views/service-settings.html index 84c63a5226..0725630dea 100644 --- a/app/templates/views/service-settings.html +++ b/app/templates/views/service-settings.html @@ -192,12 +192,7 @@

{{ _('Your service is in trial mode') }}

{% call settings_row(if_has_permission='email') %} {% set txt = _('Annual maximum
(April 1 to March 31)') %} {{ text_field(txt) }} - {# TODO: FF_ANNUAL_LIMIT removal #} - {% if config["FF_ANNUAL_LIMIT"] %} - {% set annual_limit = _('{} emails').format((current_service.email_annual_limit) | format_number) %} - {% else %} - {% set annual_limit = _('{} million emails').format((limits.free_yearly_email//1000000) | format_number) %} - {% endif%} + {% set annual_limit = _('{} emails').format((current_service.email_annual_limit) | format_number) %} {{ text_field(annual_limit, attributes="data-testid=email-annual-limit") }} {{ text_field('')}} {% endcall %} @@ -271,12 +266,7 @@

{{ _('Your service is in trial mode') }}

{% call settings_row(if_has_permission='sms') %} {% set txt = _('Annual maximum
(April 1 to March 31)') %} {{ text_field(txt) }} - {# TODO: FF_ANNUAL_LIMIT removal #} - {% if config["FF_ANNUAL_LIMIT"] %} - {% set annual_sms_limit = _('{} text messages').format((current_service.sms_annual_limit) | format_number) %} - {% else %} - {% set annual_sms_limit = _('{} text messages').format((limits.free_yearly_sms) | format_number) %} - {% endif%} + {% set annual_sms_limit = _('{} text messages').format((current_service.sms_annual_limit) | format_number) %} {{ text_field(annual_sms_limit, attributes="data-testid=sms-annual-limit") }} {{ text_field('')}} {% endcall %} @@ -383,8 +373,6 @@

{{ _('Platform admin settings') }}

) }} {% endcall %} - {# TODO: FF_ANNUAL_LIMIT removal #} - {% if config["FF_ANNUAL_LIMIT"] %} {% call row() %} {% set txt = _('Annual email limit') %} {{ text_field(txt)}} @@ -408,7 +396,6 @@

{{ _('Platform admin settings') }}

for=txt ) }} {% endcall %} - {% endif %} {% call row() %} {% set txt = _('API rate limit per minute') %} diff --git a/gunicorn_config.py b/gunicorn_config.py index 63325ca386..cf6e5e7112 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -51,6 +51,7 @@ # Start timer for total running time start_time = time.time() + def on_starting(server): server.log.info("Starting Notifications Admin") From b5a3ea60632af5de5010bdafb5535d4fc7493300 Mon Sep 17 00:00:00 2001 From: Jumana B Date: Thu, 5 Dec 2024 11:12:15 -0500 Subject: [PATCH 14/20] When the FF is off, the call to the API isn't made. Removed the FF (#2012) * Remove FF that was causing an issue * formatting fix --- app/main/views/service_settings.py | 3 --- tests/app/main/views/test_dashboard.py | 8 +++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/main/views/service_settings.py b/app/main/views/service_settings.py index 677344fb94..fd7dd363fe 100644 --- a/app/main/views/service_settings.py +++ b/app/main/views/service_settings.py @@ -16,7 +16,6 @@ from flask_babel import lazy_gettext as _l from flask_login import current_user from notifications_python_client.errors import HTTPError -from notifications_utils.decorators import requires_feature from app import ( billing_api_client, @@ -1131,7 +1130,6 @@ def set_sms_message_limit(service_id): @main.route("/service//service_settings/set-sms-annual-limit", methods=["GET", "POST"]) @user_is_platform_admin -@requires_feature("FF_ANNUAL_LIMIT") # TODO: FF_ANNUAL_LIMIT removal def set_sms_annual_limit(service_id): form = SMSAnnualMessageLimit(message_limit=current_service.sms_annual_limit) @@ -1150,7 +1148,6 @@ def set_sms_annual_limit(service_id): @main.route("/service//service_settings/set-email-annual.html", methods=["GET", "POST"]) @user_is_platform_admin -@requires_feature("FF_ANNUAL_LIMIT") # TODO: FF_ANNUAL_LIMIT removal def set_email_annual_limit(service_id): form = EmailAnnualMessageLimit(message_limit=current_service.email_annual_limit) diff --git a/tests/app/main/views/test_dashboard.py b/tests/app/main/views/test_dashboard.py index 8d5651cd01..e972baebe8 100644 --- a/tests/app/main/views/test_dashboard.py +++ b/tests/app/main/views/test_dashboard.py @@ -348,7 +348,13 @@ def test_monthly_shows_letters_in_breakdown( ) @freeze_time("2015-01-01 15:15:15.000000") def test_stats_pages_show_last_3_years( - client_request, endpoint, service_one, mocker, mock_get_monthly_notification_stats, mock_get_monthly_template_usage, mock_get_service_statistics + client_request, + endpoint, + service_one, + mocker, + mock_get_monthly_notification_stats, + mock_get_monthly_template_usage, + mock_get_service_statistics, ): mocker.patch("app.main.views.dashboard.annual_limit_client.get_all_notification_counts", return_value={"data": service_one}) page = client_request.get( From d7f02c09c19c9b9ea513ebfc5d3cf77334612ac7 Mon Sep 17 00:00:00 2001 From: Jumana B Date: Fri, 6 Dec 2024 09:18:01 -0500 Subject: [PATCH 15/20] Remove FF for updating annual limits (#2014) --- app/notify_client/service_api_client.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/notify_client/service_api_client.py b/app/notify_client/service_api_client.py index b1274337c7..4cb0f56494 100644 --- a/app/notify_client/service_api_client.py +++ b/app/notify_client/service_api_client.py @@ -3,7 +3,6 @@ from flask import current_app from flask_login import current_user -from notifications_utils.decorators import requires_feature from app.extensions import redis_client from app.notify_client import NotifyAdminAPIClient, _attach_current_user, cache @@ -168,7 +167,6 @@ def update_sms_message_limit(self, service_id, sms_daily_limit): sms_daily_limit=sms_daily_limit, ) - @requires_feature("FF_ANNUAL_LIMIT") # TODO: FF_ANNUAL_LIMIT removal @cache.delete("service-{service_id}") def update_sms_annual_limit(self, service_id, sms_annual_limit): return self.update_service( @@ -176,7 +174,6 @@ def update_sms_annual_limit(self, service_id, sms_annual_limit): sms_annual_limit=sms_annual_limit, ) - @requires_feature("FF_ANNUAL_LIMIT") # TODO: FF_ANNUAL_LIMIT removal @cache.delete("service-{service_id}") def update_email_annual_limit(self, service_id, email_annual_limit): return self.update_service( From 51b10a9f3ddee8887ae481d7628523419ccb6c64 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 9 Dec 2024 11:01:09 -0400 Subject: [PATCH 16/20] Annual limit sending validation + budget component on ready to send screen (#2010) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: validate annual limit * fix: modify wrapper to work with older redis keys * test: add tests! * chore: `utcnow` is deprecated, use `now` instead * feat(annual limits): show error if user is over annual limits * fix: move daily limit message into the appropriate template * feat: add a client to get daily/yearly sent stats using caching where possible * add translations * chore: formatting * fix send to work without FF too * feat: fix sending; write tests; 🙏 * fix: only check for `send_exceeds_annual_limit` when FF is on * fix(tests): only run the new tests with the FF on (they cant pass with it off!) * chore: remove unused imports * fix: move secondary message outside banner * fix: undo ruff change made by accident * feat(error message): split annual into its own * feat(send): refactor limit checker to use `notification_counts_client` class when sending * feat(remaining messages summary): add heading mode; fix missing translations * feat(ready to send): show RMS componont on page, prevent users from moving forward if service is at either limit * feat(ready to send): update view template * feat(annual error mesage email): refactor page a bit, add case for remaining = 0 * fix(notifications_count_client): cast bytes to int (why are they stored as bytes?) * fix: refactor error message views to make them re-usable * chore: translations * tests: add tests for new send code, template code, and changes to notification_counts_client * chore: fix translation * chore: add missing translations * chore: fix failing tests due to merge * fix: remove commented out code * test fixes * chore: refactor a bit so other methods that render _template.html all have the info needed for budget component * fix failing tests; remove letter test * fix(test_templates): update mocks in tests * fix(ready to send): always show the buttons when the FF is off; if the FF is on, show them as long as they have remaining cap today and this year * fix(test_template): only run the test with the FF on * fix(rms): update text only versions to use new signature --------- Co-authored-by: William B <7444334+whabanks@users.noreply.github.com> --- app/main/views/send.py | 38 +- app/main/views/templates.py | 29 ++ .../notification_counts_client.py | 149 +++++++ .../remaining-messages-summary.html | 12 +- .../check/too-many-email-messages.html | 2 +- .../check/too-many-messages-annual.html | 24 ++ .../check/too-many-sms-message-parts.html | 4 +- app/templates/views/check/column-errors.html | 28 +- app/templates/views/notifications/check.html | 7 + .../storybook/remaining-messages-summary.html | 12 +- app/templates/views/templates/_template.html | 26 +- app/translations/csv/fr.csv | 6 +- tests/app/main/views/test_send.py | 364 +++++++++++++++--- tests/app/main/views/test_templates.py | 198 +++++----- .../test_notification_counts_client.py | 202 ++++++++++ tests/conftest.py | 61 ++- 16 files changed, 971 insertions(+), 191 deletions(-) create mode 100644 app/notify_client/notification_counts_client.py create mode 100644 app/templates/partials/check/too-many-messages-annual.html create mode 100644 tests/app/notify_client/test_notification_counts_client.py diff --git a/app/main/views/send.py b/app/main/views/send.py index 2a5720e38b..53acc9ba63 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -53,6 +53,7 @@ ) from app.main.views.dashboard import aggregate_notifications_stats from app.models.user import Users +from app.notify_client.notification_counts_client import notification_counts_client from app.s3_client.s3_csv_client import ( copy_bulk_send_file_to_uploads, list_bulk_send_uploads, @@ -649,8 +650,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_ sms_fragments_sent_today = daily_sms_fragment_count(service_id) emails_sent_today = daily_email_count(service_id) - remaining_sms_message_fragments = current_service.sms_daily_limit - sms_fragments_sent_today - remaining_email_messages = current_service.message_limit - emails_sent_today + remaining_sms_message_fragments_today = current_service.sms_daily_limit - sms_fragments_sent_today + remaining_email_messages_today = current_service.message_limit - emails_sent_today contents = s3download(service_id, upload_id) @@ -659,7 +660,7 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_ email_reply_to = None sms_sender = None recipients_remaining_messages = ( - remaining_email_messages if db_template["template_type"] == "email" else remaining_sms_message_fragments + remaining_email_messages_today if db_template["template_type"] == "email" else remaining_sms_message_fragments_today ) if db_template["template_type"] == "email": @@ -743,8 +744,8 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_ original_file_name=request.args.get("original_file_name", ""), upload_id=upload_id, form=CsvUploadForm(), - remaining_messages=remaining_email_messages, - remaining_sms_message_fragments=remaining_sms_message_fragments, + remaining_messages=remaining_email_messages_today, + remaining_sms_message_fragments=remaining_sms_message_fragments_today, sms_parts_to_send=sms_parts_to_send, is_sms_parts_estimated=is_sms_parts_estimated, choose_time_form=choose_time_form, @@ -783,7 +784,24 @@ def check_messages(service_id, template_id, upload_id, row_index=2): data["original_file_name"] = SanitiseASCII.encode(data.get("original_file_name", "")) data["sms_parts_requested"] = data["stats_daily"]["sms"]["requested"] data["sms_parts_remaining"] = current_service.sms_daily_limit - daily_sms_fragment_count(service_id) - data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"] + + if current_app.config["FF_ANNUAL_LIMIT"]: + data["send_exceeds_annual_limit"] = False + data["send_exceeds_daily_limit"] = False + # determine the remaining sends for daily + annual + limit_stats = notification_counts_client.get_limit_stats(current_service) + remaining_annual = limit_stats[data["template"].template_type]["annual"]["remaining"] + + if remaining_annual < data["count_of_recipients"]: + data["recipients_remaining_messages"] = remaining_annual + data["send_exceeds_annual_limit"] = True + else: + # if they arent over their limit, and its sms, check if they are over their daily limit + if data["template"].template_type == "sms": + data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"] + + else: + data["send_exceeds_daily_limit"] = data["recipients"].sms_fragment_count > data["sms_parts_remaining"] if ( data["recipients"].too_many_rows @@ -804,6 +822,10 @@ def check_messages(service_id, template_id, upload_id, row_index=2): if data["send_exceeds_daily_limit"]: return render_template("views/check/column-errors.html", **data) + if current_app.config["FF_ANNUAL_LIMIT"]: + if data["send_exceeds_annual_limit"]: + return render_template("views/check/column-errors.html", **data) + metadata_kwargs = { "notification_count": data["count_of_recipients"], "template_id": str(template_id), @@ -1085,6 +1107,10 @@ def get_template_error_dict(exception): error = "too-many-sms-messages" elif "Content for template has a character count greater than the limit of" in exception.message: error = "message-too-long" + elif "Exceeded annual email sending limit" in exception.message: + error = "too-many-email-annual" + elif "Exceeded annual SMS sending limit" in exception.message: + error = "too-many-sms-annual" else: raise exception diff --git a/app/main/views/templates.py b/app/main/views/templates.py index 6c7f8a9dce..28079cca77 100644 --- a/app/main/views/templates.py +++ b/app/main/views/templates.py @@ -57,6 +57,7 @@ TemplateList, TemplateLists, ) +from app.notify_client.notification_counts_client import notification_counts_client from app.template_previews import TemplatePreview, get_page_count_for_letter from app.utils import ( email_or_sms_not_enabled, @@ -125,6 +126,31 @@ def get_char_limit_error_msg(): return _("Too many characters") +def get_limit_stats(notification_type): + # get the limit stats for the current service + limit_stats = notification_counts_client.get_limit_stats(current_service) + + # transform the stats into a format that can be used in the template + limit_stats = { + "dailyLimit": limit_stats[notification_type]["daily"]["limit"], + "dailyUsed": limit_stats[notification_type]["daily"]["sent"], + "dailyRemaining": limit_stats[notification_type]["daily"]["remaining"], + "yearlyLimit": limit_stats[notification_type]["annual"]["limit"], + "yearlyUsed": limit_stats[notification_type]["annual"]["sent"], + "yearlyRemaining": limit_stats[notification_type]["annual"]["remaining"], + "notification_type": notification_type, + "heading": _("Ready to send?"), + } + + # determine ready to send heading + if limit_stats["yearlyRemaining"] == 0: + limit_stats["heading"] = _("Sending paused until annual limit resets") + elif limit_stats["dailyRemaining"] == 0: + limit_stats["heading"] = _("Sending paused until 7pm ET. You can schedule more messages to send later.") + + return limit_stats + + @main.route("/services//templates/") @user_has_permissions() def view_template(service_id, template_id): @@ -142,6 +168,7 @@ def view_template(service_id, template_id): template=get_email_preview_template(template, template_id, service_id), template_postage=template["postage"], user_has_template_permission=user_has_template_permission, + **get_limit_stats(template["template_type"]), ) @@ -1072,6 +1099,7 @@ def delete_service_template(service_id, template_id): "views/templates/template.html", template=get_email_preview_template(template, template["id"], service_id), user_has_template_permission=True, + **get_limit_stats(template["template_type"]), ) @@ -1085,6 +1113,7 @@ def confirm_redact_template(service_id, template_id): template=get_email_preview_template(template, template["id"], service_id), user_has_template_permission=True, show_redaction_message=True, + **get_limit_stats(template["template_type"]), ) diff --git a/app/notify_client/notification_counts_client.py b/app/notify_client/notification_counts_client.py new file mode 100644 index 0000000000..d97511efd0 --- /dev/null +++ b/app/notify_client/notification_counts_client.py @@ -0,0 +1,149 @@ +from datetime import datetime + +from notifications_utils.clients.redis import ( + email_daily_count_cache_key, + sms_daily_count_cache_key, +) + +from app import redis_client, service_api_client, template_statistics_client +from app.models.service import Service + + +class NotificationCounts: + def get_all_notification_counts_for_today(self, service_id): + # try to get today's stats from redis + todays_sms = redis_client.get(sms_daily_count_cache_key(service_id)) + todays_sms = int(todays_sms) if todays_sms is not None else None + + todays_email = redis_client.get(email_daily_count_cache_key(service_id)) + todays_email = int(todays_email) if todays_email is not None else None + + if todays_sms is not None and todays_email is not None: + return {"sms": todays_sms, "email": todays_email} + # fallback to the API if the stats are not in redis + else: + stats = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1) + transformed_stats = _aggregate_notifications_stats(stats) + + return transformed_stats + + def get_all_notification_counts_for_year(self, service_id, year): + """ + Get total number of notifications by type for the current service for the current year + + Return value: + { + 'sms': int, + 'email': int + } + + """ + stats_today = self.get_all_notification_counts_for_today(service_id) + stats_this_year = service_api_client.get_monthly_notification_stats(service_id, year)["data"] + stats_this_year = _aggregate_stats_from_service_api(stats_this_year) + # aggregate stats_today and stats_this_year + for template_type in ["sms", "email"]: + stats_this_year[template_type] += stats_today[template_type] + + return stats_this_year + + def get_limit_stats(self, service: Service): + """ + Get the limit stats for the current service, by notification type, including: + - how many notifications were sent today and this year + - the monthy and daily limits + - the number of notifications remaining today and this year + Returns: + dict: A dictionary containing the limit stats for email and SMS notifications. The structure is as follows: + { + "email": { + "annual": { + "limit": int, # The annual limit for email notifications + "sent": int, # The number of email notifications sent this year + "remaining": int, # The number of email notifications remaining this year + }, + "daily": { + "limit": int, # The daily limit for email notifications + "sent": int, # The number of email notifications sent today + "remaining": int, # The number of email notifications remaining today + }, + }, + "sms": { + "annual": { + "limit": int, # The annual limit for SMS notifications + "sent": int, # The number of SMS notifications sent this year + "remaining": int, # The number of SMS notifications remaining this year + }, + "daily": { + "limit": int, # The daily limit for SMS notifications + "sent": int, # The number of SMS notifications sent today + "remaining": int, # The number of SMS notifications remaining today + }, + } + } + """ + + sent_today = self.get_all_notification_counts_for_today(service.id) + sent_thisyear = self.get_all_notification_counts_for_year(service.id, datetime.now().year) + + limit_stats = { + "email": { + "annual": { + "limit": service.email_annual_limit, + "sent": sent_thisyear["email"], + "remaining": service.email_annual_limit - sent_thisyear["email"], + }, + "daily": { + "limit": service.message_limit, + "sent": sent_today["email"], + "remaining": service.message_limit - sent_today["email"], + }, + }, + "sms": { + "annual": { + "limit": service.sms_annual_limit, + "sent": sent_thisyear["sms"], + "remaining": service.sms_annual_limit - sent_thisyear["sms"], + }, + "daily": { + "limit": service.sms_daily_limit, + "sent": sent_today["sms"], + "remaining": service.sms_daily_limit - sent_today["sms"], + }, + }, + } + + return limit_stats + + +# TODO: consolidate this function and other functions that transform the results of template_statistics_client calls +def _aggregate_notifications_stats(template_statistics): + template_statistics = _filter_out_cancelled_stats(template_statistics) + notifications = {"sms": 0, "email": 0} + for stat in template_statistics: + notifications[stat["template_type"]] += stat["count"] + + return notifications + + +def _filter_out_cancelled_stats(template_statistics): + return [s for s in template_statistics if s["status"] != "cancelled"] + + +def _aggregate_stats_from_service_api(stats): + """Aggregate monthly notification stats excluding cancelled""" + total_stats = {"sms": {}, "email": {}} + + for month_data in stats.values(): + for msg_type in ["sms", "email"]: + if msg_type in month_data: + for status, count in month_data[msg_type].items(): + if status != "cancelled": + if status not in total_stats[msg_type]: + total_stats[msg_type][status] = 0 + total_stats[msg_type][status] += count + + return {msg_type: sum(counts.values()) for msg_type, counts in total_stats.items()} + + +notification_counts_client = NotificationCounts() diff --git a/app/templates/components/remaining-messages-summary.html b/app/templates/components/remaining-messages-summary.html index 42175d47fd..b685e03aff 100644 --- a/app/templates/components/remaining-messages-summary.html +++ b/app/templates/components/remaining-messages-summary.html @@ -1,4 +1,4 @@ -{% macro remaining_messages_summary(dailyLimit, dailyUsed, yearlyLimit, yearlyUsed, notification_type, textOnly=None) %} +{% macro remaining_messages_summary(dailyLimit, dailyUsed, yearlyLimit, yearlyUsed, notification_type, headingMode=False, textOnly=None) %} {% set textOnly_allowed_values = ['text', 'emoji'] %} {% if textOnly not in textOnly_allowed_values %} @@ -95,14 +95,14 @@ {% endif %} - {% if sections[0].skip %} + {% if not headingMode and sections[0].skip %}

- Sending paused until annual limit resets + {{ _('Sending paused until annual limit resets') }}

- {% elif sections[0].remaining == "0" %} + {% elif not headingMode and sections[0].remaining == "0" %}

- Sending paused until 7pm ET. You can schedule more messages to send later. + {{ _('Sending paused until 7pm ET. You can schedule more messages to send later.') }}

{% endif %} -{% endmacro %} +{% endmacro %} \ No newline at end of file diff --git a/app/templates/partials/check/too-many-email-messages.html b/app/templates/partials/check/too-many-email-messages.html index 987846c472..9f66acd9c3 100644 --- a/app/templates/partials/check/too-many-email-messages.html +++ b/app/templates/partials/check/too-many-email-messages.html @@ -1,6 +1,6 @@ {% from "components/links.html" import content_link %} -

+

{%- if current_service.trial_mode %} {{ _("Your service is in trial mode. To send more messages, request to go live").format(url_for('main.request_to_go_live', service_id=current_service.id)) }} {% else %} diff --git a/app/templates/partials/check/too-many-messages-annual.html b/app/templates/partials/check/too-many-messages-annual.html new file mode 100644 index 0000000000..86e4cb3ae9 --- /dev/null +++ b/app/templates/partials/check/too-many-messages-annual.html @@ -0,0 +1,24 @@ +{% from "components/links.html" import content_link %} + +{% if template.template_type == 'email' %} + {% set units = _('email messages') %} +{% else %} + {% set units = _('text messages') %} +{% endif %} + +

+ {%- if current_service.trial_mode %} + {{ _("Your service is in trial mode. To send more messages, request to go live").format(url_for('main.request_to_go_live', service_id=current_service.id)) }} + {% else %} + {% if recipients_remaining_messages > 0 %} +

{{ _('{} can only send {} more {} until annual limit resets'.format(current_service.name, recipients_remaining_messages, units)) }}

+

+ {{ _('To send some of these messages now, edit the spreadsheet to {} recipients maximum. '.format(recipients_remaining_messages)) }} + {{ _('To send to recipients you removed, wait until April 1, {} or contact them some other way.'.format(now().year)) }} +

+ {% else %} +

{{ _('{} cannot send any more {} until April 1, {}'.format(current_service.name, units, now().year)) }}

+

{{ _('For more information, visit the usage report for {}.'.format(url_for('.monthly', service_id=current_service.id), current_service.name)) }}

+ {% endif %} + {%- endif -%} +

\ No newline at end of file diff --git a/app/templates/partials/check/too-many-sms-message-parts.html b/app/templates/partials/check/too-many-sms-message-parts.html index 1000fe1ac5..71cc5693a0 100644 --- a/app/templates/partials/check/too-many-sms-message-parts.html +++ b/app/templates/partials/check/too-many-sms-message-parts.html @@ -1,9 +1,9 @@ {% from "components/links.html" import content_link %} -

+

{%- if current_service.trial_mode %} {{ _("Your service is in trial mode. To send more messages, request to go live").format(url_for('main.request_to_go_live', service_id=current_service.id)) }} {% else %} {{ _("To request a daily limit above {} text messages, {}").format(current_service.sms_daily_limit, content_link(_("contact us"), url_for('main.contact'), is_external_link=true)) }} {%- endif -%} -

\ No newline at end of file +

diff --git a/app/templates/views/check/column-errors.html b/app/templates/views/check/column-errors.html index 32f2ae9e8e..1e46c28333 100644 --- a/app/templates/views/check/column-errors.html +++ b/app/templates/views/check/column-errors.html @@ -10,7 +10,9 @@ {% set prefix_txt = _('a column called') %} {% set prefix_plural_txt = _('columns called') %} -{% if send_exceeds_daily_limit and (sms_parts_remaining <= 0) %} +{% if send_exceeds_annual_limit %} + {% set page_title = _('These messages exceed the annual limit') %} +{% elif send_exceeds_daily_limit and (sms_parts_remaining <= 0) %} {% set page_title = _('These messages exceed your daily limit') %} {% elif send_exceeds_daily_limit or recipients.more_rows_than_can_send %} {% set page_title = _('These messages exceed your daily limit') %} @@ -164,24 +166,20 @@

{{ _('You cannot send all these text messages today') {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}}

- {% elif recipients.more_rows_than_can_send and false %} + {% elif send_exceeds_annual_limit %} {% call banner_wrapper(type='dangerous') %} - {% include "partials/check/too-many-email-messages.html" %} + {% include "partials/check/too-many-messages-annual.html" %} {% endcall %} {% elif recipients.more_rows_than_can_send %} - {% call banner_wrapper(type='dangerous') %} - {% include "partials/check/too-many-email-messages.html" %} - {% endcall %} -

{{ _('You cannot send all these email messages today') }}

-

- {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], - content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}} -

- - + {% call banner_wrapper(type='dangerous') %} + {% include "partials/check/too-many-email-messages.html" %} + {% endcall %} +

{{ _('You cannot send all these email messages today') }}

+

+ {{ _("You can try sending these messages after {} Eastern Time. Check {}.").format(time_to_reset[current_lang], + content_link(_("your current local time"), _('https://nrc.canada.ca/en/web-clock/'), is_external_link=true))}} +

{% endif %} - - {% if not send_exceeds_daily_limit %} diff --git a/app/templates/views/notifications/check.html b/app/templates/views/notifications/check.html index 17c668b0c8..0161a74f17 100644 --- a/app/templates/views/notifications/check.html +++ b/app/templates/views/notifications/check.html @@ -66,6 +66,13 @@

{{_('You cannot send this email message today') }} + {% elif error == 'too-many-email-annual' or error == 'too-many-sms-annual' %} + {{ page_header(_('These messages exceed the annual limit'), back_link=back_link) }} +
+ {% call banner_wrapper(type='dangerous') %} + {% set recipients_remaining_messages = 0 %} + {% include "partials/check/too-many-messages-annual.html" %} + {% endcall %} {% elif error == 'message-too-long' %} {# the only row_errors we can get when sending one off messages is that the message is too long #} {{ govuk_back_link(back_link) }} diff --git a/app/templates/views/storybook/remaining-messages-summary.html b/app/templates/views/storybook/remaining-messages-summary.html index 694ec40b4c..f5baec4cdd 100644 --- a/app/templates/views/storybook/remaining-messages-summary.html +++ b/app/templates/views/storybook/remaining-messages-summary.html @@ -53,33 +53,33 @@

Mixed

Text only

below limit
- {{ remaining_messages_summary(10000, 700, 10000, 750, "email", "text") }} + {{ remaining_messages_summary(10000, 700, 10000, 750, "email", False, "text") }}
near limit
- {{ remaining_messages_summary(1000, 800, 1000, 900, "email", "text") }} + {{ remaining_messages_summary(1000, 800, 1000, 900, "email", False, "text") }}
at limit
- {{ remaining_messages_summary(1000, 1000, 1000, 1000, "email", "text") }} + {{ remaining_messages_summary(1000, 1000, 1000, 1000, "email", False, "text") }}

Text only emoji

below limit
- {{ remaining_messages_summary(1000, 700, 1000, 750, "email", "emoji") }} + {{ remaining_messages_summary(1000, 700, 1000, 750, "email", False, "emoji") }}
near limit
- {{ remaining_messages_summary(1000, 800, 1000, 900, "email", "emoji") }} + {{ remaining_messages_summary(1000, 800, 1000, 900, "email", False, "emoji") }}
at limit
- {{ remaining_messages_summary(1000, 1000, 1000, 1000, "email", "emoji") }} + {{ remaining_messages_summary(1000, 1000, 1000, 1000, "email", False, "emoji") }}
diff --git a/app/templates/views/templates/_template.html b/app/templates/views/templates/_template.html index 86c19a127e..918f52efc9 100644 --- a/app/templates/views/templates/_template.html +++ b/app/templates/views/templates/_template.html @@ -1,4 +1,5 @@ {% from 'components/message-count-label.html' import message_count_label %} +{% from 'components/remaining-messages-summary.html' import remaining_messages_summary with context %}
{% if template._template.archived %} @@ -13,20 +14,23 @@

{% else %} {% if current_user.has_permissions('send_messages', restrict_admin_usage=True) %} -

{{ _('Ready to send?') }}

+

{{ heading }}

-
- - {{ _('Yes, add recipients') }} - - {{ _('No, send yourself this message') }} -
+ {% if config["FF_ANNUAL_LIMIT"] %} + {{ remaining_messages_summary(dailyLimit, dailyUsed, yearlyLimit, yearlyUsed, notification_type, yearlyRemaining == 0 or dailyRemaining == 0) }} + {% endif %} + {% if not config["FF_ANNUAL_LIMIT"] or (yearlyRemaining > 0 and dailyRemaining > 0) %} +
+ + {{ _('Yes, add recipients') }} + + {{ _('No, send yourself this message') }} +
+ {% endif %} {% endif %} {% endif %}
-
+
{{ template|string|translate_preview_template }} -
- - +
\ No newline at end of file diff --git a/app/translations/csv/fr.csv b/app/translations/csv/fr.csv index 57aead4f21..5c61bbcea0 100644 --- a/app/translations/csv/fr.csv +++ b/app/translations/csv/fr.csv @@ -2027,4 +2027,8 @@ "Annual usage","Utilisation annuelle" "resets at 7pm Eastern Time","Réinitialisation à 19 h, heure de l’Est" "Visit usage report","Consulter le rapport d’utilisation" -"Month by month totals","Totaux mensuels" \ No newline at end of file +"Month by month totals","Totaux mensuels" +"email messages","courriels" +"Sending paused until 7pm ET. You can schedule more messages to send later.","FR: Sending paused until 7pm ET. You can schedule more messages to send later." +"Sending paused until annual limit resets","FR: Sending paused until annual limit resets" +"These messages exceed the annual limit","FR: These messages exceed the annual limit" \ No newline at end of file diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 78df4b10cc..a5aeb2db72 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -6,6 +6,7 @@ from io import BytesIO from itertools import repeat from os import path +from unittest.mock import patch from uuid import uuid4 from zipfile import BadZipFile @@ -41,6 +42,7 @@ mock_get_service_letter_template, mock_get_service_template, normalize_spaces, + set_config, ) template_types = ["email", "sms"] @@ -2543,6 +2545,7 @@ def test_check_messages_shows_too_many_sms_messages_errors( mock_get_jobs, mock_s3_download, mock_s3_set_metadata, + mock_get_limit_stats, fake_uuid, num_requested, expected_msg, @@ -2584,6 +2587,30 @@ def test_check_messages_shows_too_many_sms_messages_errors( assert details == expected_msg +@pytest.fixture +def mock_notification_counts_client(): + with patch("app.main.views.send.notification_counts_client") as mock: + yield mock + + +@pytest.fixture +def mock_daily_sms_fragment_count(): + with patch("app.main.views.send.daily_sms_fragment_count") as mock: + yield mock + + +@pytest.fixture +def mock_daily_email_count(): + with patch("app.main.views.send.daily_email_count") as mock: + yield mock + + +@pytest.fixture +def mock_get_service_template_annual_limits(): + with patch("app.service_api_client.get_service_template") as mock: + yield mock + + @pytest.mark.parametrize( "num_requested,expected_msg", [ @@ -2601,6 +2628,7 @@ def test_check_messages_shows_too_many_email_messages_errors( mock_get_template_statistics, mock_get_job_doesnt_exist, mock_get_jobs, + mock_get_limit_stats, fake_uuid, num_requested, expected_msg, @@ -2723,49 +2751,6 @@ def test_warns_if_file_sent_already( mock_get_jobs.assert_called_once_with(SERVICE_ONE_ID, limit_days=0) -def test_check_messages_column_error_doesnt_show_optional_columns( - mocker, - client_request, - mock_get_service_letter_template, - mock_has_permissions, - fake_uuid, - mock_get_users_by_service, - mock_get_service_statistics, - mock_get_template_statistics, - mock_get_job_doesnt_exist, - mock_get_jobs, -): - mocker.patch( - "app.main.views.send.s3download", - return_value="\n".join(["address_line_1,address_line_2,foo"] + ["First Lastname,1 Example Road,SW1 1AA"]), - ) - - mocker.patch( - "app.main.views.send.get_page_count_for_letter", - return_value=5, - ) - - with client_request.session_transaction() as session: - session["file_uploads"] = { - fake_uuid: { - "template_id": "", - "original_file_name": "", - } - } - - page = client_request.get( - "main.check_messages", - service_id=SERVICE_ONE_ID, - template_id=fake_uuid, - upload_id=fake_uuid, - _test_page_title=False, - ) - - assert normalize_spaces(page.select_one(".banner-dangerous").text) == ( - "Your spreadsheet is missing a column called ‘postcode’. " "Add the missing column." - ) - - def test_check_messages_adds_sender_id_in_session_to_metadata( client_request, mocker, @@ -3401,3 +3386,296 @@ class Object(object): multiple_choise_options = [x.text.strip() for x in options] assert multiple_choise_options == expected_filenames + + +class TestAnnualLimitsSend: + @pytest.mark.parametrize( + "num_being_sent, num_sent_today, num_sent_this_year, expect_to_see_annual_limit_msg, expect_to_see_daily_limit_msg", + [ + # annual limit for mock_get_live_service is 10,000email/10,000sms + # daily limit for mock_get_live_service is 1,000email/1,000sms + # 1000 have already been sent today, trying to send 100 more [over both limits] + (100, 1000, 10000, True, False), + # No sent yet today or this year, trying to send 1001 [over both limits] + (10001, 0, 0, True, False), + # 600 have already been sent this year, trying to send 500 more [over annual limit but not daily] + (500, 0, 9600, True, False), + # No sent yet today or this year, trying to send 1001 [over daily limit but not annual] + (1001, 0, 0, False, True), + # No sent yet today or this year, trying to send 100 [over neither limit] + (100, 0, 0, False, False), + ], + ids=[ + "email_over_both_limits", + "email_over_both_limits2", + "email_over_annual_but_not_daily", + "email_over_daily_but_not_annual", + "email_over_neither", + ], + ) + def test_email_send_fails_approrpiately_when_over_limits( + self, + mocker, + client_request, + mock_get_live_service, # set email_annual_limit and sms_annual_limit to 1000 + mock_get_users_by_service, + mock_get_service_email_template_without_placeholders, + mock_get_template_statistics, + mock_get_job_doesnt_exist, + mock_get_jobs, + mock_s3_set_metadata, + mock_notification_counts_client, + mock_daily_sms_fragment_count, + mock_daily_email_count, + fake_uuid, + num_being_sent, + num_sent_today, + num_sent_this_year, + expect_to_see_annual_limit_msg, + expect_to_see_daily_limit_msg, + app_, + ): + with set_config(app_, "FF_ANNUAL_LIMIT", True): + mocker.patch( + "app.main.views.send.s3download", + return_value=",\n".join( + ["email address"] + ([mock_get_users_by_service(None)[0]["email_address"]] * num_being_sent) + ), + ) + + mock_notification_counts_client.get_limit_stats.return_value = { + "email": { + "annual": { + "limit": 1, # doesn't matter for our test + "sent": 1, # doesn't matter for our test + "remaining": 10000 + - num_sent_this_year + - num_sent_today, # The number of email notifications remaining this year + }, + "daily": { + "limit": 1, # doesn't matter for our test + "sent": 1, # doesn't matter for our test + "remaining": 1000 - num_sent_today, # The number of email notifications remaining today + }, + } + } + + # mock that we've already sent `emails_sent_today` emails today + mock_daily_email_count.return_value = num_sent_today + mock_daily_sms_fragment_count.return_value = 900 # not used in test but needs a value + + with client_request.session_transaction() as session: + session["file_uploads"] = { + fake_uuid: { + "template_id": fake_uuid, + "notification_count": 1, + "valid": True, + } + } + + page = client_request.get( + "main.check_messages", + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + upload_id=fake_uuid, + original_file_name="valid.csv", + _test_page_title=False, + ) + + if expect_to_see_annual_limit_msg: + assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None + else: + assert page.find(attrs={"data-testid": "exceeds-annual"}) is None + + if expect_to_see_daily_limit_msg: + assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None + else: + assert page.find(attrs={"data-testid": "exceeds-daily"}) is None + + @pytest.mark.parametrize( + "num_being_sent, num_sent_today, num_sent_this_year, expect_to_see_annual_limit_msg, expect_to_see_daily_limit_msg", + [ + # annual limit for mock_get_live_service is 10,000email/10,000sms + # daily limit for mock_get_live_service is 1,000email/1,000sms + # 1000 have already been sent today, trying to send 100 more [over both limits] + (100, 1000, 10000, True, False), + # No sent yet today or this year, trying to send 1001 [over both limits] + (10001, 0, 0, True, False), + # 600 have already been sent this year, trying to send 500 more [over annual limit but not daily] + (500, 0, 9600, True, False), + # No sent yet today or this year, trying to send 1001 [over daily limit but not annual] + (1001, 0, 0, False, True), + # No sent yet today or this year, trying to send 100 [over neither limit] + (100, 0, 0, False, False), + ], + ids=[ + "sms_over_both_limits", + "sms_over_both_limits2", + "sms_over_annual_but_not_daily", + "sms_over_daily_but_not_annual", + "sms_over_neither", + ], + ) + def test_sms_send_fails_approrpiately_when_over_limits( + self, + mocker, + client_request, + mock_get_live_service, # set email_annual_limit and sms_annual_limit to 1000 + mock_get_users_by_service, + mock_get_service_sms_template_without_placeholders, + mock_get_template_statistics, + mock_get_job_doesnt_exist, + mock_get_jobs, + mock_s3_set_metadata, + mock_notification_counts_client, + mock_daily_sms_fragment_count, + mock_daily_email_count, + fake_uuid, + num_being_sent, + num_sent_today, + num_sent_this_year, + expect_to_see_annual_limit_msg, + expect_to_see_daily_limit_msg, + app_, + ): + with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED + mocker.patch( + "app.main.views.send.s3download", + return_value=",\n".join( + ["phone number"] + ([mock_get_users_by_service(None)[0]["mobile_number"]] * num_being_sent) + ), + ) + mock_notification_counts_client.get_limit_stats.return_value = { + "sms": { + "annual": { + "limit": 1, # doesn't matter for our test + "sent": 1, # doesn't matter for our test + "remaining": 10000 + - num_sent_this_year + - num_sent_today, # The number of email notifications remaining this year + }, + "daily": { + "limit": 1, # doesn't matter for our test + "sent": 1, # doesn't matter for our test + "remaining": 1000 - num_sent_today, # The number of email notifications remaining today + }, + } + } + # mock that we've already sent `num_sent_today` emails today + mock_daily_email_count.return_value = 900 # not used in test but needs a value + mock_daily_sms_fragment_count.return_value = num_sent_today + + with client_request.session_transaction() as session: + session["file_uploads"] = { + fake_uuid: { + "template_id": fake_uuid, + "notification_count": 1, + "valid": True, + } + } + + page = client_request.get( + "main.check_messages", + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + upload_id=fake_uuid, + original_file_name="valid.csv", + _test_page_title=False, + ) + + if expect_to_see_annual_limit_msg: + assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None + else: + assert page.find(attrs={"data-testid": "exceeds-annual"}) is None + + if expect_to_see_daily_limit_msg: + assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None + else: + assert page.find(attrs={"data-testid": "exceeds-daily"}) is None + + @pytest.mark.parametrize( + "num_to_send, remaining_daily, remaining_annual, error_shown", + [ + (2, 2, 2, "none"), + (5, 5, 4, "annual"), + (5, 4, 5, "daily"), + (5, 4, 4, "annual"), + ], + ) + def test_correct_error_displayed( + self, + mocker, + client_request, + mock_get_live_service, # set email_annual_limit and sms_annual_limit to 1000 + mock_get_users_by_service, + mock_get_service_email_template_without_placeholders, + mock_get_template_statistics, + mock_get_job_doesnt_exist, + mock_get_jobs, + mock_s3_set_metadata, + mock_daily_email_count, + mock_notification_counts_client, + fake_uuid, + num_to_send, + remaining_daily, + remaining_annual, + error_shown, + app_, + ): + with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED + # mock that `num_sent_this_year` have already been sent this year + mock_notification_counts_client.get_limit_stats.return_value = { + "email": { + "annual": { + "limit": 1, # doesn't matter for our test + "sent": 1, # doesn't matter for our test + "remaining": remaining_annual, # The number of email notifications remaining this year + }, + "daily": { + "limit": 1, # doesn't matter for our test + "sent": 1, # doesn't matter for our test + "remaining": remaining_daily, # The number of email notifications remaining today + }, + } + } + + # only change this value when we're expecting an error + if error_shown != "none": + mock_daily_email_count.return_value = 1000 - ( + num_to_send - 1 + ) # svc limit is 1000 - exceeding the daily limit is calculated based off of this + else: + mock_daily_email_count.return_value = 0 # none sent + + mocker.patch( + "app.main.views.send.s3download", + return_value=",\n".join( + ["email address"] + ([mock_get_users_by_service(None)[0]["email_address"]] * num_to_send) + ), + ) + with client_request.session_transaction() as session: + session["file_uploads"] = { + fake_uuid: { + "template_id": fake_uuid, + "notification_count": 1, + "valid": True, + } + } + page = client_request.get( + "main.check_messages", + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + upload_id=fake_uuid, + original_file_name="valid.csv", + _test_page_title=False, + ) + + if error_shown == "annual": + assert page.find(attrs={"data-testid": "exceeds-annual"}) is not None + assert page.find(attrs={"data-testid": "exceeds-daily"}) is None + elif error_shown == "daily": + assert page.find(attrs={"data-testid": "exceeds-annual"}) is None + assert page.find(attrs={"data-testid": "exceeds-daily"}) is not None + elif error_shown == "none": + assert page.find(attrs={"data-testid": "exceeds-annual"}) is None + assert page.find(attrs={"data-testid": "exceeds-daily"}) is None diff --git a/tests/app/main/views/test_templates.py b/tests/app/main/views/test_templates.py index b48234af1f..333dced2a8 100644 --- a/tests/app/main/views/test_templates.py +++ b/tests/app/main/views/test_templates.py @@ -1,6 +1,6 @@ from datetime import datetime from functools import partial -from unittest.mock import ANY, MagicMock, Mock +from unittest.mock import ANY, MagicMock, Mock, patch import pytest from flask import url_for @@ -49,11 +49,18 @@ fake_uuid, mock_get_service_template_with_process_type, normalize_spaces, + set_config, ) DEFAULT_PROCESS_TYPE = TemplateProcessTypes.BULK.value +@pytest.fixture +def mock_notification_counts_client(): + with patch("app.main.views.templates.notification_counts_client") as mock: + yield mock + + class TestRedisPreviewUtilities: def test_set_get(self, fake_uuid, mocker): mock_redis_obj = MockRedis() @@ -113,6 +120,7 @@ def test_create_email_template_cat_other_to_freshdesk( mock_get_service_template_when_no_template_exists, mock_get_template_categories, mock_send_other_category_to_freshdesk, + mock_get_limit_stats, active_user_with_permissions, fake_uuid, app_, @@ -147,6 +155,7 @@ def test_edit_email_template_cat_other_to_freshdesk( mock_get_template_categories, mock_update_service_template, mock_send_other_category_to_freshdesk, + mock_get_limit_stats, active_user_with_permissions, fake_uuid, app_, @@ -490,7 +499,13 @@ def test_should_show_page_for_one_template( def test_caseworker_redirected_to_one_off( - client_request, mock_get_service_templates, mock_get_service_template, mocker, fake_uuid, active_caseworking_user + client_request, + mock_get_service_templates, + mock_get_service_template, + mock_get_limit_stats, + mocker, + fake_uuid, + active_caseworking_user, ): client_request.login(active_caseworking_user) client_request.get( @@ -510,6 +525,7 @@ def test_user_with_only_send_and_view_redirected_to_one_off( client_request, mock_get_service_templates, mock_get_service_template, + mock_get_limit_stats, active_user_with_permissions, mocker, fake_uuid, @@ -532,40 +548,6 @@ def test_user_with_only_send_and_view_redirected_to_one_off( ) -@pytest.mark.parametrize( - "permissions", - ( - {"send_messages", "view_activity"}, - {"send_messages"}, - {"view_activity"}, - {}, - ), -) -def test_user_with_only_send_and_view_sees_letter_page( - client_request, - mock_get_service_templates, - mock_get_template_folders, - mock_get_service_letter_template, - single_letter_contact_block, - mock_has_jobs, - active_user_with_permissions, - mocker, - fake_uuid, - permissions, -): - mocker.patch("app.main.views.templates.get_page_count_for_letter", return_value=1) - active_user_with_permissions["permissions"][SERVICE_ONE_ID] = permissions - client_request.login(active_user_with_permissions) - page = client_request.get( - "main.view_template", - service_id=SERVICE_ONE_ID, - template_id=fake_uuid, - _test_page_title=False, - ) - assert normalize_spaces(page.select_one("h1").text) == ("Two week reminder") - assert normalize_spaces(page.select_one("title").text) == ("Two week reminder – Templates - service one – Notify") - - @pytest.mark.parametrize( "letter_branding, expected_link, expected_link_text", ( @@ -610,46 +592,11 @@ def test_letter_with_default_branding_has_add_logo_button( assert first_edit_link.text == expected_link_text -@pytest.mark.parametrize( - "template_postage,expected_result", - [ - ("first", "Postage: first class"), - ("second", "Postage: second class"), - ], -) -def test_view_letter_template_displays_postage( - client_request, - service_one, - mock_get_service_templates, - mock_get_template_folders, - single_letter_contact_block, - mock_has_jobs, - active_user_with_permissions, - mocker, - fake_uuid, - template_postage, - expected_result, -): - mocker.patch("app.main.views.templates.get_page_count_for_letter", return_value=1) - client_request.login(active_user_with_permissions) - - template = create_letter_template(postage=template_postage) - mocker.patch("app.service_api_client.get_service_template", return_value=template) - - page = client_request.get( - "main.view_template", - service_id=SERVICE_ONE_ID, - template_id=template["data"]["id"], - _test_page_title=False, - ) - - assert normalize_spaces(page.select_one(".letter-postage").text) == expected_result - - def test_view_non_letter_template_does_not_display_postage( client_request, mock_get_service_template, mock_get_template_folders, + mock_get_limit_stats, fake_uuid, ): page = client_request.get( @@ -740,6 +687,7 @@ def test_should_be_able_to_view_a_template_with_links( client_request, mock_get_service_template, mock_get_template_folders, + mock_get_limit_stats, active_user_with_permissions, single_letter_contact_block, fake_uuid, @@ -777,6 +725,7 @@ def test_should_show_template_id_on_template_page( mock_get_service_template, mock_get_template_folders, fake_uuid, + mock_get_limit_stats, ): page = client_request.get( ".view_template", @@ -792,6 +741,7 @@ def test_should_show_logos_on_template_page( fake_uuid, mocker, service_one, + mock_get_limit_stats, app_, ): mocker.patch( @@ -817,6 +767,7 @@ def test_should_not_show_send_buttons_on_template_page_for_user_without_permissi client_request, fake_uuid, mock_get_service_template, + mock_get_limit_stats, active_user_view_permissions, ): client_request.login(active_user_view_permissions) @@ -838,6 +789,7 @@ def test_should_show_sms_template_with_downgraded_unicode_characters( service_one, single_letter_contact_block, mock_get_template_folders, + mock_get_limit_stats, fake_uuid, ): msg = "here:\tare some “fancy quotes” and zero\u200bwidth\u200bspaces" @@ -1335,6 +1287,7 @@ def test_should_redirect_when_saving_a_template( client_request, mock_get_template_categories, mock_update_service_template, + mock_get_limit_stats, fake_uuid, app_, mocker, @@ -2032,6 +1985,7 @@ def test_should_show_delete_template_page_with_time_block( client_request, mock_get_service_template, mock_get_template_folders, + mock_get_limit_stats, mocker, fake_uuid, ): @@ -2060,11 +2014,7 @@ def test_should_show_delete_template_page_with_time_block( def test_should_show_delete_template_page_with_time_block_for_empty_notification( - client_request, - mock_get_service_template, - mock_get_template_folders, - mocker, - fake_uuid, + client_request, mock_get_service_template, mock_get_template_folders, mocker, fake_uuid, mock_get_limit_stats ): with freeze_time("2012-01-08 12:00:00"): template = template_json("1234", "1234", "Test template", "sms", "Something very interesting") @@ -2095,6 +2045,7 @@ def test_should_show_delete_template_page_with_never_used_block( client_request, mock_get_service_template, mock_get_template_folders, + mock_get_limit_stats, fake_uuid, mocker, ): @@ -2168,6 +2119,7 @@ def test_should_show_page_for_a_deleted_template( mock_get_user, mock_get_user_by_email, mock_has_permissions, + mock_notification_counts_client, fake_uuid, ): template_id = fake_uuid @@ -2214,6 +2166,7 @@ def test_route_permissions( mock_get_template_folders, mock_get_template_statistics_for_template, mock_get_template_categories, + mock_get_limit_stats, fake_uuid, ): validate_route_permission( @@ -2323,6 +2276,7 @@ def test_can_create_email_template_with_emoji( mock_get_template_folders, mock_get_service_template_when_no_template_exists, mock_get_template_categories, + mock_get_limit_stats, app_, ): page = client_request.post( @@ -2365,6 +2319,7 @@ def test_create_template_with_process_types( mock_get_template_folders, mock_get_service_template_when_no_template_exists, mock_get_template_categories, + mock_get_limit_stats, app_, mocker, platform_admin_user, @@ -2478,6 +2433,7 @@ def test_should_create_sms_template_without_downgrading_unicode_characters( def test_should_show_message_before_redacting_template( client_request, mock_get_service_template, + mock_get_limit_stats, service_one, fake_uuid, ): @@ -2501,6 +2457,7 @@ def test_should_show_redact_template( mock_get_service_template, mock_get_template_folders, mock_redact_template, + mock_get_limit_stats, single_letter_contact_block, service_one, fake_uuid, @@ -2524,6 +2481,7 @@ def test_should_show_hint_once_template_redacted( mocker, service_one, mock_get_template_folders, + mock_get_limit_stats, fake_uuid, ): template = create_template(redact_personalisation=True) @@ -2539,27 +2497,6 @@ def test_should_show_hint_once_template_redacted( assert page.select(".hint")[0].text.strip() == "Recipients' information will be redacted from system" -def test_should_not_show_redaction_stuff_for_letters( - client_request, - mocker, - fake_uuid, - mock_get_service_letter_template, - mock_get_template_folders, - single_letter_contact_block, -): - mocker.patch("app.main.views.templates.get_page_count_for_letter", return_value=1) - - page = client_request.get( - "main.view_template", - service_id=SERVICE_ONE_ID, - template_id=fake_uuid, - _test_page_title=False, - ) - - assert page.select(".hint") == [] - assert "personalisation" not in " ".join(link.text.lower() for link in page.select("a")) - - def test_set_template_sender( client_request, fake_uuid, @@ -2677,6 +2614,7 @@ def test_template_should_show_email_address_in_correct_language( client_request, mock_get_service_email_template, mock_get_template_folders, + mock_get_limit_stats, fake_uuid, ): # check english @@ -2705,6 +2643,7 @@ def test_template_should_show_phone_number_in_correct_language( client_request, mock_get_service_template, mock_get_template_folders, + mock_get_limit_stats, fake_uuid, ): # check english @@ -2742,3 +2681,66 @@ def test_should_hide_category_name_from_template_list_if_marked_hidden( # assert that "HIDDEN_CATEGORY" is not found anywhere in the page using beautifulsoup assert "HIDDEN_CATEGORY" not in page.text assert not page.find(text="HIDDEN_CATEGORY") + + +class TestAnnualLimits: + @pytest.mark.parametrize( + "remaining_daily, remaining_annual, buttons_shown", + [ + (10, 100, True), # Within both limits + (0, 100, False), # Exceeds daily limit + (10, 0, False), # Exceeds annual limit + (0, 0, False), # Exceeds both limits + (1, 1, True), # Exactly at both limits + ], + ) + def test_should_hide_send_buttons_when_appropriate( + self, + client_request, + mock_get_service_template, + mock_get_template_folders, + mock_notification_counts_client, + fake_uuid, + remaining_daily, + remaining_annual, + buttons_shown, + app_, + ): + with set_config(app_, "FF_ANNUAL_LIMIT", True): # REMOVE LINE WHEN FF REMOVED + mock_notification_counts_client.get_limit_stats.return_value = { + "email": { + "annual": { + "limit": 1, # doesn't matter for our test + "sent": 1, # doesn't matter for our test + "remaining": remaining_annual, # The number of email notifications remaining this year + }, + "daily": { + "limit": 1, # doesn't matter for our test + "sent": 1, # doesn't matter for our test + "remaining": remaining_daily, # The number of email notifications remaining today + }, + }, + "sms": { + "annual": { + "limit": 1, # doesn't matter for our test + "sent": 1, # doesn't matter for our test + "remaining": remaining_annual, # The number of email notifications remaining this year + }, + "daily": { + "limit": 1, # doesn't matter for our test + "sent": 1, # doesn't matter for our test + "remaining": remaining_daily, # The number of email notifications remaining today + }, + }, + } + + page = client_request.get( + ".view_template", + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + _test_page_title=False, + ) + if buttons_shown: + assert page.find(attrs={"data-testid": "send-buttons"}) is not None + else: + assert page.find(attrs={"data-testid": "send-buttons"}) is None diff --git a/tests/app/notify_client/test_notification_counts_client.py b/tests/app/notify_client/test_notification_counts_client.py new file mode 100644 index 0000000000..3d4f510bca --- /dev/null +++ b/tests/app/notify_client/test_notification_counts_client.py @@ -0,0 +1,202 @@ +from datetime import datetime +from unittest.mock import Mock, patch + +import pytest + +from app.notify_client.notification_counts_client import NotificationCounts + + +@pytest.fixture +def mock_redis(): + with patch("app.notify_client.notification_counts_client.redis_client") as mock: + yield mock + + +@pytest.fixture +def mock_template_stats(): + with patch("app.notify_client.notification_counts_client.template_statistics_client") as mock: + yield mock + + +@pytest.fixture +def mock_service_api(): + with patch("app.notify_client.notification_counts_client.service_api_client") as mock: + yield mock + + +@pytest.fixture +def mock_get_all_notification_counts_for_today(): + with patch("app.notify_client.notification_counts_client.get_all_notification_counts_for_today") as mock: + yield mock + + +class TestNotificationCounts: + def test_get_all_notification_counts_for_today_redis_has_data(self, mock_redis): + # Setup + mock_redis.get.side_effect = [5, 10] # sms, email + wrapper = NotificationCounts() + + # Execute + result = wrapper.get_all_notification_counts_for_today("service-123") + + # Assert + assert result == {"sms": 5, "email": 10} + assert mock_redis.get.call_count == 2 + + @pytest.mark.parametrize( + "redis_side_effect, expected_result", + [ + ([None, None], {"sms": 10, "email": 10}), + ([None, 10], {"sms": 10, "email": 10}), # Falls back to API if either is None + ([10, None], {"sms": 10, "email": 10}), # Falls back to API if either is None + ([25, 25], {"sms": 25, "email": 25}), # Falls back to API if either is None + ], + ) + def test_get_all_notification_counts_for_today_redis_missing_data( + self, mock_redis, mock_template_stats, redis_side_effect, expected_result + ): + # Setup + mock_redis.get.side_effect = redis_side_effect + mock_template_stats.get_template_statistics_for_service.return_value = [ + {"template_id": "a1", "template_type": "sms", "count": 3, "status": "delivered"}, + {"template_id": "a2", "template_type": "email", "count": 7, "status": "temporary-failure"}, + {"template_id": "a3", "template_type": "email", "count": 3, "status": "delivered"}, + {"template_id": "a4", "template_type": "sms", "count": 7, "status": "delivered"}, + ] + + wrapper = NotificationCounts() + + # Execute + result = wrapper.get_all_notification_counts_for_today("service-123") + + # Assert + assert result == expected_result + + if None in redis_side_effect: + mock_template_stats.get_template_statistics_for_service.assert_called_once() + + def test_get_all_notification_counts_for_year(self, mock_service_api): + # Setup + mock_service_api.get_monthly_notification_stats.return_value = { + "data": { + "2024-01": { + "sms": {"sent": 1, "temporary-failure:": 22}, + "email": {"delivered": 1, "permanent-failure": 1, "sending": 12, "technical-failure": 1}, + }, + "2024-02": {"sms": {"sent": 1}, "email": {"delivered": 1}}, + } + } + wrapper = NotificationCounts() + + with patch.object(wrapper, "get_all_notification_counts_for_today") as mock_today: + mock_today.return_value = {"sms": 5, "email": 5} + + # Execute + result = wrapper.get_all_notification_counts_for_year("service-123", 2024) + + # Assert + assert result["sms"] == 29 # 1 + 22 + 1 + 5 + assert result["email"] == 21 # 1 + 1 + 12 + 1 + 1 + 5 + + def test_get_limit_stats(self, mocker): + # Setup + mock_service = Mock(id="service-1", email_annual_limit=1000, sms_annual_limit=500, message_limit=100, sms_daily_limit=50) + + mock_notification_client = NotificationCounts() + + # Mock the dependency methods + + mocker.patch.object( + mock_notification_client, "get_all_notification_counts_for_today", return_value={"email": 20, "sms": 10} + ) + mocker.patch.object( + mock_notification_client, "get_all_notification_counts_for_year", return_value={"email": 200, "sms": 100} + ) + + # Execute + result = mock_notification_client.get_limit_stats(mock_service) + + # Assert + assert result == { + "email": { + "annual": { + "limit": 1000, + "sent": 200, + "remaining": 800, + }, + "daily": { + "limit": 100, + "sent": 20, + "remaining": 80, + }, + }, + "sms": { + "annual": { + "limit": 500, + "sent": 100, + "remaining": 400, + }, + "daily": { + "limit": 50, + "sent": 10, + "remaining": 40, + }, + }, + } + + @pytest.mark.parametrize( + "today_counts,year_counts,expected_remaining", + [ + ( + {"email": 0, "sms": 0}, + {"email": 0, "sms": 0}, + {"email": {"annual": 1000, "daily": 100}, "sms": {"annual": 500, "daily": 50}}, + ), + ( + {"email": 100, "sms": 50}, + {"email": 1000, "sms": 500}, + {"email": {"annual": 0, "daily": 0}, "sms": {"annual": 0, "daily": 0}}, + ), + ( + {"email": 50, "sms": 25}, + {"email": 500, "sms": 250}, + {"email": {"annual": 500, "daily": 50}, "sms": {"annual": 250, "daily": 25}}, + ), + ], + ) + def test_get_limit_stats_remaining_calculations(self, mocker, today_counts, year_counts, expected_remaining): + # Setup + mock_service = Mock(id="service-1", email_annual_limit=1000, sms_annual_limit=500, message_limit=100, sms_daily_limit=50) + + mock_notification_client = NotificationCounts() + + mocker.patch.object(mock_notification_client, "get_all_notification_counts_for_today", return_value=today_counts) + mocker.patch.object(mock_notification_client, "get_all_notification_counts_for_year", return_value=year_counts) + + # Execute + result = mock_notification_client.get_limit_stats(mock_service) + + # Assert remaining counts + assert result["email"]["annual"]["remaining"] == expected_remaining["email"]["annual"] + assert result["email"]["daily"]["remaining"] == expected_remaining["email"]["daily"] + assert result["sms"]["annual"]["remaining"] == expected_remaining["sms"]["annual"] + assert result["sms"]["daily"]["remaining"] == expected_remaining["sms"]["daily"] + + def test_get_limit_stats_dependencies_called(self, mocker): + # Setup + mock_service = Mock(id="service-1", email_annual_limit=1000, sms_annual_limit=500, message_limit=100, sms_daily_limit=50) + mock_notification_client = NotificationCounts() + + mock_today = mocker.patch.object( + mock_notification_client, "get_all_notification_counts_for_today", return_value={"email": 0, "sms": 0} + ) + mock_year = mocker.patch.object( + mock_notification_client, "get_all_notification_counts_for_year", return_value={"email": 0, "sms": 0} + ) + + # Execute + mock_notification_client.get_limit_stats(mock_service) + + # Assert dependencies called + mock_today.assert_called_once_with(mock_service.id) + mock_year.assert_called_once_with(mock_service.id, datetime.now().year) diff --git a/tests/conftest.py b/tests/conftest.py index 94ff0ca5be..a491717587 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -550,7 +550,14 @@ def fake_uuid(): @pytest.fixture(scope="function") def mock_get_service(mocker, api_user_active): def _get(service_id): - service = service_json(service_id, users=[api_user_active["id"]], message_limit=50, sms_daily_limit=20) + service = service_json( + service_id, + users=[api_user_active["id"]], + message_limit=50, + sms_daily_limit=20, + email_annual_limit=1000, + sms_annual_limit=1000, + ) return {"data": service} return mocker.patch("app.service_api_client.get_service", side_effect=_get) @@ -675,7 +682,9 @@ def mock_service_email_from_is_unique(mocker): @pytest.fixture(scope="function") def mock_get_live_service(mocker, api_user_active): def _get(service_id): - service = service_json(service_id, users=[api_user_active["id"]], restricted=False) + service = service_json( + service_id, users=[api_user_active["id"]], restricted=False, sms_annual_limit=10000, email_annual_limit=10000 + ) return {"data": service} return mocker.patch("app.service_api_client.get_service", side_effect=_get) @@ -971,6 +980,21 @@ def _get(service_id, template_id, version=None): return mocker.patch("app.service_api_client.get_service_template", side_effect=_get) +@pytest.fixture(scope="function") +def mock_get_service_sms_template_without_placeholders(mocker): + def _get(service_id, template_id, version=None): + template = template_json( + service_id, + template_id, + "Two week reminder", + "sms", + "Yo.", + ) + return {"data": template} + + return mocker.patch("app.service_api_client.get_service_template", side_effect=_get) + + @pytest.fixture(scope="function") def mock_get_service_letter_template(mocker, content=None, subject=None, postage="second"): def _get(service_id, template_id, version=None, postage=postage): @@ -1123,6 +1147,39 @@ def _update( return mocker.patch("app.service_api_client.update_service_template", side_effect=_update) +@pytest.fixture(scope="function") +def mock_get_limit_stats(mocker): + def _get_data(svc): + return { + "email": { + "annual": { + "limit": 1000, + "sent": 10, + "remaining": 990, + }, + "daily": { + "limit": 100, + "sent": 5, + "remaining": 95, + }, + }, + "sms": { + "annual": { + "limit": 1000, + "sent": 10, + "remaining": 990, + }, + "daily": { + "limit": 100, + "sent": 5, + "remaining": 95, + }, + }, + } + + return mocker.patch("app.main.views.templates.notification_counts_client.get_limit_stats", side_effect=_get_data) + + def create_template( service_id=SERVICE_ONE_ID, template_id=None, From ff236ccc1e4bb6371611609c296a80b2dc286bd9 Mon Sep 17 00:00:00 2001 From: Andrew Date: Tue, 10 Dec 2024 11:17:51 -0400 Subject: [PATCH 17/20] fix(dashboard): update check to see if redis has today's data (#2017) * fix(dashboard): update check to see if redis has today's data * fix: update test for redis has no data --- app/main/views/dashboard.py | 2 +- tests/app/main/views/test_dashboard.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main/views/dashboard.py b/app/main/views/dashboard.py index 54c141a82d..2c40d1a2a1 100644 --- a/app/main/views/dashboard.py +++ b/app/main/views/dashboard.py @@ -282,7 +282,7 @@ def aggregate_by_type(notification_data): todays_data = annual_limit_client.get_all_notification_counts(current_service.id) # if redis is empty, query the db - if todays_data is None: + if all(value == 0 for value in todays_data.values()): todays_data = service_api_client.get_service_statistics(service_id, limit_days=1, today_only=False) annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "db") diff --git a/tests/app/main/views/test_dashboard.py b/tests/app/main/views/test_dashboard.py index e972baebe8..3be02e81d6 100644 --- a/tests/app/main/views/test_dashboard.py +++ b/tests/app/main/views/test_dashboard.py @@ -1582,7 +1582,7 @@ def test_usage_report_aggregates_calculated_properly_without_redis( # mock annual_limit_client.get_all_notification_counts mocker.patch( "app.main.views.dashboard.annual_limit_client.get_all_notification_counts", - return_value=None, + return_value={"sms_delivered": 0, "email_delivered": 0, "sms_failed": 0, "email_failed": 0}, ) mocker.patch( From 3a5482a862d85d258cc8ba817e56909ba7799787 Mon Sep 17 00:00:00 2001 From: Mike Pond <32133001+P0NDER0SA@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:12:09 -0500 Subject: [PATCH 18/20] Update Staging Deployment Name (#2019) * Update Staging Deployment Name * Update docker.yaml --- .github/workflows/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 71358535af..904b8a97c9 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -95,7 +95,7 @@ jobs: - name: Update images in staging run: | DOCKER_TAG=${GITHUB_SHA::7} - kubectl set image deployment.apps/admin admin=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config + kubectl set image deployment.apps/notify-admin notify-admin=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config - name: my-app-install token id: notify-pr-bot From 7863163cca71f89332735cd8a04910fc4e5fb6b0 Mon Sep 17 00:00:00 2001 From: William B <7444334+whabanks@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:19:58 -0400 Subject: [PATCH 19/20] Ensure Ruff checks formatting during CI checks (#2021) * Ensure make test runs ruff format --check * Update ruff extension cfg - The VSCode ruff extension will no longer automatically remove unused imports but will still notify you they're incorrect - Updated ruff to 0.8.2 --- .devcontainer/devcontainer.json | 13 +++++------ Makefile | 2 +- poetry.lock | 40 ++++++++++++++++----------------- pyproject.toml | 2 +- scripts/run_tests.sh | 3 +++ 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ce94cdc09a..c419dcea02 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -33,15 +33,12 @@ "vsliveshare.vsliveshare" ], "settings": { + "ruff.lint.ignore": ["F401"], + "ruff.lint.run": "onSave", + "ruff.organizeImports": false, "[python]": { - "ruff.lint.run": "onSave", - "editor.defaultFormatter": "charliermarsh.ruff", - "editor.formatOnSave": false, - "editor.codeActionsOnSave": { - "source.fixAll": "explicit" - } - }, - "ruff.configurationPreference": "filesystemFirst" + "editor.defaultFormatter": "charliermarsh.ruff" + } } } }, diff --git a/Makefile b/Makefile index 71d608ebf6..2e622e27b6 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ run-gunicorn: .PHONY: format format: - ruff check --select I --fix . + ruff check --fix . ruff check ruff format . mypy ./ diff --git a/poetry.lock b/poetry.lock index 1acc726313..0c6ff0d2c0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2186,29 +2186,29 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.6.9" +version = "0.8.2" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.6.9-py3-none-linux_armv6l.whl", hash = "sha256:064df58d84ccc0ac0fcd63bc3090b251d90e2a372558c0f057c3f75ed73e1ccd"}, - {file = "ruff-0.6.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:140d4b5c9f5fc7a7b074908a78ab8d384dd7f6510402267bc76c37195c02a7ec"}, - {file = "ruff-0.6.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:53fd8ca5e82bdee8da7f506d7b03a261f24cd43d090ea9db9a1dc59d9313914c"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645d7d8761f915e48a00d4ecc3686969761df69fb561dd914a773c1a8266e14e"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eae02b700763e3847595b9d2891488989cac00214da7f845f4bcf2989007d577"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d5ccc9e58112441de8ad4b29dcb7a86dc25c5f770e3c06a9d57e0e5eba48829"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:417b81aa1c9b60b2f8edc463c58363075412866ae4e2b9ab0f690dc1e87ac1b5"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c866b631f5fbce896a74a6e4383407ba7507b815ccc52bcedabb6810fdb3ef7"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7b118afbb3202f5911486ad52da86d1d52305b59e7ef2031cea3425142b97d6f"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a67267654edc23c97335586774790cde402fb6bbdb3c2314f1fc087dee320bfa"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3ef0cc774b00fec123f635ce5c547dac263f6ee9fb9cc83437c5904183b55ceb"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:12edd2af0c60fa61ff31cefb90aef4288ac4d372b4962c2864aeea3a1a2460c0"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:55bb01caeaf3a60b2b2bba07308a02fca6ab56233302406ed5245180a05c5625"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:925d26471fa24b0ce5a6cdfab1bb526fb4159952385f386bdcc643813d472039"}, - {file = "ruff-0.6.9-py3-none-win32.whl", hash = "sha256:eb61ec9bdb2506cffd492e05ac40e5bc6284873aceb605503d8494180d6fc84d"}, - {file = "ruff-0.6.9-py3-none-win_amd64.whl", hash = "sha256:785d31851c1ae91f45b3d8fe23b8ae4b5170089021fbb42402d811135f0b7117"}, - {file = "ruff-0.6.9-py3-none-win_arm64.whl", hash = "sha256:a9641e31476d601f83cd602608739a0840e348bda93fec9f1ee816f8b6798b93"}, - {file = "ruff-0.6.9.tar.gz", hash = "sha256:b076ef717a8e5bc819514ee1d602bbdca5b4420ae13a9cf61a0c0a4f53a2baa2"}, + {file = "ruff-0.8.2-py3-none-linux_armv6l.whl", hash = "sha256:c49ab4da37e7c457105aadfd2725e24305ff9bc908487a9bf8d548c6dad8bb3d"}, + {file = "ruff-0.8.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ec016beb69ac16be416c435828be702ee694c0d722505f9c1f35e1b9c0cc1bf5"}, + {file = "ruff-0.8.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f05cdf8d050b30e2ba55c9b09330b51f9f97d36d4673213679b965d25a785f3c"}, + {file = "ruff-0.8.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60f578c11feb1d3d257b2fb043ddb47501ab4816e7e221fbb0077f0d5d4e7b6f"}, + {file = "ruff-0.8.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbd5cf9b0ae8f30eebc7b360171bd50f59ab29d39f06a670b3e4501a36ba5897"}, + {file = "ruff-0.8.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b402ddee3d777683de60ff76da801fa7e5e8a71038f57ee53e903afbcefdaa58"}, + {file = "ruff-0.8.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:705832cd7d85605cb7858d8a13d75993c8f3ef1397b0831289109e953d833d29"}, + {file = "ruff-0.8.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:32096b41aaf7a5cc095fa45b4167b890e4c8d3fd217603f3634c92a541de7248"}, + {file = "ruff-0.8.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e769083da9439508833cfc7c23e351e1809e67f47c50248250ce1ac52c21fb93"}, + {file = "ruff-0.8.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fe716592ae8a376c2673fdfc1f5c0c193a6d0411f90a496863c99cd9e2ae25d"}, + {file = "ruff-0.8.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:81c148825277e737493242b44c5388a300584d73d5774defa9245aaef55448b0"}, + {file = "ruff-0.8.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d261d7850c8367704874847d95febc698a950bf061c9475d4a8b7689adc4f7fa"}, + {file = "ruff-0.8.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1ca4e3a87496dc07d2427b7dd7ffa88a1e597c28dad65ae6433ecb9f2e4f022f"}, + {file = "ruff-0.8.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:729850feed82ef2440aa27946ab39c18cb4a8889c1128a6d589ffa028ddcfc22"}, + {file = "ruff-0.8.2-py3-none-win32.whl", hash = "sha256:ac42caaa0411d6a7d9594363294416e0e48fc1279e1b0e948391695db2b3d5b1"}, + {file = "ruff-0.8.2-py3-none-win_amd64.whl", hash = "sha256:2aae99ec70abf43372612a838d97bfe77d45146254568d94926e8ed5bbb409ea"}, + {file = "ruff-0.8.2-py3-none-win_arm64.whl", hash = "sha256:fb88e2a506b70cfbc2de6fae6681c4f944f7dd5f2fe87233a7233d888bad73e8"}, + {file = "ruff-0.8.2.tar.gz", hash = "sha256:b84f4f414dda8ac7f75075c1fa0b905ac0ff25361f42e6d5da681a465e0f78e5"}, ] [[package]] @@ -2750,4 +2750,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "~3.12.7" -content-hash = "981af1136bc73bbdab78ac127a508c97561d557aa8b3c521de03c3af14ff8c06" +content-hash = "8dde551218693f0c7fbcf66efd7c34b508f5d3007d00a91f14d41058e23a09ab" diff --git a/pyproject.toml b/pyproject.toml index b2680bca2c..e102337b19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,7 @@ mypy = "1.11.2" monkeytype = "23.3.0" poethepoet = "^0.24.4" pre-commit = "^3.7.1" -ruff = "^0.6.9" +ruff = "^0.8.2" # stubs libraries to keep mypy happy types-python-dateutil = "2.9.0.20241003" diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 97cca72678..9dac814985 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -31,6 +31,9 @@ display_result $? 1 "Code style check" ruff check --select I . display_result $? 1 "Import order check" +ruff format --check . +display_result $? 1 "Code format check" + mypy ./ display_result $? 1 "Type check" From 245407836b99dd103d85fe3b0736ecf8dd78e416 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 18 Dec 2024 14:28:10 -0400 Subject: [PATCH 20/20] Fix/handle api exception bulk send (#2022) * fix(start_job): add exception handling in case API throws an annual limits error at this point; update the expected error message strings * fix(check.html): don't render the template preview in the case of an API exception * chore: add tests * fix: pass time_to_reset into the view --- app/main/views/send.py | 16 +++- app/templates/views/notifications/check.html | 4 +- tests/app/main/views/test_send.py | 98 ++++++++++++++++++++ 3 files changed, 113 insertions(+), 5 deletions(-) diff --git a/app/main/views/send.py b/app/main/views/send.py index 53acc9ba63..f0e4e86406 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -888,8 +888,15 @@ def check_notification_preview(service_id, template_id, filetype): @main.route("/services//start-job/", methods=["POST"]) @user_has_permissions("send_messages", restrict_admin_usage=True) def start_job(service_id, upload_id): - job_api_client.create_job(upload_id, service_id, scheduled_for=request.form.get("scheduled_for", "")) - + try: + job_api_client.create_job(upload_id, service_id, scheduled_for=request.form.get("scheduled_for", "")) + except HTTPError as exception: + return render_template( + "views/notifications/check.html", + time_to_reset=get_limit_reset_time_et(), + **(get_template_error_dict(exception) if exception else {}), + template=None, + ) session.pop("sender_id", None) return redirect( @@ -1107,11 +1114,12 @@ def get_template_error_dict(exception): error = "too-many-sms-messages" elif "Content for template has a character count greater than the limit of" in exception.message: error = "message-too-long" - elif "Exceeded annual email sending limit" in exception.message: + elif "Exceeded annual email sending" in exception.message: error = "too-many-email-annual" - elif "Exceeded annual SMS sending limit" in exception.message: + elif "Exceeded annual SMS sending" in exception.message: error = "too-many-sms-annual" else: + current_app.logger.error("Unhandled exception from API: {}".format(exception)) raise exception return { diff --git a/app/templates/views/notifications/check.html b/app/templates/views/notifications/check.html index 0161a74f17..e9d61b7f94 100644 --- a/app/templates/views/notifications/check.html +++ b/app/templates/views/notifications/check.html @@ -88,7 +88,9 @@

{{_('You cannot send this email message today') }}