From d2081d1219c8afd44434aff2d11033603b0df8d1 Mon Sep 17 00:00:00 2001 From: Jumana B Date: Wed, 20 Nov 2024 15:22:24 -0500 Subject: [PATCH] Func to find service annual stats and send an email (#2359) * Func to find service annual stats and send an email * Fix changes as we now have the template * Added remaining translations + testing * We are going to add q4 data even if we don't send it * PR fix * format fr and en numbers correctly * fix --- app/celery/reporting_tasks.py | 93 +++++++++++++++++++- app/config.py | 50 +++++++++++ app/dao/annual_limits_data_dao.py | 45 ++++++++++ app/dao/users_dao.py | 24 ++++- app/user/rest.py | 33 +++++++ tests/app/celery/test_reporting_tasks.py | 30 +++++++ tests/app/dao/test_annual_limits_data_dao.py | 32 ++++++- tests/app/dao/test_users_dao.py | 27 ++++++ 8 files changed, 331 insertions(+), 3 deletions(-) diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index 8090d42bb7..4e2b486d57 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -8,14 +8,21 @@ from app import notify_celery from app.config import QueueNames from app.cronitor import cronitor -from app.dao.annual_limits_data_dao import get_previous_quarter, insert_quarter_data +from app.dao.annual_limits_data_dao import ( + fetch_quarter_cummulative_stats, + get_all_quarters, + get_previous_quarter, + insert_quarter_data, +) from app.dao.fact_billing_dao import fetch_billing_data_for_day, update_fact_billing from app.dao.fact_notification_status_dao import ( fetch_notification_status_for_day, fetch_quarter_data, update_fact_notification_status, ) +from app.dao.users_dao import get_services_for_all_users from app.models import Service +from app.user.rest import send_annual_usage_data @notify_celery.task(name="create-nightly-billing") @@ -177,3 +184,87 @@ def insert_quarter_data_for_annual_limits(process_day): start_date, end_date, chunk, e ) ) + + +def _format_number(number, use_space=False): + if use_space: + return "{:,}".format(number).replace(",", " ") + return "{:,}".format(number) + + +def _create_quarterly_email_markdown_list(service_info, service_ids, cummulative_data_dict): + """ + This function creates a markdown list of the service names and their email and sms usage + + Example: + ## Notify + Emails: you’ve sent 5,000 out of 100,000 (5%) + Text messages: you’ve sent 1,000 out of 2,000 (50%) + """ + markdown_list_en = "" + markdown_list_fr = "" + for service_id in service_ids: + service_data = cummulative_data_dict.get(str(service_id), {}) + service_name, email_annual_limit, sms_annual_limit = service_info[service_id] + email_count = service_data.get("email", 0) + sms_count = service_data.get("sms", 0) + + markdown_list_en += f"## {service_name} \n" + markdown_list_fr += f"## {service_name} \n" + + email_percentage = round(float(email_count / email_annual_limit), 4) * 100 if email_count else 0 + email_count_en = _format_number(email_count) + email_annual_limit_en = _format_number(email_annual_limit) + email_count_fr = _format_number(email_count, use_space=True) + email_annual_limit_fr = _format_number(email_annual_limit, use_space=True) + markdown_list_en += f"Emails: you've sent {email_count_en} out of {email_annual_limit_en} ({email_percentage}%)\n" + markdown_list_fr += f"Courriels: {email_count_fr} envoyés sur {email_annual_limit_fr} ({email_percentage}%)\n" + + sms_percentage = round(float(sms_count / sms_annual_limit), 4) * 100 if sms_count else 0 + sms_count_en = _format_number(sms_count) + sms_annual_limit_en = _format_number(sms_annual_limit) + sms_count_fr = _format_number(sms_count, use_space=True) + sms_annual_limit_fr = _format_number(sms_annual_limit, use_space=True) + markdown_list_en += f"Text messages: you've sent {sms_count_en} out of {sms_annual_limit_en} ({sms_percentage}%)\n" + markdown_list_fr += f"Messages texte : {sms_count_fr} envoyés sur {sms_annual_limit_fr} ({sms_percentage}%)\n" + + markdown_list_en += "\n" + markdown_list_fr += "\n" + return markdown_list_en, markdown_list_fr + + +@notify_celery.task(name="send-quarterly-email") +@statsd(namespace="tasks") +def send_quarter_email(process_date): + service_info = {x.id: (x.name, x.email_annual_limit, x.sms_annual_limit) for x in Service.query.all()} + + user_service_array = get_services_for_all_users() + quarters_list = get_all_quarters(process_date) + chunk_size = 50 + iter_user_service_array = iter(user_service_array) + start_year = int(quarters_list[0].split("-")[-1]) + end_year = start_year + 1 + + while True: + chunk = list(islice(iter_user_service_array, chunk_size)) + + if not chunk: + current_app.logger.info("send_quarter_email completed {} ".format(datetime.now(timezone.utc).date())) + break + + try: + all_service_ids = set() + for _, _, services in chunk: + all_service_ids.update(services) + all_service_ids = list(all_service_ids) + cummulative_data = fetch_quarter_cummulative_stats(quarters_list, all_service_ids) + cummulative_data_dict = {str(c_data_id): c_data for c_data_id, c_data in cummulative_data} + for user_id, _, service_ids in chunk: + markdown_list_en, markdown_list_fr = _create_quarterly_email_markdown_list( + service_info, service_ids, cummulative_data_dict + ) + send_annual_usage_data(user_id, start_year, end_year, markdown_list_en, markdown_list_fr) + current_app.logger.info("send_quarter_email task completed for user {} ".format(user_id)) + except Exception as e: + current_app.logger.error("send_quarter_email task failed for for user {} . Error: {}".format(user_id, e)) + continue diff --git a/app/config.py b/app/config.py index 7430066e46..9e3edb0219 100644 --- a/app/config.py +++ b/app/config.py @@ -503,6 +503,56 @@ class Config(object): "schedule": crontab(hour=9, minute=0), # 4:00 EST in UTC "options": {"queue": QueueNames.PERIODIC}, }, + # quarterly queue + "insert-quarter-data-for-annual-limits-q1": { + "task": "insert-quarter-data-for-annual-limits", + "schedule": crontab( + minute=0, hour=23, day_of_month=1, month_of_year=7 + ), # Running this at the end of the day on 1st July + "options": {"queue": QueueNames.PERIODIC}, + }, + "insert-quarter-data-for-annual-limits-q2": { + "task": "insert-quarter-data-for-annual-limits", + "schedule": crontab( + minute=0, hour=23, day_of_month=1, month_of_year=10 + ), # Running this at the end of the day on 1st Oct + "options": {"queue": QueueNames.PERIODIC}, + }, + "insert-quarter-data-for-annual-limits-q3": { + "task": "insert-quarter-data-for-annual-limits", + "schedule": crontab( + minute=0, hour=23, day_of_month=1, month_of_year=1 + ), # Running this at the end of the day on 1st Jan + "options": {"queue": QueueNames.PERIODIC}, + }, + "insert-quarter-data-for-annual-limits-q4": { + "task": "insert-quarter-data-for-annual-limits", + "schedule": crontab( + minute=0, hour=23, day_of_month=1, month_of_year=1 + ), # Running this at the end of the day on 1st April + "options": {"queue": QueueNames.PERIODIC}, + }, + "send-quarterly-email-q1": { + "task": "send-quarterly-email", + "schedule": crontab( + minute=0, hour=23, day_of_month=2, month_of_year=7 + ), # Running this at the end of the day on 2nd July + "options": {"queue": QueueNames.PERIODIC}, + }, + "send-quarterly-email-q2": { + "task": "send-quarterly-email", + "schedule": crontab( + minute=0, hour=23, day_of_month=2, month_of_year=10 + ), # Running this at the end of the day on 2nd Oct + "options": {"queue": QueueNames.PERIODIC}, + }, + "send-quarterly-email-q3": { + "task": "send-quarterly-email", + "schedule": crontab( + minute=0, hour=23, day_of_month=3, month_of_year=1 + ), # Running this at the end of the day on 2nd Jan + "options": {"queue": QueueNames.PERIODIC}, + }, } CELERY_QUEUES: List[Any] = [] CELERY_DELIVER_SMS_RATE_LIMIT = os.getenv("CELERY_DELIVER_SMS_RATE_LIMIT", "1/s") diff --git a/app/dao/annual_limits_data_dao.py b/app/dao/annual_limits_data_dao.py index 91aecf004f..65a9d4e779 100644 --- a/app/dao/annual_limits_data_dao.py +++ b/app/dao/annual_limits_data_dao.py @@ -1,5 +1,6 @@ from datetime import datetime +from sqlalchemy import func from sqlalchemy.dialects.postgresql import insert from app import db @@ -36,6 +37,20 @@ def get_previous_quarter(date_to_check): return quarter_name, (start_date, end_date) +def get_all_quarters(process_day): + previous_quarter, _ = get_previous_quarter(process_day) + quarter, year = previous_quarter.split("-") + + quarter_mapping = { + "Q1": [f"Q1-{year}"], + "Q2": [f"Q1-{year}", f"Q2-{year}"], + "Q3": [f"Q1-{year}", f"Q2-{year}", f"Q3-{year}"], + "Q4": [f"Q1-{year}", f"Q2-{year}", f"Q3-{year}", f"Q4-{year}"], + } + + return quarter_mapping[quarter] + + def insert_quarter_data(data, quarter, service_info): """ Insert data for each quarter into the database. @@ -70,3 +85,33 @@ def insert_quarter_data(data, quarter, service_info): ) db.session.connection().execute(stmt) db.session.commit() + + +def fetch_quarter_cummulative_stats(quarters, service_ids): + """ + Fetch notification status data for a list of quarters and service_ids. + + This function returns a list of namedtuples with the following fields: + - service_id, + - notification_type, + - notification_count + """ + subquery = ( + db.session.query( + AnnualLimitsData.service_id, + AnnualLimitsData.notification_type, + func.sum(AnnualLimitsData.notification_count).label("notification_count"), + ) + .filter(AnnualLimitsData.service_id.in_(service_ids), AnnualLimitsData.time_period.in_(quarters)) + .group_by(AnnualLimitsData.service_id, AnnualLimitsData.notification_type) + .subquery() + ) + + return ( + db.session.query( + subquery.c.service_id, + func.json_object_agg(subquery.c.notification_type, subquery.c.notification_count).label("notification_counts"), + ) + .group_by(subquery.c.service_id) + .all() + ) diff --git a/app/dao/users_dao.py b/app/dao/users_dao.py index 456648933b..21f469d5d1 100644 --- a/app/dao/users_dao.py +++ b/app/dao/users_dao.py @@ -10,7 +10,7 @@ from app.dao.permissions_dao import permission_dao from app.dao.service_user_dao import dao_get_service_users_by_user_id from app.errors import InvalidRequest -from app.models import EMAIL_AUTH_TYPE, User, VerifyCode +from app.models import EMAIL_AUTH_TYPE, Service, ServiceUser, User, VerifyCode from app.utils import escape_special_characters @@ -200,3 +200,25 @@ def user_can_be_archived(user): def get_archived_email_address(email_address): date = datetime.utcnow().strftime("%Y-%m-%d") return "_archived_{}_{}".format(date, email_address) + + +def get_services_for_all_users(): + """ + Return (user_id, email_address, [service_id1, service_id2]...] for all users + where both the user and the service are active. + + """ + result = ( + db.session.query( + User.id.label("user_id"), + User.email_address.label("email_address"), + func.array_agg(Service.id).label("service_ids"), + ) + .join(ServiceUser, User.id == ServiceUser.user_id) + .join(Service, Service.id == ServiceUser.service_id) + .filter(User.state == "active", Service.active.is_(True), Service.restricted.is_(False), Service.research_mode.is_(False)) + .group_by(User.id, User.email_address) + .all() + ) + + return result diff --git a/app/user/rest.py b/app/user/rest.py index f9e288fce3..afd1556a26 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -886,3 +886,36 @@ def _update_alert(user_to_update, changes=None): ) send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) + + +def send_annual_usage_data(user_id, start_year, end_year, markdown_en, markdown_fr): + """ + We are sending a notification to the user to inform them that their annual usage + per service. + + """ + user = get_user_by_id(user_id=user_id) + template = dao_get_template_by_id(current_app.config["ANNUAL_LIMIT_QUARTERLY_USAGE_TEMPLATE_ID"]) + service = Service.query.get(current_app.config["NOTIFY_SERVICE_ID"]) + + saved_notification = persist_notification( + template_id=template.id, + template_version=template.version, + recipient=user.email_address, + service=service, + personalisation={ + "name": user.name, + "start_year": start_year, + "end_year": end_year, + "data_for_each_service_en": markdown_en, + "data_for_each_service_fr": markdown_fr, + }, + notification_type=template.template_type, + api_key_id=None, + key_type=KEY_TYPE_NORMAL, + reply_to_text=service.get_default_reply_to_email_address(), + ) + + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) + + return jsonify({}), 204 diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index 53ceb547ca..0f5854734f 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -11,6 +11,7 @@ create_nightly_notification_status, create_nightly_notification_status_for_day, insert_quarter_data_for_annual_limits, + send_quarter_email, ) from app.dao.fact_billing_dao import get_rate from app.models import ( @@ -605,3 +606,32 @@ def test_insert_quarter_data(self, notify_db_session): # Data for Q1 2018 insert_quarter_data_for_annual_limits(datetime(2018, 7, 1)) assert AnnualLimitsData.query.filter_by(service_id=service_1.id, time_period="Q1-2018").first().notification_count == 10 + + +class TestSendQuarterEmail: + def test_send_quarter_email(self, sample_user, mocker, notify_db_session): + service_1 = create_service(service_name="service_1") + service_2 = create_service(service_name="service_2") + + create_ft_notification_status(date(2018, 1, 1), "sms", service_1, count=4) + create_ft_notification_status(date(2018, 5, 2), "sms", service_1, count=10) + create_ft_notification_status(date(2018, 3, 20), "sms", service_2, count=100) + create_ft_notification_status(date(2018, 2, 1), "sms", service_2, count=1000) + + # Data for Q4 2017 + insert_quarter_data_for_annual_limits(datetime(2018, 4, 1)) + + service_1.users = [sample_user] + service_2.users = [sample_user] + send_mock = mocker.patch("app.celery.reporting_tasks.send_annual_usage_data") + + markdown_list_en = "## service_1 \nText messages: you've sent 4 out of 25,000 (0.0%)\n\n## service_2 \nText messages: you've sent 1 100 out of 25 000 (4.0%)\n\n" + markdown_list_fr = "## service_1 \n\n## service_2 \n\n" + send_quarter_email(datetime(2018, 4, 1)) + assert send_mock.call_args( + sample_user.id, + 2018, + 2019, + markdown_list_en, + markdown_list_fr, + ) diff --git a/tests/app/dao/test_annual_limits_data_dao.py b/tests/app/dao/test_annual_limits_data_dao.py index 546f671946..f86d32da97 100644 --- a/tests/app/dao/test_annual_limits_data_dao.py +++ b/tests/app/dao/test_annual_limits_data_dao.py @@ -3,7 +3,11 @@ import pytest -from app.dao.annual_limits_data_dao import get_previous_quarter, insert_quarter_data +from app.dao.annual_limits_data_dao import ( + fetch_quarter_cummulative_stats, + get_previous_quarter, + insert_quarter_data, +) from app.models import AnnualLimitsData, Service from tests.app.db import create_service @@ -53,3 +57,29 @@ def test_insert_quarter_data(self, notify_db_session): service_info, ) assert AnnualLimitsData.query.filter_by(service_id=service_2.id).first().notification_count == 500 + + +class TestFetchCummulativeStats: + def test_fetch_quarter_cummulative_stats(self, notify_db_session): + service_1 = create_service(service_name="service_1") + service_2 = create_service(service_name="service_2") + + service_info = {x.id: (x.email_annual_limit, x.sms_annual_limit) for x in Service.query.all()} + NotificationData = namedtuple("NotificationData", ["service_id", "notification_type", "notification_count"]) + + data = [ + NotificationData(service_1.id, "sms", 4), + NotificationData(service_2.id, "sms", 1100), + NotificationData(service_1.id, "email", 2), + ] + insert_quarter_data(data, "Q1-2018", service_info) + + data2 = [NotificationData(service_1.id, "sms", 5)] + insert_quarter_data(data2, "Q2-2018", service_info) + + result = fetch_quarter_cummulative_stats(["Q1-2018", "Q2-2018"], [service_1.id, service_2.id]) + for service_id, counts in result: + if service_id == service_1.id: + assert counts == {"sms": 9, "email": 2} + if service_id == service_2.id: + assert counts == {"sms": 1100} diff --git a/tests/app/dao/test_users_dao.py b/tests/app/dao/test_users_dao.py index 5e792ce8a6..bd934b8290 100644 --- a/tests/app/dao/test_users_dao.py +++ b/tests/app/dao/test_users_dao.py @@ -14,6 +14,7 @@ dao_archive_user, delete_codes_older_created_more_than_a_day_ago, delete_model_user, + get_services_for_all_users, get_user_by_email, get_user_by_id, increment_failed_login_count, @@ -324,3 +325,29 @@ def test_check_password_for_blocked_user(notify_api, notify_db, notify_db_sessio def test_check_password_for_allowed_user(notify_api, notify_db, notify_db_session, sample_user): allowed_user = create_user(email="allowed@test.com", blocked=False) assert allowed_user.check_password("password") + + +class TestGetServicesAllUsers: + def test_get_services_for_all_users(self, sample_user, fake_uuid, notify_db_session): + sample_user.current_session_id = fake_uuid + + service_1 = create_service(service_name="Service 1") + service_1_user = create_user(email="1@test.com") + service_1.users = [sample_user, service_1_user] + + service_2 = create_service(service_name="Service 2") + service_2_user = create_user(email="2@test.com") + service_2.users = [sample_user, service_2_user] + + results = get_services_for_all_users() + assert len(results) == 3 + for user_id, email_address, services in results: + if user_id == sample_user.id: + assert email_address == sample_user.email_address + assert set(services) == set([service_1.id, service_2.id]) + if user_id == service_1_user.id: + assert email_address == service_1_user.email_address + assert services == [service_1.id] + if user_id == service_2_user.id: + assert email_address == service_2_user.email_address + assert services == [service_2.id]