Task: Filter Heartbeats (#2108)

* Add filter for heartbeat template

* fix formatting

* Edit and add a test

* test for rest endpoint

* Add filter heartbeats for live service data

app/dao/fact_notification_status_dao.py

@@ -148,8 +148,8 @@ def fetch_notification_status_for_service_by_month(start_date, end_date, service
     )
 
 
-def fetch_delivered_notification_stats_by_month():
-    return (
+def fetch_delivered_notification_stats_by_month(filter_heartbeats=None):
+    query = (
         db.session.query(
             func.date_trunc("month", FactNotificationStatus.bst_date).cast(db.Text).label("month"),
             FactNotificationStatus.notification_type,
@@ -168,8 +168,17 @@ def fetch_delivered_notification_stats_by_month():
             func.date_trunc("month", FactNotificationStatus.bst_date).desc(),
             FactNotificationStatus.notification_type,
         )
-        .all()
     )
+    if filter_heartbeats:
+        query = query.filter(
+            FactNotificationStatus.template_id != current_app.config["HEARTBEAT_TEMPLATE_EMAIL_LOW"],
+            FactNotificationStatus.template_id != current_app.config["HEARTBEAT_TEMPLATE_EMAIL_MEDIUM"],
+            FactNotificationStatus.template_id != current_app.config["HEARTBEAT_TEMPLATE_EMAIL_HIGH"],
+            FactNotificationStatus.template_id != current_app.config["HEARTBEAT_TEMPLATE_SMS_LOW"],
+            FactNotificationStatus.template_id != current_app.config["HEARTBEAT_TEMPLATE_SMS_MEDIUM"],
+            FactNotificationStatus.template_id != current_app.config["HEARTBEAT_TEMPLATE_SMS_HIGH"],
+        )
+    return query.all()
 
 
 def fetch_notification_stats_for_trial_services():

app/dao/services_dao.py

@@ -84,7 +84,7 @@ def dao_count_live_services():
     ).count()
 
 
-def dao_fetch_live_services_data():
+def dao_fetch_live_services_data(filter_heartbeats=None):
     year_start_date, year_end_date = get_current_financial_year()
 
     most_recent_annual_billing = (
@@ -176,8 +176,17 @@ def dao_fetch_live_services_data():
             AnnualBilling.free_sms_fragment_limit,
         )
         .order_by(asc(Service.go_live_at))
-        .all()
     )
+    if filter_heartbeats:
+        data = data.join(Template, Service.id == Template.service_id).filter(
+            Template.id != current_app.config["HEARTBEAT_TEMPLATE_EMAIL_LOW"],
+            Template.id != current_app.config["HEARTBEAT_TEMPLATE_EMAIL_MEDIUM"],
+            Template.id != current_app.config["HEARTBEAT_TEMPLATE_EMAIL_HIGH"],
+            Template.id != current_app.config["HEARTBEAT_TEMPLATE_SMS_LOW"],
+            Template.id != current_app.config["HEARTBEAT_TEMPLATE_SMS_MEDIUM"],
+            Template.id != current_app.config["HEARTBEAT_TEMPLATE_SMS_HIGH"],
+        )
+    data = data.all()
     results = []
     for row in data:
         existing_service = next((x for x in results if x["service_id"] == row.service_id), None)

app/service/rest.py

@@ -209,13 +209,15 @@ def find_services_by_name():
 
 @service_blueprint.route("/live-services-data", methods=["GET"])
 def get_live_services_data():
-    data = dao_fetch_live_services_data()
+    filter_heartbeats = request.args.get("filter_heartbeats", None) == "True"
+    data = dao_fetch_live_services_data(filter_heartbeats=filter_heartbeats)
     return jsonify(data=data)
 
 
 @service_blueprint.route("/delivered-notifications-stats-by-month-data", methods=["GET"])
 def get_delivered_notification_stats_by_month_data():
-    return jsonify(data=fetch_delivered_notification_stats_by_month())
+    filter_heartbeats = request.args.get("filter_heartbeats", None) == "True"
+    return jsonify(data=fetch_delivered_notification_stats_by_month(filter_heartbeats=filter_heartbeats))
 
 
 @service_blueprint.route("/<uuid:service_id>", methods=["GET"])

tests/app/dao/test_fact_notification_status_dao.py

@@ -51,6 +51,7 @@
     create_template,
     save_notification,
 )
+from tests.conftest import set_config
 
 
 def test_update_fact_notification_status(notify_db_session):
@@ -761,6 +762,62 @@ def test_fetch_delivered_notification_stats_by_month(sample_service):
     assert results[3].count == 6
 
 
+@freeze_time("2020-11-02 14:00")
+def test_fetch_delivered_notification_stats_by_month_filter_heartbeats(notify_api, sample_service):
+    sms_template = create_template(service=sample_service, template_type="sms", template_name="a")
+    email_template = create_template(service=sample_service, template_type="email", template_name="b")
+
+    # Not counted: before GC Notify started
+    create_ft_notification_status(
+        utc_date=date(2019, 10, 10),
+        service=sample_service,
+        template=email_template,
+        count=3,
+    )
+
+    create_ft_notification_status(
+        utc_date=date(2019, 12, 10),
+        service=sample_service,
+        template=email_template,
+        count=3,
+    )
+
+    create_ft_notification_status(
+        utc_date=date(2019, 12, 5),
+        service=sample_service,
+        template=sms_template,
+        notification_status=NOTIFICATION_DELIVERED,
+        count=6,
+    )
+
+    create_ft_notification_status(
+        utc_date=date(2020, 1, 1),
+        service=sample_service,
+        template=sms_template,
+        notification_status=NOTIFICATION_SENT,
+        count=4,
+    )
+
+    # Not counted: failed notifications
+    create_ft_notification_status(
+        utc_date=date(2020, 1, 1),
+        service=sample_service,
+        template=sms_template,
+        notification_status=NOTIFICATION_FAILED,
+        count=10,
+    )
+
+    create_ft_notification_status(
+        utc_date=date(2020, 3, 1),
+        service=sample_service,
+        template=email_template,
+        count=5,
+    )
+    with set_config(notify_api, "HEARTBEAT_TEMPLATE_EMAIL_LOW", email_template.id):
+        results = fetch_delivered_notification_stats_by_month(filter_heartbeats=True)
+        assert len(results) == 2
+
+
 def test_fetch_delivered_notification_stats_by_month_empty():
     assert fetch_delivered_notification_stats_by_month() == []
 

tests/app/dao/test_services_dao.py

@@ -89,6 +89,7 @@
     create_user,
     save_notification,
 )
+from tests.conftest import set_config
 
 # from unittest import mock
 
@@ -495,7 +496,8 @@ def test_get_all_user_services_should_return_empty_list_if_no_services_for_user(
 
 
 @freeze_time("2019-04-23T10:00:00")
-def test_dao_fetch_live_services_data(sample_user):
+@pytest.mark.parametrize("filter_heartbeats", [True, False])
+def test_dao_fetch_live_services_data_filter_heartbeats(notify_api, sample_user, filter_heartbeats):
     org = create_organisation(organisation_type="nhs_central")
     service = create_service(go_live_user=sample_user, go_live_at="2014-04-20T10:00:00")
     template = create_template(service=service)
@@ -563,8 +565,12 @@ def test_dao_fetch_live_services_data(sample_user):
     # 3rd service: billing from 2019
     create_annual_billing(service_3.id, 200, 2019)
 
-    results = dao_fetch_live_services_data()
-    assert len(results) == 3
+    with set_config(notify_api, "HEARTBEAT_TEMPLATE_EMAIL_LOW", template.id):
+        results = dao_fetch_live_services_data(filter_heartbeats=filter_heartbeats)
+        if not filter_heartbeats:
+            assert len(results) == 3
+        else:
+            assert len(results) == 2
     # checks the results and that they are ordered by date:
     # @todo: this test is temporarily forced to pass until we can add the fiscal year back into
     # the query and create a new endpoint for the homepage stats

tests/app/service/test_rest.py

@@ -70,6 +70,7 @@
     create_user,
     save_notification,
 )
+from tests.conftest import set_config
 
 
 def test_get_service_list(client, service_factory):
@@ -253,6 +254,20 @@ def test_get_delivered_notification_stats_by_month_data(admin_request, sample_se
     assert first["count"] == 3
 
 
+def test_get_delivered_notification_stats_by_month_data_without_heartbeat(notify_api, admin_request, sample_service):
+    email_template = create_template(service=sample_service, template_type="email", template_name="b")
+
+    create_ft_notification_status(
+        utc_date=date(2019, 12, 10),
+        service=sample_service,
+        template=email_template,
+        count=3,
+    )
+    with set_config(notify_api, "HEARTBEAT_TEMPLATE_EMAIL_LOW", email_template.id):
+        response = admin_request.get("service.get_delivered_notification_stats_by_month_data", filter_heartbeats=True)["data"]
+        assert len(response) == 0
+
+
 def test_get_service_by_id(admin_request, sample_service):
     json_resp = admin_request.get("service.get_service_by_id", service_id=sample_service.id)
     assert json_resp["data"]["name"] == sample_service.name