Revert 1989: Add annual limits to dashboard and usage reports (#2000)

* annual-limits-error-msg

* chore: fix poetry.lock

app/config.py

@@ -80,7 +80,7 @@ class Config(object):
     # FEATURE FLAGS
     FF_SALESFORCE_CONTACT = env.bool("FF_SALESFORCE_CONTACT", True)
     FF_RTL = env.bool("FF_RTL", True)
-    FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", False)
+    FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", True)
 
     FREE_YEARLY_EMAIL_LIMIT = env.int("FREE_YEARLY_EMAIL_LIMIT", 20_000_000)
     FREE_YEARLY_SMS_LIMIT = env.int("FREE_YEARLY_SMS_LIMIT", 100_000)
@@ -215,7 +215,7 @@ class Test(Development):
     NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef"
 
     FF_RTL = True
-    FF_ANNUAL_LIMIT = True
+    FF_ANNUAL_LIMIT = False
 
 
 class ProductionFF(Config):

app/extensions.py

@@ -1,6 +1,5 @@
 from flask_caching import Cache
 from notifications_utils.clients.antivirus.antivirus_client import AntivirusClient
-from notifications_utils.clients.redis.annual_limit import RedisAnnualLimit
 from notifications_utils.clients.redis.bounce_rate import RedisBounceRate
 from notifications_utils.clients.redis.redis_client import RedisClient
 from notifications_utils.clients.statsd.statsd_client import StatsdClient
@@ -11,6 +10,4 @@
 zendesk_client = ZendeskClient()
 redis_client = RedisClient()
 bounce_rate_client = RedisBounceRate(redis_client)
-annual_limit_client = RedisAnnualLimit(redis_client)
-
 cache = Cache(config={"CACHE_TYPE": "simple"})  # TODO: pull config out to config.py later

app/main/views/dashboard.py

@@ -24,7 +24,7 @@
     service_api_client,
     template_statistics_client,
 )
-from app.extensions import annual_limit_client, bounce_rate_client
+from app.extensions import bounce_rate_client
 from app.main import main
 from app.models.enum.bounce_rate_status import BounceRateStatus
 from app.models.enum.notification_statuses import NotificationStatuses
@@ -229,83 +229,16 @@ def usage(service_id):
 @main.route("/services/<service_id>/monthly")
 @user_has_permissions("view_activity")
 def monthly(service_id):
-    def combine_daily_to_annual(daily, annual, mode):
-        if mode == "redis":
-            # the redis client omits properties if there are no counts yet, so account for this here\
-            daily_redis = {
-                field: daily.get(field, 0) for field in ["sms_delivered", "sms_failed", "email_delivered", "email_failed"]
-            }
-            annual["sms"] += daily_redis["sms_delivered"] + daily_redis["sms_failed"]
-            annual["email"] += daily_redis["email_delivered"] + daily_redis["email_failed"]
-        elif mode == "db":
-            annual["sms"] += daily["sms"]["requested"]
-            annual["email"] += daily["email"]["requested"]
-
-        return annual
-
-    def combine_daily_to_monthly(daily, monthly, mode):
-        if mode == "redis":
-            # the redis client omits properties if there are no counts yet, so account for this here\
-            daily_redis = {
-                field: daily.get(field, 0) for field in ["sms_delivered", "sms_failed", "email_delivered", "email_failed"]
-            }
-
-            monthly[0]["sms_counts"]["failed"] += daily_redis["sms_failed"]
-            monthly[0]["sms_counts"]["requested"] += daily_redis["sms_failed"] + daily_redis["sms_delivered"]
-            monthly[0]["email_counts"]["failed"] += daily_redis["email_failed"]
-            monthly[0]["email_counts"]["requested"] += daily_redis["email_failed"] + daily_redis["email_delivered"]
-        elif mode == "db":
-            monthly[0]["sms_counts"]["failed"] += daily["sms"]["failed"]
-            monthly[0]["sms_counts"]["requested"] += daily["sms"]["requested"]
-            monthly[0]["email_counts"]["failed"] += daily["email"]["failed"]
-            monthly[0]["email_counts"]["requested"] += daily["email"]["requested"]
-
-        return monthly
-
-    def aggregate_by_type(notification_data):
-        counts = {"sms": 0, "email": 0, "letter": 0}
-        for month_data in notification_data["data"].values():
-            for message_type, message_counts in month_data.items():
-                if isinstance(message_counts, dict):
-                    counts[message_type] += sum(message_counts.values())
-
-        # return the result
-        return counts
-
     year, current_financial_year = requested_and_current_financial_year(request)
-    monthly_data = service_api_client.get_monthly_notification_stats(service_id, year)
-    annual_data = aggregate_by_type(monthly_data)
-
-    todays_data = annual_limit_client.get_all_notification_counts(current_service.id)
-
-    # if redis is empty, query the db
-    if todays_data is None:
-        todays_data = service_api_client.get_service_statistics(service_id, limit_days=1, today_only=False)
-        annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "db")
-
-        months = (format_monthly_stats_to_list(monthly_data["data"]),)
-        monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "db")
-    else:
-        # aggregate daily + annual
-        current_app.logger.info("todays data" + str(todays_data))
-        annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "redis")
-
-        months = (format_monthly_stats_to_list(monthly_data["data"]),)
-        monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "redis")
-
-    # add today's data to monthly data
-
     return render_template(
         "views/dashboard/monthly.html",
-        months=monthly_data_aggregate,
+        months=format_monthly_stats_to_list(service_api_client.get_monthly_notification_stats(service_id, year)["data"]),
         years=get_tuples_of_financial_years(
             partial_url=partial(url_for, ".monthly", service_id=service_id),
             start=current_financial_year - 2,
             end=current_financial_year,
         ),
-        annual_data=annual_data_aggregate,
         selected_year=year,
-        current_financial_year=current_financial_year,
     )
 
 
@@ -351,21 +284,6 @@ def aggregate_notifications_stats(template_statistics):
 
 
 def get_dashboard_partials(service_id):
-    def aggregate_by_type(data, daily_data):
-        counts = {"sms": 0, "email": 0, "letter": 0}
-        # flatten out this structure to match the above
-        for month_data in data["data"].values():
-            for message_type, message_counts in month_data.items():
-                if isinstance(message_counts, dict):
-                    counts[message_type] += sum(message_counts.values())
-
-        # add todays data to the annual data
-        counts = {
-            "sms": counts["sms"] + daily_data["sms"]["requested"],
-            "email": counts["email"] + daily_data["email"]["requested"],
-        }
-        return counts
-
     all_statistics_weekly = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=7)
     template_statistics_weekly = aggregate_template_usage(all_statistics_weekly)
 
@@ -382,10 +300,6 @@ def aggregate_by_type(data, daily_data):
     dashboard_totals_weekly = (get_dashboard_totals(stats_weekly),)
     bounce_rate_data = get_bounce_rate_data_from_redis(service_id)
 
-    # get annual data from fact table (all data this year except today)
-    annual_data = service_api_client.get_monthly_notification_stats(service_id, year=get_current_financial_year())
-    annual_data = aggregate_by_type(annual_data, dashboard_totals_daily[0])
-
     return {
         "upcoming": render_template("views/dashboard/_upcoming.html", scheduled_jobs=scheduled_jobs),
         "daily_totals": render_template(
@@ -394,13 +308,6 @@ def aggregate_by_type(data, daily_data):
             statistics=dashboard_totals_daily[0],
             column_width=column_width,
         ),
-        "annual_totals": render_template(
-            "views/dashboard/_totals_annual.html",
-            service_id=service_id,
-            statistics=dashboard_totals_daily[0],
-            statistics_annual=annual_data,
-            column_width=column_width,
-        ),
         "weekly_totals": render_template(
             "views/dashboard/_totals.html",
             service_id=service_id,
@@ -422,7 +329,6 @@ def aggregate_by_type(data, daily_data):
 
 
 def _get_daily_stats(service_id):
-    # TODO: get from redis, else fallback to template_statistics_client.get_template_statistics_for_service
     all_statistics_daily = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1)
     stats_daily = aggregate_notifications_stats(all_statistics_daily)
     dashboard_totals_daily = (get_dashboard_totals(stats_daily),)

app/notify_client/service_api_client.py

@@ -1,5 +1,5 @@
 import json
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 
 from flask import current_app
 from flask_login import current_user
@@ -9,12 +9,6 @@
 from app.notify_client import NotifyAdminAPIClient, _attach_current_user, cache
 
 
-def _seconds_until_midnight():
-    now = datetime.now(timezone.utc)
-    midnight = datetime.combine(now + timedelta(days=1), datetime.min.time())
-    return int((midnight - now).total_seconds())
-
-
 class ServiceAPIClient(NotifyAdminAPIClient):
     @cache.delete("user-{user_id}")
     def create_service(
@@ -383,15 +377,8 @@ def is_service_email_from_unique(self, service_id, email_from):
     def get_service_history(self, service_id):
         return self.get("/service/{0}/history".format(service_id))
 
-    # TODO: cache this once the backend is updated to exlude data from the current day
-    # @flask_cache.memoize(timeout=_seconds_until_midnight())
     def get_monthly_notification_stats(self, service_id, year):
-        return self.get(
-            url="/service/{}/notifications/monthly?year={}".format(
-                service_id,
-                year,
-            )
-        )
+        return self.get(url="/service/{}/notifications/monthly?year={}".format(service_id, year))
 
     def get_safelist(self, service_id):
         return self.get(url="/service/{}/safelist".format(service_id))
@@ -635,15 +622,5 @@ def _use_case_data_name(self, service_id):
     def _tos_key_name(self, service_id):
         return f"tos-accepted-{service_id}"
 
-    def aggregate_by_type(self, notification_data):
-        counts = {"sms": 0, "email": 0, "letter": 0}
-        for month_data in notification_data["data"].values():
-            for message_type, message_counts in month_data.items():
-                if isinstance(message_counts, dict):
-                    counts[message_type] += sum(message_counts.values())
-
-        # return the result
-        return counts
-
 
 service_api_client = ServiceAPIClient()

app/templates/views/dashboard/_totals_daily.html

@@ -1,29 +1,8 @@
 {% from "components/big-number.html" import big_number %}
 {% from "components/message-count-label.html" import message_count_label %}
 {% from 'components/remaining-messages.html' import remaining_messages %}
-{% from "components/show-more.html" import show_more %}
 
-{% if config["FF_ANNUAL_LIMIT"] %}
-  <div class="ajax-block-container" data-testid="daily-usage">
-      <h2 class="heading-medium mt-8">
-        {{ _('Daily usage') }}
-        <br />
-        <small class="text-gray-600 text-small font-normal" style="color: #5E6975">
-          {{ _('resets at 7pm Eastern Time') }}
-        </small> 
-      </h2>
-      <div class="grid-row contain-floats mb-10">
-        <div class="{{column_width}}">
-          {{ remaining_messages(header=_('emails'), total=current_service.message_limit, used=statistics['email']['requested'], muted=true) }}
-        </div>
-        <div class="{{column_width}}">
-          {{ remaining_messages(header=_('text messages'), total=current_service.sms_daily_limit, used=statistics['sms']['requested'], muted=true) }}
-        </div>
-      </div>
-      {{ show_more(url_for('main.contact'), _('Request a daily limit increase')) }}  
-  </div>
-{% else %}
-  <div class="ajax-block-container">
+<div class="ajax-block-container">
     <h2 class="heading-medium mt-8">
       {{ _('Daily usage') }}
       <br />
@@ -39,5 +18,4 @@ <h2 class="heading-medium mt-8">
         {{ remaining_messages(header=_('text messages'), total=current_service.sms_daily_limit, used=statistics['sms']['requested']) }}
       </div>
     </div>
-  </div>
-{% endif %}
+</div>

app/templates/views/dashboard/dashboard.html

@@ -26,10 +26,7 @@ <h2 class="heading-medium mt-8">{{ _("Scheduled sends") }}</h2>
 
     {{ ajax_block(partials, updates_url, 'weekly_totals', interval=5) }}
     {{ ajax_block(partials, updates_url, 'daily_totals', interval=5) }}
-    {% if config["FF_ANNUAL_LIMIT"] %}
-      {{ ajax_block(partials, updates_url, 'annual_totals', interval=5) }}
-    {% endif %}
-
+
     <hr />
 
     {% if partials['has_template_statistics'] %}

app/templates/views/dashboard/monthly.html

@@ -1,23 +1,22 @@
-{% from "components/big-number.html" import big_number_with_status, big_number, big_number_simple %}
+{% from "components/big-number.html" import big_number_with_status, big_number %}
 {% from "components/pill.html" import pill %}
 {% from "components/table.html" import list_table, field, hidden_field_heading, right_aligned_field_heading, row_heading %}
 {% from "components/message-count-label.html" import message_count_label %}
-{% from 'components/remaining-messages.html' import remaining_messages %}
 
 {% extends "admin_template.html" %}
 
 {% block service_page_title %}
-  {{ _('Usage report') }}
+  {{ _('Messages sent,') }}
   {{ selected_year }} {{ _('to') }} {{ selected_year + 1 }} {{ _('fiscal year') }}
 {% endblock %}
 
 {% block maincolumn_content %}
 
   <h1 class="heading-large">
-    {{ _('Usage report') }}
+    {{ _('Messages sent') }}
   </h1>
 
-  <div class="mb-12">
+  <div class="mb-6">
     {{ pill(
       items=years,
       current_value=selected_year,
@@ -26,50 +25,6 @@ <h1 class="heading-large">
     ) }}
   </div>
 
-  {% if config["FF_ANNUAL_LIMIT"] %}
-    <h2 class="heading-medium mt-12">
-      {% if selected_year == current_financial_year %}
-        {{ _('Annual limit overview') }}
-      {% else %}
-        {{ _('Annual overview') }}
-      {% endif %}
-      <br />
-      <small class="text-gray-600 text-small font-normal" style="color: #5E6975">
-        {{ _('Fiscal year begins April 1, ') ~ selected_year ~ _(' and ends March 31, ') ~ (selected_year + 1) }}
-      </small> 
-    </h2>
-    <div class="grid-row contain-floats mb-10">
-      {% if selected_year == current_financial_year %}
-        <div class="w-1/2 float-left py-0 px-0 px-gutterHalf box-border">
-          {{ remaining_messages(header=_('emails'), total=current_service.email_annual_limit, used=annual_data['email']) }}
-        </div>
-        <div class="w-1/2 float-left py-0 px-0 px-gutterHalf box-border">
-          {{ remaining_messages(header=_('text messages'), total=current_service.sms_annual_limit, used=annual_data['sms']) }}
-        </div>
-      {% else %}
-        <div class="w-1/2 float-left py-0 px-0 px-gutterHalf box-border">
-          {{ big_number_simple(
-              annual_data['email'],
-              _('emails'),
-
-            )
-          }}
-        </div>
-        <div class="w-1/2 float-left py-0 px-0 px-gutterHalf box-border">
-          {{ big_number_simple(
-              annual_data['sms'],
-              _('text messages'),
-
-            )
-          }}
-        </div>
-      {% endif %}
-    </div>
-    <h2 class="heading-medium mt-12">
-      {{ _('Month by month totals') }}
-    </h2>
-  {% endif %}
-
   {% if months %}
    {% set spend_txt = _('Total spend') %}
    {% set heading_1 = _('Month') %}