From e8e42ca679f1db5d771d1b5232291fe966aa313b Mon Sep 17 00:00:00 2001 From: Tim DiLauro Date: Mon, 18 Sep 2023 14:59:50 -0400 Subject: [PATCH] Add `start` and `until` date arguments to playback time reporting. (PP-273) (#1376) * Add `start` and `until` date arguments to playback time reporting. * Make report tests pass. * WIP before previous months bug * Handle out of order dates and parse into UTC. - Dates are parsed into UTC, rather than naive, `datetime`s. - Factored out the query into a helper function to improve readability. * Add tests for new functionality. --- core/jobs/playtime_entries.py | 129 +++++++++++++++---- tests/core/jobs/test_playtime_entries.py | 156 ++++++++++++++++++++++- 2 files changed, 256 insertions(+), 29 deletions(-) diff --git a/core/jobs/playtime_entries.py b/core/jobs/playtime_entries.py index 2b271fc3c0..b68378446b 100644 --- a/core/jobs/playtime_entries.py +++ b/core/jobs/playtime_entries.py @@ -1,9 +1,14 @@ +from __future__ import annotations + +import argparse import csv import os from collections import defaultdict from datetime import datetime, timedelta from tempfile import TemporaryFile +from typing import TYPE_CHECKING +import dateutil.parser import pytz from sqlalchemy.sql.functions import sum @@ -15,6 +20,9 @@ from core.util.email import EmailManager from scripts import Script +if TYPE_CHECKING: + from sqlalchemy.orm import Query + class PlaytimeEntriesSummationScript(Script): def do_run(self): @@ -64,42 +72,85 @@ def do_run(self): class PlaytimeEntriesEmailReportsScript(Script): - def do_run(self): - """Send a quarterly report with aggregated playtimes via email""" - # 3 months prior, shifted to the 1st of the month - start, until = previous_months(number_of_months=3) + REPORT_DATE_FORMAT = "%Y-%m-%d" - # Let the database do the math for us - result = ( - self._db.query(PlaytimeSummary) - .with_entities( - PlaytimeSummary.identifier_str, - PlaytimeSummary.collection_name, - PlaytimeSummary.library_name, - PlaytimeSummary.identifier_id, - sum(PlaytimeSummary.total_seconds_played), - ) - .filter( - PlaytimeSummary.timestamp >= start, - PlaytimeSummary.timestamp < until, - ) - .group_by( - PlaytimeSummary.identifier_str, - PlaytimeSummary.collection_name, - PlaytimeSummary.library_name, - PlaytimeSummary.identifier_id, + @classmethod + def arg_parser(cls): + # The default `start` and `until` dates encompass the previous three months. + # We convert them to strings here so that they are handled the same way + # as non-default dates specified as arguments. + default_start, default_until = ( + date.isoformat() for date in previous_months(number_of_months=3) + ) + + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "--start", + metavar="YYYY-MM-DD", + default=default_start, + type=dateutil.parser.isoparse, + help="Start date for report in ISO 8601 'yyyy-mm-dd' format.", + ) + parser.add_argument( + "--until", + metavar="YYYY-MM-DD", + default=default_until, + type=dateutil.parser.isoparse, + help="'Until' date for report in ISO 8601 'yyyy-mm-dd' format." + " The report will represent entries from the 'start' date up until," + " but not including, this date.", + ) + return parser + + @classmethod + def parse_command_line(cls, _db=None, cmd_args=None, *args, **kwargs): + parsed = super().parse_command_line(_db=_db, cmd_args=cmd_args, *args, **kwargs) + utc_start = pytz.utc.localize(parsed.start) + utc_until = pytz.utc.localize(parsed.until) + if utc_start >= utc_until: + cls.arg_parser().error( + f"start date ({utc_start.strftime(cls.REPORT_DATE_FORMAT)}) must be before " + f"until date ({utc_until.strftime(cls.REPORT_DATE_FORMAT)})." ) + return argparse.Namespace( + **{**vars(parsed), **dict(start=utc_start, until=utc_until)} + ) + + def do_run(self): + """Produce a report for the given (or default) date range.""" + parsed = self.parse_command_line() + start = parsed.start + until = parsed.until + + formatted_start_date = start.strftime(self.REPORT_DATE_FORMAT) + formatted_until_date = until.strftime(self.REPORT_DATE_FORMAT) + report_date_label = f"{formatted_start_date} - {formatted_until_date}" + email_subject = ( + f"Playtime Summaries {formatted_start_date} - {formatted_until_date}" + ) + attachment_name = ( + f"playtime-summary-{formatted_start_date}-{formatted_until_date}" ) # Write to a temporary file so we don't overflow the memory - with TemporaryFile("w+", prefix=f"playtimereport{until}", suffix="csv") as temp: + with TemporaryFile( + "w+", prefix=f"playtimereport{formatted_until_date}", suffix="csv" + ) as temp: # Write the data as a CSV writer = csv.writer(temp) writer.writerow( ["date", "urn", "collection", "library", "title", "total seconds"] ) - for urn, collection_name, library_name, identifier_id, total in result: + for ( + urn, + collection_name, + library_name, + identifier_id, + total, + ) in self._fetch_report_records(start=start, until=until): edition = None if identifier_id: edition = get_one( @@ -107,7 +158,7 @@ def do_run(self): ) title = edition and edition.title row = ( - f"{start} - {until}", + report_date_label, urn, collection_name, library_name, @@ -124,11 +175,33 @@ def do_run(self): ) if recipient: EmailManager.send_email( - f"Playtime Summaries {start} - {until}", + email_subject, receivers=[recipient], text="", - attachments={f"playtime-summary-{start}-{until}": temp.read()}, + attachments={attachment_name: temp.read()}, ) else: self.log.error("No reporting email found, logging complete report.") self.log.warning(temp.read()) + + def _fetch_report_records(self, start: datetime, until: datetime) -> Query: + return ( + self._db.query(PlaytimeSummary) + .with_entities( + PlaytimeSummary.identifier_str, + PlaytimeSummary.collection_name, + PlaytimeSummary.library_name, + PlaytimeSummary.identifier_id, + sum(PlaytimeSummary.total_seconds_played), + ) + .filter( + PlaytimeSummary.timestamp >= start, + PlaytimeSummary.timestamp < until, + ) + .group_by( + PlaytimeSummary.identifier_str, + PlaytimeSummary.collection_name, + PlaytimeSummary.library_name, + PlaytimeSummary.identifier_id, + ) + ) diff --git a/tests/core/jobs/test_playtime_entries.py b/tests/core/jobs/test_playtime_entries.py index ae14633f29..0cef68e379 100644 --- a/tests/core/jobs/test_playtime_entries.py +++ b/tests/core/jobs/test_playtime_entries.py @@ -1,8 +1,13 @@ +from __future__ import annotations + +import re from datetime import datetime, timedelta from typing import List from unittest.mock import MagicMock, call, patch +import pytest import pytz +from freezegun import freeze_time from api.model.time_tracking import PlaytimeTimeEntry from core.config import Configuration @@ -15,7 +20,7 @@ from core.model.identifier import Identifier from core.model.library import Library from core.model.time_tracking import PlaytimeEntry, PlaytimeSummary -from core.util.datetime_helpers import previous_months, utc_now +from core.util.datetime_helpers import datetime_utc, previous_months, utc_now from tests.fixtures.database import DatabaseTransactionFixture @@ -307,3 +312,152 @@ def test_no_reporting_email(self, db: DatabaseTransactionFixture): assert script._log.error.call_count == 1 assert script._log.warning.call_count == 1 assert "date,urn,collection," in script._log.warning.call_args[0][0] + + @pytest.mark.parametrize( + "current_utc_time, start_arg, expected_start, until_arg, expected_until", + [ + # Default values from two dates within the same month (next two cases). + [ + datetime(2020, 1, 1, 0, 0, 0), + None, + datetime_utc(2019, 10, 1, 0, 0, 0), + None, + datetime_utc(2020, 1, 1, 0, 0, 0), + ], + [ + datetime(2020, 1, 31, 0, 0, 0), + None, + datetime_utc(2019, 10, 1, 0, 0, 0), + None, + datetime_utc(2020, 1, 1, 0, 0, 0), + ], + # `start` specified, `until` defaulted. + [ + datetime(2020, 1, 31, 0, 0, 0), + "2019-06-11", + datetime_utc(2019, 6, 11, 0, 0, 0), + None, + datetime_utc(2020, 1, 1, 0, 0, 0), + ], + # `start` defaulted, `until` specified. + [ + datetime(2020, 1, 31, 0, 0, 0), + None, + datetime_utc(2019, 10, 1, 0, 0, 0), + "2019-11-20", + datetime_utc(2019, 11, 20, 0, 0, 0), + ], + # When both dates are specified, the current datetime doesn't matter. + # Both dates specified, but we test at a specific time here anyway. + [ + datetime(2020, 1, 31, 0, 0, 0), + "2018-07-03", + datetime_utc(2018, 7, 3, 0, 0, 0), + "2019-04-30", + datetime_utc(2019, 4, 30, 0, 0, 0), + ], + # The same dates are specified, but we test at the actual current time. + [ + utc_now(), + "2018-07-03", + datetime_utc(2018, 7, 3, 0, 0, 0), + "2019-04-30", + datetime_utc(2019, 4, 30, 0, 0, 0), + ], + # The same dates are specified, but we test at the actual current time. + [ + utc_now(), + "4099-07-03", + datetime_utc(4099, 7, 3, 0, 0, 0), + "4150-04-30", + datetime_utc(4150, 4, 30, 0, 0, 0), + ], + ], + ) + def test_parse_command_line( + self, + current_utc_time: datetime, + start_arg: str | None, + expected_start: datetime, + until_arg: str | None, + expected_until: datetime, + ): + start_args = ["--start", start_arg] if start_arg else [] + until_args = ["--until", until_arg] if until_arg else [] + cmd_args = start_args + until_args + + with freeze_time(current_utc_time): + parsed = PlaytimeEntriesEmailReportsScript.parse_command_line( + cmd_args=cmd_args + ) + assert expected_start == parsed.start + assert expected_until == parsed.until + assert pytz.UTC == parsed.start.tzinfo + assert pytz.UTC == parsed.until.tzinfo + + @pytest.mark.parametrize( + "current_utc_time, start_arg, expected_start, until_arg, expected_until", + [ + # `start` specified, `until` defaulted. + [ + datetime(2020, 1, 31, 0, 0, 0), + "2020-02-01", + datetime_utc(2020, 2, 1, 0, 0, 0), + None, + datetime_utc(2020, 1, 1, 0, 0, 0), + ], + # `start` defaulted, `until` specified. + [ + datetime(2020, 1, 31, 0, 0, 0), + None, + datetime_utc(2019, 10, 1, 0, 0, 0), + "2019-06-11", + datetime_utc(2019, 6, 11, 0, 0, 0), + ], + # When both dates are specified, the current datetime doesn't matter. + # Both dates specified, but we test at a specific time here anyway. + [ + datetime(2020, 1, 31, 0, 0, 0), + "2019-04-30", + datetime_utc(2019, 4, 30, 0, 0, 0), + "2018-07-03", + datetime_utc(2018, 7, 3, 0, 0, 0), + ], + # The same dates are specified, but we test at the actual current time. + [ + utc_now(), + "2019-04-30", + datetime_utc(2019, 4, 30, 0, 0, 0), + "2018-07-03", + datetime_utc(2018, 7, 3, 0, 0, 0), + ], + # The same dates are specified, but we test at the actual current time. + [ + utc_now(), + "4150-04-30", + datetime_utc(4150, 4, 30, 0, 0, 0), + "4099-07-03", + datetime_utc(4099, 7, 3, 0, 0, 0), + ], + ], + ) + def test_parse_command_line_start_not_before_until( + self, + capsys, + current_utc_time: datetime, + start_arg: str | None, + expected_start: datetime, + until_arg: str | None, + expected_until: datetime, + ): + start_args = ["--start", start_arg] if start_arg else [] + until_args = ["--until", until_arg] if until_arg else [] + cmd_args = start_args + until_args + + with freeze_time(current_utc_time), pytest.raises(SystemExit) as excinfo: + parsed = PlaytimeEntriesEmailReportsScript.parse_command_line( + cmd_args=cmd_args + ) + _, err = capsys.readouterr() + assert 2 == excinfo.value.code + assert re.search(r"start date \(.*\) must be before until date \(.*\).", err)