diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py index 34e5016..aadc0af 100644 --- a/aim/cli/digifeeds.py +++ b/aim/cli/digifeeds.py @@ -169,7 +169,8 @@ def process_barcodes( @app.command() def generate_barcodes_in_s3_report(): """ - Generates a report of barcodes that have been added to the s3 bucket in the - last week. This report is sent to a dropbox folder. + Generates a report of barcodes that have been moved to the google pickup + location in the last two weeks. It is based on the files in the processed + location in the s3 bucket. This report is sent to a dropbox folder. """ functions.generate_barcodes_added_in_last_two_weeks_report() diff --git a/aim/digifeeds/functions.py b/aim/digifeeds/functions.py index 428534c..e2c901d 100644 --- a/aim/digifeeds/functions.py +++ b/aim/digifeeds/functions.py @@ -2,7 +2,7 @@ import boto3 from pathlib import Path from rclone_python import rclone -from datetime import datetime +from datetime import datetime, timedelta import csv import tempfile @@ -19,15 +19,30 @@ def list_barcodes_in_input_bucket(): return barcodes +def last_two_weeks_rclone_filter(start_date: datetime = datetime.today()): + day_count = 14 + dates = [] + for single_date in (start_date - timedelta(n) for n in range(day_count)): + formatted_date = single_date.strftime("%Y-%m-%d") + dates.append(f"{formatted_date}*") + joined = ",".join(dates) + return f"{{{joined}}}" + + def barcodes_added_in_last_two_weeks(): files = rclone.ls( - path=f"{S.digifeeds_s3_rclone_remote}:{S.digifeeds_s3_input_path}", - args=["--max-age 14d"], + path=f"{S.digifeeds_s3_bucket}:{S.digifeeds_s3_processed_path}", + args=[f'--include "{last_two_weeks_rclone_filter()}"'], ) output = [] for file in files: - barcode = file["Name"].split(".")[0] - date = datetime.fromisoformat(file["ModTime"]).strftime("%Y-%m-%d") + barcode = file["Name"].split("_")[2].split(".")[0] + date = file["Name"].split("_")[0] + S.logger.info( + "added_to_barcode_report", + barcode=barcode, + message="Added to barcode report", + ) output.append([date, barcode]) return output @@ -36,6 +51,7 @@ def barcodes_added_in_last_two_weeks(): def write_barcodes_added_in_last_two_weeks_report(outfile): output = barcodes_added_in_last_two_weeks() writer = csv.writer(outfile, delimiter="\t", lineterminator="\n") + S.logger.info("writing_report_rows_to_file") writer.writerows(output) @@ -45,7 +61,8 @@ def generate_barcodes_added_in_last_two_weeks_report(): write_barcodes_added_in_last_two_weeks_report(rf) today = datetime.today().strftime("%Y-%m-%d") + S.logger.info("writing report to dropbox") rclone.copyto( in_path=report_file.name, - out_path=f"{S.digifeeds_reports_rclone_remote}:{today}_barcodes_in_s3_bucket.tsv", + out_path=f"{S.digifeeds_reports_rclone_remote}:{today}_barcodes_in_s3_processed.tsv", ) diff --git a/tests/digifeeds/test_functions.py b/tests/digifeeds/test_functions.py index d78cc85..f971fd2 100644 --- a/tests/digifeeds/test_functions.py +++ b/tests/digifeeds/test_functions.py @@ -1,19 +1,33 @@ +from datetime import datetime import json from aim.digifeeds.functions import ( rclone, barcodes_added_in_last_two_weeks, write_barcodes_added_in_last_two_weeks_report, generate_barcodes_added_in_last_two_weeks_report, + last_two_weeks_rclone_filter, ) from io import StringIO +def test_last_two_weeks_rclone_filter(): + filters = last_two_weeks_rclone_filter( + start_date=datetime.strptime("2025-01-02", "%Y-%m-%d") + ) + expected_filter_string = ( + "{2025-01-02*,2025-01-01*,2024-12-31*,2024-12-30*,2024-12-29*" + ",2024-12-28*,2024-12-27*,2024-12-26*,2024-12-25*,2024-12-24*" + ",2024-12-23*,2024-12-22*,2024-12-21*,2024-12-20*}" + ) + assert filters == expected_filter_string + + def test_barcodes_added_in_last_two_weeks(mocker): ls_data_raw = """ [ { - "Path": "35112203951670.zip", - "Name": "35112203951670.zip", + "Path": "2024-12-01_07-10-02_35112203951670.zip", + "Name": "2024-12-01_07-10-02_35112203951670.zip", "Size": 554562627, "MimeType": "application/zip", "ModTime": "2024-12-14T02:01:05.093051502-05:00", @@ -21,8 +35,8 @@ def test_barcodes_added_in_last_two_weeks(mocker): "Tier": "STANDARD" }, { - "Path": "39015004707009.zip", - "Name": "39015004707009.zip", + "Path": "2024-12-01_07-10-02_39015004707009.zip", + "Name": "2024-12-01_07-10-02_39015004707009.zip", "Size": 232895588, "MimeType": "application/zip", "ModTime": "2024-12-14T02:02:29.111076546-05:00", @@ -34,8 +48,8 @@ def test_barcodes_added_in_last_two_weeks(mocker): mocker.patch.object(rclone, "ls", return_value=json.loads(ls_data_raw)) output = barcodes_added_in_last_two_weeks() assert output == [ - ["2024-12-14", "35112203951670"], - ["2024-12-14", "39015004707009"], + ["2024-12-01", "35112203951670"], + ["2024-12-01", "39015004707009"], ]