uses the files in the processed folder for the report; adds logs

aim/cli/digifeeds.py

@@ -169,7 +169,8 @@ def process_barcodes(
 @app.command()
 def generate_barcodes_in_s3_report():
     """
-    Generates a report of barcodes that have been added to the s3 bucket in the
-    last week. This report is sent to a dropbox folder.
+    Generates a report of barcodes that have been moved to the google pickup
+    location in the last two weeks. It is based on the files in the processed
+    location in the s3 bucket. This report is sent to a dropbox folder.
     """
     functions.generate_barcodes_added_in_last_two_weeks_report()

aim/digifeeds/functions.py

@@ -2,7 +2,7 @@
 import boto3
 from pathlib import Path
 from rclone_python import rclone
-from datetime import datetime
+from datetime import datetime, timedelta
 import csv
 import tempfile
 
@@ -19,15 +19,30 @@ def list_barcodes_in_input_bucket():
     return barcodes
 
 
+def last_two_weeks_rclone_filter(start_date: datetime = datetime.today()):
+    day_count = 14
+    dates = []
+    for single_date in (start_date - timedelta(n) for n in range(day_count)):
+        formatted_date = single_date.strftime("%Y-%m-%d")
+        dates.append(f"{formatted_date}*")
+    joined = ",".join(dates)
+    return f"{{{joined}}}"
+
+
 def barcodes_added_in_last_two_weeks():
     files = rclone.ls(
-        path=f"{S.digifeeds_s3_rclone_remote}:{S.digifeeds_s3_input_path}",
-        args=["--max-age 14d"],
+        path=f"{S.digifeeds_s3_bucket}:{S.digifeeds_s3_processed_path}",
+        args=[f'--include "{last_two_weeks_rclone_filter()}"'],
     )
     output = []
     for file in files:
-        barcode = file["Name"].split(".")[0]
-        date = datetime.fromisoformat(file["ModTime"]).strftime("%Y-%m-%d")
+        barcode = file["Name"].split("_")[2].split(".")[0]
+        date = file["Name"].split("_")[0]
+        S.logger.info(
+            "added_to_barcode_report",
+            barcode=barcode,
+            message="Added to barcode report",
+        )
         output.append([date, barcode])
 
     return output
@@ -36,6 +51,7 @@ def barcodes_added_in_last_two_weeks():
 def write_barcodes_added_in_last_two_weeks_report(outfile):
     output = barcodes_added_in_last_two_weeks()
     writer = csv.writer(outfile, delimiter="\t", lineterminator="\n")
+    S.logger.info("writing_report_rows_to_file")
     writer.writerows(output)
 
 
@@ -45,7 +61,8 @@ def generate_barcodes_added_in_last_two_weeks_report():
         write_barcodes_added_in_last_two_weeks_report(rf)
 
     today = datetime.today().strftime("%Y-%m-%d")
+    S.logger.info("writing report to dropbox")
     rclone.copyto(
         in_path=report_file.name,
-        out_path=f"{S.digifeeds_reports_rclone_remote}:{today}_barcodes_in_s3_bucket.tsv",
+        out_path=f"{S.digifeeds_reports_rclone_remote}:{today}_barcodes_in_s3_processed.tsv",
     )

tests/digifeeds/test_functions.py

@@ -1,28 +1,42 @@
+from datetime import datetime
 import json
 from aim.digifeeds.functions import (
     rclone,
     barcodes_added_in_last_two_weeks,
     write_barcodes_added_in_last_two_weeks_report,
     generate_barcodes_added_in_last_two_weeks_report,
+    last_two_weeks_rclone_filter,
 )
 from io import StringIO
 
 
+def test_last_two_weeks_rclone_filter():
+    filters = last_two_weeks_rclone_filter(
+        start_date=datetime.strptime("2025-01-02", "%Y-%m-%d")
+    )
+    expected_filter_string = (
+        "{2025-01-02*,2025-01-01*,2024-12-31*,2024-12-30*,2024-12-29*"
+        ",2024-12-28*,2024-12-27*,2024-12-26*,2024-12-25*,2024-12-24*"
+        ",2024-12-23*,2024-12-22*,2024-12-21*,2024-12-20*}"
+    )
+    assert filters == expected_filter_string
+
+
 def test_barcodes_added_in_last_two_weeks(mocker):
     ls_data_raw = """
     [
       {
-        "Path": "35112203951670.zip",
-        "Name": "35112203951670.zip",
+        "Path": "2024-12-01_07-10-02_35112203951670.zip",
+        "Name": "2024-12-01_07-10-02_35112203951670.zip",
         "Size": 554562627,
         "MimeType": "application/zip",
         "ModTime": "2024-12-14T02:01:05.093051502-05:00",
         "IsDir": false,
         "Tier": "STANDARD"
       },
       {
-        "Path": "39015004707009.zip",
-        "Name": "39015004707009.zip",
+        "Path": "2024-12-01_07-10-02_39015004707009.zip",
+        "Name": "2024-12-01_07-10-02_39015004707009.zip",
         "Size": 232895588,
         "MimeType": "application/zip",
         "ModTime": "2024-12-14T02:02:29.111076546-05:00",
@@ -34,8 +48,8 @@ def test_barcodes_added_in_last_two_weeks(mocker):
     mocker.patch.object(rclone, "ls", return_value=json.loads(ls_data_raw))
     output = barcodes_added_in_last_two_weeks()
     assert output == [
-        ["2024-12-14", "35112203951670"],
-        ["2024-12-14", "39015004707009"],
+        ["2024-12-01", "35112203951670"],
+        ["2024-12-01", "39015004707009"],
     ]