From 7af45c2e563c0c953ea269a639854cc47beafa92 Mon Sep 17 00:00:00 2001
From: javex <florian.ruechel@inexplicity.de>
Date: Tue, 26 Sep 2023 18:21:49 +0930
Subject: [PATCH] Add timestamped upload

---
 .github/workflows/scrape-woolies.yml |  2 +-
 woolies.py                           | 15 +++++++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/scrape-woolies.yml b/.github/workflows/scrape-woolies.yml
index 413bc3c..df561b2 100644
--- a/.github/workflows/scrape-woolies.yml
+++ b/.github/workflows/scrape-woolies.yml
@@ -24,4 +24,4 @@ jobs:
         with:
           role-to-assume: "${{ vars.TARGET_ROLE_ARN }}"
           aws-region: ap-southeast-2
-      - run: aws s3 cp ./woolies_all.json s3://grocery-scrape-au/woolies_all.json
\ No newline at end of file
+      - run: aws s3 sync ./woolies/ s3://grocery-scrape-au/wooles/
\ No newline at end of file
diff --git a/woolies.py b/woolies.py
index ae99941..a533fff 100644
--- a/woolies.py
+++ b/woolies.py
@@ -1,6 +1,8 @@
 import requests
 import json
 import sys
+import pathlib
+from datetime import datetime
 
 
 class WooliesAPI:
@@ -97,6 +99,16 @@ def save_cache(cache_data):
         f.write(json.dumps(cache_data))
 
 
+def save_data(categories):
+    now = datetime.now()
+    date_str = now.strftime("%Y-%m-%d")
+    fname = f"{date_str}.json"
+    save_dir = pathlib.Path(f"woolies")
+    save_dir.mkdir(exist_ok=True)
+    fpath = save_dir / fname
+    fpath.write_text(json.dumps(categories))
+
+
 def main():
     quick = False
     if len(sys.argv) > 1 and sys.argv[1] == "--quick":
@@ -126,8 +138,7 @@ def main():
         if quick:
             break
         #save_cache(categories)
-    with open('woolies_all.json', 'w') as f:
-        f.write(json.dumps(categories))
+    save_data(categories)
     #print(json.dumps(category, indent=4))