Add method to generate fresh price history

This adds the option to re-regenerate the data from source, reading the
whole history. Good if some changes need this or if the data was
corrupted in some way.

hotprices_au/analysis.py

@@ -130,4 +130,22 @@ def transform_data(day, output_dir, data_dir, store_filter=None):
         fp.write(json.dumps(all_items))
 
     copy_items_to_site(latest_canonical_file, data_dir)
-    return all_items
+    return all_items
+
+def parse_full_history(output_dir: pathlib.Path, data_dir, store_filter=None):
+    # First remote canonical data
+    latest_canonical_file = output_dir / "latest-canonical.json.gz"
+    if latest_canonical_file.exists():
+        latest_canonical_file.unlink()
+    # List all stores in output_dir first
+    for store in output_dir.iterdir():
+        if store.name not in sites.sites:
+            # Files we can ignore
+            continue
+        if store_filter is not None and store.name != store_filter:
+            # We filter here so we do one store at a time
+            continue
+        for data_file in sorted(store.iterdir()):
+            fname = data_file.name
+            day = datetime.strptime(fname.split('.')[0], '%Y-%m-%d')
+            transform_data(day, output_dir, data_dir, store_filter=store.name)

main.py

@@ -11,7 +11,10 @@ def main_sync(args):
 
 def main_analysis(args):
     data_dir = pathlib.Path('static/data')
-    analysis.transform_data(args.day, args.output_dir, data_dir, args.store)
+    if args.history:
+        analysis.parse_full_history(args.output_dir, data_dir, args.store)
+    else:
+        analysis.transform_data(args.day, args.output_dir, data_dir, args.store)
 
 
 def parse_date(date):
@@ -33,6 +36,10 @@ def main():
     analysis_parser = subparsers.add_parser('analysis')
     analysis_parser.add_argument('--day', type=parse_date, default=datetime.now())
     analysis_parser.add_argument('--store', choices=list(sites.sites))
+    analysis_parser.add_argument(
+        '--history', action='store_true', default=False,
+        help="Read the entire history and re-generate information from source",
+    )
     analysis_parser.set_defaults(func=main_analysis)
 
     args = parser.parse_args()