From cec2a0290bff8d204e6368e9e9dc3258dfff69b2 Mon Sep 17 00:00:00 2001 From: javex Date: Wed, 27 Sep 2023 19:06:06 +0930 Subject: [PATCH] Add compression to reduce file size --- hotprices_au/output.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/hotprices_au/output.py b/hotprices_au/output.py index aae03aa..35672f7 100644 --- a/hotprices_au/output.py +++ b/hotprices_au/output.py @@ -1,13 +1,22 @@ import pathlib import json +import gzip from datetime import datetime -def save_data(store, categories): +def save_data(store, categories, compression='gzip'): now = datetime.now() date_str = now.strftime("%Y-%m-%d") fname = f"{date_str}.json" save_dir = pathlib.Path(f"output/{store}") save_dir.mkdir(parents=True, exist_ok=True) fpath = save_dir / fname - fpath.write_text(json.dumps(categories)) \ No newline at end of file + + if compression == 'gzip': + fpath = fpath.with_suffix('.json.gz') + with gzip.open(fpath, 'wt') as fp: + fp.write(json.dumps(categories)) + elif compression is None: + fpath.write_text(json.dumps(categories)) + else: + raise RuntimeError(f"Unsupported compression '{compression}'") \ No newline at end of file