Merge pull request #5 from openclimatefix/download-nwp

download nwp data at the start
openclimatefix · Sep 1, 2023 · aad5e94 · aad5e94
2 parents c6dc858 + 2b9d707
commit aad5e94
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 4 deletions.
diff --git a/configs/data_configuration.yaml b/configs/data_configuration.yaml
@@ -16,7 +16,7 @@ input_data:
     metadata_only: false
 
   nwp:
-    nwp_zarr_path: !ENV ${NWP_ZARR_PATH}
+    nwp_zarr_path: nwp.zarr
     history_minutes: 120
     forecast_minutes: 480
     time_resolution_minutes: 60
@@ -27,7 +27,7 @@ input_data:
     nwp_image_size_pixels_width: 24
 
   satellite:
-    satellite_zarr_path: latest.zarr.zip
+    satellite_zarr_path: sat.zarr.zip
     history_minutes: 90
     forecast_minutes: 0
     live_delay_minutes: 60

diff --git a/pvnet_app/app.py b/pvnet_app/app.py
@@ -267,13 +267,18 @@ def app(
     # Download satellite data - can't load zipped zarr straight from s3 bucket
     logger.info("Downloading zipped satellite data")
     fs = fsspec.open(os.environ["SATELLITE_ZARR_PATH"]).fs
-    fs.get(os.environ["SATELLITE_ZARR_PATH"], "latest.zarr.zip")
+    fs.get(os.environ["SATELLITE_ZARR_PATH"], "sat.zarr.zip")
 
     # Also download 15-minute satellite if it exists
     sat_latest_15 = os.environ["SATELLITE_ZARR_PATH"].replace(".zarr.zip", "_15.zarr.zip")
     if fs.exists(sat_latest_15):
         logger.info("Downloading 15-minute satellite data")
-        fs.get(sat_latest_15, "latest_15.zarr.zip")
+        fs.get(sat_latest_15, "sat_15.zarr.zip")
+
+    # Download nwp data - can't load zipped zarr straight from s3 bucket
+    logger.info("Downloading nwp data")
+    fs = fsspec.open(os.environ["NWP_ZARR_PATH"]).fs
+    fs.get(os.environ["NWP_ZARR_PATH"], "nwp.zarr", recursive=True)
 
     # ---------------------------------------------------------------------------
     # 2. Set up data loader