Merge pull request #33 from openclimatefix/datapipes_v3

Update to datapipes v3

pvnet_app/app.py

@@ -25,15 +25,14 @@
 from nowcasting_datamodel.models.base import Base_Forecast
 from ocf_datapipes.load import OpenGSPFromDatabase
 from ocf_datapipes.training.pvnet import construct_sliced_data_pipeline
-from ocf_datapipes.transform.numpy.batch.sun_position import ELEVATION_MEAN, ELEVATION_STD
-from ocf_datapipes.utils.consts import BatchKey
-from ocf_datapipes.utils.utils import stack_np_examples_into_batch
+from ocf_datapipes.utils.consts import ELEVATION_MEAN, ELEVATION_STD
+from ocf_datapipes.batch import BatchKey, stack_np_examples_into_batch
 from pvnet_summation.models.base_model import BaseModel as SummationBaseModel
 from torch.utils.data import DataLoader
 from torch.utils.data.datapipes.iter import IterableWrapper
 
 import pvnet
-from pvnet.data.datamodule import batch_to_tensor, copy_batch_to_device
+from pvnet.data.utils import batch_to_tensor, copy_batch_to_device
 from pvnet.models.base_model import BaseModel as PVNetBaseModel
 from pvnet.utils import GSPLocationLookup
 
@@ -62,12 +61,12 @@
 
 # Huggingfacehub model repo and commit for PVNet (GSP-level model)
 default_model_name = "openclimatefix/pvnet_v2"
-default_model_version = "805ca9b2ee3120592b0b70b7c75a454e2b4e4bec"
+default_model_version = "22e577100d55787eb2547d701275b9bb48f7bfa0"
 
 # Huggingfacehub model repo and commit for PVNet summation (GSP sum to national model)
 # If summation_model_name is set to None, a simple sum is computed instead
 default_summation_model_name = "openclimatefix/pvnet_v2_summation"
-default_summation_model_version = "6c5361101b461ae991662bdff05f7a0b77b4040b"
+default_summation_model_version = "e14bc98039511b2e383100a27f0c8e3b558d6c36"
 
 model_name_ocf_db = "pvnet_v2"
 use_adjuster = os.getenv("USE_ADJUSTER", "True").lower() == "true"

pvnet_app/utils.py

@@ -52,18 +52,26 @@ def populate_data_config_sources(input_path, output_path):
 
     production_paths = {
         "gsp": os.environ["DB_URL"],
-        "nwp": nwp_path,
+        "nwp": {"ukv": nwp_path},
         "satellite": sat_path,
         # TODO: include hrvsatellite
     }        
 
     # Replace data sources
-    for source in ["gsp", "nwp", "satellite", "hrvsatellite"]:
-        if source in config["input_data"]:
-            # If not empty - i.e. if used
+    for source in ["gsp", "satellite", "hrvsatellite"]:
+        if source in config["input_data"] :
             if config["input_data"][source][f"{source}_zarr_path"]!="":
                 assert source in production_paths, f"Missing production path: {source}"
                 config["input_data"][source][f"{source}_zarr_path"] = production_paths[source]
+
+    # NWP is nested so much be treated separately
+    if "nwp" in config["input_data"]:
+        nwp_config = config["input_data"]["nwp"]
+        for nwp_source in nwp_config.keys():
+            if nwp_config[nwp_source][f"nwp_zarr_path"]!="":
+                assert "nwp" in production_paths, f"Missing production path: nwp"
+                assert nwp_source in production_paths["nwp"], f"Missing NWP path: {nwp_source}"
+                nwp_config[nwp_source][f"nwp_zarr_path"] = production_paths["nwp"][nwp_source]
 
     # We do not need to set PV path right now. This currently done through datapipes
     # TODO - Move the PV path to here

requirements.txt

@@ -1,7 +1,7 @@
-torch[cpu]>=2.1.1
-PVNet-summation>=0.0.9
-pvnet==2.4.0
-ocf_datapipes==2.2.14
+torch[cpu]>=2.0.0
+PVNet-summation==0.1.*
+pvnet>=2.6.7,<2.7
+ocf_datapipes>=3.0.1,<3.1
 nowcasting_datamodel>=1.5.22
 fsspec[s3]
 xarray