diff --git a/pvnet_app/app.py b/pvnet_app/app.py index 84b8184..4a7bc35 100644 --- a/pvnet_app/app.py +++ b/pvnet_app/app.py @@ -25,15 +25,14 @@ from nowcasting_datamodel.models.base import Base_Forecast from ocf_datapipes.load import OpenGSPFromDatabase from ocf_datapipes.training.pvnet import construct_sliced_data_pipeline -from ocf_datapipes.transform.numpy.batch.sun_position import ELEVATION_MEAN, ELEVATION_STD -from ocf_datapipes.utils.consts import BatchKey -from ocf_datapipes.utils.utils import stack_np_examples_into_batch +from ocf_datapipes.utils.consts import ELEVATION_MEAN, ELEVATION_STD +from ocf_datapipes.batch import BatchKey, stack_np_examples_into_batch from pvnet_summation.models.base_model import BaseModel as SummationBaseModel from torch.utils.data import DataLoader from torch.utils.data.datapipes.iter import IterableWrapper import pvnet -from pvnet.data.datamodule import batch_to_tensor, copy_batch_to_device +from pvnet.data.utils import batch_to_tensor, copy_batch_to_device from pvnet.models.base_model import BaseModel as PVNetBaseModel from pvnet.utils import GSPLocationLookup @@ -62,12 +61,12 @@ # Huggingfacehub model repo and commit for PVNet (GSP-level model) default_model_name = "openclimatefix/pvnet_v2" -default_model_version = "805ca9b2ee3120592b0b70b7c75a454e2b4e4bec" +default_model_version = "22e577100d55787eb2547d701275b9bb48f7bfa0" # Huggingfacehub model repo and commit for PVNet summation (GSP sum to national model) # If summation_model_name is set to None, a simple sum is computed instead default_summation_model_name = "openclimatefix/pvnet_v2_summation" -default_summation_model_version = "6c5361101b461ae991662bdff05f7a0b77b4040b" +default_summation_model_version = "e14bc98039511b2e383100a27f0c8e3b558d6c36" model_name_ocf_db = "pvnet_v2" use_adjuster = os.getenv("USE_ADJUSTER", "True").lower() == "true" diff --git a/pvnet_app/utils.py b/pvnet_app/utils.py index 2d5e42b..0978499 100644 --- a/pvnet_app/utils.py +++ b/pvnet_app/utils.py @@ -52,18 +52,26 @@ def populate_data_config_sources(input_path, output_path): production_paths = { "gsp": os.environ["DB_URL"], - "nwp": nwp_path, + "nwp": {"ukv": nwp_path}, "satellite": sat_path, # TODO: include hrvsatellite } # Replace data sources - for source in ["gsp", "nwp", "satellite", "hrvsatellite"]: - if source in config["input_data"]: - # If not empty - i.e. if used + for source in ["gsp", "satellite", "hrvsatellite"]: + if source in config["input_data"] : if config["input_data"][source][f"{source}_zarr_path"]!="": assert source in production_paths, f"Missing production path: {source}" config["input_data"][source][f"{source}_zarr_path"] = production_paths[source] + + # NWP is nested so much be treated separately + if "nwp" in config["input_data"]: + nwp_config = config["input_data"]["nwp"] + for nwp_source in nwp_config.keys(): + if nwp_config[nwp_source][f"nwp_zarr_path"]!="": + assert "nwp" in production_paths, f"Missing production path: nwp" + assert nwp_source in production_paths["nwp"], f"Missing NWP path: {nwp_source}" + nwp_config[nwp_source][f"nwp_zarr_path"] = production_paths["nwp"][nwp_source] # We do not need to set PV path right now. This currently done through datapipes # TODO - Move the PV path to here diff --git a/requirements.txt b/requirements.txt index cab86fd..7ed90a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -torch[cpu]>=2.1.1 -PVNet-summation>=0.0.9 -pvnet==2.4.0 -ocf_datapipes==2.2.14 +torch[cpu]>=2.0.0 +PVNet-summation==0.1.* +pvnet>=2.6.7,<2.7 +ocf_datapipes>=3.0.1,<3.1 nowcasting_datamodel>=1.5.22 fsspec[s3] xarray