From 88e79a16f1bcd06b8728da57b4e0536b1b0701e5 Mon Sep 17 00:00:00 2001 From: Sukhil Patel Date: Thu, 2 May 2024 17:30:11 +0100 Subject: [PATCH 1/3] Use the streaming site datapipe --- pvnet/data/pv_site_datamodule.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pvnet/data/pv_site_datamodule.py b/pvnet/data/pv_site_datamodule.py index 87b80360..6e8f94d3 100644 --- a/pvnet/data/pv_site_datamodule.py +++ b/pvnet/data/pv_site_datamodule.py @@ -2,7 +2,7 @@ import glob from ocf_datapipes.batch import BatchKey, batch_to_tensor, stack_np_examples_into_batch -from ocf_datapipes.training.pvnet_site import pvnet_site_netcdf_datapipe +from ocf_datapipes.training.pvnet_site import pvnet_site_netcdf_datapipe, pvnet_site_datapipe from pvnet.data.base import BaseDataModule @@ -11,8 +11,10 @@ class PVSiteDataModule(BaseDataModule): """Datamodule for training pvnet site and using pvnet site pipeline in `ocf_datapipes`.""" def _get_datapipe(self, start_time, end_time): - data_pipeline = pvnet_site_netcdf_datapipe( - keys=["pv", "nwp"], + data_pipeline = pvnet_site_datapipe( + self.configuration, + start_time=start_time, + end_time=end_time ) data_pipeline = ( From d2fa46cf7088005a35d6dde4ec82d095b0a56e85 Mon Sep 17 00:00:00 2001 From: Sukhil Patel Date: Fri, 3 May 2024 14:24:59 +0100 Subject: [PATCH 2/3] Add extra datapipe transforms needed --- pvnet/data/pv_site_datamodule.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pvnet/data/pv_site_datamodule.py b/pvnet/data/pv_site_datamodule.py index 6e8f94d3..031e5aa1 100644 --- a/pvnet/data/pv_site_datamodule.py +++ b/pvnet/data/pv_site_datamodule.py @@ -2,7 +2,7 @@ import glob from ocf_datapipes.batch import BatchKey, batch_to_tensor, stack_np_examples_into_batch -from ocf_datapipes.training.pvnet_site import pvnet_site_netcdf_datapipe, pvnet_site_datapipe +from ocf_datapipes.training.pvnet_site import pvnet_site_netcdf_datapipe, pvnet_site_datapipe, uncombine_from_single_dataset, split_dataset_dict_dp, ConvertToNumpyBatchIterDataPipe from pvnet.data.base import BaseDataModule @@ -12,10 +12,12 @@ class PVSiteDataModule(BaseDataModule): def _get_datapipe(self, start_time, end_time): data_pipeline = pvnet_site_datapipe( - self.configuration, + self.configuration, start_time=start_time, - end_time=end_time + end_time=end_time, ) + data_pipeline = data_pipeline.map(uncombine_from_single_dataset).map(split_dataset_dict_dp) + data_pipeline = data_pipeline.pvnet_site_convert_to_numpy_batch() data_pipeline = ( data_pipeline.batch(self.batch_size) From 9f78a0fca24e0a12d122b0a4f4e6020e9769654e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 May 2024 13:34:05 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pvnet/data/pv_site_datamodule.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pvnet/data/pv_site_datamodule.py b/pvnet/data/pv_site_datamodule.py index 031e5aa1..1e0499d0 100644 --- a/pvnet/data/pv_site_datamodule.py +++ b/pvnet/data/pv_site_datamodule.py @@ -2,7 +2,12 @@ import glob from ocf_datapipes.batch import BatchKey, batch_to_tensor, stack_np_examples_into_batch -from ocf_datapipes.training.pvnet_site import pvnet_site_netcdf_datapipe, pvnet_site_datapipe, uncombine_from_single_dataset, split_dataset_dict_dp, ConvertToNumpyBatchIterDataPipe +from ocf_datapipes.training.pvnet_site import ( + pvnet_site_datapipe, + pvnet_site_netcdf_datapipe, + split_dataset_dict_dp, + uncombine_from_single_dataset, +) from pvnet.data.base import BaseDataModule