import fixes

pvnet/data/datamodule.py

@@ -5,6 +5,7 @@
 import torch
 from torch.utils.data import DataLoader
 from torch.utils.data.datapipes.datapipe import IterDataPipe
+from torch.utils.data.datapipes.iter import FileLister
 from torch.utils.data.datapipes._decorator import functional_datapipe
 
 from lightning.pytorch import LightningDataModule
@@ -184,7 +185,6 @@ def train_dataloader(self):
             datapipe = self._get_premade_batches_datapipe("train", shuffle=True)
         else:
             datapipe = self._get_datapipe(*self.train_period)
-        rs = MultiProcessingReadingService(**self.readingservice_config)
         return DataLoader(datapipe, **self._common_dataloader_kwargs)
 
     def val_dataloader(self):

scripts/save_batches.py

@@ -25,11 +25,13 @@
 
 import hydra
 import torch
+from torch.utils.data import DataLoader
+from torch.utils.data.datapipes.datapipe import IterDataPipe
 from ocf_datapipes.training.pvnet import pvnet_datapipe
 from ocf_datapipes.utils.utils import stack_np_examples_into_batch
 from omegaconf import DictConfig, OmegaConf
 from sqlalchemy import exc as sa_exc
-from torch.utils.data import DataLoader
+
 from tqdm import tqdm
 
 from pvnet.data.datamodule import batch_to_tensor

scripts/save_concurrent_batches.py

@@ -23,6 +23,8 @@
 import hydra
 import numpy as np
 import torch
+from torch.utils.data import DataLoader
+from torch.utils.data.datapipes.datapipe import IterDataPipe
 from ocf_datapipes.training.common import (
     open_and_return_datapipes,
 )
@@ -31,7 +33,7 @@
 from ocf_datapipes.utils.utils import stack_np_examples_into_batch
 from omegaconf import DictConfig, OmegaConf
 from sqlalchemy import exc as sa_exc
-from torch.utils.data import DataLoader
+
 from tqdm import tqdm
 
 from pvnet.data.datamodule import batch_to_tensor