diff --git a/ocf_datapipes/config/model.py b/ocf_datapipes/config/model.py index fc7b4713d..9c35ea0d0 100644 --- a/ocf_datapipes/config/model.py +++ b/ocf_datapipes/config/model.py @@ -319,9 +319,9 @@ class Sensor(DataSourceMixin, StartEndDatetimeMixin, TimeResolutionMixin, XYDime get_center: bool = Field( False, description="If the batches are centered on one Sensor system (or not). " - "The other options is to have one GSP at the center of a batch. " - "Typically, get_center would be set to true if and only if " - "SensorDataSource is used to define the geospatial positions of each example.", + "The other options is to have one GSP at the center of a batch. " + "Typically, get_center would be set to true if and only if " + "SensorDataSource is used to define the geospatial positions of each example.", ) sensor_filename: str = Field( @@ -344,10 +344,11 @@ class Sensor(DataSourceMixin, StartEndDatetimeMixin, TimeResolutionMixin, XYDime live_load_extra_minutes: int = Field( 0, description="The number of extra minutes in the past we should load. Then the recent " - "values can be interpolated, and the extra minutes removed. This is " - "because some live data takes ~1 hour to come in.", + "values can be interpolated, and the extra minutes removed. This is " + "because some live data takes ~1 hour to come in.", ) + class Satellite(DataSourceMixin, TimeResolutionMixin): """Satellite configuration model""" diff --git a/ocf_datapipes/load/sensor/awos.py b/ocf_datapipes/load/sensor/awos.py index 98b742680..15e4069bd 100644 --- a/ocf_datapipes/load/sensor/awos.py +++ b/ocf_datapipes/load/sensor/awos.py @@ -1,20 +1,16 @@ """Load ASOS data from local files for training/inference""" -import io import logging -from datetime import datetime -from pathlib import Path -from typing import Optional, Union import fsspec -import numpy as np -import pandas as pd import xarray as xr from torchdata.datapipes import functional_datapipe from torchdata.datapipes.iter import IterDataPipe + from ocf_datapipes.config.model import Sensor _log = logging.getLogger(__name__) + @functional_datapipe("OpenAWOS") class OpenAWOSFromNetCDFIterDataPipe(IterDataPipe): """OpenAWOSFromNetCDFIterDataPipe""" diff --git a/ocf_datapipes/validation/check_for_nans.py b/ocf_datapipes/validation/check_for_nans.py index 830625d9a..421ce2a83 100644 --- a/ocf_datapipes/validation/check_for_nans.py +++ b/ocf_datapipes/validation/check_for_nans.py @@ -12,7 +12,11 @@ class CheckNaNsIterDataPipe(IterDataPipe): """Checks, and optionally fills, NaNs in Xarray Dataset""" def __init__( - self, source_datapipe: IterDataPipe, dataset_name: str = None, fill_nans: bool = False, fill_value: float = 0.0 + self, + source_datapipe: IterDataPipe, + dataset_name: str = None, + fill_nans: bool = False, + fill_value: float = 0.0, ): """ Checks and optionally fills NaNs in the data @@ -41,7 +45,8 @@ def __iter__(self) -> Union[xr.DataArray, xr.Dataset]: xr_data = self.fill_nan(data=xr_data, fill_value=self.fill_value) else: xr_data[self.dataset_name] = self.fill_nan( - data=xr_data[self.dataset_name], fill_value=self.fill_value, + data=xr_data[self.dataset_name], + fill_value=self.fill_value, ) self.check_nan_and_inf( data=xr_data if self.dataset_name is None else xr_data[self.dataset_name], diff --git a/ocf_datapipes/visualization/batch.py b/ocf_datapipes/visualization/batch.py index a842df8e5..f5005e28b 100644 --- a/ocf_datapipes/visualization/batch.py +++ b/ocf_datapipes/visualization/batch.py @@ -1,2 +1 @@ """Visualization of batch data.""" -import numpy as np