Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update PVNet datapipe to support ECMWF #229

Merged
merged 13 commits into from
Oct 26, 2023
Merged

Update PVNet datapipe to support ECMWF #229

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
494a427
Add passing in NWP provider to OpenNWP in config
jacobbieker Oct 23, 2023
49f62ef
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 23, 2023
5d23f6a
Add provider to test config
jacobbieker Oct 23, 2023
71a6d4b
Merge remote-tracking branch 'origin/jacob/india-pvnet' into jacob/in…
jacobbieker Oct 23, 2023
2fb1445
Merge tag 'v2.0.3' into jacob/india-pvnet
jacobbieker Oct 24, 2023
ca38ea6
Merge tag 'v2.0.4' into jacob/india-pvnet
jacobbieker Oct 24, 2023
2a19c0e
Fix test
jacobbieker Oct 24, 2023
0f55056
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 24, 2023
eb90d61
Add option to open GFS
jacobbieker Oct 24, 2023
3d5ac60
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 24, 2023
39f098f
Update datapipe
jacobbieker Oct 24, 2023
39d7bbb
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 24, 2023
d17bdc7
Update nwp_pv.py
jacobbieker Oct 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions ocf_datapipes/load/nwp/nwp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from torchdata.datapipes.iter import IterDataPipe

from ocf_datapipes.load.nwp.providers.ecmwf import open_ifs
from ocf_datapipes.load.nwp.providers.gfs import open_gfs
from ocf_datapipes.load.nwp.providers.icon import open_icon_eu, open_icon_global
from ocf_datapipes.load.nwp.providers.ukv import open_ukv

Expand All @@ -34,14 +35,16 @@ def __init__(
i.e. OSGB for UKV, Lat/Lon for ICON EU, Icoshedral grid for ICON Global
"""
self.zarr_path = zarr_path
if provider == "ukv":
if provider.lower() == "ukv" or provider == "UKMetOffice":
self.open_nwp = open_ukv
elif provider == "icon-eu":
elif provider.lower() == "icon-eu":
self.open_nwp = open_icon_eu
elif provider == "icon-global":
elif provider.lower() == "icon-global":
self.open_nwp = open_icon_global
elif provider == "ecmwf":
elif provider.lower() == "ecmwf":
self.open_nwp = open_ifs
elif provider.lower() == "gfs":
self.open_nwp = open_gfs
else:
raise ValueError(f"Unknown provider: {provider}")

Expand Down
5 changes: 4 additions & 1 deletion ocf_datapipes/training/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,10 @@ def open_and_return_datapipes(
if use_nwp:
logger.debug("Opening NWP Data")
nwp_datapipe = (
OpenNWP(configuration.input_data.nwp.nwp_zarr_path)
OpenNWP(
configuration.input_data.nwp.nwp_zarr_path,
provider=configuration.input_data.nwp.nwp_provider,
)
.select_channels(configuration.input_data.nwp.nwp_channels)
.add_t0_idx_and_sample_period_duration(
sample_period_duration=timedelta(hours=1),
Expand Down
10 changes: 8 additions & 2 deletions ocf_datapipes/training/example/nwp_pv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,10 @@ def nwp_pv_datapipe(
.fork(2, buffer_size=BUFFER_SIZE)
)

if configuration.input_data.nwp.nwp_provider == "UKMetOffice":
if (
configuration.input_data.nwp.nwp_provider == "UKMetOffice"
or configuration.input_data.nwp.nwp_provider == "ukv"
):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could make this slightly tidier using:
if configuration.input_data.nwp.nwp_provider in [ "UKMetOffice", "ukv"]?

nwp_datapipe = OpenNWPID(configuration.input_data.nwp.nwp_zarr_path)
elif configuration.input_data.nwp.nwp_provider == "GFS":
nwp_datapipe = OpenGFSForecast(configuration.input_data.nwp.nwp_zarr_path)
Expand Down Expand Up @@ -178,7 +181,10 @@ def nwp_pv_datapipe(
forecast_duration=timedelta(minutes=configuration.input_data.nwp.forecast_minutes),
)

if configuration.input_data.nwp.nwp_provider == "UKMetOffice":
if (
configuration.input_data.nwp.nwp_provider == "UKMetOffice"
or configuration.input_data.nwp.nwp_provider == "ukv"
):
nwp_datapipe = nwp_datapipe.normalize(mean=NWP_MEAN, std=NWP_STD)
else:
nwp_datapipe = nwp_datapipe.normalize(mean=NWP_GFS_MEAN, std=NWP_GFS_STD)
Expand Down
1 change: 1 addition & 0 deletions tests/config/test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ input_data:
nwp_image_size_pixels_height: 2
nwp_image_size_pixels_width: 2
nwp_zarr_path: tests/data/nwp_data/test.zarr
nwp_provider: "ukv"
history_minutes: 60
forecast_minutes: 120
time_resolution_minutes: 60
Expand Down