Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Add test for checking physical limits and zeroes in NWP data #… #340

Open
wants to merge 18 commits into
base: main
Choose a base branch
from
Open

chore: Add test for checking physical limits and zeroes in NWP data #… #340

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
3ee287c
chore: Add test for checking physical limits and zeroes in NWP data #…
glitch401 Jul 3, 2024
1e2df80
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 3, 2024
8105b91
changes to generate test data on the go. remove unnecessary zarr file…
glitch401 Jul 4, 2024
1eafe49
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 4, 2024
d5bc6cf
Fix ValueError message for NWP data containing zeros and outside phys…
glitch401 Jul 4, 2024
d8cfa9d
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 4, 2024
5e68173
Fix ValueError message coding style
glitch401 Jul 4, 2024
466b710
update physical limits in according to pvnet_uk_region/data_config.yaml
glitch401 Jul 5, 2024
692500c
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 5, 2024
0667bab
Update temperature physical limits in OpenNWPIterDataPipe
glitch401 Jul 5, 2024
246d898
Fix NaN check in stack_np_examples_into_batch function
glitch401 Jul 11, 2024
55627eb
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 11, 2024
7ba254d
changes made to adapt for lazy loading
glitch401 Jul 16, 2024
c6ee33d
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 16, 2024
d0c4f6f
moved limits to a constant file
glitch401 Jul 24, 2024
19050c7
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 24, 2024
3fe89fc
Refactor test_merge_numpy_examples_to_batch.py and test_load_nwp.py t…
glitch401 Aug 15, 2024
ace0259
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Aug 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,6 @@ coverage.xml
.pytest_cache/
test.nc

#test data generator
tests/load/nwp/test_data_generator.py

# Translations
*.mo
*.pot
Expand Down
36 changes: 36 additions & 0 deletions ocf_datapipes/load/nwp/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
"""
Module for defining limits for NWP data.
"""

# limits for NWP data in accordance with https://huggingface.co/openclimatefix/pvnet_uk_region/blob/main/data_config.yaml
NWP_LIMITS = {
"t2m": (200, 350), # Temperature in Kelvin (-100°C to 60°C)
"dswrf": (0, 1500), # Downward short-wave radiation flux, W/m^2
"dlwrf": (0, 750), # Downward long-wave radiation flux, W/m^2
"hcc": (0, 100), # High cloud cover, %
"mcc": (0, 100), # Medium cloud cover, %
"lcc": (0, 100), # Low cloud cover, %
"tcc": (0, 100), # Total cloud cover, %
"sde": (0, 1000), # Snowfall depth, meters
"duvrs": (0, 500), # Direct UV radiation at surface, W/m^2 (positive values only)
"u10": (-200, 200), # U component of 10m wind, m/s
"v10": (-200, 200), # V component of 10m wind, m/s
# UKV NWP channels (additional to ECMWF)
"prate": (0, 2000), # Precipitation rate, , kg/m^2/s (equivalent to 0-2000 mm/day)
"r": (0, 100), # Relative humidity, %
"si10": (0, 250), # Wind speed at 10m, m/s
"t": (200, 350), # Temperature in Kelvin (-100°C to 60°C)
"vis": (0, 100000), # Visibility, meters
# Satellite channels (no direct mapping to physical limits, using placeholder values)
"IR_016": (0, 1000), # Infrared channel
"IR_039": (0, 1000), # Infrared channel
"IR_087": (0, 1000), # Infrared channel
"IR_097": (0, 1000), # Infrared channel
"IR_108": (0, 1000), # Infrared channel
"IR_120": (0, 1000), # Infrared channel
"IR_134": (0, 1000), # Infrared channel
"VIS006": (0, 1000), # Visible channel
"VIS008": (0, 1000), # Visible channel
"WV_062": (0, 1000), # Water vapor channel
"WV_073": (0, 1000), # Water vapor channel
}
34 changes: 2 additions & 32 deletions ocf_datapipes/load/nwp/nwp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import dask
import xarray as xr
from constants import NWP_LIMITS
from ocf_blosc2 import Blosc2 # noqa: F401
from torch.utils.data import IterDataPipe, functional_datapipe

Expand Down Expand Up @@ -42,39 +43,8 @@ def __init__(
self.zarr_path = zarr_path
self.check_for_zeros = check_for_zeros
self.check_physical_limits = check_physical_limits
self.limits = NWP_LIMITS
glitch401 marked this conversation as resolved.
Show resolved Hide resolved

# limits for NWP data in accordance with https://huggingface.co/openclimatefix/pvnet_uk_region/blob/main/data_config.yaml
self.limits = {
"t2m": (200, 350), # Temperature in Kelvin (-100°C to 60°C)
"dswrf": (0, 1500), # Downward short-wave radiation flux, W/m^2
"dlwrf": (0, 750), # Downward long-wave radiation flux, W/m^2
"hcc": (0, 100), # High cloud cover, %
"mcc": (0, 100), # Medium cloud cover, %
"lcc": (0, 100), # Low cloud cover, %
"tcc": (0, 100), # Total cloud cover, %
"sde": (0, 1000), # Snowfall depth, meters
"duvrs": (0, 500), # Direct UV radiation at surface, W/m^2 (positive values only)
"u10": (-200, 200), # U component of 10m wind, m/s
"v10": (-200, 200), # V component of 10m wind, m/s
# UKV NWP channels (additional to ECMWF)
"prate": (0, 2000), # Precipitation rate, , kg/m^2/s (equivalent to 0-2000 mm/day)
"r": (0, 100), # Relative humidity, %
"si10": (0, 250), # Wind speed at 10m, m/s
"t": (200, 350), # Temperature in Kelvin (-100°C to 60°C)
"vis": (0, 100000), # Visibility, meters
# Satellite channels (no direct mapping to physical limits, using placeholder values)
"IR_016": (0, 1000), # Infrared channel
"IR_039": (0, 1000), # Infrared channel
"IR_087": (0, 1000), # Infrared channel
"IR_097": (0, 1000), # Infrared channel
"IR_108": (0, 1000), # Infrared channel
"IR_120": (0, 1000), # Infrared channel
"IR_134": (0, 1000), # Infrared channel
"VIS006": (0, 1000), # Visible channel
"VIS008": (0, 1000), # Visible channel
"WV_062": (0, 1000), # Water vapor channel
"WV_073": (0, 1000), # Water vapor channel
}
logger.info(f"Using {provider.lower()}")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

very much just a suggestion, but it would be nice to have some control over which variables receive the checks. Intuitively, that should probably be possible by just passing a list of keys to be checked instead of True to check_for_zeroes/check_physical_limits

if provider.lower() == "ukv":
self.open_nwp = open_ukv
Expand Down