Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add datetime features #89

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
+70 −0
Open

add datetime features #89

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
abc3485
add datetime features
peterdudfield Dec 6, 2024
a3d8046
only make in site dataset
peterdudfield Dec 6, 2024
bd75790
Merge commit 'aae2af3818f2f1371097be834554981e2c2f032d' into datetime…
peterdudfield Dec 20, 2024
f327ab9
PR comments
peterdudfield Dec 20, 2024
e8cc5e2
remove from proces and combine, as it wont be used there
peterdudfield Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ocf_data_sampler/numpy_batch/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Conversion from Xarray to NumpyBatch"""

from .datetime_features import make_datetime_numpy_batch
from .gsp import convert_gsp_to_numpy_batch, GSPBatchKey
from .nwp import convert_nwp_to_numpy_batch, NWPBatchKey
from .satellite import convert_satellite_to_numpy_batch, SatelliteBatchKey
Expand Down
42 changes: 42 additions & 0 deletions ocf_data_sampler/numpy_batch/datetime_features.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
"""Datapipes to trigonometric date and time to NumpyBatch"""

import numpy as np
import pandas as pd
from numpy.typing import NDArray


def _get_date_time_in_pi(
dt: pd.DatetimeIndex,
) -> tuple[NDArray[np.float64], NDArray[np.float64]]:
"""
Change the datetimes, into time and date scaled in radians
"""

day_of_year = dt.dayofyear
minute_of_day = dt.minute + dt.hour * 60

# converting into positions on sin-cos circle
time_in_pi = (2 * np.pi) * (minute_of_day / (24 * 3600))
peterdudfield marked this conversation as resolved.
Show resolved Hide resolved
date_in_pi = (2 * np.pi) * (day_of_year / (365 * 24 * 3600))
peterdudfield marked this conversation as resolved.
Show resolved Hide resolved

return date_in_pi, time_in_pi


def make_datetime_numpy_batch(datetimes: pd.DatetimeIndex, key_prefix: str = "wind") -> dict:
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Potentially update main function to include input validation - also to simplify the dict construction via literal. Feel this improves maintenance ease a little overall. As example:

def make_datetime_numpy_batch(datetimes: pd.DatetimeIndex, key_prefix: str = "wind") -> dict:
    """
    Make a dict of datetime features using sin/cos transformations
    """
    if datetimes.empty:
        raise ValueError("Input datetimes is empty")

    # Obtain scaled radians for date and time
    date_in_pi, time_in_pi = _get_date_time_in_pi(datetimes)

    # Create dict
    return {
        f"{key_prefix}_date_sin": np.sin(date_in_pi),
        f"{key_prefix}_date_cos": np.cos(date_in_pi),
        f"{key_prefix}_time_sin": np.sin(time_in_pi),
        f"{key_prefix}_time_cos": np.cos(time_in_pi),
    }

""" Make dictionary of datetime features"""
time_numpy_batch = {}

date_in_pi, time_in_pi = _get_date_time_in_pi(datetimes)

# Store
date_sin_batch_key = key_prefix + "_date_sin"
date_cos_batch_key = key_prefix + "_date_cos"
time_sin_batch_key = key_prefix + "_time_sin"
time_cos_batch_key = key_prefix + "_time_cos"

time_numpy_batch[date_sin_batch_key] = np.sin(date_in_pi)
time_numpy_batch[date_cos_batch_key] = np.cos(date_in_pi)
time_numpy_batch[time_sin_batch_key] = np.sin(time_in_pi)
time_numpy_batch[time_cos_batch_key] = np.cos(time_in_pi)

return time_numpy_batch
6 changes: 6 additions & 0 deletions ocf_data_sampler/torch_datasets/process_and_combine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
convert_nwp_to_numpy_batch,
convert_satellite_to_numpy_batch,
convert_gsp_to_numpy_batch,
make_datetime_numpy_batch,
make_sun_position_numpy_batch,
convert_site_to_numpy_batch,
)
Expand Down Expand Up @@ -109,6 +110,11 @@ def process_and_combine_datasets(
make_sun_position_numpy_batch(datetimes, lon, lat, key_prefix=target_key)
)

if "site" in dataset_dict:
numpy_modalities.append(
make_datetime_numpy_batch(datetimes, key_prefix=target_key)
)

# Combine all the modalities and fill NaNs
combined_sample = merge_dicts(numpy_modalities)
combined_sample = fill_nans_in_arrays(combined_sample)
Expand Down
26 changes: 26 additions & 0 deletions tests/numpy_batch/test_datetime_features.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import numpy as np
import pandas as pd

from ocf_data_sampler.numpy_batch.datetime_features import make_datetime_numpy_batch

from ocf_data_sampler.numpy_batch import GSPBatchKey


def test_calculate_azimuth_and_elevation():

# Pick the day of the summer solstice
datetimes = pd.to_datetime(["2024-06-20 12:00", "2024-06-20 12:30", "2024-06-20 13:00"])

# Calculate sun angles
datetime_features = make_datetime_numpy_batch(datetimes)

assert len(datetime_features) == 4

assert len(datetime_features["wind_date_sin"]) == len(datetimes)
assert (datetime_features["wind_date_cos"] != datetime_features["wind_date_sin"]).all()

# assert all values are between -1 and 1
assert all(np.abs(datetime_features["wind_date_sin"]) <= 1)
assert all(np.abs(datetime_features["wind_date_cos"]) <= 1)
assert all(np.abs(datetime_features["wind_time_sin"]) <= 1)
assert all(np.abs(datetime_features["wind_time_cos"]) <= 1)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Testing wise - perhaps update to additionaly verify if a custom prefix is sufficiently applied to all dict keys that the function generates and a further basic check if an empty input corresponds to error raising as implemented in main. As example:

def test_make_datetime_numpy_batch_custom_key_prefix():
    # Test function correctly applies custom prefix to dict keys
    datetimes = pd.to_datetime(["2024-06-20 12:00", "2024-06-20 12:30", "2024-06-20 13:00"])
    key_prefix = "solar"

    datetime_features = make_datetime_numpy_batch(datetimes, key_prefix=key_prefix)

    # Assert dict contains expected quantity of keys and verify starting with custom prefix
    assert len(datetime_features) == 4
    assert all(key.startswith(key_prefix) for key in datetime_features.keys())


def test_make_datetime_numpy_batch_empty_input():
    # Verification that function raises error for empty input
    datetimes = pd.DatetimeIndex([])
    
    with pytest.raises(ValueError, match="Input datetimes is empty"):
        make_datetime_numpy_batch(datetimes)

Loading