Skip to content

Commit

Permalink
using ocf_datapipes in Open-Source-Quartz-Solar-Forecast (#76)
Browse files Browse the repository at this point in the history
* accessing ocf_datapipes in quartz project

* refactoring of code

* moved datapipes to scripts
  • Loading branch information
roshnaeem authored Mar 8, 2024
1 parent 0cda6fa commit d8a9897
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 1 deletion.
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
.env
.env
scripts/datapipes/test.nc
scripts/datapipes/UK_PV_metadata.csv
scripts/datapipes/nwp_data
Empty file added scripts/datapipes/__init__.py
Empty file.
56 changes: 56 additions & 0 deletions scripts/datapipes/ocf_datapipe.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import ocf_datapipes # noqa
from ocf_datapipes.training.example.simple_pv import simple_pv_datapipe
from ocf_datapipes.training.example.pv_nwp import pv_nwp_datapipe
from ocf_datapipes.batch import BatchKey
from datetime import datetime
import os
import certifi
import ssl
import warnings

# Suppress warnings and configure SSL certificate for secure connections
warnings.filterwarnings("ignore")
os.environ['SSL_CERT_FILE'] = certifi.where()
ssl._create_default_https_context = ssl._create_unverified_context

def process_ocf_datapipes(config_file):

"""
This function demonstrates the use of Open Climate Fix's datapipes with the Open Source Quartz Solar Forecast Project.
It is an exploratory integration and not currently utilized in the Quartz Solar Forecast production pipeline.
It processes solar power (PV) and numerical weather prediction (NWP) data to prepare it for forecasting tasks.
:param config_file: The config file that specifies the paths to the data files, data preprocessing parameters, and other configurations necessary for the datapipes to function correctly.
- 'pv_and_nwp_config.yaml' specifes the paths to necessary data files (e.g., PV output data, NWP data).
- The YAML file needs to be edited to reflect the correct paths to your data files and any specific preprocessing requirements for your project.
This script is meant as a starting point for integrating Open Climate Fix datapipes into the Quartz Solar Forecast Project, serving as an example of how to preprocess and load data for solar power forecasting.
"""

pv_data_pipe = simple_pv_datapipe(configuration_filename=config_file)
pv_nwp_data_pipe = pv_nwp_datapipe(configuration_filename=config_file)

## Pv datapipe: store the first batch of processed data for inspection
pv_batch = next(iter(pv_data_pipe))

# Access the batch elements through batch keys: Convert the first set of timestamps to readable dates
pv_times_readable = [datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') for ts in pv_batch[BatchKey.pv_time_utc][0]]
print("Readable PV Times:", pv_times_readable)

# Check Observed Capacity (Wp), and Nominal Capacity (Wp) of the PV system
observed_capacity = pv_batch[BatchKey.pv_observed_capacity_wp]
nominal_capacity = pv_batch[BatchKey.pv_nominal_capacity_wp]

print("Observed Capacity (Wp):", observed_capacity)
print("Nominal Capacity (Wp):", nominal_capacity)

# NWP_PV DataPipe: Retrieve and print NWP data from the batch
pv_nwp_batch = next(iter(pv_nwp_data_pipe))
nwp_data = pv_nwp_batch[BatchKey.nwp]
print("NWP data", nwp_data)


# Load configuration from YAML specifying data sources and processing parameters for datapipes
config_file = 'pv_and_nwp_config.yaml'
process_ocf_datapipes(config_file)
23 changes: 23 additions & 0 deletions scripts/datapipes/pv_and_nwp_config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
input_data:
pv:
pv_files_groups:
- label: solar_sheffield_passiv
pv_filename: 'test.nc' # Available at https://github.com/openclimatefix/ocf_datapipes/blob/main/tests/data/pv/passiv/test.nc
pv_metadata_filename: 'UK_PV_metadata.csv' # Available at https://github.com/openclimatefix/ocf_datapipes/blob/main/tests/data/pv/passiv/UK_PV_metadata.csv
get_center: false
pv_image_size_meters_height: 10000000
pv_image_size_meters_width: 10000000
nwp:
ukv:
nwp_channels:
- t
nwp_image_size_pixels_height: 2
nwp_image_size_pixels_width: 2
nwp_zarr_path: nwp_data/test.zarr # Available at https://github.com/openclimatefix/ocf_datapipes/tree/main/tests/data/nwp_data/test.zarr
nwp_provider: "ukv"
history_minutes: 60
forecast_minutes: 120
time_resolution_minutes: 60
index_by_id: True
dropout_timedeltas_minutes: [-180]
dropout_fraction: 1.0

0 comments on commit d8a9897

Please sign in to comment.