merge branch

configs/config.yaml

@@ -10,8 +10,12 @@ defaults:
   - experiment: null
   - hparams_search: null
   - hydra: default.yaml
-  - renewable: "pv"
 
+renewable: "wind"
+batch_output_dir: "/mnt/storage_ssd_4tb/windnet_batches"
+num_val_batches: 3200
+num_train_batches: 64000
+num_test_batches: 100
   # enable color logging
 #  - override hydra/hydra_logging: colorlog
 #  - override hydra/job_logging: colorlog

configs/datamodule/configuration/leonardo_wind_configuration.yaml

@@ -7,28 +7,32 @@ input_data:
   default_history_minutes: 60
   data_source_which_defines_geospatial_locations: "sensor"
   nwp:
-    nwp_zarr_path: "/mnt/storage_b/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/NWP/ECMWF/nw-india/zarr/*.zarr.zip"
+    nwp_zarr_path: "/mnt/storage_b/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/NWP/ECMWF/nw-india/zarr/2022*.zarr.zip"
     history_minutes: 60
     forecast_minutes: 480
     time_resolution_minutes: 60
     nwp_channels:
-      - t # live = t2m
-      - dswrf
-      - dlwrf
       - hcc
-      - mcc
       - lcc
-      - vis
-      - r # live = r2
-      - prate # live ~= rprate
-      - si10 # 10-metre wind speed | live = unknown
-    nwp_image_size_pixels_height: 96
-    nwp_image_size_pixels_width: 96
+      - mcc
+      - prate
+      - sde
+      - sr
+      - t2m
+      - tcc
+      - u10
+      - u100
+      - u200
+      - v10
+      - v100
+      - v200
+    nwp_image_size_pixels_height: 16
+    nwp_image_size_pixels_width: 16
     x_dim_name: "longitude"
     y_dim_name: "latitude"
     nwp_provider: "ecmwf"
   sensor:
-    sensor_filename: "/mnt/storage_ssd_4tb/ASOS/asos_india.nc"
+    sensor_filename: "/mnt/storage_ssd_4tb/ASOS/asos_india_new_all_nw_india.nc"
     sensor_variables:
       - wind_speed_knots
     x_dim_name: "longitude"

configs/datamodule/ocf_datapipes.yaml

@@ -1,15 +1,15 @@
 _target_: pvnet.data.wind_datamodule.WindDataModule
 configuration: "/home/jacob/PVNet/configs/datamodule/configuration/leonardo_wind_configuration.yaml"
-num_workers: 20
+num_workers: 1
 prefetch_factor: 2
 batch_size: 8
 batch_dir: "/mnt/storage_ssd_4tb/windnet_batches"
 train_period:
-  - null
-  - 2021-12-31
-val_period:
   - 2022-01-01
+  - 2022-09-29
+val_period:
+  - 2022-09-30
   - 2022-12-31
 test_period:
-  - 2023-01-01
-  - 2023-11-28
+  - 2022-09-01
+  - 2022-12-31

scripts/save_batches.py

@@ -19,7 +19,8 @@
 import os
 import shutil
 import sys
-
+import dask
+dask.config.set(scheduler='single-threaded')
 # Tired of seeing these warnings
 import warnings
 
@@ -54,7 +55,7 @@ def __call__(self, input):
         if self.output_format == "torch":
             torch.save(batch, f"{self.batch_dir}/{i:06}.pt")
         elif self.output_format == "netcdf":
-            batch.to_netcdf(f"{self.batch_dir}/{i:06}.nc", mode="w")
+            batch.to_netcdf(f"{self.batch_dir}/{i:06}.nc", mode="w", engine="h5netcdf")
 
 
 def _get_datapipe(config_path, start_time, end_time, batch_size, renewable: str = "pv"):
@@ -70,9 +71,9 @@ def _get_datapipe(config_path, start_time, end_time, batch_size, renewable: str
         end_time=end_time,
     )
 
-    data_pipeline = (
-        data_pipeline.batch(batch_size).map(stack_np_examples_into_batch).map(batch_to_tensor)
-    )
+    #data_pipeline = (
+    #    data_pipeline.batch(batch_size).map(stack_np_examples_into_batch).map(batch_to_tensor)
+    #)
     return data_pipeline
 
 
@@ -115,13 +116,13 @@ def main(config: DictConfig):
         batch_size=None,  # batched in datapipe step
         sampler=None,
         batch_sampler=None,
-        num_workers=config_dm.num_workers,
+        num_workers=0, #config_dm.num_workers,
         collate_fn=None,
         pin_memory=False,
         drop_last=False,
         timeout=0,
         worker_init_fn=None,
-        prefetch_factor=config_dm.prefetch_factor,
+        prefetch_factor=None, #config_dm.prefetch_factor,
         persistent_workers=False,
     )