debugging wind data

india_forecast_app/models/pvnet/model.py

@@ -162,7 +162,7 @@ def _create_dataloader(self):
         populate_data_config_sources(data_config_filename, populated_data_config_filename)
 
         # Location and time datapipes
-        # TODO not sure what to use here for the location pipe - site uuid/location?
+        # TODO not sure if this is the correct way to set these up...
         location_pipe = IterableWrapper([Location(coordinate_system="lon_lat", x=72.6399, y=26.4499)])
         t0_datapipe = IterableWrapper([self.t0])
         # t0_datapipe = IterableWrapper([self.t0]).repeat(len(location_pipe))
@@ -179,7 +179,11 @@ def _create_dataloader(self):
                     t0_datapipe=t0_datapipe
                 )
             )
-            log.error(base_datapipe_dict)
+            log.info(next(iter(base_datapipe_dict["nwp"]["ecmwf"])))
+
+            # TODO figure out why this is an empty dataset
+            log.info(next(iter(base_datapipe_dict["wind"])).to_pandas())
+
             base_datapipe = DictDatasetIterDataPipe(
                 {k: v for k, v in base_datapipe_dict.items() if k != "config"},
             ).map(combine_to_single_dataset)

tests/conftest.py

@@ -109,16 +109,27 @@ def forecast_values():
 
 
 @pytest.fixture(scope="session")
-def nwp_data(tmp_path_factory):
+def time_before_present():
+    """Returns a fixed time in the past with specified offset"""
+
+    now = pd.Timestamp.now(tz=None)
+
+    def _time_before_present(dt: dt.timedelta):
+        return now - dt
+
+    return _time_before_present
+
+@pytest.fixture(scope="session")
+def nwp_data(tmp_path_factory, time_before_present):
     """Dummy NWP data"""
 
     # Load dataset which only contains coordinates, but no data
     ds = xr.open_zarr(
         f"{os.path.dirname(os.path.abspath(__file__))}/test_data/nwp.zarr"
     )
 
-    # Last init time was at least 2 hours ago and floor to 3-hour interval
-    t0_datetime_utc = ((pd.Timestamp.now(tz=None) - dt.timedelta(hours=2))
+    # Last t0 to at least 2 hours ago and floor to 3-hour interval
+    t0_datetime_utc = (time_before_present(dt.timedelta(hours=2))
                        .floor(dt.timedelta(hours=3)))
     ds.init_time.values[:] = pd.date_range(
         t0_datetime_utc - dt.timedelta(hours=3 * (len(ds.init_time) - 1)),
@@ -156,7 +167,7 @@ def nwp_data(tmp_path_factory):
 
 
 @pytest.fixture(scope="session")
-def wind_data(tmp_path_factory):
+def wind_data(tmp_path_factory, time_before_present):
     """Dummy wind data"""
 
     # AS wind data is loaded by the app from environment variable,
@@ -168,8 +179,33 @@ def wind_data(tmp_path_factory):
     netcdf_source_path = f"{root_source_path}/test_data/wind/wind_data.nc"
     temp_netcdf_path = f"{root_path}/wind_data.nc"
     os.environ["WIND_NETCDF_PATH"] = temp_netcdf_path
-    fs = fsspec.open(netcdf_source_path).fs
-    fs.copy(netcdf_source_path, temp_netcdf_path)
+    ds = xr.open_dataset(netcdf_source_path)
+
+    # Set t0 to at least 2 hours ago and floor to 15-min interval
+    t0_datetime_utc = (time_before_present(dt.timedelta(hours=2))
+                       .floor(dt.timedelta(minutes=15)))
+    ds.time_utc.values[:] = pd.date_range(
+        t0_datetime_utc - dt.timedelta(minutes=15 * (len(ds.time_utc) - 1)),
+        t0_datetime_utc,
+        freq=dt.timedelta(minutes=15),
+    )
+
+    # This is important to avoid saving errors
+    for v in list(ds.coords.keys()):
+        if ds.coords[v].dtype == object:
+            ds[v].encoding.clear()
+
+    for v in list(ds.variables.keys()):
+        if ds[v].dtype == object:
+            ds[v].encoding.clear()
+
+    # Add data to dataset
+    # ds["wind"] = xr.DataArray(
+    #     np.zeros([len(ds[c]) for c in ds.xindexes]),
+    #     coords=[ds[c] for c in ds.xindexes],
+    # )
+
+    ds.to_netcdf(temp_netcdf_path, engine="h5netcdf")
 
     metadata_source_path = f"{root_source_path}/test_data/wind/wind_metadata.csv"
     temp_metadata_path = f"{root_path}/wind_metadata.csv"

tests/test_app.py

@@ -29,11 +29,10 @@ def test_get_sites(db_session):
 
 
 @pytest.mark.parametrize("asset_type", ["pv", "wind"])
-def test_get_model(asset_type, nwp_data, wind_data, caplog):
+def test_get_model(asset_type, nwp_data, wind_data):
     """Test for getting valid model"""
 
-    caplog.set_level('INFO')
-    model = get_model(asset_type, timestamp=dt.datetime.now(tz=dt.UTC))
+    model = get_model(asset_type, timestamp=dt.datetime.now(tz=None))
 
     assert hasattr(model, 'version')
     assert isinstance(model.version, str)
@@ -42,9 +41,11 @@ def test_get_model(asset_type, nwp_data, wind_data, caplog):
 
 # @pytest.mark.skip(reason="Temporarily disabled while integrating Windnet")
 @pytest.mark.parametrize("asset_type", ["pv", "wind"])
-def test_run_model(db_session, asset_type, nwp_data, wind_data):
+def test_run_model(db_session, asset_type, nwp_data, wind_data, caplog):
     """Test for running PV and wind models"""
 
+    caplog.set_level('INFO')
+
     model = PVNetModel if asset_type == "wind" else DummyModel
 
     forecast = run_model(

tests/test_data/inspect.py

@@ -0,0 +1,11 @@
+import xarray as xr
+
+
+def run():
+    ds = xr.open_dataset("data/wind/wind_data.nc")
+    # ds = xr.open_dataset("tests/test_data/wind/wind_data.nc")
+    print(ds["time_utc"])
+
+    ds = xr.open_zarr("data/nwp.zarr")
+    # ds = xr.open_zarr("tests/test_data/nwp.zarr")
+    print(ds['init_time'])

tests/test_data/wind/wind_metadata.csv

@@ -0,0 +1,2 @@
+system_id,latitude,longitude,capacity_megawatts
+0,26.4499,72.6399,3381.0