Merge pull request #24 from openclimatefix/less_ram_regrid

Regrid more RAM efficiently
openclimatefix · Nov 6, 2023 · f26cf35 · f26cf35
2 parents c845b21 + c0e44f7
commit f26cf35
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/pvnet_app/app.py b/pvnet_app/app.py
@@ -124,12 +124,15 @@ def regrid_nwp_data(nwp_path):
         return
 
     logger.info("Regridding NWP to expected grid")
-    # Its more efficient to regrid eagerly
+
+    # Pull the raw data into RAM
     ds_raw = ds_raw.compute()
 
-    # Regrid
+    # Regrid in RAM efficient way by chunking first. Each step is regridded separately
     regridder = xe.Regridder(ds_raw, ds_target_coords, method="bilinear")
-    ds_regridded = regridder(ds_raw)
+    ds_regridded = regridder(
+        ds_raw.chunk(dict(x=-1, y=-1, step=1))
+    ).compute(scheduler="single-threaded")
 
     # Re-save - including rechunking
     os.system(f"rm -fr {nwp_path}")