From c0e44f7e6932e78ac628a0aa16b24651b9797261 Mon Sep 17 00:00:00 2001
From: James Fulton <djamesfulton@yahoo.co.uk>
Date: Mon, 6 Nov 2023 17:11:11 +0000
Subject: [PATCH] regrid more RAM efficiently

---
 pvnet_app/app.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/pvnet_app/app.py b/pvnet_app/app.py
index 3f9f35e..c2a3cfe 100644
--- a/pvnet_app/app.py
+++ b/pvnet_app/app.py
@@ -124,12 +124,15 @@ def regrid_nwp_data(nwp_path):
         return
     
     logger.info("Regridding NWP to expected grid")
-    # Its more efficient to regrid eagerly
+    
+    # Pull the raw data into RAM
     ds_raw = ds_raw.compute()
     
-    # Regrid
+    # Regrid in RAM efficient way by chunking first. Each step is regridded separately
     regridder = xe.Regridder(ds_raw, ds_target_coords, method="bilinear")
-    ds_regridded = regridder(ds_raw)
+    ds_regridded = regridder(
+        ds_raw.chunk(dict(x=-1, y=-1, step=1))
+    ).compute(scheduler="single-threaded")
 
     # Re-save - including rechunking
     os.system(f"rm -fr {nwp_path}")