more dask workers, missing CICD import

MeteoSwiss · Mar 27, 2024 · db1b0d1 · db1b0d1
1 parent e789882
commit db1b0d1
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 4 deletions.
diff --git a/create_zarr_archive.py b/create_zarr_archive.py
@@ -129,9 +129,9 @@ def main(data_in, data_out, indexpath, selected_vars, selected_vars_2):
     cluster = SLURMCluster(
         queue="postproc",
         account="s83",
-        processes=1,
-        cores=256,
-        memory="446GB",
+        processes=4,
+        cores=64,
+        memory="111GB",
         local_directory="/scratch/mch/sadamov/temp",
         shared_temp_directory="/scratch/mch/sadamov/temp",
         log_directory="lightning_logs",

diff --git a/requirements.txt b/requirements.txt
@@ -2,7 +2,8 @@
 numpy>=1.24.2
 wandb>=0.13.10
 matplotlib>=3.7.0
-dask>=2022.1.1
+dask
+dask_jobqueue
 scipy>=1.10.0
 pytorch-lightning>=2.0.3
 shapely>=2.0.1