From 91fd3a32ff9290ed542503eee0dcc1d6ffbbb106 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Wed, 1 Nov 2023 12:31:20 -0400 Subject: [PATCH] Use srun, but fix environment export and buffered output. --- flow/environments/umich.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flow/environments/umich.py b/flow/environments/umich.py index 30e362213..4402b529e 100644 --- a/flow/environments/umich.py +++ b/flow/environments/umich.py @@ -17,7 +17,11 @@ class GreatLakesEnvironment(DefaultSlurmEnvironment): _gpus_per_node = {"default": 2} _shared_partitions = {"standard", "gpu"} - mpi_cmd = "mpirun" + # For unknown reasons, srun fails to export environment variables such as + # PATH on Great Lakes unless explicitly requested to with --export=ALL. + # On Great Lakes, srun also fails to flush the buffer until the end of + # the job without explicitly setting -u. + mpi_cmd = "srun -u --export=ALL" @classmethod def add_args(cls, parser):