From 73f8a58919445bd772288fdb8286c8810168f37a Mon Sep 17 00:00:00 2001 From: Yu Chin Fabian Lim Date: Fri, 11 Oct 2024 03:51:03 +0000 Subject: [PATCH] make accelerate.yaml 1.0 compatible Signed-off-by: Yu Chin Fabian Lim --- scripts/benchmarks/accelerate.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/benchmarks/accelerate.yaml b/scripts/benchmarks/accelerate.yaml index f70d74fa..7923e624 100644 --- a/scripts/benchmarks/accelerate.yaml +++ b/scripts/benchmarks/accelerate.yaml @@ -14,9 +14,10 @@ fsdp_config: fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP # this controls the FSDP pipelining - fsdp_backward_prefetch_policy: BACKWARD_PRE # set to BACKWARD_PRE for the most time-efficient pipeline + fsdp_backward_prefetch: BACKWARD_PRE # set to BACKWARD_PRE for the most time-efficient pipeline # but requires the most memory. BACKWARD_POST is the less # memory intensive option + fsdp_backward_prefetch_policy: BACKWARD_PRE # for backward compatibility accelerate<1.0 # setting this to true will increase forward memory by prefetching the next FSDP all-gather, while performing # the current forward pass.