From 317fa2555ad622f5a64f7a1108fc4ce96df548f6 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Wed, 17 Jan 2024 22:41:23 -0500 Subject: [PATCH] fix bf16 check when preprocessing data (#1140) --- src/axolotl/utils/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/axolotl/utils/config.py b/src/axolotl/utils/config.py index 7ea9581a9f..0df6136c6f 100644 --- a/src/axolotl/utils/config.py +++ b/src/axolotl/utils/config.py @@ -182,7 +182,11 @@ def validate_config(cfg): if not cfg.bf16 and not cfg.bfloat16: LOG.info("bf16 support detected, but not enabled for this configuration.") else: - if not cfg.merge_lora and (cfg.bf16 or cfg.bfloat16): + if ( + not cfg.merge_lora + and not cfg.is_preprocess + and (cfg.bf16 is True or cfg.bfloat16 is True) + ): raise ValueError( "bf16 requested, but AMP is not supported on this GPU. Requires Ampere series or above." )