From 7858e823f4d144598774abf760e86e4d06325079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine?= Date: Thu, 11 Jul 2024 17:11:57 +0200 Subject: [PATCH] fix --- src/lighteval/models/model_config.py | 29 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/lighteval/models/model_config.py b/src/lighteval/models/model_config.py index 419bd6149..75a29d02c 100644 --- a/src/lighteval/models/model_config.py +++ b/src/lighteval/models/model_config.py @@ -153,20 +153,21 @@ def _init_configs(self, model_name: str, env_config: EnvConfig) -> PretrainedCon except (AttributeError, KeyError): model_auto_quantization_config = None - # We don't load models quantized by default with a different user provided conf - if model_auto_quantization_config is not None and self.quantization_config is not None: - raise ValueError("You manually requested quantization on a model already quantized!") - - # We add the quantization to the model params we store - if model_auto_quantization_config["quant_method"] == "gptq": - if not is_autogptq_available(): - raise ImportError(NO_AUTOGPTQ_ERROR_MSG) - auto_config.quantization_config["use_exllama"] = None - self.quantization_config = GPTQConfig(**auto_config.quantization_config, disable_exllama=True) - elif model_auto_quantization_config["quant_method"] == "bitsandbytes": - if not is_bnb_available(): - raise ImportError(NO_BNB_ERROR_MSG) - self.quantization_config = BitsAndBytesConfig(**auto_config.quantization_config) + if model_auto_quantization_config is not None: + if self.quantization_config is not None: + # We don't load models quantized by default with a different user provided conf + raise ValueError("You manually requested quantization on a model already quantized!") + + # We add the quantization to the model params we store + if model_auto_quantization_config["quant_method"] == "gptq": + if not is_autogptq_available(): + raise ImportError(NO_AUTOGPTQ_ERROR_MSG) + auto_config.quantization_config["use_exllama"] = None + self.quantization_config = GPTQConfig(**auto_config.quantization_config, disable_exllama=True) + elif model_auto_quantization_config["quant_method"] == "bitsandbytes": + if not is_bnb_available(): + raise ImportError(NO_BNB_ERROR_MSG) + self.quantization_config = BitsAndBytesConfig(**auto_config.quantization_config) return auto_config