From 25afd35842ec005ff18aadf3e54dd9db9e72548f Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Wed, 27 Mar 2024 10:16:47 -0400 Subject: [PATCH 1/2] support layer replication for peft and fix rslora integration (#1445) --- src/axolotl/utils/config/models/input/v0_4_1/__init__.py | 5 +++-- src/axolotl/utils/models.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/axolotl/utils/config/models/input/v0_4_1/__init__.py b/src/axolotl/utils/config/models/input/v0_4_1/__init__.py index 442f61e34a..cce0cbc76a 100644 --- a/src/axolotl/utils/config/models/input/v0_4_1/__init__.py +++ b/src/axolotl/utils/config/models/input/v0_4_1/__init__.py @@ -6,7 +6,7 @@ import logging import os from enum import Enum -from typing import Any, Dict, List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Tuple, Union from pydantic import BaseModel, Field, conlist, field_validator, model_validator from transformers import SchedulerType @@ -179,7 +179,8 @@ class LoraConfig(BaseModel): peft_layers_to_transform: Optional[List[int]] = None peft: Optional[PeftConfig] = None peft_use_dora: Optional[bool] = None - peft_use_relora: Optional[bool] = None + peft_use_rslora: Optional[bool] = None + peft_layer_replication: Optional[List[Tuple[int, int]]] = None lora_on_cpu: Optional[bool] = None gptq: Optional[bool] = None diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py index 0a59eb2a4d..e2d33036b0 100644 --- a/src/axolotl/utils/models.py +++ b/src/axolotl/utils/models.py @@ -849,7 +849,9 @@ def load_lora(model, cfg, inference=False, config_only=False): if cfg.peft_use_dora: lora_config_kwargs["use_dora"] = cfg.peft_use_dora if cfg.peft_use_rslora: - lora_config_kwargs["use_rslora"] = cfg.use_rslora + lora_config_kwargs["use_rslora"] = cfg.peft_use_rslora + if cfg.peft_layer_replication: + lora_config_kwargs["peft_layer_replication"] = cfg.peft_layer_replication lora_config = LoraConfig( r=cfg.lora_r, From 4155e9988f6d03e1dc6d5b24a9d241358e15c2ad Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Wed, 27 Mar 2024 10:18:56 -0400 Subject: [PATCH 2/2] fix layer_replication arg to peft (#1446) --- src/axolotl/utils/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py index e2d33036b0..c40a15c48f 100644 --- a/src/axolotl/utils/models.py +++ b/src/axolotl/utils/models.py @@ -851,7 +851,7 @@ def load_lora(model, cfg, inference=False, config_only=False): if cfg.peft_use_rslora: lora_config_kwargs["use_rslora"] = cfg.peft_use_rslora if cfg.peft_layer_replication: - lora_config_kwargs["peft_layer_replication"] = cfg.peft_layer_replication + lora_config_kwargs["layer_replication"] = cfg.peft_layer_replication lora_config = LoraConfig( r=cfg.lora_r,