Mixtral official (#942)

* multipack support for official mixtral implementation * fix patch to load multipack for mixtral * chore: lint
axolotl-ai-cloud · Dec 12, 2023 · 7fabc4d · 7fabc4d
1 parent 9a5eb39
commit 7fabc4d
Show file tree

Hide file tree

Showing 8 changed files with 443 additions and 1,703 deletions.
diff --git a/examples/mistral/mixtral.yml b/examples/mistral/mixtral.yml
@@ -1,5 +1,5 @@
-base_model: DiscoResearch/mixtral-7b-8expert
-model_type: MixtralForCausalLM
+base_model: mistralai/Mixtral-8x7B-v0.1
+model_type: AutoModelForCausalLM
 tokenizer_type: LlamaTokenizer
 trust_remote_code: true
 

diff --git a/requirements.txt b/requirements.txt
@@ -2,7 +2,7 @@
 auto-gptq==0.5.1
 packaging
 peft==0.6.0
-transformers @ git+https://github.com/huggingface/transformers.git@df5c5c62ae253055336f5bb0828ca8e3e15ab6bd
+transformers @ git+https://github.com/huggingface/transformers.git@e5079b0b2abcef11ecbdae60ba4a6636c57b725d
 tokenizers==0.15.0
 bitsandbytes>=0.41.1
 accelerate==0.24.1

diff --git a/src/axolotl/models/mixtral/__init__.py b/src/axolotl/models/mixtral/__init__.py
diff --git a/src/axolotl/models/mixtral/configuration_moe_mistral.py b/src/axolotl/models/mixtral/configuration_moe_mistral.py