update

VainF · Nov 19, 2024 · cd997ba · cd997ba
1 parent 9d1f740
commit cd997ba
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/examples/LLMs/prune_llm.py b/examples/LLMs/prune_llm.py
@@ -363,7 +363,7 @@ def main():
                 model.config.intermediate_size = m.gate_proj.out_features
             elif hasattr(m, "gate_up_proj"):
                 m.hidden_size = m.gate_up_proj.in_features
-                model.config.intermediate_size = m.gate_up_proj.out_features
+                model.config.intermediate_size = m.gate_up_proj.out_features // 2
             else:
                 raise ValueError("Unknown mlp layer")
 

diff --git a/examples/LLMs/readme.md b/examples/LLMs/readme.md
@@ -450,7 +450,7 @@ Phi3Config {
   "hidden_act": "silu",
   "hidden_size": 1536,
   "initializer_range": 0.02,
-  "intermediate_size": 8192,
+  "intermediate_size": 4096,
   "max_position_embeddings": 4096,
   "model_type": "phi3",
   "num_attention_heads": 16,
@@ -465,7 +465,7 @@ Phi3Config {
   "sliding_window": 2047,
   "tie_word_embeddings": false,
   "torch_dtype": "float16",
-  "transformers_version": "4.46.2",
+  "transformers_version": "4.46.3",
   "use_cache": true,
   "vocab_size": 32064
 }