fix max_seq_len

DesmonDay · Dec 29, 2023 · 7eda0a4 · 7eda0a4
1 parent b055be6
commit 7eda0a4
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/llm/llama/pretrain-linly_llama2_7b-tp2sd4_stage2.json b/llm/llama/pretrain-linly_llama2_7b-tp2sd4_stage2.json
@@ -14,7 +14,7 @@
     "use_flash_attention": true,
     "use_fused_rms_norm": true,
     "use_fused_rope": true,
-    "max_seq_length": 4096,
+    "max_seq_length": 2048,
     "learning_rate": 3e-05,
     "min_learning_rate": 3e-06,
     "warmup_steps": 30,

diff --git a/llm/llama/pretrain-llama_7b-tp2sd4_stage2.json b/llm/llama/pretrain-llama_7b-tp2sd4_stage2.json
@@ -14,7 +14,7 @@
     "use_flash_attention": true,
     "use_fused_rms_norm": true,
     "use_fused_rope": true,
-    "max_seq_length": 4096,
+    "max_seq_length": 2048,
     "learning_rate": 3e-05,
     "min_learning_rate": 3e-06,
     "warmup_steps": 30,