feat(configs): update configs

yingtongxiong · Oct 25, 2023 · 6389984 · 6389984
1 parent ab97724
commit 6389984
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 8 deletions.
diff --git a/configs/13B_template.py b/configs/13B_template.py
@@ -1,7 +1,7 @@
 DO_ALERT = False
 
-SEQ_LEN = {seq_len}
-JOB_NAME = "13b_train_" + str(SEQ_LEN) + "_" + str({sp}) + "_" + str({intern_overlap}) + "_" + str({checkpoint})
+SEQ_LEN = 4096
+JOB_NAME = "13b_train_" + str({micro_bsz}) + "_" + str({sp}) + "_" + str({intern_overlap}) + "_" + str({checkpoint})
 HIDDEN_SIZE = 5120
 NUM_ATTENTION_HEAD = 40
 MLP_RATIO = 8 / 3

diff --git a/configs/30B_template.py b/configs/30B_template.py
@@ -1,7 +1,7 @@
 DO_ALERT = False
 
-SEQ_LEN = {seq_len}
-JOB_NAME = "7b_train_" + str(SEQ_LEN) + "_" + str({sp}) + "_" + str({intern_overlap}) + "_" + str({checkpoint})
+SEQ_LEN = 4096
+JOB_NAME = "7b_train_" + str({micro_bsz}) + "_" + str({sp}) + "_" + str({intern_overlap}) + "_" + str({checkpoint})
 HIDDEN_SIZE = 6144
 NUM_ATTENTION_HEAD = 48
 MLP_RATIO = 8 / 3

diff --git a/configs/7B_template.py b/configs/7B_template.py
@@ -1,8 +1,8 @@
 # JOB_NAME = "7b_train"
 DO_ALERT = False
 
-SEQ_LEN = {seq_len}
-JOB_NAME = "7b_train_" + str(SEQ_LEN) + "_" + str({sp}) + "_" + str({intern_overlap}) + "_" + str({checkpoint})
+SEQ_LEN = 4096
+JOB_NAME = "7b_train_" + str({micro_bsz}) + "_" + str({sp}) + "_" + str({intern_overlap}) + "_" + str({checkpoint})
 HIDDEN_SIZE = 4096
 NUM_ATTENTION_HEAD = 32
 MLP_RATIO = 8 / 3

diff --git a/configs/generate.py b/configs/generate.py
@@ -5,7 +5,7 @@
 name = "./configs/"
 root_names = ["7B_train_", "13B_train_", "30B_train_"]
 model_size = ["7B", "13B", "30B"]
-seq_length = [4096, 8192, 16384, 32768, 65536, 131072, 262144]
+micro_bsz = [1, 2, 4, 8, 16, 32, 64]
 sp = ["none", "megatron", "flash-attn", "intern", "intern"]
 intern_overlap = [False, False, False, True, False]
 checkpoint = [False, True]
@@ -32,7 +32,7 @@
                     line = line.replace("{intern_overlap}", str(intern_overlap[i]))
                     line = line.replace("{checkpoint}", str(ckpt))
                     output_file_name = (
-                        str(seq)
+                        str(mb)
                         + "_"
                         + str(sp_mode)
                         + "_overlap_"