add replication

mosaicml · Sep 12, 2024 · 76adc48 · 76adc48
1 parent 3b5f935
commit 76adc48
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/scripts/train/yamls/pretrain/mpt-125m.yaml b/scripts/train/yamls/pretrain/mpt-125m.yaml
@@ -35,7 +35,7 @@ train_loader:
   dataset:
     local: ${variables.data_local}
     remote: ${variables.data_remote}
-    split: train
+    split: train_small
     shuffle: true
     max_seq_len: ${variables.max_seq_len}
     shuffle_seed: ${variables.global_seed}
@@ -47,10 +47,11 @@ eval_loader:
   dataset:
     local: ${variables.data_local}
     remote: ${variables.data_remote}
-    split: val
+    split: val_small
     shuffle: false
     max_seq_len: ${variables.max_seq_len}
     shuffle_seed: ${variables.global_seed}
+    replication: 2
   drop_last: false
   num_workers: 8
 
@@ -98,7 +99,7 @@ fsdp_config:
 
 # TP
 tp_config:
-  strategy: ffn
+  strategy: megatron
   tensor_parallel_degree: 2
 
 # Logging