Add a config not to shuffle merged dataset

README.md

@@ -662,6 +662,10 @@ datasets:
       # For `completion` datsets only, uses the provided field instead of `text` column
       field:
 
+# If true, the datasets will not be shuffled and will keep their original order in `datasets`.
+# The same applies to the `test_datasets` option and the `pretraining_dataset` option. Default is false.
+not_shuffle_merged_datasets: false
+
 # A list of one or more datasets to eval the model with.
 # You can use either test_datasets, or val_set_size, but not both.
 test_datasets:

src/axolotl/utils/config/models/input/v0_4_1/__init__.py

@@ -411,6 +411,7 @@ class Config:
 
     datasets: Optional[conlist(Union[SFTDataset, DPODataset], min_length=1)] = None  # type: ignore
     test_datasets: Optional[conlist(Union[SFTDataset, DPODataset], min_length=1)] = None  # type: ignore
+    not_shuffle_merged_datasets: Optional[bool] = None
     dataset_prepared_path: Optional[str] = None
     dataset_shard_num: Optional[int] = None
     dataset_shard_idx: Optional[int] = None

src/axolotl/utils/data.py

@@ -415,8 +415,11 @@ def for_d_in_datasets(dataset_configs):
         dataset = concatenate_datasets(datasets)
 
         if len(datasets) > 1:
-            LOG.info("shuffle merged datasets")
-            dataset = dataset.shuffle(seed=seed)
+            if cfg.not_shuffle_merged_datasets:
+                LOG.info("NOT shuffling merged datasets")
+            else:
+                LOG.info("shuffle merged datasets")
+                dataset = dataset.shuffle(seed=seed)
 
         dataset, _ = process_datasets_for_packing(cfg, dataset, None)
 
@@ -819,7 +822,11 @@ def wrap_pretraining_dataset(
     else:
         encode = functools.partial(encode_pretraining, tokenizer, max_tokens)
 
-    dataset = dataset.shuffle(seed=seed, buffer_size=buffer_size)
+    if cfg.not_shuffle_merged_datasets:
+        LOG.info("NOT shuffling merged pretraining datasets")
+    else:
+        dataset = dataset.shuffle(seed=seed, buffer_size=buffer_size)
+
     dataset = dataset.map(
         encode,
         batched=True,