lint and fix broken code

axolotl-ai-cloud · Oct 28, 2023 · 8f11779 · 8f11779
1 parent a898808
commit 8f11779
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 6 deletions.
diff --git a/gitbook/README.md b/gitbook/README.md
@@ -1,2 +1 @@
 # Page
-
diff --git a/src/axolotl/utils/data.py b/src/axolotl/utils/data.py
@@ -251,10 +251,10 @@ def for_d_in_datasets(dataset_configs):
                 raise ValueError("unhandled dataset load")
             # support for using a subset of the data
             if config_dataset.shards:
-                shard_split = d.split if d.split else split
+                shard_split = config_dataset.split if config_dataset.split else split
                 if shard_split in ds:
                     ds = ds.shuffle(seed=seed)[shard_split].shard(
-                        num_shards=d.shards, index=0
+                        num_shards=config_dataset.shards, index=0
                     )
                 else:
                     ds = ds.shuffle(seed=seed).shard(
@@ -268,13 +268,13 @@ def for_d_in_datasets(dataset_configs):
                 d_base_type = d_type_split[0]
                 d_prompt_style = d_type_split[1] if len(d_type_split) > 1 else None
 
-            if d.split and d.split in ds:
-                ds = ds[d.split]
+            if config_dataset.split and config_dataset.split in ds:
+                ds = ds[config_dataset.split]
             elif split in ds:
                 ds = ds[split]
             elif isinstance(ds, DatasetDict):
                 raise ValueError(
-                    f"no {split} split found for dataset {d.path}, you may specify a split with 'split: ...`"
+                    f"no {split} split found for dataset {config_dataset.path}, you may specify a split with 'split: ...`"
                 )
 
             dataset_wrapper, dataset_prompter = get_dataset_wrapper(