Set persistent_workers = False for packing profiling (#718)

mosaicml · Nov 7, 2023 · dd15791 · dd15791
1 parent 2b74cb2
commit dd15791
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 0 deletions.
diff --git a/llmfoundry/data/finetuning/dataloader.py b/llmfoundry/data/finetuning/dataloader.py
@@ -400,6 +400,13 @@ def _build_collate_fn(
         packing_ratio = auto_packing_ratio(dataloader_cfg, tokenizer,
                                            device_batch_size)
 
+    if isinstance(packing_ratio, str):
+        raise ValueError(
+            'dataset.packing_ratio must be a float or "auto", but it was set to '
+            + f'{packing_ratio}.')
+
+    log.info(f'Using packing ratio {packing_ratio}')
+
     if packing_ratio == 1.0:
         return collate_fn, device_batch_size
     elif packing_ratio < 1.0:

diff --git a/llmfoundry/data/packing.py b/llmfoundry/data/packing.py
@@ -348,6 +348,7 @@ def profile_packing(
     dataloader_cfg.drop_last = False
     dataloader_cfg.num_workers = 0
     dataloader_cfg.prefetch_factor = None
+    dataloader_cfg.persistent_workers = False
 
     # Determine the packing_ratio values we'll try
     packing_ratios, raw_batch_sizes = [], []