Validation

.pre-commit-config.yaml

@@ -66,6 +66,7 @@ repos:
     - --comment-style
     - '#'
     types: [python]
+    exclude: scripts/data_prep/validate_and_tokenize_data.py
 - repo: https://github.com/PyCQA/docformatter
   rev: v1.5.0
   hooks:

llmfoundry/data/finetuning/tasks.py

@@ -434,7 +434,7 @@ def dataset_mapper(example: Dict):
 
             detected_cpu_count = os.cpu_count() or 1
             detected_cpus_with_margin = detected_cpu_count - 8
-            num_cpus_to_use = max(1, detected_cpus_with_margin)
+            num_cpus_to_use = detected_cpu_count # Hack for Valiation instead of max(1, detected_cpus_with_margin)
 
             columns_to_remove = list(dataset[0].keys())
             tokenized_dataset = dataset.map(

llmfoundry/utils/__init__.py

@@ -13,6 +13,13 @@
                                                update_batch_size_info)
     from llmfoundry.utils.model_download_utils import (
         download_from_cache_server, download_from_hf_hub)
+
+    from llmfoundry.utils.validation_utils import (
+        create_om_cfg, token_counts_and_validation, token_counts,
+        check_HF_datasets, is_hf_dataset_path, is_uc_delta_table,
+        pandas_processing_fn, integrity_check, convert_text_to_mds,
+        parse_args, _args_str, plot_hist, dataframe_to_mds)
+
 except ImportError as e:
     raise ImportError(
         'Please make sure to pip install . to get requirements for llm-foundry.'
@@ -34,4 +41,17 @@
     'update_batch_size_info',
     'log_config',
     'pop_config',
+    'create_om_cfg',
+    'token_counts_and_validation',
+    'token_counts',
+    'check_HF_datasets',
+    'is_hf_dataset_path',
+    'is_uc_delta_table',
+    'pandas_processing_fn',
+    'integrity_check',
+    'convert_text_to_mds',
+    'parse_args',
+    '_args_str',
+    'plot_hist',
+    'dataframe_to_mds',
 ]