From 32465b29c49a638daaa767fc69aefee18f3fa2f0 Mon Sep 17 00:00:00 2001 From: Daniel King Date: Mon, 30 Oct 2023 22:52:25 -0700 Subject: [PATCH] pr nit --- llmfoundry/data/finetuning/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llmfoundry/data/finetuning/tasks.py b/llmfoundry/data/finetuning/tasks.py index 42a9c0da02..edbfcc28c7 100644 --- a/llmfoundry/data/finetuning/tasks.py +++ b/llmfoundry/data/finetuning/tasks.py @@ -347,7 +347,7 @@ def dataset_mapper(example: Dict): dataset_mapper, batched=False, remove_columns=columns_to_remove, - num_proc=num_cpus_to_use), + num_proc=num_cpus_to_use, ) prompt_length_filtered_dataset = tokenized_dataset.filter( lambda example: len(example['input_ids']) < max_seq_len,