From 24ffeb44b1611fc09267369a468bb01acface9d4 Mon Sep 17 00:00:00 2001 From: Eitan Turok Date: Fri, 27 Sep 2024 13:55:01 +0000 Subject: [PATCH] type checking --- llmfoundry/data/finetuning/dataloader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llmfoundry/data/finetuning/dataloader.py b/llmfoundry/data/finetuning/dataloader.py index 69051a2d51..181528c51f 100644 --- a/llmfoundry/data/finetuning/dataloader.py +++ b/llmfoundry/data/finetuning/dataloader.py @@ -286,6 +286,7 @@ def build_finetuning_dataloader( # Get the preprocessing function. proto_preprocessing_fn = dataset_cfg.get('preprocessing_fn') + assert proto_preprocessing_fn is not None if isinstance(proto_preprocessing_fn, (dict, DictConfig)): preprocessing_fn = dataset_constructor.get_preprocessing_fn_from_dict( dict(proto_preprocessing_fn),