From 50588ac00d43ee87798c41959b379502a86263da Mon Sep 17 00:00:00 2001 From: Irene Dea Date: Wed, 27 Dec 2023 16:44:25 -0800 Subject: [PATCH] pyright fixes --- scripts/data_prep/convert_text_to_mds.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/data_prep/convert_text_to_mds.py b/scripts/data_prep/convert_text_to_mds.py index dd36c60658..d3679c309d 100644 --- a/scripts/data_prep/convert_text_to_mds.py +++ b/scripts/data_prep/convert_text_to_mds.py @@ -351,6 +351,7 @@ def convert_text_to_mds( tokenizer = AutoTokenizer.from_pretrained(tokenizer_name) bos_text = tokenizer.bos_token if bos_text is None else bos_text eos_text = tokenizer.eos_token if eos_text is None else eos_text + assert bos_text is not None and eos_text is not None # for pyright is_remote_output = is_remote_path(output_folder)