From abea2f59be876bb688a97df44446a05975ff9b9b Mon Sep 17 00:00:00 2001 From: Saaketh Date: Thu, 6 Jun 2024 10:00:39 -0700 Subject: [PATCH] import --- llmfoundry/data/data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llmfoundry/data/data.py b/llmfoundry/data/data.py index 4133e0b322..bde68a6998 100644 --- a/llmfoundry/data/data.py +++ b/llmfoundry/data/data.py @@ -14,6 +14,7 @@ from transformers import PreTrainedTokenizerBase __all__ = [ + 'AbstractConcatTokensDataset', 'ConcatTokensDataset', 'NoConcatDataset', 'stream_remote_local_validate',