From 33524af6ae41e70e8a87f9208caacdf54689f6a4 Mon Sep 17 00:00:00 2001 From: Samaneh Saadat Date: Mon, 22 Apr 2024 17:30:46 +0000 Subject: [PATCH] Update tokenizer test. --- keras_nlp/models/preprocessor_test.py | 2 +- keras_nlp/tokenizers/tokenizer_test.py | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/keras_nlp/models/preprocessor_test.py b/keras_nlp/models/preprocessor_test.py index b09a05c847..8ecc5665b2 100644 --- a/keras_nlp/models/preprocessor_test.py +++ b/keras_nlp/models/preprocessor_test.py @@ -91,7 +91,7 @@ def test_save_to_preset(self, cls, preset_name, tokenizer_type): vocab_filename = "vocabulary.txt" expected_assets = ["vocabulary.txt"] - # Check existence of files. + # Check existence of vocab file. vocab_path = os.path.join( save_dir, os.path.join(TOKENIZER_ASSET_DIR, vocab_filename) ) diff --git a/keras_nlp/tokenizers/tokenizer_test.py b/keras_nlp/tokenizers/tokenizer_test.py index d84762dfb2..9591038663 100644 --- a/keras_nlp/tokenizers/tokenizer_test.py +++ b/keras_nlp/tokenizers/tokenizer_test.py @@ -28,7 +28,6 @@ from keras_nlp.utils.preset_utils import TOKENIZER_ASSET_DIR from keras_nlp.utils.preset_utils import TOKENIZER_CONFIG_FILE from keras_nlp.utils.preset_utils import check_config_class -from keras_nlp.utils.preset_utils import get_file class SimpleTokenizer(Tokenizer): @@ -113,14 +112,11 @@ def test_save_to_preset(self, cls, preset_name, tokenizer_type): vocab_filename = "vocabulary.txt" expected_assets = ["vocabulary.txt"] - # Check existence of files. - self.assertTrue( - os.path.exists( - get_file( - save_dir, os.path.join(TOKENIZER_ASSET_DIR, vocab_filename) - ) - ) + # Check existence of vocab file. + vocab_path = os.path.join( + save_dir, os.path.join(TOKENIZER_ASSET_DIR, vocab_filename) ) + self.assertTrue(os.path.exists(vocab_path)) # Check assets. self.assertEqual(set(tokenizer.file_assets), set(expected_assets))