Rename LLAMATokenizer to SPieceTokenizer.

MoonRide303 · Jul 22, 2024 · 14764aa · 14764aa
1 parent b2c995f
commit 14764aa
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/comfy/text_encoders/aura_t5.py b/comfy/text_encoders/aura_t5.py
@@ -1,5 +1,5 @@
 from comfy import sd1_clip
-from .llama_tokenizer import LLAMATokenizer
+from .spiece_tokenizer import SPieceTokenizer
 import comfy.text_encoders.t5
 import os
 
@@ -11,7 +11,7 @@ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None):
 class PT5XlTokenizer(sd1_clip.SDTokenizer):
     def __init__(self, embedding_directory=None):
         tokenizer_path = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer"), "tokenizer.model")
-        super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=LLAMATokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)
+        super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=SPieceTokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)
 
 class AuraT5Tokenizer(sd1_clip.SD1Tokenizer):
     def __init__(self, embedding_directory=None):

diff --git a/comfy/text_encoders/llama_tokenizer.py → comfy/text_encoders/spiece_tokenizer.py b/comfy/text_encoders/llama_tokenizer.py → comfy/text_encoders/spiece_tokenizer.py
@@ -1,9 +1,9 @@
 import os
 
-class LLAMATokenizer:
+class SPieceTokenizer:
     @staticmethod
     def from_pretrained(path):
-        return LLAMATokenizer(path)
+        return SPieceTokenizer(path)
 
     def __init__(self, tokenizer_path):
         import sentencepiece