From 29c2e26724d4982a3e33114eb9064f1a11f4f4ed Mon Sep 17 00:00:00 2001
From: comfyanonymous <comfyanonymous@protonmail.com>
Date: Fri, 12 Jul 2024 01:08:45 -0400
Subject: [PATCH] Better tokenizing code for AuraFlow.

---
 comfy/text_encoders/aura_t5.py                |   6 +-
 comfy/text_encoders/llama_tokenizer.py        |  22 +
 .../t5_pile_tokenizer/added_tokens.json       | 102 --
 .../t5_pile_tokenizer/special_tokens_map.json | 125 ---
 .../t5_pile_tokenizer/tokenizer_config.json   | 945 ------------------
 5 files changed, 25 insertions(+), 1175 deletions(-)
 create mode 100644 comfy/text_encoders/llama_tokenizer.py
 delete mode 100644 comfy/text_encoders/t5_pile_tokenizer/added_tokens.json
 delete mode 100644 comfy/text_encoders/t5_pile_tokenizer/special_tokens_map.json
 delete mode 100644 comfy/text_encoders/t5_pile_tokenizer/tokenizer_config.json

diff --git a/comfy/text_encoders/aura_t5.py b/comfy/text_encoders/aura_t5.py
index 0e84189aa13..95f942ef578 100644
--- a/comfy/text_encoders/aura_t5.py
+++ b/comfy/text_encoders/aura_t5.py
@@ -1,5 +1,5 @@
 from comfy import sd1_clip
-from transformers import LlamaTokenizerFast
+from .llama_tokenizer import LLAMATokenizer
 import comfy.t5
 import os
 
@@ -10,8 +10,8 @@ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None):
 
 class PT5XlTokenizer(sd1_clip.SDTokenizer):
     def __init__(self, embedding_directory=None):
-        tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer")
-        super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=LlamaTokenizerFast, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)
+        tokenizer_path = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer"), "tokenizer.model")
+        super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=LLAMATokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)
 
 class AuraT5Tokenizer(sd1_clip.SD1Tokenizer):
     def __init__(self, embedding_directory=None):
diff --git a/comfy/text_encoders/llama_tokenizer.py b/comfy/text_encoders/llama_tokenizer.py
new file mode 100644
index 00000000000..a6db1da629c
--- /dev/null
+++ b/comfy/text_encoders/llama_tokenizer.py
@@ -0,0 +1,22 @@
+import os
+
+class LLAMATokenizer:
+    @staticmethod
+    def from_pretrained(path):
+        return LLAMATokenizer(path)
+
+    def __init__(self, tokenizer_path):
+        import sentencepiece
+        self.tokenizer = sentencepiece.SentencePieceProcessor(model_file=tokenizer_path)
+        self.end = self.tokenizer.eos_id()
+
+    def get_vocab(self):
+        out = {}
+        for i in range(self.tokenizer.get_piece_size()):
+            out[self.tokenizer.id_to_piece(i)] = i
+        return out
+
+    def __call__(self, string):
+        out = self.tokenizer.encode(string)
+        out += [self.end]
+        return {"input_ids": out}
diff --git a/comfy/text_encoders/t5_pile_tokenizer/added_tokens.json b/comfy/text_encoders/t5_pile_tokenizer/added_tokens.json
deleted file mode 100644
index 3f5132007c4..00000000000
--- a/comfy/text_encoders/t5_pile_tokenizer/added_tokens.json
+++ /dev/null
@@ -1,102 +0,0 @@
-{
-  "<extra_id_0>": 32099,
-  "<extra_id_10>": 32089,
-  "<extra_id_11>": 32088,
-  "<extra_id_12>": 32087,
-  "<extra_id_13>": 32086,
-  "<extra_id_14>": 32085,
-  "<extra_id_15>": 32084,
-  "<extra_id_16>": 32083,
-  "<extra_id_17>": 32082,
-  "<extra_id_18>": 32081,
-  "<extra_id_19>": 32080,
-  "<extra_id_1>": 32098,
-  "<extra_id_20>": 32079,
-  "<extra_id_21>": 32078,
-  "<extra_id_22>": 32077,
-  "<extra_id_23>": 32076,
-  "<extra_id_24>": 32075,
-  "<extra_id_25>": 32074,
-  "<extra_id_26>": 32073,
-  "<extra_id_27>": 32072,
-  "<extra_id_28>": 32071,
-  "<extra_id_29>": 32070,
-  "<extra_id_2>": 32097,
-  "<extra_id_30>": 32069,
-  "<extra_id_31>": 32068,
-  "<extra_id_32>": 32067,
-  "<extra_id_33>": 32066,
-  "<extra_id_34>": 32065,
-  "<extra_id_35>": 32064,
-  "<extra_id_36>": 32063,
-  "<extra_id_37>": 32062,
-  "<extra_id_38>": 32061,
-  "<extra_id_39>": 32060,
-  "<extra_id_3>": 32096,
-  "<extra_id_40>": 32059,
-  "<extra_id_41>": 32058,
-  "<extra_id_42>": 32057,
-  "<extra_id_43>": 32056,
-  "<extra_id_44>": 32055,
-  "<extra_id_45>": 32054,
-  "<extra_id_46>": 32053,
-  "<extra_id_47>": 32052,
-  "<extra_id_48>": 32051,
-  "<extra_id_49>": 32050,
-  "<extra_id_4>": 32095,
-  "<extra_id_50>": 32049,
-  "<extra_id_51>": 32048,
-  "<extra_id_52>": 32047,
-  "<extra_id_53>": 32046,
-  "<extra_id_54>": 32045,
-  "<extra_id_55>": 32044,
-  "<extra_id_56>": 32043,
-  "<extra_id_57>": 32042,
-  "<extra_id_58>": 32041,
-  "<extra_id_59>": 32040,
-  "<extra_id_5>": 32094,
-  "<extra_id_60>": 32039,
-  "<extra_id_61>": 32038,
-  "<extra_id_62>": 32037,
-  "<extra_id_63>": 32036,
-  "<extra_id_64>": 32035,
-  "<extra_id_65>": 32034,
-  "<extra_id_66>": 32033,
-  "<extra_id_67>": 32032,
-  "<extra_id_68>": 32031,
-  "<extra_id_69>": 32030,
-  "<extra_id_6>": 32093,
-  "<extra_id_70>": 32029,
-  "<extra_id_71>": 32028,
-  "<extra_id_72>": 32027,
-  "<extra_id_73>": 32026,
-  "<extra_id_74>": 32025,
-  "<extra_id_75>": 32024,
-  "<extra_id_76>": 32023,
-  "<extra_id_77>": 32022,
-  "<extra_id_78>": 32021,
-  "<extra_id_79>": 32020,
-  "<extra_id_7>": 32092,
-  "<extra_id_80>": 32019,
-  "<extra_id_81>": 32018,
-  "<extra_id_82>": 32017,
-  "<extra_id_83>": 32016,
-  "<extra_id_84>": 32015,
-  "<extra_id_85>": 32014,
-  "<extra_id_86>": 32013,
-  "<extra_id_87>": 32012,
-  "<extra_id_88>": 32011,
-  "<extra_id_89>": 32010,
-  "<extra_id_8>": 32091,
-  "<extra_id_90>": 32009,
-  "<extra_id_91>": 32008,
-  "<extra_id_92>": 32007,
-  "<extra_id_93>": 32006,
-  "<extra_id_94>": 32005,
-  "<extra_id_95>": 32004,
-  "<extra_id_96>": 32003,
-  "<extra_id_97>": 32002,
-  "<extra_id_98>": 32001,
-  "<extra_id_99>": 32000,
-  "<extra_id_9>": 32090
-}
diff --git a/comfy/text_encoders/t5_pile_tokenizer/special_tokens_map.json b/comfy/text_encoders/t5_pile_tokenizer/special_tokens_map.json
deleted file mode 100644
index 19fb1d5f4f7..00000000000
--- a/comfy/text_encoders/t5_pile_tokenizer/special_tokens_map.json
+++ /dev/null
@@ -1,125 +0,0 @@
-{
-  "additional_special_tokens": [
-    "<extra_id_99>",
-    "<extra_id_98>",
-    "<extra_id_97>",
-    "<extra_id_96>",
-    "<extra_id_95>",
-    "<extra_id_94>",
-    "<extra_id_93>",
-    "<extra_id_92>",
-    "<extra_id_91>",
-    "<extra_id_90>",
-    "<extra_id_89>",
-    "<extra_id_88>",
-    "<extra_id_87>",
-    "<extra_id_86>",
-    "<extra_id_85>",
-    "<extra_id_84>",
-    "<extra_id_83>",
-    "<extra_id_82>",
-    "<extra_id_81>",
-    "<extra_id_80>",
-    "<extra_id_79>",
-    "<extra_id_78>",
-    "<extra_id_77>",
-    "<extra_id_76>",
-    "<extra_id_75>",
-    "<extra_id_74>",
-    "<extra_id_73>",
-    "<extra_id_72>",
-    "<extra_id_71>",
-    "<extra_id_70>",
-    "<extra_id_69>",
-    "<extra_id_68>",
-    "<extra_id_67>",
-    "<extra_id_66>",
-    "<extra_id_65>",
-    "<extra_id_64>",
-    "<extra_id_63>",
-    "<extra_id_62>",
-    "<extra_id_61>",
-    "<extra_id_60>",
-    "<extra_id_59>",
-    "<extra_id_58>",
-    "<extra_id_57>",
-    "<extra_id_56>",
-    "<extra_id_55>",
-    "<extra_id_54>",
-    "<extra_id_53>",
-    "<extra_id_52>",
-    "<extra_id_51>",
-    "<extra_id_50>",
-    "<extra_id_49>",
-    "<extra_id_48>",
-    "<extra_id_47>",
-    "<extra_id_46>",
-    "<extra_id_45>",
-    "<extra_id_44>",
-    "<extra_id_43>",
-    "<extra_id_42>",
-    "<extra_id_41>",
-    "<extra_id_40>",
-    "<extra_id_39>",
-    "<extra_id_38>",
-    "<extra_id_37>",
-    "<extra_id_36>",
-    "<extra_id_35>",
-    "<extra_id_34>",
-    "<extra_id_33>",
-    "<extra_id_32>",
-    "<extra_id_31>",
-    "<extra_id_30>",
-    "<extra_id_29>",
-    "<extra_id_28>",
-    "<extra_id_27>",
-    "<extra_id_26>",
-    "<extra_id_25>",
-    "<extra_id_24>",
-    "<extra_id_23>",
-    "<extra_id_22>",
-    "<extra_id_21>",
-    "<extra_id_20>",
-    "<extra_id_19>",
-    "<extra_id_18>",
-    "<extra_id_17>",
-    "<extra_id_16>",
-    "<extra_id_15>",
-    "<extra_id_14>",
-    "<extra_id_13>",
-    "<extra_id_12>",
-    "<extra_id_11>",
-    "<extra_id_10>",
-    "<extra_id_9>",
-    "<extra_id_8>",
-    "<extra_id_7>",
-    "<extra_id_6>",
-    "<extra_id_5>",
-    "<extra_id_4>",
-    "<extra_id_3>",
-    "<extra_id_2>",
-    "<extra_id_1>",
-    "<extra_id_0>"
-  ],
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}
diff --git a/comfy/text_encoders/t5_pile_tokenizer/tokenizer_config.json b/comfy/text_encoders/t5_pile_tokenizer/tokenizer_config.json
deleted file mode 100644
index 81f8e11e9cd..00000000000
--- a/comfy/text_encoders/t5_pile_tokenizer/tokenizer_config.json
+++ /dev/null
@@ -1,945 +0,0 @@
-{
-  "add_bos_token": false,
-  "add_eos_token": true,
-  "add_prefix_space": true,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32000": {
-      "content": "<extra_id_99>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32001": {
-      "content": "<extra_id_98>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32002": {
-      "content": "<extra_id_97>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32003": {
-      "content": "<extra_id_96>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32004": {
-      "content": "<extra_id_95>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32005": {
-      "content": "<extra_id_94>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32006": {
-      "content": "<extra_id_93>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32007": {
-      "content": "<extra_id_92>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32008": {
-      "content": "<extra_id_91>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32009": {
-      "content": "<extra_id_90>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32010": {
-      "content": "<extra_id_89>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32011": {
-      "content": "<extra_id_88>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32012": {
-      "content": "<extra_id_87>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32013": {
-      "content": "<extra_id_86>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32014": {
-      "content": "<extra_id_85>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32015": {
-      "content": "<extra_id_84>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32016": {
-      "content": "<extra_id_83>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32017": {
-      "content": "<extra_id_82>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32018": {
-      "content": "<extra_id_81>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32019": {
-      "content": "<extra_id_80>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32020": {
-      "content": "<extra_id_79>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32021": {
-      "content": "<extra_id_78>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32022": {
-      "content": "<extra_id_77>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32023": {
-      "content": "<extra_id_76>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32024": {
-      "content": "<extra_id_75>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32025": {
-      "content": "<extra_id_74>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32026": {
-      "content": "<extra_id_73>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32027": {
-      "content": "<extra_id_72>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32028": {
-      "content": "<extra_id_71>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32029": {
-      "content": "<extra_id_70>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32030": {
-      "content": "<extra_id_69>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32031": {
-      "content": "<extra_id_68>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32032": {
-      "content": "<extra_id_67>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32033": {
-      "content": "<extra_id_66>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32034": {
-      "content": "<extra_id_65>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32035": {
-      "content": "<extra_id_64>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32036": {
-      "content": "<extra_id_63>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32037": {
-      "content": "<extra_id_62>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32038": {
-      "content": "<extra_id_61>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32039": {
-      "content": "<extra_id_60>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32040": {
-      "content": "<extra_id_59>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32041": {
-      "content": "<extra_id_58>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32042": {
-      "content": "<extra_id_57>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32043": {
-      "content": "<extra_id_56>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32044": {
-      "content": "<extra_id_55>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32045": {
-      "content": "<extra_id_54>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32046": {
-      "content": "<extra_id_53>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32047": {
-      "content": "<extra_id_52>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32048": {
-      "content": "<extra_id_51>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32049": {
-      "content": "<extra_id_50>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32050": {
-      "content": "<extra_id_49>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32051": {
-      "content": "<extra_id_48>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32052": {
-      "content": "<extra_id_47>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32053": {
-      "content": "<extra_id_46>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32054": {
-      "content": "<extra_id_45>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32055": {
-      "content": "<extra_id_44>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32056": {
-      "content": "<extra_id_43>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32057": {
-      "content": "<extra_id_42>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32058": {
-      "content": "<extra_id_41>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32059": {
-      "content": "<extra_id_40>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32060": {
-      "content": "<extra_id_39>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32061": {
-      "content": "<extra_id_38>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32062": {
-      "content": "<extra_id_37>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32063": {
-      "content": "<extra_id_36>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32064": {
-      "content": "<extra_id_35>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32065": {
-      "content": "<extra_id_34>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32066": {
-      "content": "<extra_id_33>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32067": {
-      "content": "<extra_id_32>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32068": {
-      "content": "<extra_id_31>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32069": {
-      "content": "<extra_id_30>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32070": {
-      "content": "<extra_id_29>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32071": {
-      "content": "<extra_id_28>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32072": {
-      "content": "<extra_id_27>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32073": {
-      "content": "<extra_id_26>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32074": {
-      "content": "<extra_id_25>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32075": {
-      "content": "<extra_id_24>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32076": {
-      "content": "<extra_id_23>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32077": {
-      "content": "<extra_id_22>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32078": {
-      "content": "<extra_id_21>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32079": {
-      "content": "<extra_id_20>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32080": {
-      "content": "<extra_id_19>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32081": {
-      "content": "<extra_id_18>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32082": {
-      "content": "<extra_id_17>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32083": {
-      "content": "<extra_id_16>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32084": {
-      "content": "<extra_id_15>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32085": {
-      "content": "<extra_id_14>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32086": {
-      "content": "<extra_id_13>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32087": {
-      "content": "<extra_id_12>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32088": {
-      "content": "<extra_id_11>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32089": {
-      "content": "<extra_id_10>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32090": {
-      "content": "<extra_id_9>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32091": {
-      "content": "<extra_id_8>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32092": {
-      "content": "<extra_id_7>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32093": {
-      "content": "<extra_id_6>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32094": {
-      "content": "<extra_id_5>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32095": {
-      "content": "<extra_id_4>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32096": {
-      "content": "<extra_id_3>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32097": {
-      "content": "<extra_id_2>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32098": {
-      "content": "<extra_id_1>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32099": {
-      "content": "<extra_id_0>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "additional_special_tokens": [
-    "<extra_id_99>",
-    "<extra_id_98>",
-    "<extra_id_97>",
-    "<extra_id_96>",
-    "<extra_id_95>",
-    "<extra_id_94>",
-    "<extra_id_93>",
-    "<extra_id_92>",
-    "<extra_id_91>",
-    "<extra_id_90>",
-    "<extra_id_89>",
-    "<extra_id_88>",
-    "<extra_id_87>",
-    "<extra_id_86>",
-    "<extra_id_85>",
-    "<extra_id_84>",
-    "<extra_id_83>",
-    "<extra_id_82>",
-    "<extra_id_81>",
-    "<extra_id_80>",
-    "<extra_id_79>",
-    "<extra_id_78>",
-    "<extra_id_77>",
-    "<extra_id_76>",
-    "<extra_id_75>",
-    "<extra_id_74>",
-    "<extra_id_73>",
-    "<extra_id_72>",
-    "<extra_id_71>",
-    "<extra_id_70>",
-    "<extra_id_69>",
-    "<extra_id_68>",
-    "<extra_id_67>",
-    "<extra_id_66>",
-    "<extra_id_65>",
-    "<extra_id_64>",
-    "<extra_id_63>",
-    "<extra_id_62>",
-    "<extra_id_61>",
-    "<extra_id_60>",
-    "<extra_id_59>",
-    "<extra_id_58>",
-    "<extra_id_57>",
-    "<extra_id_56>",
-    "<extra_id_55>",
-    "<extra_id_54>",
-    "<extra_id_53>",
-    "<extra_id_52>",
-    "<extra_id_51>",
-    "<extra_id_50>",
-    "<extra_id_49>",
-    "<extra_id_48>",
-    "<extra_id_47>",
-    "<extra_id_46>",
-    "<extra_id_45>",
-    "<extra_id_44>",
-    "<extra_id_43>",
-    "<extra_id_42>",
-    "<extra_id_41>",
-    "<extra_id_40>",
-    "<extra_id_39>",
-    "<extra_id_38>",
-    "<extra_id_37>",
-    "<extra_id_36>",
-    "<extra_id_35>",
-    "<extra_id_34>",
-    "<extra_id_33>",
-    "<extra_id_32>",
-    "<extra_id_31>",
-    "<extra_id_30>",
-    "<extra_id_29>",
-    "<extra_id_28>",
-    "<extra_id_27>",
-    "<extra_id_26>",
-    "<extra_id_25>",
-    "<extra_id_24>",
-    "<extra_id_23>",
-    "<extra_id_22>",
-    "<extra_id_21>",
-    "<extra_id_20>",
-    "<extra_id_19>",
-    "<extra_id_18>",
-    "<extra_id_17>",
-    "<extra_id_16>",
-    "<extra_id_15>",
-    "<extra_id_14>",
-    "<extra_id_13>",
-    "<extra_id_12>",
-    "<extra_id_11>",
-    "<extra_id_10>",
-    "<extra_id_9>",
-    "<extra_id_8>",
-    "<extra_id_7>",
-    "<extra_id_6>",
-    "<extra_id_5>",
-    "<extra_id_4>",
-    "<extra_id_3>",
-    "<extra_id_2>",
-    "<extra_id_1>",
-    "<extra_id_0>"
-  ],
-  "bos_token": "<s>",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "</s>",
-  "legacy": false,
-  "model_max_length": 512,
-  "pad_token": null,
-  "padding_side": "right",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
-}