Merge pull request #93 from SilasMarvin/fix-user-defined-tokens

Process user defined tokens
utilityai · Feb 21, 2024 · 68534b6 · 68534b6
2 parents 2b294f4 + 0adb2cc
commit 68534b6
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/llama-cpp-2/src/model.rs b/llama-cpp-2/src/model.rs
@@ -210,7 +210,7 @@ impl LlamaModel {
         }
 
         match self.token_type(token) {
-            LlamaTokenType::Normal => {}
+            LlamaTokenType::Normal | LlamaTokenType::UserDefined => {}
             LlamaTokenType::Control => {
                 if token == self.token_bos() || token == self.token_eos() {
                     return Ok(String::new());
@@ -219,7 +219,6 @@ impl LlamaModel {
             LlamaTokenType::Unknown
             | LlamaTokenType::Undefined
             | LlamaTokenType::Byte
-            | LlamaTokenType::UserDefined
             | LlamaTokenType::Unused => {
                 return Ok(String::new());
             }