From 0adb2cc12d5ae6cb066bd9f5ae067b279ff1dc04 Mon Sep 17 00:00:00 2001
From: Silas Marvin <19626586+SilasMarvin@users.noreply.github.com>
Date: Tue, 20 Feb 2024 19:44:56 -1000
Subject: [PATCH] Process user defined tokens

---
 llama-cpp-2/src/model.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llama-cpp-2/src/model.rs b/llama-cpp-2/src/model.rs
index c908c9a7..fa74ff11 100644
--- a/llama-cpp-2/src/model.rs
+++ b/llama-cpp-2/src/model.rs
@@ -210,7 +210,7 @@ impl LlamaModel {
         }
 
         match self.token_type(token) {
-            LlamaTokenType::Normal => {}
+            LlamaTokenType::Normal | LlamaTokenType::UserDefined => {}
             LlamaTokenType::Control => {
                 if token == self.token_bos() || token == self.token_eos() {
                     return Ok(String::new());
@@ -219,7 +219,6 @@ impl LlamaModel {
             LlamaTokenType::Unknown
             | LlamaTokenType::Undefined
             | LlamaTokenType::Byte
-            | LlamaTokenType::UserDefined
             | LlamaTokenType::Unused => {
                 return Ok(String::new());
             }