From c9ee5f14eda60b971b7303fbf7823b66e598c303 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Thu, 19 Dec 2024 21:53:11 +0000
Subject: [PATCH] fix: include add_special_tokens in kserve request

---
 router/src/kserve.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/router/src/kserve.rs b/router/src/kserve.rs
index c53fa481901..ea85eb8c933 100644
--- a/router/src/kserve.rs
+++ b/router/src/kserve.rs
@@ -205,6 +205,7 @@ pub async fn kserve_model_infer(
             let generate_request = GenerateRequest {
                 inputs: str_input.to_string(),
                 parameters: payload.parameters.clone(),
+                add_special_tokens: true,
             };
             let infer = infer.clone();
             let compute_type = compute_type.clone();
@@ -212,7 +213,7 @@ pub async fn kserve_model_infer(
             async move {
                 generate_internal(infer, compute_type, Json(generate_request), span)
                     .await
-                    .map(|(_, Json(generation))| {
+                    .map(|(_, _, Json(generation))| {
                         let generation_as_bytes = generation.generated_text.as_bytes().to_vec();
                         OutputChunk {
                             name: output.name.clone(),