Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[backport -> release/3.9.x] fix(ai-proxy): fix ai-proxy plugin processing gzip response content l… #13964

Merged
merged 2 commits into from
Dec 2, 2024
Merged

[backport -> release/3.9.x] fix(ai-proxy): fix ai-proxy plugin processing gzip response content l… #13964

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
93b3bde
fix(ai-proxy): set content-length for non compressed response
fffonion Dec 2, 2024
f0c6106
fix(ai-proxy): fix content-encoding for non 200 responses
fffonion Dec 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion kong/llm/drivers/shared.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ _M.operation_map = {
}

_M.clear_response_headers = {
shared = {
shared = { -- deprecared, not using
"Content-Length",
},
openai = {
Expand Down
5 changes: 5 additions & 0 deletions kong/llm/plugin/base.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,12 @@ function MetaPlugin:header_filter(sub_plugin, conf)
-- and seems nginx doesn't support it

elseif get_global_ctx("accept_gzip") then
-- for gzip response, don't set content-length at all to align with upstream
kong.response.clear_header("Content-Length")
kong.response.set_header("Content-Encoding", "gzip")

else
kong.response.clear_header("Content-Encoding")
end

else
Expand Down
15 changes: 8 additions & 7 deletions kong/llm/plugin/shared-filters/normalize-json-response.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ local cjson = require("cjson")

local ai_plugin_ctx = require("kong.llm.plugin.ctx")
local ai_plugin_o11y = require("kong.llm.plugin.observability")
local ai_shared = require("kong.llm.drivers.shared")

local _M = {
NAME = "normalize-json-response",
Expand Down Expand Up @@ -57,6 +56,8 @@ local function transform_body(conf)
end

set_global_ctx("response_body", response_body) -- to be sent out later or consumed by other plugins

return #response_body
end

function _M:run(conf)
Expand All @@ -81,8 +82,9 @@ function _M:run(conf)
-- if not streaming, prepare the response body buffer
-- this must be called before sending any response headers so that
-- we can modify status code if needed
local body_length
if not get_global_ctx("stream_mode") then
transform_body(conf)
body_length = transform_body(conf)
end

-- populate cost
Expand All @@ -94,12 +96,11 @@ function _M:run(conf)
ai_plugin_o11y.metrics_set("llm_usage_cost", 0)
end

-- clear shared restricted headers
for _, v in ipairs(ai_shared.clear_response_headers.shared) do
kong.response.clear_header(v)
if not get_global_ctx("accept_gzip") and not get_global_ctx("stream_mode") then
-- otherwise use our transformed body length
kong.response.set_header("Content-Length", body_length)
end


if ngx.var.http_kong_debug or conf.model_name_header then
local model_t = ai_plugin_ctx.get_request_model_table_inuse()
assert(model_t and model_t.name, "model name is missing")
Expand All @@ -109,4 +110,4 @@ function _M:run(conf)
return true
end

return _M
return _M
Loading