From c88ea857b7b9ef747b2d652013649f3edaefc93a Mon Sep 17 00:00:00 2001 From: Michael Martin Date: Thu, 21 Sep 2023 09:51:58 -0700 Subject: [PATCH 1/2] chore(deps): add lua-resty-ljsonschema 1.1.6 This rock takes lua-cjson as a dependency, so now we have an additional cjson.so file in the manifest. It's looking like about 42K in size, so it shouldn't bloat our packages/images --- kong-3.5.0-0.rockspec | 1 + scripts/explain_manifest/fixtures/alpine-amd64.txt | 5 +++++ scripts/explain_manifest/fixtures/alpine-arm64.txt | 5 +++++ scripts/explain_manifest/fixtures/amazonlinux-2-amd64.txt | 5 +++++ scripts/explain_manifest/fixtures/amazonlinux-2023-amd64.txt | 5 +++++ scripts/explain_manifest/fixtures/amazonlinux-2023-arm64.txt | 5 +++++ scripts/explain_manifest/fixtures/debian-10-amd64.txt | 5 +++++ scripts/explain_manifest/fixtures/debian-11-amd64.txt | 5 +++++ scripts/explain_manifest/fixtures/el7-amd64.txt | 5 +++++ scripts/explain_manifest/fixtures/el8-amd64.txt | 5 +++++ scripts/explain_manifest/fixtures/el9-amd64.txt | 5 +++++ scripts/explain_manifest/fixtures/el9-arm64.txt | 5 +++++ scripts/explain_manifest/fixtures/ubuntu-20.04-amd64.txt | 5 +++++ scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt | 5 +++++ scripts/explain_manifest/fixtures/ubuntu-22.04-arm64.txt | 5 +++++ 15 files changed, 71 insertions(+) diff --git a/kong-3.5.0-0.rockspec b/kong-3.5.0-0.rockspec index 88f8b605ca7d..c0eadeabde66 100644 --- a/kong-3.5.0-0.rockspec +++ b/kong-3.5.0-0.rockspec @@ -41,6 +41,7 @@ dependencies = { "lua-resty-session == 4.0.5", "lua-resty-timer-ng == 0.2.5", "lpeg == 1.0.2", + "lua-resty-ljsonschema == 1.1.6", } build = { type = "builtin", diff --git a/scripts/explain_manifest/fixtures/alpine-amd64.txt b/scripts/explain_manifest/fixtures/alpine-amd64.txt index b5bf1a0fa465..6446c3030599 100644 --- a/scripts/explain_manifest/fixtures/alpine-amd64.txt +++ b/scripts/explain_manifest/fixtures/alpine-amd64.txt @@ -31,6 +31,11 @@ Needed : - libc.so +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/kong/lib/libssl.so.1.1 Needed : - libcrypto.so.1.1 diff --git a/scripts/explain_manifest/fixtures/alpine-arm64.txt b/scripts/explain_manifest/fixtures/alpine-arm64.txt index b5bf1a0fa465..512b8d8ead11 100644 --- a/scripts/explain_manifest/fixtures/alpine-arm64.txt +++ b/scripts/explain_manifest/fixtures/alpine-arm64.txt @@ -37,6 +37,11 @@ - libc.so Rpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so diff --git a/scripts/explain_manifest/fixtures/amazonlinux-2-amd64.txt b/scripts/explain_manifest/fixtures/amazonlinux-2-amd64.txt index c8cbf3e5bd32..70104b231b27 100644 --- a/scripts/explain_manifest/fixtures/amazonlinux-2-amd64.txt +++ b/scripts/explain_manifest/fixtures/amazonlinux-2-amd64.txt @@ -79,6 +79,11 @@ - libc.so.6 Runpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 diff --git a/scripts/explain_manifest/fixtures/amazonlinux-2023-amd64.txt b/scripts/explain_manifest/fixtures/amazonlinux-2023-amd64.txt index 95eb40ea4ba9..ab0dde1598d1 100644 --- a/scripts/explain_manifest/fixtures/amazonlinux-2023-amd64.txt +++ b/scripts/explain_manifest/fixtures/amazonlinux-2023-amd64.txt @@ -72,6 +72,11 @@ - libc.so.6 Runpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 diff --git a/scripts/explain_manifest/fixtures/amazonlinux-2023-arm64.txt b/scripts/explain_manifest/fixtures/amazonlinux-2023-arm64.txt index e352ddf9485a..b877bd1be733 100644 --- a/scripts/explain_manifest/fixtures/amazonlinux-2023-arm64.txt +++ b/scripts/explain_manifest/fixtures/amazonlinux-2023-arm64.txt @@ -57,6 +57,11 @@ - libc.so.6 Rpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 diff --git a/scripts/explain_manifest/fixtures/debian-10-amd64.txt b/scripts/explain_manifest/fixtures/debian-10-amd64.txt index 95d532bef36b..6d1121a05612 100644 --- a/scripts/explain_manifest/fixtures/debian-10-amd64.txt +++ b/scripts/explain_manifest/fixtures/debian-10-amd64.txt @@ -79,6 +79,11 @@ - libc.so.6 Runpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 diff --git a/scripts/explain_manifest/fixtures/debian-11-amd64.txt b/scripts/explain_manifest/fixtures/debian-11-amd64.txt index 253e43cd2a53..fff523b65df1 100644 --- a/scripts/explain_manifest/fixtures/debian-11-amd64.txt +++ b/scripts/explain_manifest/fixtures/debian-11-amd64.txt @@ -77,6 +77,11 @@ - libc.so.6 Runpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 diff --git a/scripts/explain_manifest/fixtures/el7-amd64.txt b/scripts/explain_manifest/fixtures/el7-amd64.txt index c8cbf3e5bd32..70104b231b27 100644 --- a/scripts/explain_manifest/fixtures/el7-amd64.txt +++ b/scripts/explain_manifest/fixtures/el7-amd64.txt @@ -79,6 +79,11 @@ - libc.so.6 Runpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 diff --git a/scripts/explain_manifest/fixtures/el8-amd64.txt b/scripts/explain_manifest/fixtures/el8-amd64.txt index 7bbdad456097..2a545419d2cb 100644 --- a/scripts/explain_manifest/fixtures/el8-amd64.txt +++ b/scripts/explain_manifest/fixtures/el8-amd64.txt @@ -79,6 +79,11 @@ - libc.so.6 Runpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 diff --git a/scripts/explain_manifest/fixtures/el9-amd64.txt b/scripts/explain_manifest/fixtures/el9-amd64.txt index eca28e4a403f..e0866a846b13 100644 --- a/scripts/explain_manifest/fixtures/el9-amd64.txt +++ b/scripts/explain_manifest/fixtures/el9-amd64.txt @@ -72,6 +72,11 @@ - libc.so.6 Runpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 diff --git a/scripts/explain_manifest/fixtures/el9-arm64.txt b/scripts/explain_manifest/fixtures/el9-arm64.txt index e352ddf9485a..b877bd1be733 100644 --- a/scripts/explain_manifest/fixtures/el9-arm64.txt +++ b/scripts/explain_manifest/fixtures/el9-arm64.txt @@ -57,6 +57,11 @@ - libc.so.6 Rpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 diff --git a/scripts/explain_manifest/fixtures/ubuntu-20.04-amd64.txt b/scripts/explain_manifest/fixtures/ubuntu-20.04-amd64.txt index a7184560750f..a034e1c9b39c 100644 --- a/scripts/explain_manifest/fixtures/ubuntu-20.04-amd64.txt +++ b/scripts/explain_manifest/fixtures/ubuntu-20.04-amd64.txt @@ -77,6 +77,11 @@ - libc.so.6 Runpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 diff --git a/scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt b/scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt index 68de4cc4203f..185d054b0770 100644 --- a/scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt +++ b/scripts/explain_manifest/fixtures/ubuntu-22.04-amd64.txt @@ -70,6 +70,11 @@ - libc.so.6 Runpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 diff --git a/scripts/explain_manifest/fixtures/ubuntu-22.04-arm64.txt b/scripts/explain_manifest/fixtures/ubuntu-22.04-arm64.txt index b66889974bd0..cd8c9628f642 100644 --- a/scripts/explain_manifest/fixtures/ubuntu-22.04-arm64.txt +++ b/scripts/explain_manifest/fixtures/ubuntu-22.04-arm64.txt @@ -56,6 +56,11 @@ - ld-linux-aarch64.so.1 Runpath : /usr/local/kong/lib +- Path : /usr/local/lib/lua/5.1/cjson.so + Needed : + - libc.so.6 + Runpath : /usr/local/kong/lib + - Path : /usr/local/lib/lua/5.1/lfs.so Needed : - libc.so.6 From 5dea9ef67aca60df75b35f882cffd388a9879319 Mon Sep 17 00:00:00 2001 From: Michael Martin Date: Mon, 21 Aug 2023 13:22:04 -0700 Subject: [PATCH 2/2] feat(*): add optional wasm filter config validation --- .../kong/wasm-filter-config-schemas.yml | 7 + kong-3.5.0-0.rockspec | 1 + kong/constants.lua | 4 + kong/db/schema/entities/filter_chains.lua | 37 +- kong/db/schema/init.lua | 49 ++- kong/db/schema/json.lua | 181 ++++++++++ kong/db/schema/metaschema.lua | 130 ++++++- kong/db/strategies/postgres/init.lua | 4 +- kong/runloop/wasm.lua | 103 +++++- .../01-db/01-schema/01-schema_spec.lua | 13 +- .../01-db/01-schema/02-metaschema_spec.lua | 289 +++++++++++++++ .../20-wasm/01-admin-api_spec.lua | 18 +- spec/02-integration/20-wasm/02-db_spec.lua | 140 +++++++- .../20-wasm/03-runtime_spec.lua | 9 +- .../20-wasm/04-proxy-wasm_spec.lua | 4 +- .../20-wasm/05-cache-invalidation_spec.lua | 9 +- .../20-wasm/07-reports_spec.lua | 4 +- .../20-wasm/09-filter-meta_spec.lua | 332 ++++++++++++++++++ .../response_transformer/src/filter.rs | 14 +- 19 files changed, 1307 insertions(+), 41 deletions(-) create mode 100644 changelog/unreleased/kong/wasm-filter-config-schemas.yml create mode 100644 kong/db/schema/json.lua create mode 100644 spec/02-integration/20-wasm/09-filter-meta_spec.lua diff --git a/changelog/unreleased/kong/wasm-filter-config-schemas.yml b/changelog/unreleased/kong/wasm-filter-config-schemas.yml new file mode 100644 index 000000000000..daaa21ff7f60 --- /dev/null +++ b/changelog/unreleased/kong/wasm-filter-config-schemas.yml @@ -0,0 +1,7 @@ +message: Add support for optional Wasm filter configuration schemas +type: feature +scope: Core +prs: + - 11568 +jiras: + - KAG-662 diff --git a/kong-3.5.0-0.rockspec b/kong-3.5.0-0.rockspec index c0eadeabde66..cf357401cf6b 100644 --- a/kong-3.5.0-0.rockspec +++ b/kong-3.5.0-0.rockspec @@ -223,6 +223,7 @@ build = { ["kong.db.schema.entities.clustering_data_planes"] = "kong/db/schema/entities/clustering_data_planes.lua", ["kong.db.schema.entities.parameters"] = "kong/db/schema/entities/parameters.lua", ["kong.db.schema.entities.filter_chains"] = "kong/db/schema/entities/filter_chains.lua", + ["kong.db.schema.json"] = "kong/db/schema/json.lua", ["kong.db.schema.others.migrations"] = "kong/db/schema/others/migrations.lua", ["kong.db.schema.others.declarative_config"] = "kong/db/schema/others/declarative_config.lua", ["kong.db.schema.entity"] = "kong/db/schema/entity.lua", diff --git a/kong/constants.lua b/kong/constants.lua index f3ece205461a..46a16fcac2a1 100644 --- a/kong/constants.lua +++ b/kong/constants.lua @@ -249,6 +249,10 @@ local constants = { REQUEST_DEBUG_TOKEN_FILE = ".request_debug_token", REQUEST_DEBUG_LOG_PREFIX = "[request-debug]", + + SCHEMA_NAMESPACES = { + PROXY_WASM_FILTERS = "proxy-wasm-filters", + }, } for _, v in ipairs(constants.CLUSTERING_SYNC_STATUS) do diff --git a/kong/db/schema/entities/filter_chains.lua b/kong/db/schema/entities/filter_chains.lua index 001aea9782dd..c83d59a82f60 100644 --- a/kong/db/schema/entities/filter_chains.lua +++ b/kong/db/schema/entities/filter_chains.lua @@ -1,5 +1,6 @@ local typedefs = require "kong.db.schema.typedefs" local wasm = require "kong.runloop.wasm" +local constants = require "kong.constants" ---@class kong.db.schema.entities.filter_chain : table @@ -9,7 +10,6 @@ local wasm = require "kong.runloop.wasm" ---@field enabled boolean ---@field route table|nil ---@field service table|nil ----@field protocols table|nil ---@field created_at number ---@field updated_at number ---@field tags string[] @@ -18,22 +18,41 @@ local wasm = require "kong.runloop.wasm" ---@class kong.db.schema.entities.wasm_filter : table --- ----@field name string ----@field enabled boolean ----@field config string|table|nil +---@field name string +---@field enabled boolean +---@field config string|nil +---@field json_config any|nil local filter = { type = "record", fields = { - { name = { type = "string", required = true, one_of = wasm.filter_names, - err = "no such filter", }, }, - { config = { type = "string", required = false, }, }, - { enabled = { type = "boolean", default = true, required = true, }, }, + { name = { type = "string", required = true, one_of = wasm.filter_names, + err = "no such filter", }, }, + { config = { type = "string", required = false, }, }, + { enabled = { type = "boolean", default = true, required = true, }, }, + + { json_config = { + type = "json", + required = false, + json_schema = { + parent_subschema_key = "name", + namespace = constants.SCHEMA_NAMESPACES.PROXY_WASM_FILTERS, + optional = true, + }, + }, + }, + + }, + entity_checks = { + { mutually_exclusive = { + "config", + "json_config", + }, + }, }, } - return { name = "filter_chains", primary_key = { "id" }, diff --git a/kong/db/schema/init.lua b/kong/db/schema/init.lua index 99594d21f9a9..ce29a41038d4 100644 --- a/kong/db/schema/init.lua +++ b/kong/db/schema/init.lua @@ -5,6 +5,8 @@ local cjson = require "cjson" local new_tab = require "table.new" local nkeys = require "table.nkeys" local is_reference = require "kong.pdk.vault".is_reference +local json = require "kong.db.schema.json" +local cjson_safe = require "cjson.safe" local setmetatable = setmetatable @@ -30,10 +32,12 @@ local find = string.find local null = ngx.null local max = math.max local sub = string.sub +local safe_decode = cjson_safe.decode local random_string = utils.random_string local uuid = utils.uuid +local json_validate = json.validate local Schema = {} @@ -115,6 +119,12 @@ local validation_errors = { SUBSCHEMA_ABSTRACT_FIELD = "error in schema definition: abstract field was not specialized", -- transformations TRANSFORMATION_ERROR = "transformation failed: %s", + -- json + JSON_ENCODE_ERROR = "value could not be JSON-encoded: %s", + JSON_DECODE_ERROR = "value could not be JSON-decoded: %s", + JSON_SCHEMA_ERROR = "value failed JSON-schema validation: %s", + JSON_PARENT_KEY_MISSING = "validation of %s depends on the parent attribute %s, but it is not set", + JSON_SCHEMA_NOT_FOUND = "mandatory json schema for field (%s) not found" } @@ -129,6 +139,7 @@ Schema.valid_types = { map = true, record = true, ["function"] = true, + json = true, } @@ -1110,7 +1121,35 @@ validate_fields = function(self, input) for k, v in pairs(input) do local err local field = self.fields[tostring(k)] - if field and field.type == "self" then + + if field and field.type == "json" then + local json_schema = field.json_schema + local inline_schema = json_schema.inline + + if inline_schema then + _, errors[k] = json_validate(v, inline_schema) + + else + local parent_key = json_schema.parent_subschema_key + local json_subschema_key = input[parent_key] + + if json_subschema_key then + local schema_name = json_schema.namespace .. "/" .. json_subschema_key + inline_schema = json.get_schema(schema_name) + + if inline_schema then + _, errors[k] = json_validate(v, inline_schema) + + elseif not json_schema.optional then + errors[k] = validation_errors.JSON_SCHEMA_NOT_FOUND:format(schema_name) + end + + elseif not json_schema.optional then + errors[k] = validation_errors.JSON_PARENT_KEY_MISSING:format(k, parent_key) + end + end + + elseif field and field.type == "self" then local pok pok, err, errors[k] = pcall(self.validate_field, self, input, v) if not pok then @@ -2261,6 +2300,14 @@ end local function run_transformations(self, transformations, input, original_input, context) + if self.type == "json" and context == "select" then + local decoded, err = safe_decode(input) + if err then + return nil, validation_errors.JSON_DECODE_ERROR:format(err) + end + input = decoded + end + local output for i = 1, #transformations do local transformation = transformations[i] diff --git a/kong/db/schema/json.lua b/kong/db/schema/json.lua new file mode 100644 index 000000000000..d8de8dde9286 --- /dev/null +++ b/kong/db/schema/json.lua @@ -0,0 +1,181 @@ +--- +-- JSON schema validation. +-- +-- +local _M = {} + +local lrucache = require "resty.lrucache" +local jsonschema = require "resty.ljsonschema" +local metaschema = require "resty.ljsonschema.metaschema" +local utils = require "kong.tools.utils" +local cjson = require "cjson" + +local type = type +local cjson_encode = cjson.encode +local sha256_hex = utils.sha256_hex + + +---@class kong.db.schema.json.schema_doc : table +--- +---@field id string|nil +---@field ["$id"] string|nil +---@field ["$schema"] string|nil +---@field type string + + +-- The correct identifier for draft-4 is 'http://json-schema.org/draft-04/schema#' +-- with the the fragment (#) intact. Newer editions use an identifier _without_ +-- the fragment (e.g. 'https://json-schema.org/draft/2020-12/schema'), so we +-- will be lenient when comparing these strings. +assert(type(metaschema.id) == "string", + "JSON metaschema .id not defined or not a string") +local DRAFT_4_NO_FRAGMENT = metaschema.id:gsub("#$", "") +local DRAFT_4 = DRAFT_4_NO_FRAGMENT .. "#" + + +---@type table +local schemas = {} + + +-- Creating a json schema validator is somewhat expensive as it requires +-- generating and evaluating some Lua code, so we memoize this step with +-- a local LRU cache. +local cache = lrucache.new(1000) + +local schema_cache_key +do + local cache_keys = setmetatable({}, { __mode = "k" }) + + --- + -- Generate a unique cache key for a schema document. + -- + ---@param schema kong.db.schema.json.schema_doc + ---@return string + function schema_cache_key(schema) + local cache_key = cache_keys[schema] + + if not cache_key then + cache_key = "hash://" .. sha256_hex(cjson_encode(schema)) + cache_keys[schema] = cache_key + end + + return cache_key + end +end + + +---@param id any +---@return boolean +local function is_draft_4(id) + return id + and type(id) == "string" + and (id == DRAFT_4 or id == DRAFT_4_NO_FRAGMENT) +end + + +---@param id any +---@return boolean +local function is_non_draft_4(id) + return id + and type(id) == "string" + and (id ~= DRAFT_4 and id ~= DRAFT_4_NO_FRAGMENT) +end + + +--- +-- Validate input according to a JSON schema document. +-- +---@param input any +---@param schema kong.db.schema.json.schema_doc +---@return boolean? ok +---@return string? error +local function validate(input, schema) + assert(type(schema) == "table") + + -- we are validating a JSON schema document and need to ensure that it is + -- not using supported JSON schema draft/version + if is_draft_4(schema.id or schema["$id"]) + and is_non_draft_4(input["$schema"]) + then + return nil, "unsupported document $schema: '" .. input["$schema"] .. + "', expected: " .. DRAFT_4 + end + + local cache_key = schema_cache_key(schema) + + local validator = cache:get(cache_key) + + if not validator then + validator = assert(jsonschema.generate_validator(schema, { + name = cache_key, + -- lua-resty-ljsonschema's default behavior for detecting an array type + -- is to compare its metatable against `cjson.array_mt`. This is + -- efficient, but we can't assume that all inputs will necessarily + -- conform to this, so we opt to use the heuristic approach instead + -- (determining object/array type based on the table contents). + array_mt = false, + })) + cache:set(cache_key, validator) + end + + return validator(input) +end + + +---@type table +_M.metaschema = metaschema + + +_M.validate = validate + + +--- +-- Validate a JSON schema document. +-- +-- This is primarily for use in `kong.db.schema.metaschema` +-- +---@param input kong.db.schema.json.schema_doc +---@return boolean? ok +---@return string? error +function _M.validate_schema(input) + local typ = type(input) + + if typ ~= "table" then + return nil, "schema must be a table" + end + + return validate(input, _M.metaschema) +end + + +--- +-- Add a JSON schema document to the local registry. +-- +---@param name string +---@param schema kong.db.schema.json.schema_doc +function _M.add_schema(name, schema) + schemas[name] = schema +end + + +--- +-- Retrieve a schema from local storage by name. +-- +---@param name string +---@return table|nil schema +function _M.get_schema(name) + return schemas[name] +end + + +--- +-- Remove a schema from local storage by name (if it exists). +-- +---@param name string +---@return table|nil schema +function _M.remove_schema(name) + schemas[name] = nil +end + + +return _M diff --git a/kong/db/schema/metaschema.lua b/kong/db/schema/metaschema.lua index 50fa65ca2ace..8d829daf9ec3 100644 --- a/kong/db/schema/metaschema.lua +++ b/kong/db/schema/metaschema.lua @@ -2,6 +2,8 @@ -- @module kong.db.schema.metaschema local Schema = require "kong.db.schema" +local json_lib = require "kong.db.schema.json" +local constants = require "kong.constants" local setmetatable = setmetatable @@ -12,6 +14,7 @@ local find = string.find local type = type local next = next local keys = require("pl.tablex").keys +local values = require("pl.tablex").values local sub = string.sub local fmt = string.format @@ -40,7 +43,6 @@ local match_any_list = { } } - -- Field attributes which match a validator function in the Schema class local validators = { { between = { type = "array", elements = { type = "number" }, len_eq = 2 }, }, @@ -66,6 +68,85 @@ local validators = { { mutually_exclusive_subsets = { type = "array", elements = { type = "array", elements = { type = "string" } } } }, } +-- JSON schema is supported in two different methods: +-- +-- * inline: the JSON schema is defined in the field itself +-- * dynamic/reference: the JSON schema is stored in the database +-- +-- Inline schemas have the JSON schema definied statically within +-- the typedef's `json_schema.inline` field. Example: +-- +-- ```lua +-- local field = { +-- type = "json", +-- json_schema = { +-- inline = { +-- type = "object", +-- properties = { +-- foo = { type = "string" }, +-- }, +-- }, +-- } +-- } +-- +-- local record = { +-- type = "record", +-- fields = { +-- { name = { type = "string" } }, +-- { config = field }, +-- }, +-- } +-- +-- ``` +-- +-- Fields with dynamic schemas function similarly to Lua subschemas, wherein +-- the contents of the input are used to generate a string key that is used +-- to lookup the schema from the schema storage. Example: +-- +-- ```lua +-- local record = { +-- type = "record", +-- fields = { +-- { name = { type = "string" } }, +-- { config = { +-- type = "json", +-- json_schema = { +-- namespace = "my-record-type", +-- parent_subschema_key = "name", +-- optional = true, +-- }, +-- }, +-- }, +-- }, +-- } +-- ``` +-- +-- In this case, an input value of `{ name = "foo", config = "foo config" }` +-- will cause the validation engine to lookup a schema by the name of +-- `my-record-type/foo`. The `optional` field determines what will happen if +-- the schema does not exist. When `optional` is `false`, a missing schema +-- means that input validation will fail. When `optional` is `true`, the input +-- is always accepted. +-- +local json_metaschema = { + type = "record", + fields = { + { namespace = { type = "string", one_of = values(constants.SCHEMA_NAMESPACES), }, }, + { parent_subschema_key = { type = "string" }, }, + { optional = { type = "boolean", }, }, + { inline = { type = "any", custom_validator = json_lib.validate_schema, }, }, + }, + entity_checks = { + { at_least_one_of = { "inline", "namespace", "parent_subschema_key" }, }, + { mutually_required = { "namespace", "parent_subschema_key" }, }, + { mutually_exclusive_sets = { + set1 = { "inline" }, + set2 = { "namespace", "parent_subschema_key", "optional" }, + }, + }, + }, +} + -- Other field attributes, that do not correspond to validators local field_schema = { @@ -84,6 +165,7 @@ local field_schema = { { err = { type = "string" } }, { encrypted = { type = "boolean" }, }, { referenceable = { type = "boolean" }, }, + { json_schema = json_metaschema }, } @@ -297,6 +379,8 @@ local meta_errors = { TTL_RESERVED = "ttl is a reserved field name when ttl is enabled", SUBSCHEMA_KEY = "value must be a field name", SUBSCHEMA_KEY_TYPE = "must be a string or set field", + JSON_PARENT_KEY = "value must be a field name of the parent schema", + JSON_PARENT_KEY_TYPE = "value must be a string field of the parent schema", } @@ -305,6 +389,7 @@ local required_attributes = { set = { "elements" }, map = { "keys", "values" }, record = { "fields" }, + json = { "json_schema" }, } @@ -362,6 +447,9 @@ local attribute_types = { ["set"] = true, ["map"] = true, }, + json_schema = { + ["json"] = true, + }, } @@ -459,7 +547,7 @@ local check_fields = function(schema, errors) end local field = item[k] if type(field) == "table" then - check_field(k, field, errors) + check_field(k, field, errors, schema) else errors[k] = meta_errors.TABLE:format(k) end @@ -471,7 +559,7 @@ local check_fields = function(schema, errors) end -check_field = function(k, field, errors) +check_field = function(k, field, errors, parent_schema) if not field.type then errors[k] = meta_errors.TYPE return nil @@ -496,12 +584,46 @@ check_field = function(k, field, errors) for name, _ in pairs(nested_attributes) do if field[name] then if type(field[name]) == "table" then - check_field(k, field[name], errors) + check_field(k, field[name], errors, field) else errors[k] = meta_errors.TABLE:format(name) end end end + + if field.type == "json" + and field.json_schema + and field.json_schema.parent_subschema_key + then + local parent_subschema_key = field.json_schema.parent_subschema_key + local found = false + + for i = 1, #parent_schema.fields do + local item = parent_schema.fields[i] + local parent_field_name = next(item) + local parent_field = item[parent_field_name] + + if parent_subschema_key == parent_field_name then + if parent_field.type ~= "string" then + errors[k] = errors[k] or {} + errors[k].json_schema = { + parent_subschema_key = meta_errors.JSON_PARENT_KEY_TYPE + } + end + found = true + break + end + end + + if not found then + errors[k] = errors[k] or {} + errors[k].json_schema = { + parent_subschema_key = meta_errors.JSON_PARENT_KEY + } + return + end + end + if field.fields then return check_fields(field, errors) end diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index 23cf52384ec6..74da93465aa6 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -208,7 +208,7 @@ local function escape_literal(connector, literal, field) return error("postgres strategy to escape multidimensional arrays of maps or records is not implemented") end - elseif et == "map" or et == "record" then + elseif et == "map" or et == "record" or et == "json" then local jsons = {} for i, v in ipairs(literal) do jsons[i] = cjson.encode(v) @@ -221,7 +221,7 @@ local function escape_literal(connector, literal, field) return encode_array(literal) - elseif field.type == "map" or field.type == "record" then + elseif field.type == "map" or field.type == "record" or field.type == "json" then return encode_json(literal) end end diff --git a/kong/runloop/wasm.lua b/kong/runloop/wasm.lua index 64502ca6b084..00a6054b4495 100644 --- a/kong/runloop/wasm.lua +++ b/kong/runloop/wasm.lua @@ -12,6 +12,9 @@ local _M = { ---@type string[] filter_names = {}, + + ---@type table + filter_meta = {}, } @@ -23,11 +26,21 @@ local _M = { ---@field name string ---@field path string +---@class kong.configuration.wasm_filter.meta +--- +---@field config_schema kong.db.schema.json.schema_doc|nil + local utils = require "kong.tools.utils" local dns = require "kong.tools.dns" local reports = require "kong.reports" local clear_tab = require "table.clear" +local cjson = require "cjson.safe" +local json_schema = require "kong.db.schema.json" +local pl_file = require "pl.file" +local pl_path = require "pl.path" +local constants = require "kong.constants" + ---@module 'resty.wasmx.proxy_wasm' local proxy_wasm @@ -45,11 +58,25 @@ local assert = assert local concat = table.concat local insert = table.insert local sha256 = utils.sha256_bin +local cjson_encode = cjson.encode +local cjson_decode = cjson.decode +local fmt = string.format local VERSION_KEY = "filter_chains:version" local TTL_ZERO = { ttl = 0 } +---@class kong.runloop.wasm.filter_meta +--- +---@field config_schema table|nil + +local FILTER_META_SCHEMA = { + type = "object", + properties = { + config_schema = json_schema.metaschema, + }, +} + --- -- Fetch the current version of the filter chain state from cache @@ -369,6 +396,16 @@ local function rebuild_state(db, version, old_state) local chain_type = service_id and TYPE_SERVICE or TYPE_ROUTE + for _, filter in ipairs(chain.filters) do + if filter.enabled then + -- serialize all JSON configurations up front + if not filter.config and filter.json_config ~= nil then + filter.config = cjson_encode(filter.json_config) + filter.json_config = nil + end + end + end + insert(all_chain_refs, { type = chain_type, @@ -526,7 +563,6 @@ local function update_in_place(new_version) end - ---@param route? { id: string } ---@param service? { id: string } ---@return kong.runloop.wasm.filter_chain_reference? @@ -541,11 +577,74 @@ local function get_filter_chain_for_request(route, service) end +---@param filters kong.configuration.wasm_filter[]|nil +local function discover_filter_metadata(filters) + if not filters then return end + + local errors = {} + + for _, filter in ipairs(filters) do + local meta_path = (filter.path:gsub("%.wasm$", "")) .. ".meta.json" + + local function add_error(reason, err) + table.insert(errors, fmt("* %s (%s) %s: %s", filter.name, meta_path, reason, err)) + end + + if pl_path.exists(meta_path) then + if pl_path.isfile(meta_path) then + local data, err = pl_file.read(meta_path) + + if data then + local meta + meta, err = cjson_decode(data) + + if err then + add_error("JSON decode error", err) + + else + local ok + ok, err = json_schema.validate(meta, FILTER_META_SCHEMA) + if ok then + _M.filter_meta[filter.name] = meta + + else + add_error("file contains invalid metadata", err) + end + end + + else + add_error("I/O error", err) + end + + else + add_error("invalid type", "path exists but is not a file") + end + end + end + + if #errors > 0 then + local err = "\nFailed to load metadata for one or more filters:\n" + .. table.concat(errors, "\n") .. "\n" + + error(err) + end + + local namespace = constants.SCHEMA_NAMESPACES.PROXY_WASM_FILTERS + for name, meta in pairs(_M.filter_meta) do + if meta.config_schema then + local schema_name = namespace .. "/" .. name + json_schema.add_schema(schema_name, meta.config_schema) + end + end +end + + ---@param filters kong.configuration.wasm_filter[]|nil local function set_available_filters(filters) clear_tab(_M.filters) clear_tab(_M.filters_by_name) clear_tab(_M.filter_names) + clear_tab(_M.filter_meta) if filters then for i, filter in ipairs(filters) do @@ -553,6 +652,8 @@ local function set_available_filters(filters) _M.filters_by_name[filter.name] = filter _M.filter_names[i] = filter.name end + + discover_filter_metadata(filters) end end diff --git a/spec/01-unit/01-db/01-schema/01-schema_spec.lua b/spec/01-unit/01-db/01-schema/01-schema_spec.lua index afa614518e64..d8d669210ff4 100644 --- a/spec/01-unit/01-db/01-schema/01-schema_spec.lua +++ b/spec/01-unit/01-db/01-schema/01-schema_spec.lua @@ -315,6 +315,8 @@ describe("schema", function() "fail" }, { { type = "function" }, "fail" }, + { { type = "json", json_schema = { inline = { type = "string" }, } }, + 123 }, } local covered_check = {} @@ -2879,6 +2881,7 @@ describe("schema", function() { g = { type = "record", fields = {} }, }, { h = { type = "map", keys = {}, values = {} }, }, { i = { type = "function" }, }, + { j = { type = "json", json_schema = { inline = { type = "string" }, } }, }, } }) check_all_types_covered(Test.fields) @@ -2893,6 +2896,7 @@ describe("schema", function() assert.same(ngx.null, data.g) assert.same(ngx.null, data.h) assert.same(ngx.null, data.i) + assert.same(ngx.null, data.j) end) it("produces nil for empty string fields with selects", function() @@ -2998,6 +3002,7 @@ describe("schema", function() { g = { type = "record", fields = {} }, }, { h = { type = "map", keys = {}, values = {} }, }, { i = { type = "function" }, }, + { j = { type = "json", json_schema = { inline = { type = "string" }, } }, }, } }) check_all_types_covered(Test.fields) @@ -3027,6 +3032,7 @@ describe("schema", function() { my_record = { type = "record", fields = { { my_field = { type = "integer" } } } } }, { my_map = { type = "map", keys = {}, values = {} }, }, { my_function = { type = "function" }, }, + { my_json = { type = "json", json_schema = { inline = { type = "string" }, } }, }, } }) check_all_types_covered(Test.fields) @@ -3040,6 +3046,7 @@ describe("schema", function() my_record = "hello", my_map = "hello", my_function = "hello", + my_json = 123, } local data, err = Test:process_auto_fields(bad_value) assert.is_nil(err) @@ -3092,7 +3099,11 @@ describe("schema", function() } } } } - } } + } }, + { j = { + type = "json", + json_schema = { inline = { type = "string" }, }, + } }, } }) check_all_types_covered(Test.fields) diff --git a/spec/01-unit/01-db/01-schema/02-metaschema_spec.lua b/spec/01-unit/01-db/01-schema/02-metaschema_spec.lua index 4eab572dc5bb..a312bdca85e3 100644 --- a/spec/01-unit/01-db/01-schema/02-metaschema_spec.lua +++ b/spec/01-unit/01-db/01-schema/02-metaschema_spec.lua @@ -1,6 +1,7 @@ local Schema = require "kong.db.schema" local helpers = require "spec.helpers" local MetaSchema = require "kong.db.schema.metaschema" +local constants = require "kong.constants" describe("metaschema", function() @@ -1405,4 +1406,292 @@ describe("metasubschema", function() }, })) end) + + describe("json fields", function() + local NS = constants.SCHEMA_NAMESPACES.PROXY_WASM_FILTERS + + it("requires the field to have a json_schema attribute", function() + local ok, err = MetaSchema:validate({ + name = "test", + primary_key = { "id" }, + fields = { + { my_field = { type = "json" } }, + { id = { type = "string" }, }, + }, + }) + + assert.falsy(ok) + assert.is_table(err) + assert.matches("field of type .json. must declare .json_schema.", err.my_field) + + assert.truthy(MetaSchema:validate({ + name = "test", + primary_key = { "id" }, + fields = { + { my_field = { + type = "json", + json_schema = { inline = { type = "string" }, }, + } + }, + { id = { type = "string" }, }, + }, + })) + end) + + it("requires at least one of `inline` or `namespace`/`parent_subschema_key`", function() + local ok, err = MetaSchema:validate({ + name = "test", + primary_key = { "id" }, + fields = { + { my_field = { + type = "json", + json_schema = { }, + } + }, + { id = { type = "string" }, }, + }, + }) + + assert.falsy(ok) + assert.is_table(err) + assert.is_table(err.fields) + assert.same({ + { + json_schema = { + ["@entity"] = { + "at least one of these fields must be non-empty: 'inline', 'namespace', 'parent_subschema_key'" + } + } + } + }, err.fields) + end) + + it("requires that inline schemas are valid", function() + local ok, err = MetaSchema:validate({ + name = "test", + primary_key = { "id" }, + fields = { + { my_field = { + type = "json", + json_schema = { + inline = { + type = "not a valid json schema type", + }, + }, + } + }, + { id = { type = "string" }, }, + }, + }) + + assert.falsy(ok) + assert.is_table(err) + assert.is_table(err.fields) + assert.same({ + { + json_schema = { + inline = "property type validation failed: object needs one of the following rectifications: 1) matches none of the enum values; 2) wrong type: expected array, got string" + } + } + }, err.fields) + + assert.truthy(MetaSchema:validate({ + name = "test", + primary_key = { "id" }, + fields = { + { my_field = { + type = "json", + json_schema = { + inline = { + type = "string", + }, + }, + } + }, + { id = { type = "string" }, }, + }, + })) + end) + + it("only allows currently-supported versions of JSON schema", function() + local invalid = { + "http://json-schema.org/draft-07/schema#", + "https://json-schema.org/draft/2019-09/schema", + "https://json-schema.org/draft/2020-12/schema", + } + + local inline_schema = { + type = "string", + } + + local schema = { + name = "test", + primary_key = { "id" }, + fields = { + { my_field = { + type = "json", + json_schema = { + inline = inline_schema, + } + } + }, + { id = { type = "string" }, }, + } + } + + for _, version in ipairs(invalid) do + inline_schema["$schema"] = version + local ok, err = MetaSchema:validate(schema) + assert.is_nil(ok) + assert.is_table(err) + assert.is_table(err.fields) + assert.is_table(err.fields[1]) + assert.is_table(err.fields[1].json_schema) + assert.matches('unsupported document $schema', + err.fields[1].json_schema.inline, nil, true) + end + + -- with fragment + inline_schema["$schema"] = "http://json-schema.org/draft-04/schema#" + assert.truthy(MetaSchema:validate(schema)) + + -- sans fragment + inline_schema["$schema"] = "http://json-schema.org/draft-04/schema" + assert.truthy(MetaSchema:validate(schema)) + + -- $schema is ultimately optional + inline_schema["$schema"] = nil + assert.truthy(MetaSchema:validate(schema)) + end) + + it("mutually requires `namespace` and `parent_subschema_key`", function() + local ok, err = MetaSchema:validate({ + name = "test", + primary_key = { "id" }, + fields = { + { my_field = { + type = "json", + json_schema = { + namespace = NS, + }, + } + }, + { id = { type = "string" }, }, + }, + }) + + assert.falsy(ok) + assert.is_table(err) + assert.is_table(err.fields) + assert.same({ + { + json_schema = { + ["@entity"] = { + "all or none of these fields must be set: 'namespace', 'parent_subschema_key'" + } + } + } + }, err.fields) + + ok, err = MetaSchema:validate({ + name = "test", + primary_key = { "id" }, + fields = { + { my_field = { + type = "json", + json_schema = { + parent_subschema_key = "id", + }, + } + }, + { id = { type = "string" }, }, + }, + }) + + assert.falsy(ok) + assert.is_table(err) + assert.is_table(err.fields) + assert.same({ + { + json_schema = { + ["@entity"] = { + "all or none of these fields must be set: 'namespace', 'parent_subschema_key'" + } + } + } + }, err.fields) + end) + + it("requires that `parent_subschema_key` is a string field of the parent schema", function() + local ok, err = MetaSchema:validate({ + name = "test", + primary_key = { "id" }, + fields = { + { my_field = { + type = "json", + json_schema = { + namespace = NS, + parent_subschema_key = "my_nonexistent_field", + }, + } + }, + { id = { type = "string" }, }, + }, + }) + + assert.falsy(ok) + assert.same({ + my_field = { + json_schema = { + parent_subschema_key = "value must be a field name of the parent schema" + } + } + }, err) + + ok, err = MetaSchema:validate({ + name = "test", + primary_key = { "id" }, + fields = { + { my_field = { + type = "json", + json_schema = { + namespace = NS, + parent_subschema_key = "my_non_string_field", + }, + } + }, + { id = { type = "string" }, }, + { my_non_string_field = { type = "number" } }, + }, + }) + + assert.falsy(ok) + assert.same({ + my_field = { + json_schema = { + parent_subschema_key = "value must be a string field of the parent schema", + } + } + }, err) + + assert.truthy(MetaSchema:validate({ + name = "test", + primary_key = { "id" }, + fields = { + { my_field = { + type = "json", + json_schema = { + namespace = NS, + parent_subschema_key = "my_string_field", + }, + } + }, + { id = { type = "string" }, }, + { my_string_field = { type = "string" }, }, + }, + })) + + end) + + end) end) diff --git a/spec/02-integration/20-wasm/01-admin-api_spec.lua b/spec/02-integration/20-wasm/01-admin-api_spec.lua index ce318f01c2d2..51ce7d5f1069 100644 --- a/spec/02-integration/20-wasm/01-admin-api_spec.lua +++ b/spec/02-integration/20-wasm/01-admin-api_spec.lua @@ -3,6 +3,8 @@ local utils = require "kong.tools.utils" local fmt = string.format +local FILTER_PATH = assert(helpers.test_conf.wasm_filters_path) +local NULL = ngx.null local function json(body) return { @@ -22,8 +24,12 @@ describe("wasm admin API [#" .. strategy .. "]", function() lazy_setup(function() require("kong.runloop.wasm").enable({ - { name = "tests" }, - { name = "response_transformer" }, + { name = "tests", + path = FILTER_PATH .. "/tests.wasm", + }, + { name = "response_transformer", + path = FILTER_PATH .. "/response_transformer.wasm", + }, }) bp, db = helpers.get_db_utils(strategy, { @@ -201,9 +207,9 @@ describe("wasm admin API [#" .. strategy .. "]", function() assert.same({ "foo", "bar" }, patched.tags) assert.is_false(patched.enabled) assert.equals(2, #patched.filters) - assert.same({ name = "tests", config = "123", enabled = true }, + assert.same({ name = "tests", config = "123", enabled = true, json_config = NULL }, patched.filters[1]) - assert.same({ name = "tests", config = "456", enabled = false }, + assert.same({ name = "tests", config = "456", enabled = false, json_config = NULL }, patched.filters[2]) end) end) @@ -360,9 +366,9 @@ describe("wasm admin API [#" .. strategy .. "]", function() assert.same({ "foo", "bar" }, patched.tags) assert.is_false(patched.enabled) assert.equals(2, #patched.filters) - assert.same({ name = "tests", config = "123", enabled = true }, + assert.same({ name = "tests", config = "123", enabled = true, json_config = NULL }, patched.filters[1]) - assert.same({ name = "tests", config = "456", enabled = false }, + assert.same({ name = "tests", config = "456", enabled = false, json_config = NULL }, patched.filters[2]) end) end) diff --git a/spec/02-integration/20-wasm/02-db_spec.lua b/spec/02-integration/20-wasm/02-db_spec.lua index 6ac7ca6c73d2..1969f4b238e6 100644 --- a/spec/02-integration/20-wasm/02-db_spec.lua +++ b/spec/02-integration/20-wasm/02-db_spec.lua @@ -1,11 +1,14 @@ local helpers = require "spec.helpers" local utils = require "kong.tools.utils" +local schema_lib = require "kong.db.schema.json" + +local FILTER_PATH = assert(helpers.test_conf.wasm_filters_path) -- no cassandra support for _, strategy in helpers.each_strategy({ "postgres" }) do describe("wasm DB entities [#" .. strategy .. "]", function() - local db, dao + local db local function reset_db() if not db then return end @@ -18,8 +21,12 @@ describe("wasm DB entities [#" .. strategy .. "]", function() lazy_setup(function() require("kong.runloop.wasm").enable({ - { name = "test" }, - { name = "other" }, + { name = "test", + path = FILTER_PATH .. "/test.wasm", + }, + { name = "other", + path = FILTER_PATH .. "/other.wasm", + }, }) local _ @@ -29,13 +36,17 @@ describe("wasm DB entities [#" .. strategy .. "]", function() "services", "filter_chains", }) - - dao = db.filter_chains end) lazy_teardown(reset_db) describe("filter_chains", function() + local dao + + lazy_setup(function() + dao = db.filter_chains + end) + local function make_service() local service = assert(db.services:insert({ url = "http://wasm.test/", @@ -360,7 +371,124 @@ describe("wasm DB entities [#" .. strategy .. "]", function() end) describe(".config", function() - pending("is validated against the filter schema") + it("is an optional string", function() + local service = assert(db.services:insert({ + url = "http://example.test", + })) + + assert.truthy(dao:insert({ + service = { id = service.id }, + filters = { + { + name = "test", + config = "foo", + } + } + })) + + service = assert(db.services:insert({ + url = "http://example.test", + })) + + assert.truthy(dao:insert({ + service = { id = service.id }, + filters = { + { + name = "test", + config = nil, + } + } + })) + end) + end) + + describe(".json_config", function() + local schema_name = "proxy-wasm-filters/test" + + lazy_teardown(function() + schema_lib.remove_schema(schema_name) + end) + + it("is validated against user schema", function() + local service = assert(db.services:insert({ + url = "http://example.test", + })) + + schema_lib.add_schema("proxy-wasm-filters/test", { + type = "object", + properties = { + foo = { type = "string" }, + bar = { type = "object" }, + }, + required = { "foo", "bar" }, + additionalProperties = false, + }) + + assert.truthy(dao:insert({ + service = { id = service.id }, + filters = { + { + name = "test", + json_config = { + foo = "foo string", + bar = { a = 1, b = 2 }, + }, + } + } + })) + + service = assert(db.services:insert({ + url = "http://example.test", + })) + + local chain, err = dao:insert({ + service = { id = service.id }, + filters = { + { + name = "test", + json_config = { + foo = 123, + bar = { a = 1, b = 2 }, + }, + } + } + }) + assert.is_nil(chain) + assert.matches("property foo validation failed", err) + + service = assert(db.services:insert({ + url = "http://example.test", + })) + + chain, err = dao:insert({ + service = { id = service.id }, + filters = { + { + name = "test", + json_config = ngx.null, + } + } + }) + assert.is_nil(chain) + assert.matches("expected object, got null", err) + + service = assert(db.services:insert({ + url = "http://example.test", + })) + + chain, err = dao:insert({ + service = { id = service.id }, + filters = { + { + name = "test", + json_config = nil, + } + } + }) + assert.is_nil(chain) + assert.matches("expected object, got null", err) + + end) end) end) diff --git a/spec/02-integration/20-wasm/03-runtime_spec.lua b/spec/02-integration/20-wasm/03-runtime_spec.lua index 4802632fb5c2..5c80da756902 100644 --- a/spec/02-integration/20-wasm/03-runtime_spec.lua +++ b/spec/02-integration/20-wasm/03-runtime_spec.lua @@ -2,6 +2,7 @@ local helpers = require "spec.helpers" local cjson = require "cjson" local HEADER = "X-Proxy-Wasm" +local FILTER_PATH = assert(helpers.test_conf.wasm_filters_path) local json = cjson.encode @@ -25,8 +26,12 @@ for _, strategy in helpers.each_strategy({ "postgres", "off" }) do describe("#wasm filter execution (#" .. strategy .. ")", function() lazy_setup(function() require("kong.runloop.wasm").enable({ - { name = "tests" }, - { name = "response_transformer" }, + { name = "tests", + path = FILTER_PATH .. "/tests.wasm", + }, + { name = "response_transformer", + path = FILTER_PATH .. "/response_transformer.wasm", + }, }) local bp = helpers.get_db_utils("postgres", { diff --git a/spec/02-integration/20-wasm/04-proxy-wasm_spec.lua b/spec/02-integration/20-wasm/04-proxy-wasm_spec.lua index e2ce7ca9fe00..473609a1c2a4 100644 --- a/spec/02-integration/20-wasm/04-proxy-wasm_spec.lua +++ b/spec/02-integration/20-wasm/04-proxy-wasm_spec.lua @@ -20,7 +20,9 @@ describe("proxy-wasm filters (#wasm)", function() lazy_setup(function() require("kong.runloop.wasm").enable({ - { name = "tests" }, + { name = "tests", + path = helpers.test_conf.wasm_filters_path .. "/tests.wasm", + }, }) local bp, db = helpers.get_db_utils(DATABASE, { diff --git a/spec/02-integration/20-wasm/05-cache-invalidation_spec.lua b/spec/02-integration/20-wasm/05-cache-invalidation_spec.lua index 82e17b939dbe..3f37e30c6c69 100644 --- a/spec/02-integration/20-wasm/05-cache-invalidation_spec.lua +++ b/spec/02-integration/20-wasm/05-cache-invalidation_spec.lua @@ -5,6 +5,7 @@ local nkeys = require "table.nkeys" local HEADER = "X-Proxy-Wasm" local TIMEOUT = 20 local STEP = 0.1 +local FILTER_PATH = assert(helpers.test_conf.wasm_filters_path) local json = cjson.encode @@ -197,8 +198,12 @@ describe("#wasm filter chain cache " .. mode_suffix, function() lazy_setup(function() require("kong.runloop.wasm").enable({ - { name = "tests" }, - { name = "response_transformer" }, + { name = "tests", + path = FILTER_PATH .. "/tests.wasm", + }, + { name = "response_transformer", + path = FILTER_PATH .. "/response_transformer.wasm", + }, }) local bp diff --git a/spec/02-integration/20-wasm/07-reports_spec.lua b/spec/02-integration/20-wasm/07-reports_spec.lua index 307dea67e461..427caa8cbea4 100644 --- a/spec/02-integration/20-wasm/07-reports_spec.lua +++ b/spec/02-integration/20-wasm/07-reports_spec.lua @@ -52,7 +52,9 @@ for _, strategy in helpers.each_strategy() do }) require("kong.runloop.wasm").enable({ - { name = "tests" }, + { name = "tests", + path = helpers.test_conf.wasm_filters_path .. "/tests.wasm", + }, }) assert(helpers.start_kong({ diff --git a/spec/02-integration/20-wasm/09-filter-meta_spec.lua b/spec/02-integration/20-wasm/09-filter-meta_spec.lua new file mode 100644 index 000000000000..84a94eaf498d --- /dev/null +++ b/spec/02-integration/20-wasm/09-filter-meta_spec.lua @@ -0,0 +1,332 @@ +local helpers = require "spec.helpers" +local utils = require "kong.tools.utils" +local cjson = require "cjson" + +local file = helpers.file + +local TEST_FILTER_SRC = "spec/fixtures/proxy_wasm_filters/build/response_transformer.wasm" + +local function json(body) + return { + headers = { ["Content-Type"] = "application/json" }, + body = body, + } +end + +local function post_config(client, config) + config._format_version = config._format_version or "3.0" + + local res = client:post("/config?flatten_errors=1", json(config)) + + assert.response(res).has.jsonbody() + + return res +end + +local function random_name() + return "test-" .. utils.random_string() +end + + +for _, strategy in helpers.each_strategy({ "postgres", "off" }) do + +describe("filter metadata [#" .. strategy .. "]", function() + local filter_path + local admin + local proxy + + lazy_setup(function() + helpers.clean_prefix() + + if strategy == "postgres" then + helpers.get_db_utils(strategy, { + "routes", + "services", + "filter_chains", + }) + end + + filter_path = helpers.make_temp_dir() + do + local name = "rt_no_validation" + assert(file.copy(TEST_FILTER_SRC, filter_path .. "/" .. name .. ".wasm")) + end + + do + local name = "rt_with_validation" + assert(file.copy(TEST_FILTER_SRC, filter_path .. "/" .. name .. ".wasm")) + + assert(file.write(filter_path .. "/" .. name .. ".meta.json", cjson.encode({ + config_schema = { + type = "object", + properties = { + add = { + type = "object", + properties = { + headers = { + type = "array", + elements = { type = "string" }, + }, + }, + required = { "headers" }, + }, + }, + required = { "add" }, + } + }))) + end + + assert(helpers.start_kong({ + database = strategy, + nginx_conf = "spec/fixtures/custom_nginx.template", + plugins = "off", + wasm = true, + wasm_filters_path = filter_path, + nginx_main_worker_processes = 1, + })) + + admin = helpers.admin_client() + proxy = helpers.proxy_client() + + helpers.clean_logfile() + end) + + lazy_teardown(function() + if admin then admin:close() end + if proxy then proxy:close() end + + helpers.stop_kong() + + if filter_path and os.getenv("KONG_DONT_CLEAN") ~= "1" then + helpers.dir.rmtree(filter_path) + end + end) + + describe("config validation -", function() + local create_filter_chain + + if strategy == "off" then + create_filter_chain = function(route_host, filter_chain) + return post_config(admin, { + services = { + { name = random_name(), + url = helpers.mock_upstream_url, + routes = { + { name = random_name(), + hosts = { route_host }, + filter_chains = { filter_chain } + }, + }, + }, + }, + }) + end + + else + create_filter_chain = function(route_host, filter_chain) + local res = admin:post("/services", json { + name = random_name(), + url = helpers.mock_upstream_url, + }) + + assert.response(res).has.status(201) + + local service = assert.response(res).has.jsonbody() + + res = admin:post("/routes", json { + name = random_name(), + hosts = { route_host }, + service = { id = service.id }, + }) + + assert.response(res).has.status(201) + + local route = assert.response(res).has.jsonbody() + + res = admin:post("/routes/" .. route.id .. "/filter-chains", + json(filter_chain)) + + assert.response(res).has.jsonbody() + + return res + end + end + + it("filters with config schemas are validated", function() + local res = create_filter_chain(random_name(), { + name = random_name(), + filters = { + { + name = "rt_with_validation", + json_config = {}, -- empty + }, + }, + }) + + assert.response(res).has.status(400) + local body = assert.response(res).has.jsonbody() + + if strategy == "off" then + assert.is_table(body.flattened_errors) + assert.same(1, #body.flattened_errors) + + local err = body.flattened_errors[1] + assert.is_table(err) + assert.same("filter_chain", err.entity_type) + assert.same({ + { + field = "filters.1.config", + message = "property add is required", + type = "field" + } + }, err.errors) + + else + assert.same({ + filters = { + { + json_config = "property add is required" + } + } + }, body.fields) + end + + local host = random_name() .. ".test" + res = create_filter_chain(host, { + name = random_name(), + filters = { + { + name = "rt_with_validation", + json_config = { + add = { + headers = { + "x-foo:123", + }, + }, + }, + }, + }, + }) + + assert.response(res).has.status(201) + + assert.eventually(function() + res = proxy:get("/status/200", { headers = { host = host } }) + assert.response(res).has.status(200) + assert.response(res).has.header("x-foo") + end).has_no_error() + end) + + it("filters without config schemas are not validated", function() + local host = random_name() .. ".test" + + local res = create_filter_chain(host, { + name = random_name(), + filters = { + { + name = "rt_no_validation", + json_config = { + add = { + headers = 1234, + }, + }, + }, + }, + }) + + assert.response(res).has.status(201) + + assert.eventually(function() + res = proxy:get("/status/200", { headers = { host = host } }) + assert.response(res).has.no.header("x-foo") + assert.logfile().has.line("failed parsing filter config", true, 0) + end).has_no_error() + end) + + end) + +end) + +describe("filter metadata [#" .. strategy .. "] startup errors -", function() + local filter_path + local filter_name = "test-filter" + local meta_path + local conf + + lazy_setup(function() + if strategy == "postgres" then + helpers.get_db_utils(strategy, { + "routes", + "services", + "filter_chains", + }) + end + end) + + before_each(function() + filter_path = helpers.make_temp_dir() + assert(file.copy(TEST_FILTER_SRC, filter_path .. "/" .. filter_name .. ".wasm")) + meta_path = filter_path .. "/" .. filter_name .. ".meta.json" + + conf = { + database = strategy, + nginx_conf = "spec/fixtures/custom_nginx.template", + plugins = "off", + wasm = true, + wasm_filters_path = filter_path, + nginx_main_worker_processes = 1, + } + + helpers.clean_prefix() + helpers.prepare_prefix() + end) + + after_each(function() + helpers.kill_all() + + if filter_path and os.getenv("KONG_DONT_CLEAN") ~= "1" then + helpers.dir.rmtree(filter_path) + end + end) + + describe("kong start", function() + it("fails when filter.meta.json is not a file", function() + assert(helpers.dir.makepath(meta_path)) + local ok, err = helpers.start_kong(conf) + assert.falsy(ok) + + assert.matches("Failed to load metadata for one or more filters", err, nil, true) + assert.matches(filter_name, err, nil, true) + assert.matches(meta_path, err, nil, true) + assert.matches("path exists but is not a file", err, nil, true) + end) + + it("fails when filter.meta.json is not vaild json", function() + assert(file.write(meta_path, "oops!")) + local ok, err = helpers.start_kong(conf) + assert.falsy(ok) + + assert.matches("Failed to load metadata for one or more filters", err, nil, true) + assert.matches(filter_name, err, nil, true) + assert.matches(meta_path, err, nil, true) + assert.matches("JSON decode error", err, nil, true) + end) + + it("fails when filter.meta.json is not semantically valid", function() + assert(file.write(meta_path, cjson.encode({ + config_schema = { + type = "i am not a valid type", + }, + }))) + local ok, err = helpers.start_kong(conf) + assert.falsy(ok) + + assert.matches("Failed to load metadata for one or more filters", err, nil, true) + assert.matches(filter_name, err, nil, true) + assert.matches(meta_path, err, nil, true) + assert.matches("file contains invalid metadata", err, nil, true) + end) + end) +end) + +end -- each strategy diff --git a/spec/fixtures/proxy_wasm_filters/response_transformer/src/filter.rs b/spec/fixtures/proxy_wasm_filters/response_transformer/src/filter.rs index fbf7555ed25f..fb23189b3ee2 100644 --- a/spec/fixtures/proxy_wasm_filters/response_transformer/src/filter.rs +++ b/spec/fixtures/proxy_wasm_filters/response_transformer/src/filter.rs @@ -24,11 +24,15 @@ impl ResponseTransformerContext { impl RootContext for ResponseTransformerContext { fn on_configure(&mut self, _: usize) -> bool { let bytes = self.get_plugin_configuration().unwrap(); - if let Ok(config) = serde_json::from_slice(bytes.as_slice()) { - self.config = config; - true - } else { - false + match serde_json::from_slice::(bytes.as_slice()) { + Ok(config) => { + self.config = config; + true + }, + Err(e) => { + error!("failed parsing filter config: {}", e); + false + } } }