fix(ca_certificates): invalidate ca store caches when a ca cert is

                      updated and prevent ca_certificates that are still
                      being referenced by other entities from being deleted.

Fix [FTI-2060](https://konghq.atlassian.net/browse/FTI-2060)

changelog/unreleased/kong/ca_certificates_reference_check.yml

@@ -0,0 +1,3 @@
+message: prevent ca to be deleted when it's still referenced by other entities and invalidate the related ca store caches when a ca cert is updated.
+type: bugfix
+scope: Core

kong-3.6.0-0.rockspec

@@ -144,6 +144,7 @@ build = {
     ["kong.api.routes.tags"] = "kong/api/routes/tags.lua",
     ["kong.api.routes.targets"] = "kong/api/routes/targets.lua",
     ["kong.api.routes.upstreams"] = "kong/api/routes/upstreams.lua",
+    ["kong.api.routes.ca_certificates"] = "kong/api/routes/ca_certificates.lua",
 
     ["kong.admin_gui"] = "kong/admin_gui/init.lua",
     ["kong.admin_gui.utils"] = "kong/admin_gui/utils.lua",

kong/api/routes/ca_certificates.lua

@@ -0,0 +1,24 @@
+local certificates = require "kong.runloop.certificate"
+local fmt = string.format
+local kong = kong
+
+return {
+  ["/ca_certificates/:ca_certificates"] = {
+    DELETE = function(self, db, helpers, parent)
+      local ca_id = self.params.ca_certificates
+      local entity, element_or_err = certificates.check_ca_references(ca_id)
+
+      if entity then
+        local msg = fmt("ca_certificate %s is still referenced by %s (id = %s)", ca_id, entity, element_or_err.id)
+        kong.log.notice(msg)
+        return kong.response.exit(400, { message = msg })
+      elseif element_or_err then
+        local msg = "failed to check_ca_references, " .. element_or_err
+        kong.log.err(msg)
+        return kong.response.exit(500, { message = msg })
+      end
+
+      return parent()
+    end,
+  },
+}

kong/runloop/certificate.lua

@@ -19,6 +19,7 @@ local set_cert = ngx_ssl.set_cert
 local set_priv_key = ngx_ssl.set_priv_key
 local tb_concat   = table.concat
 local tb_sort   = table.sort
+local tb_insert = table.insert
 local kong = kong
 local type = type
 local error = error
@@ -28,6 +29,8 @@ local ipairs = ipairs
 local ngx_md5 = ngx.md5
 local ngx_exit = ngx.exit
 local ngx_ERROR = ngx.ERROR
+local null = ngx.null
+local fmt = string.format
 
 
 local default_cert_and_key
@@ -371,11 +374,159 @@ local function get_ca_certificate_store(ca_ids)
 end
 
 
+-- ordinary entities that reference ca certificates
+-- the value denotes which cache (kong.cache or kong.core_cache) is used
+local CA_CERT_REFERENCE_ENTITIES = {
+  "services",
+}
+
+-- plugins that reference ca certificates
+-- Format:
+-- mtls-auth = true
+local CA_CERT_REFERENCE_PLUGINS = {
+}
+
+local loaded_plugins
+local reference_plugins
+
+-- Examples:
+-- gen_iterator("services")
+-- gen_iterator("plugins", { mtls-auth = true })
+-- gen_iterator("plugins", "mtls-auth")
+-- We assume the field name is always `ca_certificates`
+local function gen_iterator(entity, plugins)
+  local options = {
+    workspace = null,
+  }
+
+  local iter = kong.db[entity]:each(1000, options)
+
+  local function iterator()
+    local element, err = iter()
+    if err then
+      return nil, err
+
+    elseif element == nil then
+      return nil
+
+    else
+      if plugins then
+        if type(plugins) ~= "table" then
+          plugins = { [plugins] = true }
+        end
+
+        if plugins[element.name] and element.config.ca_certificates and
+          next(element.config.ca_certificates) then
+          return element
+        else
+          return iterator()
+        end
+      else
+        if element.ca_certificates and next(element.ca_certificates) then
+          return element
+        else
+          return iterator()
+        end
+      end
+    end
+  end
+
+  return iterator
+end
+
+
+-- returns the first encountered entity element that is referencing `ca_id`
+-- otherwise, returns nil, err
+local function check_ca_references(ca_id)
+  for _, entity in ipairs(CA_CERT_REFERENCE_ENTITIES) do
+    for element, err in gen_iterator(entity) do
+      if err then
+        local msg = fmt("failed to list %s: %s", entity, err)
+        return nil, msg
+      end
+
+      for _, id in ipairs(element.ca_certificates) do
+        if id == ca_id then
+          return entity, element
+        end
+      end
+    end
+  end
+
+  if not reference_plugins then
+    reference_plugins = {}
+    loaded_plugins = loaded_plugins or kong.configuration.loaded_plugins
+
+    for k, _ in pairs(CA_CERT_REFERENCE_PLUGINS) do
+      if loaded_plugins[k] then
+        reference_plugins[k] = true
+      end
+    end
+  end
+
+  if next(reference_plugins) then
+    local entity = "plugins"
+    for element, err in gen_iterator(entity, reference_plugins) do
+      if err then
+        local msg = fmt("failed to list plugins: %s", err)
+        return nil, msg
+      end
+
+      for _, id in ipairs(element.config.ca_certificates) do
+        if id == ca_id then
+          return entity, element
+        end
+      end
+    end
+  end
+end
+
+
+-- returns an array of entities that are referencing `ca_id`
+-- return nil, err when error
+-- Examples:
+-- get_ca_certificate_references(ca_id, "services")
+-- get_ca_certificate_references(ca_id, "plugins", "mtls-auth")
+--
+-- Note we don't invalidate the ca store caches here directly because
+-- different entities use different caches (kong.cache or kong.core_cache)
+-- and use different functions to calculate the ca store cache key.
+-- And it's not a good idea to depend on the plugin implementations in Core.
+local function get_ca_certificate_references(ca_id, entity, plugin)
+  local elements = {}
+
+  for element, err in gen_iterator(entity, plugin) do
+    if err then
+      local msg = fmt("failed to list %s: %s", entity, err)
+      return nil, msg
+    end
+
+    local ca_certificates
+    if entity == "plugins" then
+      ca_certificates = element.config.ca_certificates
+    else
+      ca_certificates = element.ca_certificates
+    end
+
+    for _, id in ipairs(ca_certificates) do
+      if id == ca_id then
+        tb_insert(elements, element)
+      end
+    end
+  end
+
+  return elements
+end
+
+
 return {
   init = init,
   find_certificate = find_certificate,
   produce_wild_snis = produce_wild_snis,
   execute = execute,
   get_certificate = get_certificate,
   get_ca_certificate_store = get_ca_certificate_store,
+  ca_ids_cache_key = ca_ids_cache_key,
+  check_ca_references = check_ca_references,
+  get_ca_certificate_references = get_ca_certificate_references,
 }

kong/runloop/events.lua

@@ -319,6 +319,32 @@ local function crud_wasm_handler(data, schema_name)
 end
 
 
+local function crud_ca_certificates_handler(data)
+  if data.operation ~= "update" then
+    return
+  end
+
+  log(DEBUG, "[events] Ca_certificates updated, invalidating ca certificate store caches for services")
+
+  local elements, err = certificate.get_ca_certificate_references(data.entity.id, "services")
+  if err then
+    log(ERR, "[events] failed to get ca certificate references, ", err)
+  end
+
+  if elements then
+    local done_keys = {}
+    for _, e in ipairs(elements) do
+      local key = certificate.ca_ids_cache_key(e.ca_certificates)
+
+      if not done_keys[key] then
+        done_keys[key] = true
+        kong.core_cache:invalidate(key)
+      end
+    end
+  end
+end
+
+
 local LOCAL_HANDLERS = {
   { "dao:crud", nil         , dao_crud_handler },
 
@@ -338,6 +364,9 @@ local LOCAL_HANDLERS = {
   { "crud"    , "filter_chains"  , crud_wasm_handler },
   { "crud"    , "services"       , crud_wasm_handler },
   { "crud"    , "routes"         , crud_wasm_handler },
+
+  -- ca certificate store caches invalidations
+  { "crud"    , "ca_certificates" , crud_ca_certificates_handler },
 }
 
 

spec/02-integration/04-admin_api/16-ca_certificates_routes_spec.lua

@@ -42,6 +42,7 @@ for _, strategy in helpers.each_strategy() do
     lazy_setup(function()
       bp, db = helpers.get_db_utils(strategy, {
         "ca_certificates",
+        "services",
       })
 
       assert(helpers.start_kong {
@@ -148,6 +149,32 @@ for _, strategy in helpers.each_strategy() do
         ca = assert(bp.ca_certificates:insert())
       end)
 
+      it("not allowed to delete if it is referenced by other entities", function()
+        -- add a service that references the ca
+        local res = client:post("/services/", {
+          body = {
+            url = "https://" .. helpers.mock_upstream_host .. ":" .. helpers.mock_upstream_port,
+            protocol = "https",
+            ca_certificates = { ca.id },
+          },
+          headers = { ["Content-Type"] = "application/json" },
+        })
+        local body = assert.res_status(201, res)
+        local service = cjson.decode(body)
+
+        helpers.wait_for_all_config_update()
+
+        local res = client:delete("/ca_certificates/" .. ca.id)
+
+        local body = assert.res_status(400, res)
+        local json = cjson.decode(body)
+
+        assert.equal("ca_certificate " .. ca.id .. " is still referenced by services (id = " .. service.id .. ")", json.message)
+
+        local res = client:delete("/services/" .. service.id)
+        assert.res_status(204, res)
+      end)
+
       it("works", function()
         local res = client:delete("/ca_certificates/" .. ca.id)
         assert.res_status(204, res)