Merge remote-tracking branch 'origin/staging' into optimize-periodic-…

…deployment-manger-db-queries

build.sbt

@@ -321,7 +321,7 @@ val dropWizardV             = "5.0.0-rc15"
 val scalaCollectionsCompatV = "2.12.0"
 val testContainersScalaV    = "0.41.4"
 val testContainersJavaV     = "1.20.1"
-val nettyV                  = "4.1.113.Final"
+val nettyV                  = "4.1.115.Final"
 val nettyReactiveStreamsV   = "2.0.12"
 
 val akkaV                     = "2.6.20"
@@ -330,8 +330,8 @@ val akkaManagementV           = "1.1.4"
 val akkaHttpCirceV            = "1.39.2"
 val slickV                    = "3.4.1"  // 3.5 drops Scala 2.12
 val slickPgV                  = "0.21.1" // 0.22.2 uses Slick 3.5
-val hikariCpV                 = "5.1.0"
-val hsqldbV                   = "2.7.3"
+val hikariCpV                 = "6.2.1"
+val hsqldbV                   = "2.7.4"
 val postgresV                 = "42.7.4"
 // Flway 10 requires Java 17
 val flywayV                   = "9.22.3"

components/openapi/src/it/scala/pl/touk/nussknacker/openapi/functional/OpenAPIServiceSpec.scala

@@ -17,7 +17,7 @@ import pl.touk.nussknacker.engine.util.service.EagerServiceWithStaticParametersA
 import pl.touk.nussknacker.http.backend.FixedAsyncHttpClientBackendProvider
 import pl.touk.nussknacker.openapi.enrichers.{SwaggerEnricherCreator, SwaggerEnrichers}
 import pl.touk.nussknacker.openapi.parser.SwaggerParser
-import pl.touk.nussknacker.openapi.{ApiKeyConfig, OpenAPIServicesConfig}
+import pl.touk.nussknacker.openapi.{ApiKeySecret, OpenAPIServicesConfig, SecurityConfig, SecuritySchemeName}
 import pl.touk.nussknacker.test.PatientScalaFutures
 
 import java.net.URL
@@ -44,10 +44,10 @@ class OpenAPIServiceSpec
     val client = new DefaultAsyncHttpClient()
     try {
       new StubService().withCustomerService { port =>
-        val securities = Map("apikey" -> ApiKeyConfig("TODO"))
+        val secretBySchemeName = Map(SecuritySchemeName("apikey") -> ApiKeySecret("TODO"))
         val config = OpenAPIServicesConfig(
           new URL("http://foo"),
-          security = Some(securities),
+          security = secretBySchemeName,
           rootUrl = Some(new URL(s"http://localhost:$port"))
         )
         val services = SwaggerParser.parse(definition, config).collect { case Valid(service) =>

components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIComponentProvider.scala

@@ -10,7 +10,7 @@ import pl.touk.nussknacker.engine.api.CirceUtil
 import pl.touk.nussknacker.engine.api.component.{ComponentDefinition, ComponentProvider, NussknackerVersion}
 import pl.touk.nussknacker.engine.api.process.ProcessObjectDependencies
 import pl.touk.nussknacker.engine.util.config.ConfigEnrichments._
-import pl.touk.nussknacker.openapi.OpenAPIsConfig._
+import pl.touk.nussknacker.openapi.OpenAPIServicesConfig._
 import pl.touk.nussknacker.openapi.discovery.SwaggerOpenApiDefinitionDiscovery
 import pl.touk.nussknacker.openapi.enrichers.{SwaggerEnricherCreator, SwaggerEnrichers}
 import pl.touk.nussknacker.openapi.parser.ServiceParseError

components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIServicesConfig.scala

@@ -0,0 +1,82 @@
+package pl.touk.nussknacker.openapi
+
+import com.typesafe.config.Config
+import io.swagger.v3.oas.models.PathItem.HttpMethod
+import net.ceedubs.ficus.readers.{ArbitraryTypeReader, ValueReader}
+import pl.touk.nussknacker.http.backend.{DefaultHttpClientConfig, HttpClientConfig}
+import sttp.model.StatusCode
+
+import java.net.URL
+import scala.util.matching.Regex
+
+final case class OpenAPIServicesConfig(
+    url: URL,
+    // by default we allow only GET, as enrichers should be idempotent and not change data
+    allowedMethods: List[String] = List(HttpMethod.GET.name()),
+    codesToInterpretAsEmpty: List[Int] = List(StatusCode.NotFound.code),
+    namePattern: Regex = ".*".r,
+    rootUrl: Option[URL] = None,
+    // For backward compatibility it is called security. We should probably rename it and bundle together with secret
+    private val security: Map[SecuritySchemeName, Secret] = Map.empty,
+    private val secret: Option[Secret] = None,
+    httpClientConfig: HttpClientConfig = DefaultHttpClientConfig()
+) {
+  def securityConfig: SecurityConfig =
+    new SecurityConfig(secretBySchemeName = security, commonSecretForAnyScheme = secret)
+}
+
+final class SecurityConfig(
+    secretBySchemeName: Map[SecuritySchemeName, Secret],
+    commonSecretForAnyScheme: Option[Secret]
+) {
+
+  def secret(schemeName: SecuritySchemeName): Option[Secret] =
+    secretBySchemeName.get(schemeName) orElse commonSecretForAnyScheme
+
+}
+
+object SecurityConfig {
+  def empty: SecurityConfig = new SecurityConfig(Map.empty, None)
+}
+
+final case class SecuritySchemeName(value: String)
+
+sealed trait Secret
+
+final case class ApiKeySecret(apiKeyValue: String) extends Secret
+
+object OpenAPIServicesConfig {
+
+  import net.ceedubs.ficus.Ficus._
+  import pl.touk.nussknacker.engine.util.config.ConfigEnrichments._
+  import HttpClientConfig._
+
+  implicit val securitySchemeNameVR: ValueReader[SecuritySchemeName] =
+    ValueReader[String].map(SecuritySchemeName(_))
+
+  implicit val regexReader: ValueReader[Regex] = (config: Config, path: String) => new Regex(config.getString(path))
+
+  implicit val apiKeyVR: ValueReader[ApiKeySecret] = ValueReader.relative { conf =>
+    ApiKeySecret(
+      apiKeyValue = conf.as[String]("apiKeyValue")
+    )
+  }
+
+  implicit val secretVR: ValueReader[Secret] = ValueReader.relative { conf =>
+    conf.as[String]("type") match {
+      case "apiKey" => conf.rootAs[ApiKeySecret]
+      case typ      => throw new Exception(s"Not supported swagger security type '$typ' in the configuration")
+    }
+  }
+
+  implicit val secretBySchemeNameVR: ValueReader[Map[SecuritySchemeName, Secret]] =
+    ValueReader[Map[String, Secret]].map { secretBySchemeName =>
+      secretBySchemeName.map { case (schemeNameString, secret) =>
+        SecuritySchemeName(schemeNameString) -> secret
+      }
+    }
+
+  implicit val openAPIServicesConfigVR: ValueReader[OpenAPIServicesConfig] =
+    ArbitraryTypeReader.arbitraryTypeValueReader[OpenAPIServicesConfig]
+
+}

components/openapi/src/main/scala/pl/touk/nussknacker/openapi/parser/ParseToSwaggerService.scala

@@ -102,8 +102,7 @@ private[parser] class ParseToSwaggerService(openapi: OpenAPI, openAPIsConfig: Op
       Option(operation.getSecurity).orElse(Option(openapi.getSecurity)).map(_.asScala.toList).getOrElse(Nil)
     val securitySchemes =
       Option(openapi.getComponents).flatMap(c => Option(c.getSecuritySchemes)).map(_.asScala.toMap)
-    val securities = openAPIsConfig.security.getOrElse(Map.empty)
-    SecuritiesParser.parseSwaggerSecurities(securityRequirements, securitySchemes, securities)
+    SecuritiesParser.parseOperationSecurities(securityRequirements, securitySchemes, openAPIsConfig.securityConfig)
   }
 
   private def prepareParameters(operation: Operation): ValidationResult[List[SwaggerParameter]] = {

components/openapi/src/main/scala/pl/touk/nussknacker/openapi/parser/SecuritiesParser.scala

@@ -7,11 +7,13 @@ import io.swagger.v3.oas.models.security.{SecurityRequirement, SecurityScheme}
 import pl.touk.nussknacker.engine.api.util.ReflectUtils
 import pl.touk.nussknacker.openapi.parser.ParseToSwaggerService.ValidationResult
 import pl.touk.nussknacker.openapi.{
-  ApiKeyConfig,
   ApiKeyInCookie,
   ApiKeyInHeader,
   ApiKeyInQuery,
-  OpenAPISecurityConfig,
+  ApiKeySecret,
+  Secret,
+  SecurityConfig,
+  SecuritySchemeName,
   SwaggerSecurity
 }
 
@@ -21,24 +23,24 @@ private[parser] object SecuritiesParser extends LazyLogging {
 
   import cats.syntax.apply._
 
-  def parseSwaggerSecurities(
-      securityRequirements: List[SecurityRequirement],
-      securitySchemes: Option[Map[String, SecurityScheme]],
-      securitiesConfigs: Map[String, OpenAPISecurityConfig]
+  def parseOperationSecurities(
+      securityRequirementsDefinition: List[SecurityRequirement],
+      securitySchemesDefinition: Option[Map[String, SecurityScheme]],
+      securityConfig: SecurityConfig
   ): ValidationResult[List[SwaggerSecurity]] =
-    securityRequirements match {
+    securityRequirementsDefinition match {
       case Nil => Nil.validNel
       case _ =>
-        securitySchemes match {
+        securitySchemesDefinition match {
           case None => "There is no security scheme definition in the openAPI definition".invalidNel
           case Some(securitySchemes) => {
             // finds the first security requirement that can be met by the config
-            securityRequirements.view
+            securityRequirementsDefinition.view
               .map { securityRequirement =>
                 matchSecuritiesForRequiredSchemes(
                   securityRequirement.asScala.keys.toList,
                   securitySchemes,
-                  securitiesConfigs
+                  securityConfig
                 )
               }
               .foldLeft("No security requirement can be met because:".invalidNel[List[SwaggerSecurity]])(_.findValid(_))
@@ -48,55 +50,59 @@ private[parser] object SecuritiesParser extends LazyLogging {
         }
     }
 
-  def matchSecuritiesForRequiredSchemes(
+  private def matchSecuritiesForRequiredSchemes(
       requiredSchemesNames: List[String],
       securitySchemes: Map[String, SecurityScheme],
-      securitiesConfigs: Map[String, OpenAPISecurityConfig]
+      securitiesConfig: SecurityConfig
   ): ValidationResult[List[SwaggerSecurity]] =
-    requiredSchemesNames
-      .map { implicit schemeName: String =>
-        {
-          val securityScheme: ValidationResult[SecurityScheme] = Validated.fromOption(
-            securitySchemes.get(schemeName),
-            NonEmptyList.of(s"""there is no security scheme definition for scheme name "$schemeName"""")
-          )
-          val securityConfig: ValidationResult[OpenAPISecurityConfig] = Validated.fromOption(
-            securitiesConfigs.get(schemeName),
-            NonEmptyList.of(s"""there is no security config for scheme name "$schemeName"""")
-          )
+    requiredSchemesNames.map { schemeName =>
+      {
+        val validatedSecurityScheme: ValidationResult[SecurityScheme] = Validated.fromOption(
+          securitySchemes.get(schemeName),
+          NonEmptyList.of(s"""there is no security scheme definition for scheme name "$schemeName"""")
+        )
+        val validatedSecuritySecretConfigured: ValidationResult[Secret] = Validated.fromOption(
+          securitiesConfig.secret(SecuritySchemeName(schemeName)),
+          NonEmptyList.of(s"""there is no security secret configured for scheme name "$schemeName"""")
+        )
 
-          (securityScheme, securityConfig).tupled.andThen(t => getSecurityFromSchemeAndConfig(t._1, t._2))
-        }
+        (validatedSecurityScheme, validatedSecuritySecretConfigured)
+          .mapN { case (securityScheme, configuredSecret) =>
+            getSecurityFromSchemeAndSecret(securityScheme, configuredSecret)
+          }
+          .andThen(identity)
       }
-      .foldLeft[ValidationResult[List[SwaggerSecurity]]](Nil.validNel)(_.combine(_))
+    }.sequence
 
-  def getSecurityFromSchemeAndConfig(securityScheme: SecurityScheme, securityConfig: OpenAPISecurityConfig)(
-      implicit schemeName: String
-  ): ValidationResult[List[SwaggerSecurity]] = {
+  private def getSecurityFromSchemeAndSecret(
+      securityScheme: SecurityScheme,
+      secret: Secret
+  ): ValidationResult[SwaggerSecurity] = {
     import SecurityScheme.Type._
-    (securityScheme.getType, securityConfig) match {
-      case (APIKEY, apiKeyConfig: ApiKeyConfig) =>
-        getApiKeySecurity(securityScheme, apiKeyConfig)
+    (securityScheme.getType, secret) match {
+      case (APIKEY, apiKeySecret: ApiKeySecret) =>
+        getApiKeySecurity(securityScheme, apiKeySecret).validNel
       case (otherType: SecurityScheme.Type, _) => {
-        val securityConfigClassName = ReflectUtils.simpleNameWithoutSuffix(securityConfig.getClass)
-        s"Security type $otherType is not supported yet or ($otherType, $securityConfigClassName) is a mismatch security scheme type and security config pair".invalidNel
+        val secretClassName = ReflectUtils.simpleNameWithoutSuffix(secret.getClass)
+        s"Security type $otherType is not supported yet or ($otherType, $secretClassName) is a mismatch security scheme type and security config pair".invalidNel
       }
     }
   }
 
-  def getApiKeySecurity(securityScheme: SecurityScheme, apiKeyConfig: ApiKeyConfig)(
-      implicit schemeName: String
-  ): ValidationResult[List[SwaggerSecurity]] = {
+  private def getApiKeySecurity(
+      securityScheme: SecurityScheme,
+      apiKeySecret: ApiKeySecret
+  ): SwaggerSecurity = {
     val name = securityScheme.getName
-    val key  = apiKeyConfig.apiKeyValue
+    val key  = apiKeySecret.apiKeyValue
     import SecurityScheme.In._
     securityScheme.getIn match {
       case QUERY =>
-        (ApiKeyInQuery(name, key) :: Nil).validNel
+        ApiKeyInQuery(name, key)
       case HEADER =>
-        (ApiKeyInHeader(name, key) :: Nil).validNel
+        ApiKeyInHeader(name, key)
       case COOKIE =>
-        (ApiKeyInCookie(name, key) :: Nil).validNel
+        ApiKeyInCookie(name, key)
     }
   }
 

components/openapi/src/test/resources/swagger/multiple-schemes-for-single-operation.yml

@@ -0,0 +1,36 @@
+openapi: "3.1.0"
+info:
+  title: Simple API overview
+  version: 2.0.0
+servers:
+  - url: http://dummy.io
+paths:
+  /:
+    get:
+      security:
+        - headerConfig: []
+        - queryConfig: []
+        - cookieConfig: []
+      responses:
+        '200':
+          description: "-"
+          content:
+            application/json:
+              schema:
+                type: object
+      operationId: root
+components:
+  schemas: {}
+  securitySchemes:
+    headerConfig:
+      type: apiKey
+      name: keyHeader
+      in: header
+    queryConfig:
+      type: apiKey
+      name: keyParam
+      in: query
+    cookieConfig:
+      type: apiKey
+      name: keyCookie
+      in: cookie