From 7aa25fbedad6a8a9cacca94a611da7f91273c283 Mon Sep 17 00:00:00 2001 From: Arek Burdach Date: Tue, 17 Dec 2024 11:14:32 +0100 Subject: [PATCH 1/7] [NU-1828] OpenAPI enricher: ability to configure common secret for any security scheme (#7346) --- .../functional/OpenAPIServiceSpec.scala | 6 +- .../openapi/OpenAPIComponentProvider.scala | 2 +- .../openapi/OpenAPIServicesConfig.scala | 82 +++++++++ .../nussknacker/openapi/OpenAPIsConfig.scala | 50 ------ .../parser/ParseToSwaggerService.scala | 3 +- .../openapi/parser/SecuritiesParser.scala | 88 +++++----- .../multiple-schemes-for-single-operation.yml | 36 ++++ .../resources/swagger/service-security.yml | 8 +- .../nussknacker/openapi/BaseOpenAPITest.scala | 6 +- .../openapi/OpenAPIServicesConfigTest.scala | 73 ++++++++ .../openapi/OpenAPIsConfigTest.scala | 26 --- .../nussknacker/openapi/SecurityTest.scala | 163 +++++++++++++----- .../openapi/parser/SwaggerParserTest.scala | 2 +- docs/Changelog.md | 1 + docs/integration/OpenAPI.md | 19 +- 15 files changed, 380 insertions(+), 185 deletions(-) create mode 100644 components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIServicesConfig.scala delete mode 100644 components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIsConfig.scala create mode 100644 components/openapi/src/test/resources/swagger/multiple-schemes-for-single-operation.yml create mode 100644 components/openapi/src/test/scala/pl/touk/nussknacker/openapi/OpenAPIServicesConfigTest.scala delete mode 100644 components/openapi/src/test/scala/pl/touk/nussknacker/openapi/OpenAPIsConfigTest.scala diff --git a/components/openapi/src/it/scala/pl/touk/nussknacker/openapi/functional/OpenAPIServiceSpec.scala b/components/openapi/src/it/scala/pl/touk/nussknacker/openapi/functional/OpenAPIServiceSpec.scala index cbc7cbe770b..2d9b6631609 100644 --- a/components/openapi/src/it/scala/pl/touk/nussknacker/openapi/functional/OpenAPIServiceSpec.scala +++ b/components/openapi/src/it/scala/pl/touk/nussknacker/openapi/functional/OpenAPIServiceSpec.scala @@ -17,7 +17,7 @@ import pl.touk.nussknacker.engine.util.service.EagerServiceWithStaticParametersA import pl.touk.nussknacker.http.backend.FixedAsyncHttpClientBackendProvider import pl.touk.nussknacker.openapi.enrichers.{SwaggerEnricherCreator, SwaggerEnrichers} import pl.touk.nussknacker.openapi.parser.SwaggerParser -import pl.touk.nussknacker.openapi.{ApiKeyConfig, OpenAPIServicesConfig} +import pl.touk.nussknacker.openapi.{ApiKeySecret, OpenAPIServicesConfig, SecurityConfig, SecuritySchemeName} import pl.touk.nussknacker.test.PatientScalaFutures import java.net.URL @@ -44,10 +44,10 @@ class OpenAPIServiceSpec val client = new DefaultAsyncHttpClient() try { new StubService().withCustomerService { port => - val securities = Map("apikey" -> ApiKeyConfig("TODO")) + val secretBySchemeName = Map(SecuritySchemeName("apikey") -> ApiKeySecret("TODO")) val config = OpenAPIServicesConfig( new URL("http://foo"), - security = Some(securities), + security = secretBySchemeName, rootUrl = Some(new URL(s"http://localhost:$port")) ) val services = SwaggerParser.parse(definition, config).collect { case Valid(service) => diff --git a/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIComponentProvider.scala b/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIComponentProvider.scala index 86c3de9f4d5..1d84f8314af 100644 --- a/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIComponentProvider.scala +++ b/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIComponentProvider.scala @@ -10,7 +10,7 @@ import pl.touk.nussknacker.engine.api.CirceUtil import pl.touk.nussknacker.engine.api.component.{ComponentDefinition, ComponentProvider, NussknackerVersion} import pl.touk.nussknacker.engine.api.process.ProcessObjectDependencies import pl.touk.nussknacker.engine.util.config.ConfigEnrichments._ -import pl.touk.nussknacker.openapi.OpenAPIsConfig._ +import pl.touk.nussknacker.openapi.OpenAPIServicesConfig._ import pl.touk.nussknacker.openapi.discovery.SwaggerOpenApiDefinitionDiscovery import pl.touk.nussknacker.openapi.enrichers.{SwaggerEnricherCreator, SwaggerEnrichers} import pl.touk.nussknacker.openapi.parser.ServiceParseError diff --git a/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIServicesConfig.scala b/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIServicesConfig.scala new file mode 100644 index 00000000000..9f88df8152f --- /dev/null +++ b/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIServicesConfig.scala @@ -0,0 +1,82 @@ +package pl.touk.nussknacker.openapi + +import com.typesafe.config.Config +import io.swagger.v3.oas.models.PathItem.HttpMethod +import net.ceedubs.ficus.readers.{ArbitraryTypeReader, ValueReader} +import pl.touk.nussknacker.http.backend.{DefaultHttpClientConfig, HttpClientConfig} +import sttp.model.StatusCode + +import java.net.URL +import scala.util.matching.Regex + +final case class OpenAPIServicesConfig( + url: URL, + // by default we allow only GET, as enrichers should be idempotent and not change data + allowedMethods: List[String] = List(HttpMethod.GET.name()), + codesToInterpretAsEmpty: List[Int] = List(StatusCode.NotFound.code), + namePattern: Regex = ".*".r, + rootUrl: Option[URL] = None, + // For backward compatibility it is called security. We should probably rename it and bundle together with secret + private val security: Map[SecuritySchemeName, Secret] = Map.empty, + private val secret: Option[Secret] = None, + httpClientConfig: HttpClientConfig = DefaultHttpClientConfig() +) { + def securityConfig: SecurityConfig = + new SecurityConfig(secretBySchemeName = security, commonSecretForAnyScheme = secret) +} + +final class SecurityConfig( + secretBySchemeName: Map[SecuritySchemeName, Secret], + commonSecretForAnyScheme: Option[Secret] +) { + + def secret(schemeName: SecuritySchemeName): Option[Secret] = + secretBySchemeName.get(schemeName) orElse commonSecretForAnyScheme + +} + +object SecurityConfig { + def empty: SecurityConfig = new SecurityConfig(Map.empty, None) +} + +final case class SecuritySchemeName(value: String) + +sealed trait Secret + +final case class ApiKeySecret(apiKeyValue: String) extends Secret + +object OpenAPIServicesConfig { + + import net.ceedubs.ficus.Ficus._ + import pl.touk.nussknacker.engine.util.config.ConfigEnrichments._ + import HttpClientConfig._ + + implicit val securitySchemeNameVR: ValueReader[SecuritySchemeName] = + ValueReader[String].map(SecuritySchemeName(_)) + + implicit val regexReader: ValueReader[Regex] = (config: Config, path: String) => new Regex(config.getString(path)) + + implicit val apiKeyVR: ValueReader[ApiKeySecret] = ValueReader.relative { conf => + ApiKeySecret( + apiKeyValue = conf.as[String]("apiKeyValue") + ) + } + + implicit val secretVR: ValueReader[Secret] = ValueReader.relative { conf => + conf.as[String]("type") match { + case "apiKey" => conf.rootAs[ApiKeySecret] + case typ => throw new Exception(s"Not supported swagger security type '$typ' in the configuration") + } + } + + implicit val secretBySchemeNameVR: ValueReader[Map[SecuritySchemeName, Secret]] = + ValueReader[Map[String, Secret]].map { secretBySchemeName => + secretBySchemeName.map { case (schemeNameString, secret) => + SecuritySchemeName(schemeNameString) -> secret + } + } + + implicit val openAPIServicesConfigVR: ValueReader[OpenAPIServicesConfig] = + ArbitraryTypeReader.arbitraryTypeValueReader[OpenAPIServicesConfig] + +} diff --git a/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIsConfig.scala b/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIsConfig.scala deleted file mode 100644 index 82d3cb553f1..00000000000 --- a/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/OpenAPIsConfig.scala +++ /dev/null @@ -1,50 +0,0 @@ -package pl.touk.nussknacker.openapi - -import com.typesafe.config.Config -import io.swagger.v3.oas.models.PathItem.HttpMethod -import net.ceedubs.ficus.readers.{ArbitraryTypeReader, ValueReader} -import pl.touk.nussknacker.http.backend.{DefaultHttpClientConfig, HttpClientConfig} -import sttp.model.StatusCode - -import java.net.URL -import scala.util.matching.Regex - -final case class OpenAPIServicesConfig( - url: URL, - // by default we allow only GET, as enrichers should be idempotent and not change data - allowedMethods: List[String] = List(HttpMethod.GET.name()), - codesToInterpretAsEmpty: List[Int] = List(StatusCode.NotFound.code), - namePattern: Regex = ".*".r, - rootUrl: Option[URL] = None, - security: Option[Map[String, OpenAPISecurityConfig]] = None, - httpClientConfig: HttpClientConfig = DefaultHttpClientConfig() -) - -sealed trait OpenAPISecurityConfig - -final case class ApiKeyConfig(apiKeyValue: String) extends OpenAPISecurityConfig - -object OpenAPIsConfig { - - import net.ceedubs.ficus.Ficus._ - import pl.touk.nussknacker.engine.util.config.ConfigEnrichments._ - - implicit val openAPIServicesConfigVR: ValueReader[OpenAPIServicesConfig] = - ArbitraryTypeReader.arbitraryTypeValueReader[OpenAPIServicesConfig] - - implicit val regexReader: ValueReader[Regex] = (config: Config, path: String) => new Regex(config.getString(path)) - - implicit val openAPISecurityConfigVR: ValueReader[OpenAPISecurityConfig] = ValueReader.relative(conf => { - conf.as[String]("type") match { - case "apiKey" => conf.rootAs[ApiKeyConfig] - case typ => throw new Exception(s"Not supported swagger security type '$typ' in the configuration") - } - }) - - implicit val apiKeyConfigVR: ValueReader[ApiKeyConfig] = ValueReader.relative(conf => { - ApiKeyConfig( - apiKeyValue = conf.as[String]("apiKeyValue") - ) - }) - -} diff --git a/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/parser/ParseToSwaggerService.scala b/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/parser/ParseToSwaggerService.scala index 546f764f4c1..040083c82f7 100644 --- a/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/parser/ParseToSwaggerService.scala +++ b/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/parser/ParseToSwaggerService.scala @@ -102,8 +102,7 @@ private[parser] class ParseToSwaggerService(openapi: OpenAPI, openAPIsConfig: Op Option(operation.getSecurity).orElse(Option(openapi.getSecurity)).map(_.asScala.toList).getOrElse(Nil) val securitySchemes = Option(openapi.getComponents).flatMap(c => Option(c.getSecuritySchemes)).map(_.asScala.toMap) - val securities = openAPIsConfig.security.getOrElse(Map.empty) - SecuritiesParser.parseSwaggerSecurities(securityRequirements, securitySchemes, securities) + SecuritiesParser.parseOperationSecurities(securityRequirements, securitySchemes, openAPIsConfig.securityConfig) } private def prepareParameters(operation: Operation): ValidationResult[List[SwaggerParameter]] = { diff --git a/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/parser/SecuritiesParser.scala b/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/parser/SecuritiesParser.scala index 148ddca2fd5..7a0e2a52699 100644 --- a/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/parser/SecuritiesParser.scala +++ b/components/openapi/src/main/scala/pl/touk/nussknacker/openapi/parser/SecuritiesParser.scala @@ -7,11 +7,13 @@ import io.swagger.v3.oas.models.security.{SecurityRequirement, SecurityScheme} import pl.touk.nussknacker.engine.api.util.ReflectUtils import pl.touk.nussknacker.openapi.parser.ParseToSwaggerService.ValidationResult import pl.touk.nussknacker.openapi.{ - ApiKeyConfig, ApiKeyInCookie, ApiKeyInHeader, ApiKeyInQuery, - OpenAPISecurityConfig, + ApiKeySecret, + Secret, + SecurityConfig, + SecuritySchemeName, SwaggerSecurity } @@ -21,24 +23,24 @@ private[parser] object SecuritiesParser extends LazyLogging { import cats.syntax.apply._ - def parseSwaggerSecurities( - securityRequirements: List[SecurityRequirement], - securitySchemes: Option[Map[String, SecurityScheme]], - securitiesConfigs: Map[String, OpenAPISecurityConfig] + def parseOperationSecurities( + securityRequirementsDefinition: List[SecurityRequirement], + securitySchemesDefinition: Option[Map[String, SecurityScheme]], + securityConfig: SecurityConfig ): ValidationResult[List[SwaggerSecurity]] = - securityRequirements match { + securityRequirementsDefinition match { case Nil => Nil.validNel case _ => - securitySchemes match { + securitySchemesDefinition match { case None => "There is no security scheme definition in the openAPI definition".invalidNel case Some(securitySchemes) => { // finds the first security requirement that can be met by the config - securityRequirements.view + securityRequirementsDefinition.view .map { securityRequirement => matchSecuritiesForRequiredSchemes( securityRequirement.asScala.keys.toList, securitySchemes, - securitiesConfigs + securityConfig ) } .foldLeft("No security requirement can be met because:".invalidNel[List[SwaggerSecurity]])(_.findValid(_)) @@ -48,55 +50,59 @@ private[parser] object SecuritiesParser extends LazyLogging { } } - def matchSecuritiesForRequiredSchemes( + private def matchSecuritiesForRequiredSchemes( requiredSchemesNames: List[String], securitySchemes: Map[String, SecurityScheme], - securitiesConfigs: Map[String, OpenAPISecurityConfig] + securitiesConfig: SecurityConfig ): ValidationResult[List[SwaggerSecurity]] = - requiredSchemesNames - .map { implicit schemeName: String => - { - val securityScheme: ValidationResult[SecurityScheme] = Validated.fromOption( - securitySchemes.get(schemeName), - NonEmptyList.of(s"""there is no security scheme definition for scheme name "$schemeName"""") - ) - val securityConfig: ValidationResult[OpenAPISecurityConfig] = Validated.fromOption( - securitiesConfigs.get(schemeName), - NonEmptyList.of(s"""there is no security config for scheme name "$schemeName"""") - ) + requiredSchemesNames.map { schemeName => + { + val validatedSecurityScheme: ValidationResult[SecurityScheme] = Validated.fromOption( + securitySchemes.get(schemeName), + NonEmptyList.of(s"""there is no security scheme definition for scheme name "$schemeName"""") + ) + val validatedSecuritySecretConfigured: ValidationResult[Secret] = Validated.fromOption( + securitiesConfig.secret(SecuritySchemeName(schemeName)), + NonEmptyList.of(s"""there is no security secret configured for scheme name "$schemeName"""") + ) - (securityScheme, securityConfig).tupled.andThen(t => getSecurityFromSchemeAndConfig(t._1, t._2)) - } + (validatedSecurityScheme, validatedSecuritySecretConfigured) + .mapN { case (securityScheme, configuredSecret) => + getSecurityFromSchemeAndSecret(securityScheme, configuredSecret) + } + .andThen(identity) } - .foldLeft[ValidationResult[List[SwaggerSecurity]]](Nil.validNel)(_.combine(_)) + }.sequence - def getSecurityFromSchemeAndConfig(securityScheme: SecurityScheme, securityConfig: OpenAPISecurityConfig)( - implicit schemeName: String - ): ValidationResult[List[SwaggerSecurity]] = { + private def getSecurityFromSchemeAndSecret( + securityScheme: SecurityScheme, + secret: Secret + ): ValidationResult[SwaggerSecurity] = { import SecurityScheme.Type._ - (securityScheme.getType, securityConfig) match { - case (APIKEY, apiKeyConfig: ApiKeyConfig) => - getApiKeySecurity(securityScheme, apiKeyConfig) + (securityScheme.getType, secret) match { + case (APIKEY, apiKeySecret: ApiKeySecret) => + getApiKeySecurity(securityScheme, apiKeySecret).validNel case (otherType: SecurityScheme.Type, _) => { - val securityConfigClassName = ReflectUtils.simpleNameWithoutSuffix(securityConfig.getClass) - s"Security type $otherType is not supported yet or ($otherType, $securityConfigClassName) is a mismatch security scheme type and security config pair".invalidNel + val secretClassName = ReflectUtils.simpleNameWithoutSuffix(secret.getClass) + s"Security type $otherType is not supported yet or ($otherType, $secretClassName) is a mismatch security scheme type and security config pair".invalidNel } } } - def getApiKeySecurity(securityScheme: SecurityScheme, apiKeyConfig: ApiKeyConfig)( - implicit schemeName: String - ): ValidationResult[List[SwaggerSecurity]] = { + private def getApiKeySecurity( + securityScheme: SecurityScheme, + apiKeySecret: ApiKeySecret + ): SwaggerSecurity = { val name = securityScheme.getName - val key = apiKeyConfig.apiKeyValue + val key = apiKeySecret.apiKeyValue import SecurityScheme.In._ securityScheme.getIn match { case QUERY => - (ApiKeyInQuery(name, key) :: Nil).validNel + ApiKeyInQuery(name, key) case HEADER => - (ApiKeyInHeader(name, key) :: Nil).validNel + ApiKeyInHeader(name, key) case COOKIE => - (ApiKeyInCookie(name, key) :: Nil).validNel + ApiKeyInCookie(name, key) } } diff --git a/components/openapi/src/test/resources/swagger/multiple-schemes-for-single-operation.yml b/components/openapi/src/test/resources/swagger/multiple-schemes-for-single-operation.yml new file mode 100644 index 00000000000..50828f46a18 --- /dev/null +++ b/components/openapi/src/test/resources/swagger/multiple-schemes-for-single-operation.yml @@ -0,0 +1,36 @@ +openapi: "3.1.0" +info: + title: Simple API overview + version: 2.0.0 +servers: + - url: http://dummy.io +paths: + /: + get: + security: + - headerConfig: [] + - queryConfig: [] + - cookieConfig: [] + responses: + '200': + description: "-" + content: + application/json: + schema: + type: object + operationId: root +components: + schemas: {} + securitySchemes: + headerConfig: + type: apiKey + name: keyHeader + in: header + queryConfig: + type: apiKey + name: keyParam + in: query + cookieConfig: + type: apiKey + name: keyCookie + in: cookie diff --git a/components/openapi/src/test/resources/swagger/service-security.yml b/components/openapi/src/test/resources/swagger/service-security.yml index b2641b3c917..db96a36d113 100644 --- a/components/openapi/src/test/resources/swagger/service-security.yml +++ b/components/openapi/src/test/resources/swagger/service-security.yml @@ -16,7 +16,7 @@ paths: application/json: schema: type: object - operationId: header + operationId: headerOperationId /queryPath: get: security: @@ -28,7 +28,7 @@ paths: application/json: schema: type: object - operationId: query + operationId: queryOperationId /cookiePath: get: security: @@ -40,7 +40,7 @@ paths: application/json: schema: type: object - operationId: cookie + operationId: cookieOperationId components: schemas: {} securitySchemes: @@ -55,4 +55,4 @@ components: cookieConfig: type: apiKey name: keyCookie - in: cookie \ No newline at end of file + in: cookie diff --git a/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/BaseOpenAPITest.scala b/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/BaseOpenAPITest.scala index 88c76368c57..e4392b42363 100644 --- a/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/BaseOpenAPITest.scala +++ b/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/BaseOpenAPITest.scala @@ -4,12 +4,12 @@ import cats.data.Validated import cats.data.Validated.{Invalid, Valid} import org.apache.commons.io.IOUtils import pl.touk.nussknacker.engine.api.process.ComponentUseCase -import pl.touk.nussknacker.engine.api.{Context, ContextId, JobData, MetaData, ProcessVersion, StreamMetaData} +import pl.touk.nussknacker.engine.api._ import pl.touk.nussknacker.engine.util.runtimecontext.TestEngineRuntimeContext import pl.touk.nussknacker.engine.util.service.EagerServiceWithStaticParametersAndReturnType import pl.touk.nussknacker.openapi.enrichers.{SwaggerEnricherCreator, SwaggerEnrichers} import pl.touk.nussknacker.openapi.parser.{ServiceParseError, SwaggerParser} -import sttp.client3.testing.SttpBackendStub +import sttp.client3.SttpBackend import java.net.URL import java.nio.charset.StandardCharsets @@ -47,7 +47,7 @@ trait BaseOpenAPITest { protected def parseToEnrichers( resource: String, - backend: SttpBackendStub[Future, Any], + backend: SttpBackend[Future, Any], config: OpenAPIServicesConfig = baseConfig ): Map[ServiceName, EagerServiceWithStaticParametersAndReturnType] = { val services = parseServicesFromResourceUnsafe(resource, config) diff --git a/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/OpenAPIServicesConfigTest.scala b/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/OpenAPIServicesConfigTest.scala new file mode 100644 index 00000000000..6262fd013a3 --- /dev/null +++ b/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/OpenAPIServicesConfigTest.scala @@ -0,0 +1,73 @@ +package pl.touk.nussknacker.openapi + +import com.typesafe.config.ConfigFactory +import org.scalatest.OptionValues +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +class OpenAPIServicesConfigTest extends AnyFunSuite with Matchers with OptionValues { + + import net.ceedubs.ficus.Ficus._ + import OpenAPIServicesConfig._ + + test("should parse apikey secret for each scheme") { + val config = ConfigFactory.parseString("""url: "http://foo" + |security { + | apikeySecurityScheme { + | type: "apiKey" + | apiKeyValue: "34534asfdasf" + | } + | apikeySecurityScheme2 { + | type: "apiKey" + | apiKeyValue: "123" + | } + |}""".stripMargin) + + val parsedConfig = config.as[OpenAPIServicesConfig] + parsedConfig.securityConfig + .secret(SecuritySchemeName("apikeySecurityScheme")) + .value shouldEqual ApiKeySecret(apiKeyValue = "34534asfdasf") + parsedConfig.securityConfig + .secret(SecuritySchemeName("apikeySecurityScheme2")) + .value shouldEqual ApiKeySecret(apiKeyValue = "123") + } + + test("should parse common apikey secret for any scheme") { + val config = ConfigFactory.parseString("""url: "http://foo" + |secret { + | type: "apiKey" + | apiKeyValue: "34534asfdasf" + |}""".stripMargin) + + val parsedConfig = config.as[OpenAPIServicesConfig] + parsedConfig.securityConfig + .secret(SecuritySchemeName("someScheme")) + .value shouldEqual ApiKeySecret(apiKeyValue = "34534asfdasf") + parsedConfig.securityConfig + .secret(SecuritySchemeName("someOtherScheme")) + .value shouldEqual ApiKeySecret(apiKeyValue = "34534asfdasf") + } + + test("should parse combined apikey secret for each scheme and common apikey secret for any scheme") { + val config = ConfigFactory.parseString("""url: "http://foo" + |security { + | someScheme { + | type: "apiKey" + | apiKeyValue: "123" + | } + |} + |secret { + | type: "apiKey" + | apiKeyValue: "234" + |}""".stripMargin) + + val parsedConfig = config.as[OpenAPIServicesConfig] + parsedConfig.securityConfig + .secret(SecuritySchemeName("someScheme")) + .value shouldEqual ApiKeySecret(apiKeyValue = "123") + parsedConfig.securityConfig + .secret(SecuritySchemeName("someOtherScheme")) + .value shouldEqual ApiKeySecret(apiKeyValue = "234") + } + +} diff --git a/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/OpenAPIsConfigTest.scala b/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/OpenAPIsConfigTest.scala deleted file mode 100644 index a698eff89f2..00000000000 --- a/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/OpenAPIsConfigTest.scala +++ /dev/null @@ -1,26 +0,0 @@ -package pl.touk.nussknacker.openapi - -import com.typesafe.config.ConfigFactory -import org.scalatest.OptionValues -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.matchers.should.Matchers - -class OpenAPIsConfigTest extends AnyFunSuite with Matchers with OptionValues { - - import net.ceedubs.ficus.Ficus._ - import OpenAPIsConfig._ - - test("should parse apikey security") { - val config = ConfigFactory.parseString("""url: "http://foo" - |security { - | apikeySecuritySchema { - | type: "apiKey" - | apiKeyValue: "34534asfdasf" - | } - |}""".stripMargin) - - val parsedConfig = config.as[OpenAPIServicesConfig] - parsedConfig.security.value.get("apikeySecuritySchema").value shouldEqual ApiKeyConfig(apiKeyValue = "34534asfdasf") - } - -} diff --git a/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/SecurityTest.scala b/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/SecurityTest.scala index 28765d28616..e9de5f99285 100644 --- a/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/SecurityTest.scala +++ b/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/SecurityTest.scala @@ -3,89 +3,162 @@ package pl.touk.nussknacker.openapi import com.typesafe.scalalogging.LazyLogging import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import org.scalatest.{Assertion, BeforeAndAfterAll} +import org.scalatest.{Assertion, BeforeAndAfterAll, LoneElement, TryValues} import pl.touk.nussknacker.engine.api.ContextId import pl.touk.nussknacker.engine.api.test.EmptyInvocationCollector.Instance import pl.touk.nussknacker.engine.api.typed.TypedMap import pl.touk.nussknacker.test.PatientScalaFutures import sttp.client3.testing.SttpBackendStub -import sttp.client3.{Request, Response} +import sttp.client3.{Request, Response, SttpBackend} import sttp.model.{HeaderNames, StatusCode} import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future +import scala.util.Try class SecurityTest extends AnyFunSuite with BeforeAndAfterAll with Matchers + with TryValues + with LoneElement with LazyLogging with PatientScalaFutures with BaseOpenAPITest { - sealed case class Config( - path: String, - securityName: String, - serviceName: String, - key: String, - assertion: Request[_, _] => Assertion - ) + private class StubbedOperationLogic( + val operationId: String, + private val path: List[String], + val securitySchemeName: SecuritySchemeName, + val expectedSecret: ApiKeySecret, + val checkSecret: (Request[_, _], ApiKeySecret) => Assertion + ) { + + def handleMatchingRequest(request: Request[_, _]): Option[Try[Assertion]] = + Option(request).filter(requestMatches).map(_ => Try(checkSecret(request, expectedSecret))) - private val configs = List[Config]( - Config( - "headerPath", - "headerConfig", - "header", - "h1", - _.headers.find(_.name == "keyHeader").map(_.value) shouldBe Some("h1") + private def requestMatches(request: Request[_, _]) = { + request.uri.path == path + } + + } + + private val stubbedSecretCheckingLogics = List[StubbedOperationLogic]( + new StubbedOperationLogic( + "headerOperationId", + "headerPath" :: Nil, + SecuritySchemeName("headerConfig"), + ApiKeySecret("h1"), + (req, expectedSecret) => + req.headers.find(_.name == "keyHeader").map(_.value) shouldBe Some(expectedSecret.apiKeyValue) + ), + new StubbedOperationLogic( + "queryOperationId", + "queryPath" :: Nil, + SecuritySchemeName("queryConfig"), + ApiKeySecret("q1"), + (req, expectedSecret) => req.uri.params.get("keyParam") shouldBe Some(expectedSecret.apiKeyValue) ), - Config("queryPath", "queryConfig", "query", "q1", _.uri.params.get("keyParam") shouldBe Some("q1")), - Config( - "cookiePath", - "cookieConfig", - "cookie", - "c1", - _.headers.find(_.name == HeaderNames.Cookie).map(_.value) shouldBe Some("keyCookie=c1") + new StubbedOperationLogic( + "cookieOperationId", + "cookiePath" :: Nil, + SecuritySchemeName("cookieConfig"), + ApiKeySecret("c1"), + (req, expectedSecret) => + req.headers.find(_.name == HeaderNames.Cookie).map(_.value) shouldBe Some( + s"keyCookie=${expectedSecret.apiKeyValue}" + ) ), ) - test("service returns customers") { - val backend = SttpBackendStub.asynchronousFuture - .whenRequestMatches { request => - val pathMatches = configs.find(_.path == request.uri.path.head) - pathMatches.foreach(_.assertion(request)) - pathMatches.isDefined + private val definitionMatchingStubbedLogic = "service-security.yml" + + val backend: SttpBackend[Future, Any] = SttpBackendStub.asynchronousFuture.whenAnyRequest + .thenRespondF { request => + Future { + val operationsLogicResults = + stubbedSecretCheckingLogics + .flatMap(logic => logic.handleMatchingRequest(request).map(logic.operationId -> _)) + .toMap + operationsLogicResults.loneElement._2.success.value + Response("{}", StatusCode.Ok) } - .thenRespond(Response("{}", StatusCode.Ok)) + } - val withCorrectConfig = - enrichersForSecurityConfig(backend, configs.map(c => c.securityName -> ApiKeyConfig(c.key)).toMap) - configs.foreach { config => - withClue(config.serviceName) { + test("secret configured for each scheme in definition") { + val enricherWithCorrectConfig = + parseToEnrichers( + definitionMatchingStubbedLogic, + backend, + baseConfig.copy( + security = stubbedSecretCheckingLogics.map(c => c.securitySchemeName -> c.expectedSecret).toMap, + ) + ) + stubbedSecretCheckingLogics.foreach { logic => + withClue(logic.operationId) { implicit val contextId: ContextId = ContextId("1") - withCorrectConfig(ServiceName(config.serviceName)) + enricherWithCorrectConfig(ServiceName(logic.operationId)) .invoke(Map.empty) .futureValue shouldBe TypedMap(Map.empty) } } - val withBadConfig = - enrichersForSecurityConfig(backend, configs.map(c => c.securityName -> ApiKeyConfig("bla")).toMap) - configs.foreach { config => - withClue(config.serviceName) { + val enricherWithBadConfig = + parseToEnrichers( + definitionMatchingStubbedLogic, + backend, + baseConfig.copy( + security = stubbedSecretCheckingLogics.map(c => c.securitySchemeName -> ApiKeySecret("bla")).toMap, + ) + ) + stubbedSecretCheckingLogics.foreach { logic => + withClue(logic.operationId) { intercept[Exception] { implicit val contextId: ContextId = ContextId("1") - withBadConfig(ServiceName(config.serviceName)).invoke(Map.empty).futureValue + enricherWithBadConfig(ServiceName(logic.operationId)).invoke(Map.empty).futureValue } } } } - private def enrichersForSecurityConfig( - backend: SttpBackendStub[Future, Any], - securities: Map[String, ApiKeyConfig] - ) = { - parseToEnrichers("service-security.yml", backend, baseConfig.copy(security = Some(securities))) + test("common secret configured for any scheme") { + stubbedSecretCheckingLogics.foreach { config => + withClue(config.operationId) { + val enricherWithSingleSecurityConfig = parseToEnrichers( + definitionMatchingStubbedLogic, + backend, + baseConfig.copy(secret = Some(config.expectedSecret)) + ) + implicit val contextId: ContextId = ContextId("1") + enricherWithSingleSecurityConfig(ServiceName(config.operationId)) + .invoke(Map.empty) + .futureValue shouldBe TypedMap(Map.empty) + } + } + } + + test("common secret configured for any scheme with one operation handling multiple security schemes") { + val secretMatchesEveryScheme = ApiKeySecret("single-secret") + val backend = SttpBackendStub.asynchronousFuture.whenAnyRequest + .thenRespondF { request => + Future { + val operationsLogicResults = + stubbedSecretCheckingLogics + .map(logic => logic.operationId -> Try(logic.checkSecret(request, secretMatchesEveryScheme))) + .toMap + operationsLogicResults.filter(_._2.isSuccess) should have size 1 + Response("{}", StatusCode.Ok) + } + } + val enricherWithSingleSecurityConfig = parseToEnrichers( + "multiple-schemes-for-single-operation.yml", + backend, + baseConfig.copy(secret = Some(secretMatchesEveryScheme)) + ) + implicit val contextId: ContextId = ContextId("1") + enricherWithSingleSecurityConfig(ServiceName("root")) + .invoke(Map.empty) + .futureValue shouldBe TypedMap(Map.empty) } } diff --git a/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/parser/SwaggerParserTest.scala b/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/parser/SwaggerParserTest.scala index 52149521331..0c9adf8bd94 100644 --- a/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/parser/SwaggerParserTest.scala +++ b/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/parser/SwaggerParserTest.scala @@ -107,7 +107,7 @@ class SwaggerParserTest extends AnyFunSuite with BaseOpenAPITest with Matchers { errorsFor("noResponseType") shouldBe List("No response with application/json or */* media types found") errorsFor("unhandledSecurity") shouldBe List( - "No security requirement can be met because: there is no security config for scheme name \"headerConfig\"" + "No security requirement can be met because: there is no security secret configured for scheme name \"headerConfig\"" ) errorsFor("unhandledFormat") shouldBe List("Type 'number' in format 'decimal' is not supported") diff --git a/docs/Changelog.md b/docs/Changelog.md index 7e0ae403204..3986d46db21 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -27,6 +27,7 @@ * [#7184](https://github.com/TouK/nussknacker/pull/7184) Improve Nu Designer API notifications endpoint, to include events related to currently displayed scenario * [#7323](https://github.com/TouK/nussknacker/pull/7323) Improve Periodic DeploymentManager db queries * [#7332](https://github.com/TouK/nussknacker/pull/7332) Handle scenario names with spaces when performing migration tests, they were ignored +* [#7346](https://github.com/TouK/nussknacker/pull/7346) OpenAPI enricher: ability to configure common secret for any security scheme ## 1.18 diff --git a/docs/integration/OpenAPI.md b/docs/integration/OpenAPI.md index 66dc2bc85d5..5cfe8442ef5 100644 --- a/docs/integration/OpenAPI.md +++ b/docs/integration/OpenAPI.md @@ -67,15 +67,16 @@ components { } ``` -| Parameter | Required | Default | Description | -|------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| url | true | | URL of the [*OpenAPI interface definition*](https://swagger.io/specification/v3/). It contains definition of the service you want to interact with. | -| rootUrl | false | | The URL of the service. If not specified, the URL of the service is taken from the *OpenAPI interface definition*. | -| allowedMethods | false | ["GET"] | Usually only GET services should be used as enrichers are meant to be idempotent and not change data | -| namePattern | false | .* | Regexp for filtering operations by operationId (i.e. enricher name) | -| security | false | | Configuration for [authentication](https://swagger.io/docs/specification/authentication/) for each `securitySchemas` defined in the *OpenAPI interface definition* | -| security.*.type | false | | Type of security configuration for a given security schema. Currently only `apiKey` is supported | -| security.*.apiKeyValue | false | | API key that will be passed into the service via header, query parameter or cookie (depending on definition provided in OpenAPI) | +| Parameter | Required | Default | Description | +|------------------------|----------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| url | true | | URL of the [*OpenAPI interface definition*](https://swagger.io/specification/v3/). It contains definition of the service you want to interact with. | +| rootUrl | false | | The URL of the service. If not specified, the URL of the service is taken from the *OpenAPI interface definition*. | +| allowedMethods | false | ["GET"] | Usually only GET services should be used as enrichers are meant to be idempotent and not change data | +| namePattern | false | .* | Regexp for filtering operations by operationId (i.e. enricher name) | +| security | false | | Configuration for [authentication](https://swagger.io/docs/specification/authentication/) for each `securitySchemas` defined in the *OpenAPI interface definition* | +| security.*.type | false | | Type of security configuration for a given security schema. Currently only `apiKey` is supported | +| security.*.apiKeyValue | false | | API key that will be passed into the service via header, query parameter or cookie (depending on definition provided in OpenAPI) | +| secret | false | | Configuration for [authentication](https://swagger.io/docs/specification/authentication/) which matches any `securitySchemas` defined in the *OpenAPI interface definition*. This config entry has the same structure as values in `security` object (see above) | ## Operations From d0f0d3eeafbe2500f37ead5eb79c4dacff2dc378 Mon Sep 17 00:00:00 2001 From: Piotr Przybylski Date: Tue, 17 Dec 2024 11:26:05 +0100 Subject: [PATCH 2/7] Use `RichFunction.open(OpenContext)` instead of `RichFunction.open(Configuration)` (#7347) --- docs/MigrationGuide.md | 2 ++ .../engine/flink/api/datastream/StatefulFunction.scala | 5 ++--- .../flink/api/exception/WithExceptionHandler.scala | 5 ++--- .../api/process/FlinkContextInitializingFunction.scala | 5 ++--- .../api/process/FlinkLazyParameterFunctionHelper.scala | 7 +++---- .../engine/flink/api/process/RichLifecycleFunction.scala | 5 ++--- .../engine/flink/api/state/EvictableState.scala | 9 ++++----- .../util/function/CoProcessFunctionInterceptor.scala | 7 +++---- .../flink/util/function/ProcessFunctionInterceptor.scala | 8 +++----- ...ExtraWindowWhenNoDataTumblingAggregatorFunction.scala | 5 ++--- .../aggregate/EmitWhenEventLeftAggregatorFunction.scala | 7 +++---- .../transformer/aggregate/EnrichingWithKeyFunction.scala | 5 ++--- .../aggregate/OnEventTriggerWindowOperator.scala | 5 ++--- .../engine/flink/util/transformer/DelayTransformer.scala | 4 ++-- .../util/transformer/PreviousValueTransformer.scala | 7 +++---- .../engine/flink/table/aggregate/TableAggregation.scala | 5 ++--- .../nussknacker/engine/process/ProcessPartFunction.scala | 9 ++++----- .../process/registrar/AsyncInterpretationFunction.scala | 6 +++--- .../engine/process/registrar/SourceMetricsFunction.scala | 4 ++-- .../engine/kafka/source/flink/FlinkKafkaSource.scala | 7 +++---- .../sample/transformer/ConstantStateTransformer.scala | 6 +++--- .../sink/flink/FlinkKafkaUniversalSink.scala | 7 +++---- 22 files changed, 57 insertions(+), 73 deletions(-) diff --git a/docs/MigrationGuide.md b/docs/MigrationGuide.md index 46857f82e8b..9137f2400d3 100644 --- a/docs/MigrationGuide.md +++ b/docs/MigrationGuide.md @@ -26,6 +26,8 @@ To see the biggest differences please consult the [changelog](Changelog.md). * `def actionTooltips(processStatus: ProcessStatus): Map[ScenarioActionName, String]` - allows to define custom tooltips for actions, if not defined the default is still used * modified method: * `def statusActions(processStatus: ProcessStatus): List[ScenarioActionName]` - changed argument, to include information about latest and deployed versions +* [#7347](https://github.com/TouK/nussknacker/pull/7347) All calls to `org.apache.flink.api.common.functions.RichFunction.open(Configuration)`, + which is deprecated, were replaced with calls to `org.apache.flink.api.common.functions.RichFunction.open(OpenContext)` ## In version 1.18.0 diff --git a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/datastream/StatefulFunction.scala b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/datastream/StatefulFunction.scala index c67806ebdb5..0f545377e01 100644 --- a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/datastream/StatefulFunction.scala +++ b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/datastream/StatefulFunction.scala @@ -18,10 +18,9 @@ package pl.touk.nussknacker.engine.flink.api.datastream import org.apache.flink.annotation.Public -import org.apache.flink.api.common.functions.RichFunction +import org.apache.flink.api.common.functions.{OpenContext, RichFunction} import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} import org.apache.flink.api.common.typeutils.TypeSerializer -import org.apache.flink.configuration.Configuration /** * Trait implementing the functionality necessary to apply stateful functions in RichFunctions @@ -44,7 +43,7 @@ trait StatefulFunction[I, O, S] extends RichFunction { o } - override def open(c: Configuration) = { + override def open(openContext: OpenContext): Unit = { val info = new ValueStateDescriptor[S]("state", stateSerializer) state = getRuntimeContext().getState(info) } diff --git a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/exception/WithExceptionHandler.scala b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/exception/WithExceptionHandler.scala index 77a506f3e3a..89de86c9e55 100644 --- a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/exception/WithExceptionHandler.scala +++ b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/exception/WithExceptionHandler.scala @@ -1,7 +1,6 @@ package pl.touk.nussknacker.engine.flink.api.exception -import org.apache.flink.api.common.functions.{RichFunction, RuntimeContext} -import org.apache.flink.configuration.Configuration +import org.apache.flink.api.common.functions.{OpenContext, RichFunction, RuntimeContext} /** * Helper for using exception handler. @@ -15,7 +14,7 @@ trait WithExceptionHandler { protected var exceptionHandler: ExceptionHandler = _ - override def open(parameters: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { exceptionHandler = exceptionHandlerPreparer(getRuntimeContext) } diff --git a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkContextInitializingFunction.scala b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkContextInitializingFunction.scala index b4eaadee252..51e1d741f31 100644 --- a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkContextInitializingFunction.scala +++ b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkContextInitializingFunction.scala @@ -1,7 +1,6 @@ package pl.touk.nussknacker.engine.flink.api.process -import org.apache.flink.api.common.functions.{RichMapFunction, RuntimeContext} -import org.apache.flink.configuration.Configuration +import org.apache.flink.api.common.functions.{OpenContext, RichMapFunction, RuntimeContext} import pl.touk.nussknacker.engine.api.Context import pl.touk.nussknacker.engine.api.process.{ContextInitializer, ContextInitializingFunction} import pl.touk.nussknacker.engine.api.runtimecontext.EngineRuntimeContext @@ -14,7 +13,7 @@ class FlinkContextInitializingFunction[Raw]( private var initializingStrategy: ContextInitializingFunction[Raw] = _ - override def open(parameters: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { val contextIdGenerator = convertToEngineRuntimeContext(getRuntimeContext).contextIdGenerator(nodeId) initializingStrategy = contextInitializer.initContext(contextIdGenerator) } diff --git a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkLazyParameterFunctionHelper.scala b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkLazyParameterFunctionHelper.scala index 3399986c49a..de801f542c0 100644 --- a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkLazyParameterFunctionHelper.scala +++ b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkLazyParameterFunctionHelper.scala @@ -1,7 +1,6 @@ package pl.touk.nussknacker.engine.flink.api.process import org.apache.flink.api.common.functions._ -import org.apache.flink.configuration.Configuration import org.apache.flink.util.Collector import pl.touk.nussknacker.engine.api.LazyParameter.Evaluate import pl.touk.nussknacker.engine.api._ @@ -89,8 +88,8 @@ trait OneParamLazyParameterFunction[T <: AnyRef] extends LazyParameterInterprete protected def evaluateParameter(ctx: Context): T = _evaluateParameter(ctx) - override def open(parameters: Configuration): Unit = { - super.open(parameters) + override def open(openContext: OpenContext): Unit = { + super.open(openContext) _evaluateParameter = toEvaluateFunctionConverter.toEvaluateFunction(parameter) } @@ -117,7 +116,7 @@ trait LazyParameterInterpreterFunction { self: RichFunction => } // TODO: how can we make sure this will invoke super.open(...) (can't do it directly...) - override def open(parameters: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { toEvaluateFunctionConverter = lazyParameterHelper.createInterpreter(getRuntimeContext) exceptionHandler = lazyParameterHelper.exceptionHandler(getRuntimeContext) } diff --git a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/RichLifecycleFunction.scala b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/RichLifecycleFunction.scala index cb73532c81e..81d692d2d32 100644 --- a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/RichLifecycleFunction.scala +++ b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/RichLifecycleFunction.scala @@ -1,7 +1,6 @@ package pl.touk.nussknacker.engine.flink.api.process -import org.apache.flink.api.common.functions.{AbstractRichFunction, MapFunction, RuntimeContext} -import org.apache.flink.configuration.Configuration +import org.apache.flink.api.common.functions.{AbstractRichFunction, MapFunction, OpenContext, RuntimeContext} import pl.touk.nussknacker.engine.api.Lifecycle import pl.touk.nussknacker.engine.api.runtimecontext.EngineRuntimeContext @@ -11,7 +10,7 @@ abstract class RichLifecycleFunction extends AbstractRichFunction { protected val convertToEngineRuntimeContext: RuntimeContext => EngineRuntimeContext - override def open(parameters: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { lifecycle.open(convertToEngineRuntimeContext(getRuntimeContext)) } diff --git a/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/api/state/EvictableState.scala b/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/api/state/EvictableState.scala index 45d17502995..0068f849483 100644 --- a/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/api/state/EvictableState.scala +++ b/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/api/state/EvictableState.scala @@ -1,8 +1,7 @@ package pl.touk.nussknacker.engine.flink.api.state -import org.apache.flink.api.common.functions.RichFunction +import org.apache.flink.api.common.functions.{OpenContext, RichFunction} import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} -import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.TimerService import org.apache.flink.streaming.api.functions.KeyedProcessFunction import org.apache.flink.streaming.api.functions.co.CoProcessFunction @@ -18,8 +17,8 @@ abstract class EvictableStateFunction[In, Out, StateType] extends KeyedProcessFu protected def stateDescriptor: ValueStateDescriptor[StateType] - override def open(parameters: Configuration): Unit = { - super.open(parameters) + override def open(openContext: OpenContext): Unit = { + super.open(openContext) lastEventTimeForKey = getRuntimeContext.getState[java.lang.Long]( new ValueStateDescriptor[java.lang.Long]("timers", classOf[java.lang.Long]) ) @@ -91,7 +90,7 @@ trait LatelyEvictableStateFunctionMixin[StateType] extends RichFunction with Sta @transient protected var state: ValueState[StateType] = _ - override def open(parameters: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { latestEvictionTimeForKey = getRuntimeContext.getState[java.lang.Long]( new ValueStateDescriptor[java.lang.Long]("timers", classOf[java.lang.Long]) ) diff --git a/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/function/CoProcessFunctionInterceptor.scala b/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/function/CoProcessFunctionInterceptor.scala index bd38add02ed..8a0b58d5561 100644 --- a/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/function/CoProcessFunctionInterceptor.scala +++ b/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/function/CoProcessFunctionInterceptor.scala @@ -1,7 +1,6 @@ package pl.touk.nussknacker.engine.flink.util.function -import org.apache.flink.api.common.functions.RuntimeContext -import org.apache.flink.configuration.Configuration +import org.apache.flink.api.common.functions.{OpenContext, RuntimeContext} import org.apache.flink.streaming.api.functions.co.CoProcessFunction import org.apache.flink.util.Collector @@ -12,8 +11,8 @@ import org.apache.flink.util.Collector abstract class CoProcessFunctionInterceptor[IN1, IN2, OUT](underlying: CoProcessFunction[IN1, IN2, OUT]) extends CoProcessFunction[IN1, IN2, OUT] { - override def open(parameters: Configuration): Unit = { - underlying.open(parameters) + override def open(openContext: OpenContext): Unit = { + underlying.open(openContext) } override def setRuntimeContext(ctx: RuntimeContext): Unit = { diff --git a/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/function/ProcessFunctionInterceptor.scala b/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/function/ProcessFunctionInterceptor.scala index bc4096fde18..41e7b1aef94 100644 --- a/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/function/ProcessFunctionInterceptor.scala +++ b/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/function/ProcessFunctionInterceptor.scala @@ -1,16 +1,14 @@ package pl.touk.nussknacker.engine.flink.util.function -import org.apache.flink.api.common.functions.RuntimeContext -import org.apache.flink.configuration.Configuration +import org.apache.flink.api.common.functions.{OpenContext, RuntimeContext} import org.apache.flink.streaming.api.functions.KeyedProcessFunction -import org.apache.flink.streaming.api.functions.co.CoProcessFunction import org.apache.flink.util.Collector abstract class ProcessFunctionInterceptor[IN, OUT](underlying: KeyedProcessFunction[String, IN, OUT]) extends KeyedProcessFunction[String, IN, OUT] { - override def open(parameters: Configuration): Unit = { - underlying.open(parameters) + override def open(openContext: OpenContext): Unit = { + underlying.open(openContext) } override def setRuntimeContext(ctx: RuntimeContext): Unit = { diff --git a/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EmitExtraWindowWhenNoDataTumblingAggregatorFunction.scala b/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EmitExtraWindowWhenNoDataTumblingAggregatorFunction.scala index c6fd168e6df..dc5fcaf0c2d 100644 --- a/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EmitExtraWindowWhenNoDataTumblingAggregatorFunction.scala +++ b/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EmitExtraWindowWhenNoDataTumblingAggregatorFunction.scala @@ -1,9 +1,8 @@ package pl.touk.nussknacker.engine.flink.util.transformer.aggregate -import org.apache.flink.api.common.functions.RuntimeContext +import org.apache.flink.api.common.functions.{OpenContext, RuntimeContext} import org.apache.flink.api.common.state.ValueState import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.TimerService import org.apache.flink.streaming.api.functions.KeyedProcessFunction import org.apache.flink.util.Collector @@ -47,7 +46,7 @@ class EmitExtraWindowWhenNoDataTumblingAggregatorFunction[MapT[K, V]]( @transient private var contextIdGenerator: ContextIdGenerator = _ - override def open(parameters: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { state = getRuntimeContext.getState(stateDescriptor) contextIdGenerator = convertToEngineRuntimeContext(getRuntimeContext).contextIdGenerator(nodeId.id) } diff --git a/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EmitWhenEventLeftAggregatorFunction.scala b/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EmitWhenEventLeftAggregatorFunction.scala index 8955783b8cb..d7605bfb92e 100644 --- a/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EmitWhenEventLeftAggregatorFunction.scala +++ b/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EmitWhenEventLeftAggregatorFunction.scala @@ -1,8 +1,7 @@ package pl.touk.nussknacker.engine.flink.util.transformer.aggregate -import org.apache.flink.api.common.functions.RuntimeContext +import org.apache.flink.api.common.functions.{OpenContext, RuntimeContext} import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.TimerService import org.apache.flink.streaming.api.functions.KeyedProcessFunction import org.apache.flink.util.Collector @@ -42,8 +41,8 @@ class EmitWhenEventLeftAggregatorFunction[MapT[K, V]]( type FlinkOnTimerCtx = KeyedProcessFunction[String, ValueWithContext[StringKeyedValue[AnyRef]], ValueWithContext[AnyRef]]#OnTimerContext - override def open(parameters: Configuration): Unit = { - super.open(parameters) + override def open(openContext: OpenContext): Unit = { + super.open(openContext) contextIdGenerator = convertToEngineRuntimeContext(getRuntimeContext).contextIdGenerator(nodeId.id) } diff --git a/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EnrichingWithKeyFunction.scala b/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EnrichingWithKeyFunction.scala index cdc45efbe7f..97824a1ccef 100644 --- a/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EnrichingWithKeyFunction.scala +++ b/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/EnrichingWithKeyFunction.scala @@ -1,7 +1,6 @@ package pl.touk.nussknacker.engine.flink.util.transformer.aggregate -import org.apache.flink.api.common.functions.RuntimeContext -import org.apache.flink.configuration.Configuration +import org.apache.flink.api.common.functions.{OpenContext, RuntimeContext} import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction import org.apache.flink.streaming.api.windowing.windows.TimeWindow import org.apache.flink.util.Collector @@ -18,7 +17,7 @@ class EnrichingWithKeyFunction(convertToEngineRuntimeContext: RuntimeContext => @transient private var contextIdGenerator: ContextIdGenerator = _ - override def open(parameters: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { contextIdGenerator = convertToEngineRuntimeContext(getRuntimeContext).contextIdGenerator(nodeId) } diff --git a/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/OnEventTriggerWindowOperator.scala b/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/OnEventTriggerWindowOperator.scala index d9026604ef2..1ca336bcdff 100644 --- a/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/OnEventTriggerWindowOperator.scala +++ b/engine/flink/components/base-unbounded/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/OnEventTriggerWindowOperator.scala @@ -1,9 +1,8 @@ package pl.touk.nussknacker.engine.flink.util.transformer.aggregate import com.github.ghik.silencer.silent -import org.apache.flink.api.common.functions.{AggregateFunction, RuntimeContext} +import org.apache.flink.api.common.functions.{AggregateFunction, OpenContext, RuntimeContext} import org.apache.flink.api.common.state.AggregatingStateDescriptor -import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.datastream.{KeyedStream, SingleOutputStreamOperator} import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction import org.apache.flink.streaming.api.windowing.assigners.WindowAssigner @@ -102,7 +101,7 @@ private class ValueEmittingWindowFunction( @transient private var contextIdGenerator: ContextIdGenerator = _ - override def open(parameters: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { contextIdGenerator = convertToEngineRuntimeContext(getRuntimeContext).contextIdGenerator(nodeId) } diff --git a/engine/flink/components/base/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/DelayTransformer.scala b/engine/flink/components/base/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/DelayTransformer.scala index 2e9cd878090..820c3bb8678 100644 --- a/engine/flink/components/base/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/DelayTransformer.scala +++ b/engine/flink/components/base/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/DelayTransformer.scala @@ -1,9 +1,9 @@ package pl.touk.nussknacker.engine.flink.util.transformer +import org.apache.flink.api.common.functions.OpenContext import org.apache.flink.api.common.state.{MapState, MapStateDescriptor} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.ListTypeInfo -import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.datastream.DataStream import org.apache.flink.streaming.api.functions.KeyedProcessFunction import org.apache.flink.util.Collector @@ -75,7 +75,7 @@ class DelayFunction(nodeCtx: FlinkCustomNodeContext, delay: Duration) @transient private var state: MapState[Long, java.util.List[api.Context]] = _ - override def open(config: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { state = getRuntimeContext.getMapState(descriptor) } diff --git a/engine/flink/components/base/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/PreviousValueTransformer.scala b/engine/flink/components/base/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/PreviousValueTransformer.scala index ad14927bf49..88f9d144bc2 100644 --- a/engine/flink/components/base/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/PreviousValueTransformer.scala +++ b/engine/flink/components/base/src/main/scala/pl/touk/nussknacker/engine/flink/util/transformer/PreviousValueTransformer.scala @@ -1,9 +1,8 @@ package pl.touk.nussknacker.engine.flink.util.transformer -import org.apache.flink.api.common.functions.RichFlatMapFunction +import org.apache.flink.api.common.functions.{OpenContext, RichFlatMapFunction} import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.datastream.DataStream import org.apache.flink.util.Collector import pl.touk.nussknacker.engine.api._ @@ -46,8 +45,8 @@ case object PreviousValueTransformer extends CustomStreamTransformer with Explic private[this] var state: ValueState[Value] = _ - override def open(c: Configuration): Unit = { - super.open(c) + override def open(openContext: OpenContext): Unit = { + super.open(openContext) val info = new ValueStateDescriptor[Value]("state", typeInformation) state = getRuntimeContext.getState(info) } diff --git a/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/aggregate/TableAggregation.scala b/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/aggregate/TableAggregation.scala index 8b21893cf9f..8c0eb79a15b 100644 --- a/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/aggregate/TableAggregation.scala +++ b/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/aggregate/TableAggregation.scala @@ -1,8 +1,7 @@ package pl.touk.nussknacker.engine.flink.table.aggregate -import org.apache.flink.api.common.functions.{FlatMapFunction, RuntimeContext} +import org.apache.flink.api.common.functions.{FlatMapFunction, OpenContext, RuntimeContext} import org.apache.flink.api.common.typeinfo.{TypeInformation, Types} -import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.datastream.DataStream import org.apache.flink.streaming.api.functions.ProcessFunction import org.apache.flink.table.api.Expressions.{$, call} @@ -116,7 +115,7 @@ class TableAggregation( @transient private var contextIdGenerator: ContextIdGenerator = _ - override def open(configuration: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { contextIdGenerator = convertToEngineRuntimeContext(getRuntimeContext).contextIdGenerator(nodeId.toString) } diff --git a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/ProcessPartFunction.scala b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/ProcessPartFunction.scala index 5270feaaccb..37cf43d7524 100644 --- a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/ProcessPartFunction.scala +++ b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/ProcessPartFunction.scala @@ -1,7 +1,6 @@ package pl.touk.nussknacker.engine.process -import org.apache.flink.api.common.functions.RichFunction -import org.apache.flink.configuration.Configuration +import org.apache.flink.api.common.functions.{OpenContext, RichFunction} import pl.touk.nussknacker.engine.graph.node.NodeData import pl.touk.nussknacker.engine.process.compiler.FlinkProcessCompilerData import pl.touk.nussknacker.engine.process.exception.FlinkExceptionHandler @@ -22,8 +21,8 @@ trait ProcessPartFunction extends ExceptionHandlerFunction { } } - override def open(parameters: Configuration): Unit = { - super.open(parameters) + override def open(openContext: OpenContext): Unit = { + super.open(openContext) compilerData.open(getRuntimeContext, nodesUsed) } @@ -46,7 +45,7 @@ trait ExceptionHandlerFunction extends RichFunction { } } - override def open(parameters: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { exceptionHandler = compilerData.prepareExceptionHandler(getRuntimeContext) } diff --git a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/AsyncInterpretationFunction.scala b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/AsyncInterpretationFunction.scala index 75f887c6c18..6901b7ccb1f 100644 --- a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/AsyncInterpretationFunction.scala +++ b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/AsyncInterpretationFunction.scala @@ -3,7 +3,7 @@ package pl.touk.nussknacker.engine.process.registrar import cats.effect.IO import cats.effect.unsafe.IORuntime import com.typesafe.scalalogging.LazyLogging -import org.apache.flink.configuration.Configuration +import org.apache.flink.api.common.functions.OpenContext import org.apache.flink.streaming.api.functions.async.{ResultFuture, RichAsyncFunction} import pl.touk.nussknacker.engine.InterpretationResult import pl.touk.nussknacker.engine.Interpreter.FutureShape @@ -35,8 +35,8 @@ private[registrar] class AsyncInterpretationFunction( private var serviceExecutionContext: ServiceExecutionContext = _ - override def open(parameters: Configuration): Unit = { - super.open(parameters) + override def open(openContext: OpenContext): Unit = { + super.open(openContext) getRuntimeContext.registerUserCodeClassLoaderReleaseHookIfAbsent( "closeAsyncExecutionContext", diff --git a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/SourceMetricsFunction.scala b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/SourceMetricsFunction.scala index 5174ba94be6..3a0cd4d9427 100644 --- a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/SourceMetricsFunction.scala +++ b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/SourceMetricsFunction.scala @@ -1,6 +1,6 @@ package pl.touk.nussknacker.engine.process.registrar -import org.apache.flink.configuration.Configuration +import org.apache.flink.api.common.functions.OpenContext import org.apache.flink.streaming.api.functions.ProcessFunction import org.apache.flink.util.Collector import pl.touk.nussknacker.engine.api.process.ComponentUseCase @@ -12,7 +12,7 @@ private[registrar] class SourceMetricsFunction[T](sourceId: String, componentUse @transient private var metrics: OneSourceMetrics = _ - override def open(parameters: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { metrics = new OneSourceMetrics(sourceId) val metricsProvider = createMetricsProvider(componentUseCase, getRuntimeContext) metrics.registerOwnMetrics(metricsProvider) diff --git a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala index 3deb771269b..ee856a4e369 100644 --- a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala +++ b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala @@ -3,8 +3,7 @@ package pl.touk.nussknacker.engine.kafka.source.flink import cats.data.NonEmptyList import com.github.ghik.silencer.silent import com.typesafe.scalalogging.LazyLogging -import org.apache.flink.api.common.functions.RuntimeContext -import org.apache.flink.configuration.Configuration +import org.apache.flink.api.common.functions.{OpenContext, RuntimeContext} import org.apache.flink.streaming.api.datastream.DataStreamSource import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment import org.apache.flink.streaming.api.functions.source.SourceFunction @@ -166,9 +165,9 @@ class FlinkKafkaConsumerHandlingExceptions[T]( protected var exceptionPurposeContextIdGenerator: ContextIdGenerator = _ - override def open(parameters: Configuration): Unit = { + override def open(openContext: OpenContext): Unit = { patchRestoredState() - super.open(parameters) + super.open(openContext) exceptionHandler = exceptionHandlerPreparer(getRuntimeContext) exceptionPurposeContextIdGenerator = convertToEngineRuntimeContext(getRuntimeContext).contextIdGenerator(nodeId.id) deserializationSchema.setExceptionHandlingData(exceptionHandler, exceptionPurposeContextIdGenerator, nodeId) diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/transformer/ConstantStateTransformer.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/transformer/ConstantStateTransformer.scala index a2f782e5bde..2b6de4ae659 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/transformer/ConstantStateTransformer.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/transformer/ConstantStateTransformer.scala @@ -1,6 +1,6 @@ package pl.touk.nussknacker.engine.management.sample.transformer -import org.apache.flink.api.common.functions.RichMapFunction +import org.apache.flink.api.common.functions.{OpenContext, RichMapFunction} import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.configuration.Configuration @@ -22,8 +22,8 @@ case class ConstantStateTransformer[T: TypeInformation](defaultValue: T) extends var constantState: ValueState[T] = _ - override def open(parameters: Configuration): Unit = { - super.open(parameters) + override def open(openContext: OpenContext): Unit = { + super.open(openContext) val descriptor = new ValueStateDescriptor[T]("constantState", implicitly[TypeInformation[T]]) constantState = getRuntimeContext.getState(descriptor) } diff --git a/engine/flink/schemed-kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/schemedkafka/sink/flink/FlinkKafkaUniversalSink.scala b/engine/flink/schemed-kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/schemedkafka/sink/flink/FlinkKafkaUniversalSink.scala index 4ef5e35d98a..7c3531147a5 100644 --- a/engine/flink/schemed-kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/schemedkafka/sink/flink/FlinkKafkaUniversalSink.scala +++ b/engine/flink/schemed-kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/schemedkafka/sink/flink/FlinkKafkaUniversalSink.scala @@ -2,9 +2,8 @@ package pl.touk.nussknacker.engine.schemedkafka.sink.flink import com.typesafe.scalalogging.LazyLogging import io.confluent.kafka.schemaregistry.ParsedSchema -import org.apache.flink.api.common.functions.{RichMapFunction, RuntimeContext} +import org.apache.flink.api.common.functions.{OpenContext, RichMapFunction, RuntimeContext} import org.apache.flink.api.common.typeinfo.{TypeInformation, Types} -import org.apache.flink.configuration.Configuration import org.apache.flink.formats.avro.typeutils.NkSerializableParsedSchema import org.apache.flink.streaming.api.datastream.{DataStream, DataStreamSink} import org.apache.flink.streaming.api.functions.sink.SinkFunction @@ -81,8 +80,8 @@ class FlinkKafkaUniversalSink( @transient private var encodeRecord: Any => AnyRef = _ - override def open(parameters: Configuration): Unit = { - super.open(parameters) + override def open(openContext: OpenContext): Unit = { + super.open(openContext) encodeRecord = schemaSupportDispatcher .forSchemaType(schema.getParsedSchema.schemaType()) .formValueEncoder(schema.getParsedSchema, validationMode) From 5069ac1a9130bfd4439416cb92c2f3ef4f153a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Tue, 17 Dec 2024 11:32:39 +0100 Subject: [PATCH 3/7] [NU-1777] "Perform single execution" action was renamed to "Run off schedule" (#7342) --- ...gle-execution.svg => run-off-schedule.svg} | 0 .../components/Process/ProcessStateUtils.ts | 6 ++-- .../client/src/components/Process/types.ts | 2 +- .../buttons/BuiltinButtonTypes.ts | 2 +- .../buttons/TOOLBAR_BUTTONS_MAP.ts | 4 +-- .../toolbarSettings/defaultToolbarsConfig.ts | 2 +- ...ionButton.tsx => RunOffScheduleButton.tsx} | 23 +++++++------ .../src/containers/event-tracking/helpers.ts | 4 +-- .../use-register-tracking-events.ts | 2 +- designer/client/src/http/HttpService.ts | 4 +-- .../client/src/reducers/selectors/graph.ts | 8 ++--- .../api/deployment/DMScenarioCommand.scala | 4 +-- .../testing/DeploymentManagerStub.scala | 2 +- ...uest.scala => RunOffScheduleRequest.scala} | 2 +- ...nse.scala => RunOffScheduleResponse.scala} | 2 +- .../main/resources/defaultDesignerConfig.conf | 2 +- .../ui/api/ManagementResources.scala | 33 ++++++++++--------- .../scenariotoolbar/ToolbarButtonConfig.scala | 8 ++--- .../deployment/DeploymentService.scala | 16 ++++----- .../process/deployment/ScenarioCommand.scala | 6 ++-- .../repository/ScenarioActionRepository.scala | 4 +-- ...tionsAndCommentsToScenarioActivities.scala | 2 +- ...DevelopmentDeploymentManagerProvider.scala | 10 +++--- .../periodic/PeriodicDeploymentManager.scala | 18 +++++----- ...eriodicProcessStateDefinitionManager.scala | 2 +- .../periodic/PeriodicStateStatus.scala | 8 ++--- ...dicProcessStateDefinitionManagerTest.scala | 4 +-- .../management/FlinkDeploymentManager.scala | 4 +-- .../embedded/EmbeddedDeploymentManager.scala | 2 +- .../k8s/manager/K8sDeploymentManager.scala | 2 +- .../engine/api/deployment/ProcessAction.scala | 17 +++++----- .../deployment/CustomActionDefinition.scala | 2 +- 32 files changed, 101 insertions(+), 106 deletions(-) rename designer/client/src/assets/img/toolbarButtons/{perform-single-execution.svg => run-off-schedule.svg} (100%) rename designer/client/src/components/toolbars/scenarioActions/buttons/{PerformSingleExecutionButton.tsx => RunOffScheduleButton.tsx} (70%) rename designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/{PerformSingleExecutionRequest.scala => RunOffScheduleRequest.scala} (65%) rename designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/{PerformSingleExecutionResponse.scala => RunOffScheduleResponse.scala} (51%) diff --git a/designer/client/src/assets/img/toolbarButtons/perform-single-execution.svg b/designer/client/src/assets/img/toolbarButtons/run-off-schedule.svg similarity index 100% rename from designer/client/src/assets/img/toolbarButtons/perform-single-execution.svg rename to designer/client/src/assets/img/toolbarButtons/run-off-schedule.svg diff --git a/designer/client/src/components/Process/ProcessStateUtils.ts b/designer/client/src/components/Process/ProcessStateUtils.ts index 520b380fed2..650ebcee894 100644 --- a/designer/client/src/components/Process/ProcessStateUtils.ts +++ b/designer/client/src/components/Process/ProcessStateUtils.ts @@ -18,11 +18,9 @@ class ProcessStateUtils { public canArchive = (state: ProcessStateType): boolean => state?.allowedActions.includes(PredefinedActionName.Archive); - public canSeePerformSingleExecution = (state: ProcessStateType): boolean => - state?.visibleActions.includes(PredefinedActionName.PerformSingleExecution); + public canSeeRunOffSchedule = (state: ProcessStateType): boolean => state?.visibleActions.includes(PredefinedActionName.RunOffSchedule); - public canPerformSingleExecution = (state: ProcessStateType): boolean => - state?.allowedActions.includes(PredefinedActionName.PerformSingleExecution); + public canRunOffSchedule = (state: ProcessStateType): boolean => state?.allowedActions.includes(PredefinedActionName.RunOffSchedule); getStateDescription({ isArchived, isFragment }: Scenario, processState: ProcessStateType): string { if (isArchived) { diff --git a/designer/client/src/components/Process/types.ts b/designer/client/src/components/Process/types.ts index f5cbd62763a..cfb7d590d65 100644 --- a/designer/client/src/components/Process/types.ts +++ b/designer/client/src/components/Process/types.ts @@ -9,7 +9,7 @@ export enum PredefinedActionName { Archive = "ARCHIVE", UnArchive = "UNARCHIVE", Pause = "PAUSE", - PerformSingleExecution = "PERFORM_SINGLE_EXECUTION", + RunOffSchedule = "RUN_OFF_SCHEDULE", } export type ActionName = string; diff --git a/designer/client/src/components/toolbarSettings/buttons/BuiltinButtonTypes.ts b/designer/client/src/components/toolbarSettings/buttons/BuiltinButtonTypes.ts index ec4816131a4..6e2f87d29bb 100644 --- a/designer/client/src/components/toolbarSettings/buttons/BuiltinButtonTypes.ts +++ b/designer/client/src/components/toolbarSettings/buttons/BuiltinButtonTypes.ts @@ -2,7 +2,7 @@ export enum BuiltinButtonTypes { processSave = "process-save", processDeploy = "process-deploy", processCancel = "process-cancel", - processPerformSingleExecution = "process-perform-single-execution", + processRunOffSchedule = "process-run-off-schedule", editUndo = "edit-undo", editRedo = "edit-redo", editCopy = "edit-copy", diff --git a/designer/client/src/components/toolbarSettings/buttons/TOOLBAR_BUTTONS_MAP.ts b/designer/client/src/components/toolbarSettings/buttons/TOOLBAR_BUTTONS_MAP.ts index 6f02c96d672..817f034173d 100644 --- a/designer/client/src/components/toolbarSettings/buttons/TOOLBAR_BUTTONS_MAP.ts +++ b/designer/client/src/components/toolbarSettings/buttons/TOOLBAR_BUTTONS_MAP.ts @@ -31,7 +31,7 @@ import { ZoomOutButton } from "../../toolbars/view/buttons/ZoomOutButton"; import { BuiltinButtonTypes } from "./BuiltinButtonTypes"; import { CustomButtonTypes } from "./CustomButtonTypes"; import { ToolbarButton, ToolbarButtonTypes } from "./types"; -import PerformSingleExecutionButton from "../../toolbars/scenarioActions/buttons/PerformSingleExecutionButton"; +import RunOffScheduleButton from "../../toolbars/scenarioActions/buttons/RunOffScheduleButton"; export type PropsOfButton = ToolbarButton & { type: T; @@ -45,7 +45,7 @@ export const TOOLBAR_BUTTONS_MAP: ToolbarButtonsMap = { [BuiltinButtonTypes.processSave]: SaveButton, [BuiltinButtonTypes.processDeploy]: DeployButton, [BuiltinButtonTypes.processCancel]: CancelDeployButton, - [BuiltinButtonTypes.processPerformSingleExecution]: PerformSingleExecutionButton, + [BuiltinButtonTypes.processRunOffSchedule]: RunOffScheduleButton, [BuiltinButtonTypes.viewZoomIn]: ZoomInButton, [BuiltinButtonTypes.viewZoomOut]: ZoomOutButton, [BuiltinButtonTypes.viewReset]: ResetViewButton, diff --git a/designer/client/src/components/toolbarSettings/defaultToolbarsConfig.ts b/designer/client/src/components/toolbarSettings/defaultToolbarsConfig.ts index ee30210c9ef..9ae6b2376a4 100644 --- a/designer/client/src/components/toolbarSettings/defaultToolbarsConfig.ts +++ b/designer/client/src/components/toolbarSettings/defaultToolbarsConfig.ts @@ -29,7 +29,7 @@ export function defaultToolbarsConfig(isFragment: boolean, isArchived: boolean): { type: BuiltinButtonTypes.processSave }, { type: BuiltinButtonTypes.processDeploy }, { type: BuiltinButtonTypes.processCancel }, - { type: BuiltinButtonTypes.processPerformSingleExecution }, + { type: BuiltinButtonTypes.processRunOffSchedule }, ], }, { diff --git a/designer/client/src/components/toolbars/scenarioActions/buttons/PerformSingleExecutionButton.tsx b/designer/client/src/components/toolbars/scenarioActions/buttons/RunOffScheduleButton.tsx similarity index 70% rename from designer/client/src/components/toolbars/scenarioActions/buttons/PerformSingleExecutionButton.tsx rename to designer/client/src/components/toolbars/scenarioActions/buttons/RunOffScheduleButton.tsx index 7dd1f13bd2a..2dec4aaa1a1 100644 --- a/designer/client/src/components/toolbars/scenarioActions/buttons/PerformSingleExecutionButton.tsx +++ b/designer/client/src/components/toolbars/scenarioActions/buttons/RunOffScheduleButton.tsx @@ -2,13 +2,13 @@ import React from "react"; import { useTranslation } from "react-i18next"; import { useDispatch, useSelector } from "react-redux"; import { loadProcessState } from "../../../../actions/nk"; -import Icon from "../../../../assets/img/toolbarButtons/perform-single-execution.svg"; +import Icon from "../../../../assets/img/toolbarButtons/run-off-schedule.svg"; import HttpService from "../../../../http/HttpService"; import { getProcessName, getProcessVersionId, - isPerformSingleExecutionPossible, - isPerformSingleExecutionVisible, + isRunOffSchedulePossible, + isRunOffScheduleVisible, } from "../../../../reducers/selectors/graph"; import { getCapabilities } from "../../../../reducers/selectors/other"; import { useWindows, WindowKind } from "../../../../windowManager"; @@ -21,30 +21,29 @@ import { RootState } from "../../../../reducers"; import { getProcessState } from "../../../../reducers/selectors/scenarioState"; import { PredefinedActionName } from "../../../Process/types"; -export default function PerformSingleExecutionButton(props: ToolbarButtonProps) { +export default function RunOffScheduleButton(props: ToolbarButtonProps) { const { t } = useTranslation(); const dispatch = useDispatch(); const { disabled, type } = props; const scenarioState = useSelector((state: RootState) => getProcessState(state)); - const isVisible = useSelector(isPerformSingleExecutionVisible); - const isPossible = useSelector(isPerformSingleExecutionPossible); + const isVisible = useSelector(isRunOffScheduleVisible); + const isPossible = useSelector(isRunOffSchedulePossible); const processName = useSelector(getProcessName); const processVersionId = useSelector(getProcessVersionId); const capabilities = useSelector(getCapabilities); const available = !disabled && isPossible && capabilities.deploy; const { open } = useWindows(); - const action = (p, c) => - HttpService.performSingleExecution(p, c).finally(() => dispatch(loadProcessState(processName, processVersionId))); - const message = t("panels.actions.perform-single-execution.dialog", "Perform single execution", { name: processName }); + const action = (p, c) => HttpService.runOffSchedule(p, c).finally(() => dispatch(loadProcessState(processName, processVersionId))); + const message = t("panels.actions.run-of-out-schedule.dialog", "Perform single execution", { name: processName }); - const defaultTooltip = t("panels.actions.perform-single-execution.tooltip", "run now"); - const tooltip = ProcessStateUtils.getActionCustomTooltip(scenarioState, PredefinedActionName.PerformSingleExecution) ?? defaultTooltip; + const defaultTooltip = t("panels.actions.run-off-schedule.tooltip", "run now"); + const tooltip = ProcessStateUtils.getActionCustomTooltip(scenarioState, PredefinedActionName.RunOffSchedule) ?? defaultTooltip; if (isVisible) { return ( } diff --git a/designer/client/src/containers/event-tracking/helpers.ts b/designer/client/src/containers/event-tracking/helpers.ts index 824538f0f53..531eeed0cf8 100644 --- a/designer/client/src/containers/event-tracking/helpers.ts +++ b/designer/client/src/containers/event-tracking/helpers.ts @@ -92,8 +92,8 @@ export const mapToolbarButtonToStatisticsEvent = ( case BuiltinButtonTypes.processCancel: { return EventTrackingSelector.ScenarioCancel; } - case BuiltinButtonTypes.processPerformSingleExecution: { - return EventTrackingSelector.ScenarioPerformSingleExecution; + case BuiltinButtonTypes.processRunOffSchedule: { + return EventTrackingSelector.ScenarioRunOffSchedule; } case BuiltinButtonTypes.processArchiveToggle: { return EventTrackingSelector.ScenarioArchiveToggle; diff --git a/designer/client/src/containers/event-tracking/use-register-tracking-events.ts b/designer/client/src/containers/event-tracking/use-register-tracking-events.ts index a11d28e25b4..dd656396ea4 100644 --- a/designer/client/src/containers/event-tracking/use-register-tracking-events.ts +++ b/designer/client/src/containers/event-tracking/use-register-tracking-events.ts @@ -51,7 +51,7 @@ enum ClickEventsSelector { ScenarioSave = "SCENARIO_SAVE", TestCounts = "TEST_COUNTS", ScenarioCancel = "SCENARIO_CANCEL", - ScenarioPerformSingleExecution = "SCENARIO_PERFORM_SINGLE_EXECUTION", + ScenarioRunOffSchedule = "SCENARIO_RUN_OFF_SCHEDULE", ScenarioArchiveToggle = "SCENARIO_ARCHIVE_TOGGLE", ScenarioUnarchive = "SCENARIO_UNARCHIVE", ScenarioCustomAction = "SCENARIO_CUSTOM_ACTION", diff --git a/designer/client/src/http/HttpService.ts b/designer/client/src/http/HttpService.ts index d5bae6579c2..375d2ec3c9b 100644 --- a/designer/client/src/http/HttpService.ts +++ b/designer/client/src/http/HttpService.ts @@ -360,12 +360,12 @@ class HttpService { }); } - performSingleExecution(processName: string, comment?: string) { + runOffSchedule(processName: string, comment?: string) { const data = { comment: comment, }; return api - .post(`/processManagement/performSingleExecution/${encodeURIComponent(processName)}`, data) + .post(`/processManagement/runOffSchedule/${encodeURIComponent(processName)}`, data) .then((res) => { const msg = res.data.msg; this.#addInfo(msg); diff --git a/designer/client/src/reducers/selectors/graph.ts b/designer/client/src/reducers/selectors/graph.ts index 5c5d92b9dc6..3fbf656d70a 100644 --- a/designer/client/src/reducers/selectors/graph.ts +++ b/designer/client/src/reducers/selectors/graph.ts @@ -57,12 +57,10 @@ export const isDeployedVersion = createSelector( (visibleVersion, deployedVersion) => visibleVersion === deployedVersion, ); export const isCancelPossible = createSelector(getProcessState, (state) => ProcessStateUtils.canCancel(state)); -export const isPerformSingleExecutionVisible = createSelector([getProcessState], (state) => - ProcessStateUtils.canSeePerformSingleExecution(state), -); -export const isPerformSingleExecutionPossible = createSelector( +export const isRunOffScheduleVisible = createSelector([getProcessState], (state) => ProcessStateUtils.canSeeRunOffSchedule(state)); +export const isRunOffSchedulePossible = createSelector( [hasError, getProcessState, isFragment], - (error, state, fragment) => !fragment && !error && ProcessStateUtils.canPerformSingleExecution(state), + (error, state, fragment) => !fragment && !error && ProcessStateUtils.canRunOffSchedule(state), ); export const isMigrationPossible = createSelector( [isSaveDisabled, hasError, getProcessState, isFragment], diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DMScenarioCommand.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DMScenarioCommand.scala index a9a4f1ffc76..a01e89e8118 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DMScenarioCommand.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DMScenarioCommand.scala @@ -81,8 +81,8 @@ case class DMCancelScenarioCommand(scenarioName: ProcessName, user: User) extend case class DMStopScenarioCommand(scenarioName: ProcessName, savepointDir: Option[String], user: User) extends DMScenarioCommand[SavepointResult] -case class DMPerformSingleExecutionCommand( +case class DMRunOffScheduleCommand( processVersion: ProcessVersion, canonicalProcess: CanonicalProcess, user: User, -) extends DMScenarioCommand[SingleExecutionResult] +) extends DMScenarioCommand[RunOffScheduleResult] diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/testing/DeploymentManagerStub.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/testing/DeploymentManagerStub.scala index 8bb56fdf704..5f483981534 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/testing/DeploymentManagerStub.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/testing/DeploymentManagerStub.scala @@ -76,7 +76,7 @@ trait StubbingCommands { self: DeploymentManager => case _: DMCancelDeploymentCommand => Future.successful(()) case _: DMCancelScenarioCommand => Future.successful(()) case _: DMMakeScenarioSavepointCommand => Future.successful(SavepointResult("")) - case _: DMPerformSingleExecutionCommand | _: DMCustomActionCommand | _: DMTestScenarioCommand => notImplemented + case _: DMRunOffScheduleCommand | _: DMCustomActionCommand | _: DMTestScenarioCommand => notImplemented } } diff --git a/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/PerformSingleExecutionRequest.scala b/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/RunOffScheduleRequest.scala similarity index 65% rename from designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/PerformSingleExecutionRequest.scala rename to designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/RunOffScheduleRequest.scala index 1a8659bedcf..94c9371b3a4 100644 --- a/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/PerformSingleExecutionRequest.scala +++ b/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/RunOffScheduleRequest.scala @@ -2,6 +2,6 @@ package pl.touk.nussknacker.restmodel import io.circe.generic.JsonCodec -@JsonCodec final case class PerformSingleExecutionRequest( +@JsonCodec final case class RunOffScheduleRequest( comment: Option[String] = None, ) diff --git a/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/PerformSingleExecutionResponse.scala b/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/RunOffScheduleResponse.scala similarity index 51% rename from designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/PerformSingleExecutionResponse.scala rename to designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/RunOffScheduleResponse.scala index 420e75e6340..ab83360cc9f 100644 --- a/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/PerformSingleExecutionResponse.scala +++ b/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/RunOffScheduleResponse.scala @@ -3,4 +3,4 @@ package pl.touk.nussknacker.restmodel import io.circe.generic.JsonCodec @JsonCodec -final case class PerformSingleExecutionResponse(isSuccess: Boolean, msg: String) +final case class RunOffScheduleResponse(isSuccess: Boolean, msg: String) diff --git a/designer/server/src/main/resources/defaultDesignerConfig.conf b/designer/server/src/main/resources/defaultDesignerConfig.conf index 2cd256d61af..e9f525c8992 100644 --- a/designer/server/src/main/resources/defaultDesignerConfig.conf +++ b/designer/server/src/main/resources/defaultDesignerConfig.conf @@ -133,7 +133,7 @@ processToolbarConfig { { type: "process-deploy", disabled: { fragment: true, archived: true, type: "oneof" } } { type: "process-cancel", disabled: { fragment: true, archived: true, type: "oneof" } } { type: "custom-link", name: "metrics", icon: "/assets/buttons/metrics.svg", url: "/metrics/$processName", disabled: { fragment: true } } - { type: "process-perform-single-execution", disabled: { fragment: true, archived: true, type: "oneof" } } + { type: "process-run-off-schedule", disabled: { fragment: true, archived: true, type: "oneof" } } ] } { diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala index b4857bb8597..89a35068ceb 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala @@ -19,8 +19,8 @@ import pl.touk.nussknacker.engine.testmode.TestProcess._ import pl.touk.nussknacker.restmodel.{ CustomActionRequest, CustomActionResponse, - PerformSingleExecutionRequest, - PerformSingleExecutionResponse + RunOffScheduleRequest, + RunOffScheduleResponse } import pl.touk.nussknacker.ui.api.ProcessesResources.ProcessUnmarshallingError import pl.touk.nussknacker.ui.api.description.NodesApiEndpoints.Dtos.AdhocTestParametersRequest @@ -275,24 +275,25 @@ class ManagementResources( ) } } - } ~ path("performSingleExecution" / ProcessNameSegment) { processName => - (post & processId(processName) & entity(as[PerformSingleExecutionRequest])) { (processIdWithName, req) => - canDeploy(processIdWithName) { - complete { - measureTime("singleExecution", metricRegistry) { - deploymentService - .processCommand( - PerformSingleExecutionCommand( - commonData = CommonCommandData(processIdWithName, req.comment.flatMap(Comment.from), user), + } ~ path(("runOffSchedule" | "performSingleExecution") / ProcessNameSegment) { + processName => // backward compatibility purpose + (post & processId(processName) & entity(as[RunOffScheduleRequest])) { (processIdWithName, req) => + canDeploy(processIdWithName) { + complete { + measureTime("singleExecution", metricRegistry) { + deploymentService + .processCommand( + RunOffScheduleCommand( + commonData = CommonCommandData(processIdWithName, req.comment.flatMap(Comment.from), user), + ) ) - ) - .flatMap(actionResult => - toHttpResponse(PerformSingleExecutionResponse(isSuccess = true, actionResult.msg))(StatusCodes.OK) - ) + .flatMap(actionResult => + toHttpResponse(RunOffScheduleResponse(isSuccess = true, actionResult.msg))(StatusCodes.OK) + ) + } } } } - } } } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/scenariotoolbar/ToolbarButtonConfig.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/scenariotoolbar/ToolbarButtonConfig.scala index c8c3ac0899c..61cfdc570bb 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/scenariotoolbar/ToolbarButtonConfig.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/scenariotoolbar/ToolbarButtonConfig.scala @@ -43,10 +43,10 @@ object ToolbarButtonConfigType extends Enumeration { CustomLink ) - val ProcessSave: Value = Value("process-save") - val ProcessCancel: Value = Value("process-cancel") - val ProcessDeploy: Value = Value("process-deploy") - val ProcessPerformSingleExecution: Value = Value("process-perform-single-execution") + val ProcessSave: Value = Value("process-save") + val ProcessCancel: Value = Value("process-cancel") + val ProcessDeploy: Value = Value("process-deploy") + val ProcessRunOffSchedule: Value = Value("process-run-off-schedule") val EditUndo: Value = Value("edit-undo") val EditRedo: Value = Value("edit-redo") diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala index 596b887f541..bd72c1c211d 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala @@ -75,10 +75,10 @@ class DeploymentService( def processCommand[Result](command: ScenarioCommand[Result]): Future[Result] = { command match { - case command: RunDeploymentCommand => runDeployment(command) - case command: CancelScenarioCommand => cancelScenario(command) - case command: PerformSingleExecutionCommand => processSingleExecution(command) - case command: CustomActionCommand => processCustomAction(command) + case command: RunDeploymentCommand => runDeployment(command) + case command: CancelScenarioCommand => cancelScenario(command) + case command: RunOffScheduleCommand => runOffSchedule(command) + case command: CustomActionCommand => processCustomAction(command) } } @@ -313,7 +313,7 @@ class DeploymentService( val fixedActionDefinitions = List( CustomActionDefinition(ScenarioActionName.Deploy, Nil, Nil, None), CustomActionDefinition(ScenarioActionName.Cancel, Nil, Nil, None), - CustomActionDefinition(ScenarioActionName.PerformSingleExecution, Nil, Nil, None) + CustomActionDefinition(ScenarioActionName.RunOffSchedule, Nil, Nil, None) ) val actionsDefinedInCustomActions = dispatcher .deploymentManagerUnsafe(processingType) @@ -731,13 +731,13 @@ class DeploymentService( Await.result(dbioRunner.run(actionRepository.deleteInProgressActions()), 10 seconds) } - private def processSingleExecution(command: PerformSingleExecutionCommand): Future[SingleExecutionResult] = { + private def runOffSchedule(command: RunOffScheduleCommand): Future[RunOffScheduleResult] = { processAction( command = command, - actionName = ScenarioActionName.PerformSingleExecution, + actionName = ScenarioActionName.RunOffSchedule, actionParams = Map.empty, dmCommandCreator = ctx => - DMPerformSingleExecutionCommand( + DMRunOffScheduleCommand( ctx.latestScenarioDetails.toEngineProcessVersion, ctx.latestScenarioDetails.json, command.commonData.user.toManagerUser, diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/ScenarioCommand.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/ScenarioCommand.scala index 7d0050d7bc4..c63ee10e87d 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/ScenarioCommand.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/ScenarioCommand.scala @@ -5,7 +5,7 @@ import pl.touk.nussknacker.engine.api.component.NodesDeploymentData import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName import pl.touk.nussknacker.engine.api.process.ProcessIdWithName -import pl.touk.nussknacker.engine.deployment.{CustomActionResult, ExternalDeploymentId, SingleExecutionResult} +import pl.touk.nussknacker.engine.deployment.{CustomActionResult, ExternalDeploymentId, RunOffScheduleResult} import pl.touk.nussknacker.ui.security.api.LoggedUser import scala.concurrent.Future @@ -34,9 +34,9 @@ case class CustomActionCommand( params: Map[String, String], ) extends ScenarioCommand[CustomActionResult] -case class PerformSingleExecutionCommand( +case class RunOffScheduleCommand( commonData: CommonCommandData, -) extends ScenarioCommand[SingleExecutionResult] +) extends ScenarioCommand[RunOffScheduleResult] // TODO CancelScenarioCommand will be legacy in some future because it operates on the scenario level instead of deployment level - // we should replace it by command operating on deployment diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala index e4afa1efc28..68aecc26a42 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala @@ -290,7 +290,7 @@ class DbScenarioActionRepository private ( ScenarioActivityType.ScenarioPaused case ScenarioActionName.Rename => ScenarioActivityType.ScenarioNameChanged - case ScenarioActionName.PerformSingleExecution => + case ScenarioActionName.RunOffSchedule => ScenarioActivityType.PerformedSingleExecution case otherCustomName => ScenarioActivityType.CustomAction(otherCustomName.value) @@ -518,7 +518,7 @@ class DbScenarioActionRepository private ( case ScenarioActivityType.OutgoingMigration => None case ScenarioActivityType.PerformedSingleExecution => - Some(ScenarioActionName.PerformSingleExecution) + Some(ScenarioActionName.RunOffSchedule) case ScenarioActivityType.PerformedScheduledExecution => None case ScenarioActivityType.AutomaticUpdate => diff --git a/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala b/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala index 4db2f05e841..38aaa699fb2 100644 --- a/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala +++ b/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala @@ -229,7 +229,7 @@ class V1_057__MigrateActionsAndCommentsToScenarioActivities } "migrate custom action 'run now' with comment to scenario_activities table" in { testMigratingActionWithComment( - scenarioActionName = ScenarioActionName.PerformSingleExecution, + scenarioActionName = ScenarioActionName.RunOffSchedule, actionComment = Some("Run now: Deployed at the request of business"), expectedActivity = (sid, sad, user, date, sv) => ScenarioActivity.PerformedSingleExecution( diff --git a/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/DevelopmentDeploymentManagerProvider.scala b/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/DevelopmentDeploymentManagerProvider.scala index c6603dceec5..fe9518f9f26 100644 --- a/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/DevelopmentDeploymentManagerProvider.scala +++ b/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/DevelopmentDeploymentManagerProvider.scala @@ -101,10 +101,10 @@ class DevelopmentDeploymentManager(actorSystem: ActorSystem, modelData: BaseMode case DMCancelDeploymentCommand(name, _, user) => // TODO: cancelling specific deployment cancelScenario(DMCancelScenarioCommand(name, user)) - case command: DMCancelScenarioCommand => cancelScenario(command) - case command: DMCustomActionCommand => invokeCustomAction(command) - case command: DMPerformSingleExecutionCommand => performSingleExecution(command) - case _: DMMakeScenarioSavepointCommand => Future.successful(SavepointResult("")) + case command: DMCancelScenarioCommand => cancelScenario(command) + case command: DMCustomActionCommand => invokeCustomAction(command) + case command: DMRunOffScheduleCommand => runOffSchedule(command) + case _: DMMakeScenarioSavepointCommand => Future.successful(SavepointResult("")) case DMTestScenarioCommand(_, canonicalProcess, scenarioTestData) => flinkTestRunner.test(canonicalProcess, scenarioTestData) // it's just for streaming e2e tests from file purposes } @@ -183,7 +183,7 @@ class DevelopmentDeploymentManager(actorSystem: ActorSystem, modelData: BaseMode } } - private def performSingleExecution(command: DMPerformSingleExecutionCommand): Future[SingleExecutionResult] = { + private def runOffSchedule(command: DMRunOffScheduleCommand): Future[RunOffScheduleResult] = { notImplemented } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala index 2f510a8de42..42b87ba0352 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala @@ -6,7 +6,7 @@ import com.typesafe.scalalogging.LazyLogging import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.process.{ProcessIdWithName, ProcessName, VersionId} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, ExternalDeploymentId, SingleExecutionResult} +import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, ExternalDeploymentId, RunOffScheduleResult} import pl.touk.nussknacker.engine.management.FlinkConfig import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.PeriodicProcessStatus import pl.touk.nussknacker.engine.management.periodic.Utils.{createActorWithRetry, runSafely} @@ -113,11 +113,11 @@ class PeriodicDeploymentManager private[periodic] ( override def processCommand[Result](command: DMScenarioCommand[Result]): Future[Result] = command match { - case command: DMValidateScenarioCommand => validate(command) - case command: DMRunDeploymentCommand => runDeployment(command) - case command: DMCancelScenarioCommand => cancelScenario(command) - case command: DMStopScenarioCommand => stopScenario(command) - case command: DMPerformSingleExecutionCommand => actionInstantBatch(command) + case command: DMValidateScenarioCommand => validate(command) + case command: DMRunDeploymentCommand => runDeployment(command) + case command: DMCancelScenarioCommand => cancelScenario(command) + case command: DMStopScenarioCommand => stopScenario(command) + case command: DMRunOffScheduleCommand => actionInstantBatch(command) case _: DMTestScenarioCommand | _: DMCancelDeploymentCommand | _: DMStopDeploymentCommand | _: DMMakeScenarioSavepointCommand | _: DMCustomActionCommand => delegate.processCommand(command) @@ -257,12 +257,12 @@ class PeriodicDeploymentManager private[periodic] ( ): Future[List[ScenarioActivity]] = service.getScenarioActivitiesSpecificToPeriodicProcess(processIdWithName, after) - private def actionInstantBatch(command: DMPerformSingleExecutionCommand): Future[SingleExecutionResult] = { + private def actionInstantBatch(command: DMRunOffScheduleCommand): Future[RunOffScheduleResult] = { val processName = command.processVersion.processName val instantScheduleResult = instantSchedule(processName) instantScheduleResult - .map(_ => SingleExecutionResult(s"Scenario ${processName.value} scheduled for immediate start")) - .getOrElse(SingleExecutionResult(s"Failed to schedule $processName to run as instant batch")) + .map(_ => RunOffScheduleResult(s"Scenario ${processName.value} scheduled for immediate start")) + .getOrElse(RunOffScheduleResult(s"Failed to schedule $processName to run as instant batch")) } // TODO: Why we don't allow running not scheduled scenario? Maybe we can try to schedule it? diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManager.scala index fdc63d939f8..008aabcca05 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManager.scala @@ -15,7 +15,7 @@ class PeriodicProcessStateDefinitionManager(delegate: ProcessStateDefinitionMana statusTooltipsPF = PeriodicStateStatus.statusTooltipsPF, statusDescriptionsPF = PeriodicStateStatus.statusDescriptionsPF, customStateDefinitions = PeriodicStateStatus.customStateDefinitions, - customVisibleActions = Some(defaultVisibleActions ::: ScenarioActionName.PerformSingleExecution :: Nil), + customVisibleActions = Some(defaultVisibleActions ::: ScenarioActionName.RunOffSchedule :: Nil), customActionTooltips = Some(PeriodicStateStatus.customActionTooltips), delegate = delegate ) { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicStateStatus.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicStateStatus.scala index e85c1e28ba8..a90c0cee3fb 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicStateStatus.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicStateStatus.scala @@ -41,11 +41,11 @@ object PeriodicStateStatus { List(ScenarioActionName.Cancel) case ProcessStatus(_: ScheduledStatus, _, deployedVersionId, Some(currentlyPresentedVersionId)) if deployedVersionId.contains(currentlyPresentedVersionId) => - List(ScenarioActionName.Cancel, ScenarioActionName.Deploy, ScenarioActionName.PerformSingleExecution) + List(ScenarioActionName.Cancel, ScenarioActionName.Deploy, ScenarioActionName.RunOffSchedule) case ProcessStatus(_: ScheduledStatus, _, _, None) => // At the moment of deployment or validation, we may not have the information about the currently displayed version // In that case we assume, that it was validated before the deployment was initiated. - List(ScenarioActionName.Cancel, ScenarioActionName.Deploy, ScenarioActionName.PerformSingleExecution) + List(ScenarioActionName.Cancel, ScenarioActionName.Deploy, ScenarioActionName.RunOffSchedule) case ProcessStatus(_: ScheduledStatus, _, _, _) => List(ScenarioActionName.Cancel, ScenarioActionName.Deploy) case ProcessStatus(WaitingForScheduleStatus, _, _, _) => @@ -88,10 +88,10 @@ object PeriodicStateStatus { case None => "[unknown]" } Map( - ScenarioActionName.PerformSingleExecution -> s"Version ${print(deployedVersionIdOpt)} is deployed, but different version ${print(currentlyPresentedVersionId)} is displayed" + ScenarioActionName.RunOffSchedule -> s"Version ${print(deployedVersionIdOpt)} is deployed, but different version ${print(currentlyPresentedVersionId)} is displayed" ) case ProcessStatus(other, _, _, _) => - Map(ScenarioActionName.PerformSingleExecution -> s"Disabled for ${other.name} status.") + Map(ScenarioActionName.RunOffSchedule -> s"Disabled for ${other.name} status.") } } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManagerTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManagerTest.scala index d00b7ae3328..77e551054a2 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManagerTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManagerTest.scala @@ -91,7 +91,7 @@ class PeriodicProcessStateDefinitionManagerTest extends AnyFunSuite with Matcher currentlyPresentedVersionId = Some(VersionId(5)), ) ) shouldEqual Map( - ScenarioActionName.PerformSingleExecution -> "Version 4 is deployed, but different version 5 is displayed" + ScenarioActionName.RunOffSchedule -> "Version 4 is deployed, but different version 5 is displayed" ) } @@ -104,7 +104,7 @@ class PeriodicProcessStateDefinitionManagerTest extends AnyFunSuite with Matcher currentlyPresentedVersionId = Some(VersionId(5)), ) ) shouldEqual Map( - ScenarioActionName.PerformSingleExecution -> "Disabled for CANCELED status." + ScenarioActionName.RunOffSchedule -> "Disabled for CANCELED status." ) } diff --git a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkDeploymentManager.scala b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkDeploymentManager.scala index adf07a67c7b..e9547c02b58 100644 --- a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkDeploymentManager.scala +++ b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkDeploymentManager.scala @@ -121,8 +121,8 @@ abstract class FlinkDeploymentManager( } case DMTestScenarioCommand(_, canonicalProcess, scenarioTestData) => testRunner.test(canonicalProcess, scenarioTestData) - case command: DMCustomActionCommand => processCustomAction(command) - case _: DMPerformSingleExecutionCommand => notImplemented + case command: DMCustomActionCommand => processCustomAction(command) + case _: DMRunOffScheduleCommand => notImplemented } private def validate(command: DMValidateScenarioCommand): Future[Unit] = { diff --git a/engine/lite/embeddedDeploymentManager/src/main/scala/pl/touk/nussknacker/engine/embedded/EmbeddedDeploymentManager.scala b/engine/lite/embeddedDeploymentManager/src/main/scala/pl/touk/nussknacker/engine/embedded/EmbeddedDeploymentManager.scala index 0b9debc2d26..76862770f0d 100644 --- a/engine/lite/embeddedDeploymentManager/src/main/scala/pl/touk/nussknacker/engine/embedded/EmbeddedDeploymentManager.scala +++ b/engine/lite/embeddedDeploymentManager/src/main/scala/pl/touk/nussknacker/engine/embedded/EmbeddedDeploymentManager.scala @@ -112,7 +112,7 @@ class EmbeddedDeploymentManager( case command: DMCancelScenarioCommand => cancelScenario(command) case command: DMTestScenarioCommand => testScenario(command) case _: DMStopDeploymentCommand | _: DMStopScenarioCommand | _: DMMakeScenarioSavepointCommand | - _: DMCustomActionCommand | _: DMPerformSingleExecutionCommand => + _: DMCustomActionCommand | _: DMRunOffScheduleCommand => notImplemented } diff --git a/engine/lite/k8sDeploymentManager/src/main/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManager.scala b/engine/lite/k8sDeploymentManager/src/main/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManager.scala index 030fb1f6e81..85e68aa9e04 100644 --- a/engine/lite/k8sDeploymentManager/src/main/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManager.scala +++ b/engine/lite/k8sDeploymentManager/src/main/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManager.scala @@ -114,7 +114,7 @@ class K8sDeploymentManager( case command: DMCancelScenarioCommand => cancelScenario(command) case command: DMTestScenarioCommand => testScenario(command) case _: DMCancelDeploymentCommand | _: DMStopDeploymentCommand | _: DMStopScenarioCommand | - _: DMMakeScenarioSavepointCommand | _: DMCustomActionCommand | _: DMPerformSingleExecutionCommand => + _: DMMakeScenarioSavepointCommand | _: DMCustomActionCommand | _: DMRunOffScheduleCommand => notImplemented } diff --git a/extensions-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/ProcessAction.scala b/extensions-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/ProcessAction.scala index 9e56e3e8426..0ff65f26373 100644 --- a/extensions-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/ProcessAction.scala +++ b/extensions-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/ProcessAction.scala @@ -70,24 +70,23 @@ object ScenarioActionName { val UnArchive: ScenarioActionName = ScenarioActionName("UNARCHIVE") val Pause: ScenarioActionName = ScenarioActionName("PAUSE") // TODO: To implement in future.. val Rename: ScenarioActionName = ScenarioActionName("RENAME") + // TODO: We kept the old name of "run now" CustomAction for compatibility reasons. + // In the future it can be changed to better name, according to convention, but that would require database migration + // In the meantime, there are methods serialize and deserialize, which operate on name RUN_OFF_SCHEDULE instead. + val RunOffSchedule: ScenarioActionName = ScenarioActionName("run now") val DefaultActions: List[ScenarioActionName] = Nil val StateActions: Set[ScenarioActionName] = Set(Cancel, Deploy, Pause) - // TODO: We kept the old name of "run now" CustomAction for compatibility reasons. - // In the future it can be changed to better name, according to convention, but that would require database migration - // In the meantime, there are methods serialize and deserialize, which operate on name PERFORM_SINGLE_EXECUTION instead. - val PerformSingleExecution: ScenarioActionName = ScenarioActionName("run now") - def serialize(name: ScenarioActionName): String = name match { - case ScenarioActionName.PerformSingleExecution => "PERFORM_SINGLE_EXECUTION" - case other => other.value + case ScenarioActionName.RunOffSchedule => "RUN_OFF_SCHEDULE" + case other => other.value } def deserialize(str: String): ScenarioActionName = str match { - case "PERFORM_SINGLE_EXECUTION" => ScenarioActionName.PerformSingleExecution - case other => ScenarioActionName(other) + case "RUN_OFF_SCHEDULE" => ScenarioActionName.RunOffSchedule + case other => ScenarioActionName(other) } } diff --git a/extensions-api/src/main/scala/pl/touk/nussknacker/engine/deployment/CustomActionDefinition.scala b/extensions-api/src/main/scala/pl/touk/nussknacker/engine/deployment/CustomActionDefinition.scala index aacbb1101ba..35da5d45ca4 100644 --- a/extensions-api/src/main/scala/pl/touk/nussknacker/engine/deployment/CustomActionDefinition.scala +++ b/extensions-api/src/main/scala/pl/touk/nussknacker/engine/deployment/CustomActionDefinition.scala @@ -33,4 +33,4 @@ case class CustomActionParameter( case class CustomActionResult(msg: String) -case class SingleExecutionResult(msg: String) +case class RunOffScheduleResult(msg: String) From 2b7211f944fcfdd39da9b0c9167df143e3d7bd0f Mon Sep 17 00:00:00 2001 From: mgoworko <37329559+mgoworko@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:38:19 +0100 Subject: [PATCH 4/7] Fetch only latest scenario activities for periodic DM (#7344) --- engine/flink/management/periodic/README.md | 1 + .../periodic/PeriodicBatchConfig.scala | 4 +- .../periodic/PeriodicDeploymentManager.scala | 1 + .../periodic/PeriodicProcessService.scala | 7 +- .../PeriodicDeploymentManagerTest.scala | 1 + ...eriodicProcessServiceIntegrationTest.scala | 115 +++++++++++------- .../periodic/PeriodicProcessServiceTest.scala | 1 + 7 files changed, 82 insertions(+), 48 deletions(-) diff --git a/engine/flink/management/periodic/README.md b/engine/flink/management/periodic/README.md index 9c0c0cf09bf..caa2148a016 100644 --- a/engine/flink/management/periodic/README.md +++ b/engine/flink/management/periodic/README.md @@ -31,3 +31,4 @@ Use `deploymentManager` with the following properties: - `deployMaxRetries` - maximum amount of retries for failed deployment. - `deployRetryPenalize` - an amount of time by which the next retry should be delayed. - `jarsDir` - directory for jars storage. +- `maxFetchedPeriodicScenarioActivities` - optional, maximum number of latest ScenarioActivities that will be fetched, by default 200 diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicBatchConfig.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicBatchConfig.scala index 3f79c3443fe..61b8f9bb1ac 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicBatchConfig.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicBatchConfig.scala @@ -13,6 +13,7 @@ import scala.concurrent.duration._ * @param deployInterval {@link DeploymentActor} check interval. * @param deploymentRetry {@link DeploymentRetryConfig} for deployment failure recovery. * @param jarsDir Directory for jars storage. + * @param maxFetchedPeriodicScenarioActivities Optional limit of number of latest periodic-related Scenario Activities that are returned by Periodic DM. */ case class PeriodicBatchConfig( db: Config, @@ -21,7 +22,8 @@ case class PeriodicBatchConfig( deployInterval: FiniteDuration = 17 seconds, deploymentRetry: DeploymentRetryConfig, executionConfig: PeriodicExecutionConfig, - jarsDir: String + jarsDir: String, + maxFetchedPeriodicScenarioActivities: Option[Int] = Some(200), ) /** diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala index 42b87ba0352..9ad4fdf8342 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala @@ -60,6 +60,7 @@ object PeriodicDeploymentManager { additionalDeploymentDataProvider, periodicBatchConfig.deploymentRetry, periodicBatchConfig.executionConfig, + periodicBatchConfig.maxFetchedPeriodicScenarioActivities, processConfigEnricher, clock, dependencies.actionService, diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala index 6fca6e85dbd..6e48530d7c6 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala @@ -48,6 +48,7 @@ class PeriodicProcessService( additionalDeploymentDataProvider: AdditionalDeploymentDataProvider, deploymentRetryConfig: DeploymentRetryConfig, executionConfig: PeriodicExecutionConfig, + maxFetchedPeriodicScenarioActivities: Option[Int], processConfigEnricher: ProcessConfigEnricher, clock: Clock, actionService: ProcessingTypeActionService, @@ -99,7 +100,11 @@ class PeriodicProcessService( retriesLeft = deployment.nextRetryAt.map(_ => deployment.retriesLeft), ) } - } yield activities + limitedActivities = maxFetchedPeriodicScenarioActivities match { + case Some(limit) => activities.sortBy(_.date).takeRight(limit) + case None => activities + } + } yield limitedActivities def schedule( schedule: ScheduleProperty, diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala index bae299e53fd..d4310d1b5ec 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala @@ -72,6 +72,7 @@ class PeriodicDeploymentManagerTest additionalDeploymentDataProvider = DefaultAdditionalDeploymentDataProvider, deploymentRetryConfig = DeploymentRetryConfig(), executionConfig = executionConfig, + maxFetchedPeriodicScenarioActivities = None, processConfigEnricher = ProcessConfigEnricher.identity, clock = Clock.systemDefaultZone(), new ProcessingTypeActionServiceStub, diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala index e8b05eb53c8..e7807319126 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -13,21 +13,10 @@ import org.scalatest.concurrent.ScalaFutures import org.scalatest.exceptions.TestFailedException import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import org.scalatest.time.{Millis, Seconds, Span} import org.testcontainers.utility.DockerImageName +import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus -import pl.touk.nussknacker.engine.api.deployment.{ - DataFreshnessPolicy, - ProcessActionId, - ProcessingTypeActionServiceStub, - ScenarioActivity, - ScenarioId, - ScenarioUser, - ScenarioVersionId, - ScheduledExecutionStatus, - UserName -} import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, ProcessName} import pl.touk.nussknacker.engine.api.{MetaData, ProcessVersion, StreamMetaData} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess @@ -82,7 +71,8 @@ class PeriodicProcessServiceIntegrationTest def withFixture( deploymentRetryConfig: DeploymentRetryConfig = DeploymentRetryConfig(), - executionConfig: PeriodicExecutionConfig = PeriodicExecutionConfig() + executionConfig: PeriodicExecutionConfig = PeriodicExecutionConfig(), + maxFetchedPeriodicScenarioActivities: Option[Int] = None, )(testCode: Fixture => Any): Unit = { val postgresConfig = ConfigFactory.parseMap( Map( @@ -105,7 +95,9 @@ class PeriodicProcessServiceIntegrationTest def runTestCodeWithDbConfig(config: Config) = { val (db: jdbc.JdbcBackend.DatabaseDef, dbProfile: JdbcProfile) = DbInitializer.init(config) try { - testCode(new Fixture(db, dbProfile, deploymentRetryConfig, executionConfig)) + testCode( + new Fixture(db, dbProfile, deploymentRetryConfig, executionConfig, maxFetchedPeriodicScenarioActivities) + ) } finally { db.close() } @@ -120,7 +112,8 @@ class PeriodicProcessServiceIntegrationTest db: JdbcBackend.DatabaseDef, dbProfile: JdbcProfile, deploymentRetryConfig: DeploymentRetryConfig, - executionConfig: PeriodicExecutionConfig + executionConfig: PeriodicExecutionConfig, + maxFetchedPeriodicScenarioActivities: Option[Int], ) { val delegateDeploymentManagerStub = new DeploymentManagerStub val jarManagerStub = new JarManagerStub @@ -144,6 +137,7 @@ class PeriodicProcessServiceIntegrationTest additionalDeploymentDataProvider = DefaultAdditionalDeploymentDataProvider, deploymentRetryConfig = deploymentRetryConfig, executionConfig = executionConfig, + maxFetchedPeriodicScenarioActivities = maxFetchedPeriodicScenarioActivities, processConfigEnricher = ProcessConfigEnricher.identity, clock = fixedClock(currentTime), new ProcessingTypeActionServiceStub, @@ -451,6 +445,66 @@ class PeriodicProcessServiceIntegrationTest } it should "handle multiple one time schedules" in withFixture() { f => + handleMultipleOneTimeSchedules(f) + def service = f.periodicProcessService(startTime) + val activities = service.getScenarioActivitiesSpecificToPeriodicProcess(processIdWithName, None).futureValue + val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] + val secondActivity = activities(1).asInstanceOf[ScenarioActivity.PerformedScheduledExecution] + activities shouldBe List( + ScenarioActivity.PerformedScheduledExecution( + scenarioId = ScenarioId(1), + scenarioActivityId = firstActivity.scenarioActivityId, + user = ScenarioUser(None, UserName("Nussknacker"), None, None), + date = firstActivity.date, + scenarioVersionId = Some(ScenarioVersionId(1)), + dateFinished = firstActivity.dateFinished, + scheduleName = "schedule1", + scheduledExecutionStatus = ScheduledExecutionStatus.Finished, + createdAt = firstActivity.createdAt, + retriesLeft = None, + nextRetryAt = None + ), + ScenarioActivity.PerformedScheduledExecution( + scenarioId = ScenarioId(1), + scenarioActivityId = secondActivity.scenarioActivityId, + user = ScenarioUser(None, UserName("Nussknacker"), None, None), + date = secondActivity.date, + scenarioVersionId = Some(ScenarioVersionId(1)), + dateFinished = secondActivity.dateFinished, + scheduleName = "schedule2", + scheduledExecutionStatus = ScheduledExecutionStatus.Finished, + createdAt = secondActivity.createdAt, + retriesLeft = None, + nextRetryAt = None + ), + ) + } + + it should "handle multiple one time schedules and return only latest activities" in withFixture( + maxFetchedPeriodicScenarioActivities = Some(1) + ) { f => + handleMultipleOneTimeSchedules(f) + def service = f.periodicProcessService(startTime) + val activities = service.getScenarioActivitiesSpecificToPeriodicProcess(processIdWithName, None).futureValue + val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] + activities shouldBe List( + ScenarioActivity.PerformedScheduledExecution( + scenarioId = ScenarioId(1), + scenarioActivityId = firstActivity.scenarioActivityId, + user = ScenarioUser(None, UserName("Nussknacker"), None, None), + date = firstActivity.date, + scenarioVersionId = Some(ScenarioVersionId(1)), + dateFinished = firstActivity.dateFinished, + scheduleName = "schedule2", + scheduledExecutionStatus = ScheduledExecutionStatus.Finished, + createdAt = firstActivity.createdAt, + retriesLeft = None, + nextRetryAt = None + ), + ) + } + + private def handleMultipleOneTimeSchedules(f: Fixture) = { var currentTime = startTime def service = f.periodicProcessService(currentTime) val timeToTriggerSchedule1 = startTime.plus(1, ChronoUnit.HOURS) @@ -549,37 +603,6 @@ class PeriodicProcessServiceIntegrationTest inactiveStates.latestDeploymentForSchedule(schedule1).state.status shouldBe PeriodicProcessDeploymentStatus.Finished inactiveStates.latestDeploymentForSchedule(schedule2).state.status shouldBe PeriodicProcessDeploymentStatus.Finished - val activities = service.getScenarioActivitiesSpecificToPeriodicProcess(processIdWithName, None).futureValue - val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] - val secondActivity = activities(1).asInstanceOf[ScenarioActivity.PerformedScheduledExecution] - activities shouldBe List( - ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), - scenarioActivityId = firstActivity.scenarioActivityId, - user = ScenarioUser(None, UserName("Nussknacker"), None, None), - date = firstActivity.date, - scenarioVersionId = Some(ScenarioVersionId(1)), - dateFinished = firstActivity.dateFinished, - scheduleName = "schedule1", - scheduledExecutionStatus = ScheduledExecutionStatus.Finished, - createdAt = firstActivity.createdAt, - retriesLeft = None, - nextRetryAt = None - ), - ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), - scenarioActivityId = secondActivity.scenarioActivityId, - user = ScenarioUser(None, UserName("Nussknacker"), None, None), - date = secondActivity.date, - scenarioVersionId = Some(ScenarioVersionId(1)), - dateFinished = secondActivity.dateFinished, - scheduleName = "schedule2", - scheduledExecutionStatus = ScheduledExecutionStatus.Finished, - createdAt = secondActivity.createdAt, - retriesLeft = None, - nextRetryAt = None - ), - ) } it should "handle failed event handler" in withFixture() { f => diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala index d35b65a0593..9b6569f972f 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala @@ -93,6 +93,7 @@ class PeriodicProcessServiceTest }, DeploymentRetryConfig(), PeriodicExecutionConfig(), + maxFetchedPeriodicScenarioActivities = None, new ProcessConfigEnricher { override def onInitialSchedule( From 7feab5903dc1086262562522c2b758b34803ae62 Mon Sep 17 00:00:00 2001 From: Piotr Przybylski Date: Tue, 17 Dec 2024 14:12:23 +0100 Subject: [PATCH 5/7] Remove uses of deprecated methods from RuntimeContext (#7348) --- .../engine/flink/api/NkGlobalParameters.scala | 11 +++-------- .../process/ExecutionConfigPreparer.scala | 3 +-- .../FlinkEngineRuntimeContextImpl.scala | 6 +++--- .../FlinkMetricsProviderForScenario.scala | 4 +--- .../registrar/FlinkProcessRegistrar.scala | 2 +- .../generic/DelayedFlinkKafkaConsumer.scala | 2 +- .../NkGlobalParametersEncoderTest.scala | 18 ++++++++---------- 7 files changed, 18 insertions(+), 28 deletions(-) diff --git a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/NkGlobalParameters.scala b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/NkGlobalParameters.scala index fabab3aec76..ddd7fbc38e2 100644 --- a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/NkGlobalParameters.scala +++ b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/NkGlobalParameters.scala @@ -1,10 +1,9 @@ package pl.touk.nussknacker.engine.flink.api import com.typesafe.config.Config -import io.circe.{Decoder, Encoder} +import io.circe.Encoder import net.ceedubs.ficus.Ficus._ import net.ceedubs.ficus.readers.ArbitraryTypeReader._ -import org.apache.flink.api.common.ExecutionConfig import org.apache.flink.api.common.ExecutionConfig.GlobalJobParameters import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.namespaces.NamingStrategy @@ -73,12 +72,8 @@ object NkGlobalParameters { NkGlobalParameters(buildInfo, processVersion, configGlobalParameters, namespaceTags, additionalInformation) } - def setInContext(ec: ExecutionConfig, globalParameters: NkGlobalParameters): Unit = { - ec.setGlobalJobParameters(globalParameters) - } - - def readFromContext(ec: ExecutionConfig): Option[NkGlobalParameters] = - NkGlobalParametersToMapEncoder.decode(ec.getGlobalJobParameters.toMap.asScala.toMap) + def fromMap(jobParameters: java.util.Map[String, String]): Option[NkGlobalParameters] = + NkGlobalParametersToMapEncoder.decode(jobParameters.asScala.toMap) private object NkGlobalParametersToMapEncoder { diff --git a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/ExecutionConfigPreparer.scala b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/ExecutionConfigPreparer.scala index d614af52f67..34c9140ac1a 100644 --- a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/ExecutionConfigPreparer.scala +++ b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/ExecutionConfigPreparer.scala @@ -53,8 +53,7 @@ object ExecutionConfigPreparer extends LazyLogging { override def prepareExecutionConfig( config: ExecutionConfig )(jobData: JobData, deploymentData: DeploymentData): Unit = { - NkGlobalParameters.setInContext( - config, + config.setGlobalJobParameters( NkGlobalParameters.create( buildInfo, jobData.processVersion, diff --git a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkEngineRuntimeContextImpl.scala b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkEngineRuntimeContextImpl.scala index ee33a6bca26..984d34a80df 100644 --- a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkEngineRuntimeContextImpl.scala +++ b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkEngineRuntimeContextImpl.scala @@ -1,6 +1,5 @@ package pl.touk.nussknacker.engine.process.compiler -import com.github.ghik.silencer.silent import org.apache.flink.api.common.functions.RuntimeContext import pl.touk.nussknacker.engine.api.JobData import pl.touk.nussknacker.engine.api.process.ComponentUseCase @@ -15,9 +14,10 @@ case class FlinkEngineRuntimeContextImpl( metricsProvider: MetricsProviderForScenario ) extends FlinkEngineRuntimeContext { - @silent("deprecated") override def contextIdGenerator(nodeId: String): ContextIdGenerator = - new IncContextIdGenerator(jobData.metaData.name.value + "-" + nodeId + "-" + runtimeContext.getIndexOfThisSubtask) + new IncContextIdGenerator( + jobData.metaData.name.value + "-" + nodeId + "-" + runtimeContext.getTaskInfo.getIndexOfThisSubtask + ) } diff --git a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkMetricsProviderForScenario.scala b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkMetricsProviderForScenario.scala index 7fbd65f16f5..ccc9708f381 100644 --- a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkMetricsProviderForScenario.scala +++ b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/compiler/FlinkMetricsProviderForScenario.scala @@ -3,7 +3,6 @@ package pl.touk.nussknacker.engine.process.compiler import cats.data.NonEmptyList import com.codahale.metrics import com.codahale.metrics.SlidingTimeWindowReservoir -import com.github.ghik.silencer.silent import org.apache.flink import org.apache.flink.api.common.functions.RuntimeContext import org.apache.flink.dropwizard.metrics.DropwizardHistogramWrapper @@ -53,9 +52,8 @@ class FlinkMetricsProviderForScenario(runtimeContext: RuntimeContext) extends Ba ??? // Shouldn't be needed because Flink jobs are recreated "from scratch" and no cleanup of metrics during cancel is needed } - @silent("deprecated") private def groupsWithName(nameParts: NonEmptyList[String], tags: Map[String, String]): (MetricGroup, String) = { - val namespaceTags = extractTags(NkGlobalParameters.readFromContext(runtimeContext.getExecutionConfig)) + val namespaceTags = extractTags(NkGlobalParameters.fromMap(runtimeContext.getGlobalJobParameters)) tagMode(nameParts, tags ++ namespaceTags) } diff --git a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/FlinkProcessRegistrar.scala b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/FlinkProcessRegistrar.scala index ef22dc586d6..408428d2453 100644 --- a/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/FlinkProcessRegistrar.scala +++ b/engine/flink/executor/src/main/scala/pl/touk/nussknacker/engine/process/registrar/FlinkProcessRegistrar.scala @@ -128,7 +128,7 @@ class FlinkProcessRegistrar( resultCollector: ResultCollector, deploymentData: DeploymentData ): Unit = { - val globalParameters = NkGlobalParameters.readFromContext(env.getConfig) + val globalParameters = NkGlobalParameters.fromMap(env.getConfig.getGlobalJobParameters.toMap) def nodeContext( nodeComponentId: NodeComponentInfo, diff --git a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/generic/DelayedFlinkKafkaConsumer.scala b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/generic/DelayedFlinkKafkaConsumer.scala index 263c9abe958..0f29f74707e 100644 --- a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/generic/DelayedFlinkKafkaConsumer.scala +++ b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/generic/DelayedFlinkKafkaConsumer.scala @@ -136,7 +136,7 @@ class DelayedFlinkKafkaConsumer[T]( runtimeContext.getProcessingTimeService, runtimeContext.getExecutionConfig.getAutoWatermarkInterval, runtimeContext.getUserCodeClassLoader, - runtimeContext.getTaskNameWithSubtasks, + runtimeContext.getTaskInfo.getTaskNameWithSubtasks, runtimeContext.getMetricGroup, consumerMetricGroup, deserializer, diff --git a/engine/flink/tests/src/test/scala/pl/touk/nussknacker/defaultmodel/NkGlobalParametersEncoderTest.scala b/engine/flink/tests/src/test/scala/pl/touk/nussknacker/defaultmodel/NkGlobalParametersEncoderTest.scala index c36d050f7da..7ad5e7528a1 100644 --- a/engine/flink/tests/src/test/scala/pl/touk/nussknacker/defaultmodel/NkGlobalParametersEncoderTest.scala +++ b/engine/flink/tests/src/test/scala/pl/touk/nussknacker/defaultmodel/NkGlobalParametersEncoderTest.scala @@ -41,20 +41,18 @@ class NkGlobalParametersEncoderTest extends AnyFunSuite with Matchers { ) List(globalParamsWithAllOptionalValues, globalParamsWithNoOptionalValues).foreach { params => - val ec = new ExecutionConfig() - ec.setGlobalJobParameters(params) - val globalParamsFromEc = NkGlobalParameters.readFromContext(ec).get - - params.buildInfo shouldBe globalParamsFromEc.buildInfo - params.processVersion shouldBe globalParamsFromEc.processVersion - params.configParameters shouldBe globalParamsFromEc.configParameters - params.namespaceParameters shouldBe globalParamsFromEc.namespaceParameters - params.additionalInformation shouldBe globalParamsFromEc.additionalInformation + val decodedParams = NkGlobalParameters.fromMap(params.toMap).get + + decodedParams.buildInfo shouldBe params.buildInfo + decodedParams.processVersion shouldBe params.processVersion + decodedParams.configParameters shouldBe params.configParameters + decodedParams.namespaceParameters shouldBe params.namespaceParameters + decodedParams.additionalInformation shouldBe params.additionalInformation } } test("returns None when context doesnt have required parameters") { - NkGlobalParameters.readFromContext(new ExecutionConfig()) shouldBe None + NkGlobalParameters.fromMap(new ExecutionConfig.GlobalJobParameters().toMap) shouldBe None } } From 5970344f5c8bc1eb10baa099c58b89773f674f3b Mon Sep 17 00:00:00 2001 From: Piotr Rudnicki Date: Tue, 17 Dec 2024 19:50:01 +0100 Subject: [PATCH 6/7] Provide more descriptive validation message for inline maps with non stringy keys (#7353) --- .../engine/spel/SpelExpressionParseError.scala | 7 +++++-- .../pl/touk/nussknacker/engine/spel/TyperSpec.scala | 11 +++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/spel/SpelExpressionParseError.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/spel/SpelExpressionParseError.scala index 449640cb2bb..49b27a7f186 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/spel/SpelExpressionParseError.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/spel/SpelExpressionParseError.scala @@ -39,8 +39,11 @@ object SpelExpressionParseError { } case object MapWithExpressionKeysError extends UnsupportedOperationError { - override def message: String = - "Currently inline maps with not literal keys (e.g. expressions as keys) are not supported" + + override def message: String = { + "Currently only string keys for inline maps are supported" + } + } case object ArrayConstructorError extends UnsupportedOperationError { diff --git a/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/TyperSpec.scala b/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/TyperSpec.scala index 9edd8839cc4..f69278af040 100644 --- a/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/TyperSpec.scala +++ b/scenario-compiler/src/test/scala/pl/touk/nussknacker/engine/spel/TyperSpec.scala @@ -1,7 +1,7 @@ package pl.touk.nussknacker.engine.spel -import cats.data.Validated.Valid -import cats.data.ValidatedNel +import cats.data.Validated.{Invalid, Valid} +import cats.data.{NonEmptyList, ValidatedNel} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.springframework.expression.common.TemplateParserContext @@ -15,6 +15,7 @@ import pl.touk.nussknacker.engine.dict.{KeysDictTyper, SimpleDictRegistry} import pl.touk.nussknacker.engine.expression.PositionRange import pl.touk.nussknacker.engine.spel.SpelExpressionParseError.IllegalOperationError.DynamicPropertyAccessError import pl.touk.nussknacker.engine.spel.SpelExpressionParseError.MissingObjectError.NoPropertyError +import pl.touk.nussknacker.engine.spel.SpelExpressionParseError.UnsupportedOperationError.MapWithExpressionKeysError import pl.touk.nussknacker.engine.spel.Typer.TypingResultWithContext import pl.touk.nussknacker.engine.spel.TyperSpecTestData.TestRecord._ import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap @@ -29,6 +30,12 @@ class TyperSpec extends AnyFunSuite with Matchers with ValidatedValuesDetailedMe private val parser: standard.SpelExpressionParser = new org.springframework.expression.spel.standard.SpelExpressionParser() + test("not allow maps with keys other than strings") { + typeExpression("{1L: 'foo'}") shouldBe Invalid( + NonEmptyList(MapWithExpressionKeysError, List()) + ) + } + test("simple expression") { typeExpression("#x + 2", "x" -> 2) shouldBe Valid( CollectedTypingResult( From a5d530503ecde04c14ecc26cd59d7785e9321b10 Mon Sep 17 00:00:00 2001 From: Arek Burdach Date: Tue, 17 Dec 2024 23:35:02 +0100 Subject: [PATCH 7/7] [NU-1828] Processing types configs loader api (#7336) --- build.sbt | 54 +++++--- .../ProcessingTypeConfigsLoader.scala | 12 ++ .../ProcessingTypeConfigsLoaderFactory.scala | 15 +++ .../ui/LocalNussknackerWithSingleModel.scala | 12 +- .../touk/nussknacker/ui/NussknackerApp.scala | 3 +- .../nussknacker/ui/NussknackerConfig.scala | 31 ----- .../ui/api/description/DictApiEndpoints.scala | 3 +- .../ui/api/description/UserApiEndpoints.scala | 2 +- .../scenarioActivity/InputOutput.scala | 4 +- .../ui/config/DesignerConfig.scala | 34 ++++++ .../ui/config/DesignerConfigLoader.scala | 53 +++++--- .../ui/db/entity/BaseEntityFactory.scala | 3 +- .../ui/db/entity/ProcessEntityFactory.scala | 3 +- .../QuestDbFEStatisticsRepository.scala | 2 +- .../ui/factory/NussknackerAppFactory.scala | 83 +++++++++---- ...sConfigBasedProcessingTypeDataLoader.scala | 92 +++++++------- .../server/AkkaHttpBasedRouteProvider.scala | 115 ++++++++---------- .../ui/server/NussknackerHttpServer.scala | 7 +- .../nussknacker/ui/server/RouteProvider.scala | 5 +- .../util/IOToFutureSttpBackendConverter.scala | 27 ++++ .../nussknacker/test/base/it/NuItTest.scala | 8 +- .../test/base/it/NuResourcesTest.scala | 15 +-- .../ui/api/NussknackerHttpServerSpec.scala | 8 +- .../ConfigurationTest.scala | 61 ++++++---- .../ProcessingTypeDataLoaderSpec.scala} | 43 ++++--- .../ProcessingTypeDataProviderSpec.scala | 5 +- .../ScenarioParametersServiceTest.scala | 42 +++---- .../engine/lite/app/NuRuntimeApp.scala | 10 +- .../engine/ConfigWithUnresolvedVersion.scala | 19 +++ .../api/AuthenticationConfiguration.scala | 4 +- .../nussknacker/engine/util/UriUtils.scala | 11 +- .../engine/util/config/ConfigFactoryExt.scala | 32 ++--- .../ConfigWithUnresolvedVersionExt.scala | 32 ----- .../util/loader/ScalaServiceLoader.scala | 11 ++ .../util/config/ConfigFactoryExtSpec.scala | 3 +- 35 files changed, 505 insertions(+), 359 deletions(-) create mode 100644 designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoader.scala create mode 100644 designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoaderFactory.scala delete mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfig.scala create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/util/IOToFutureSttpBackendConverter.scala rename designer/server/src/test/scala/pl/touk/nussknacker/ui/{integration => config}/ConfigurationTest.scala (69%) rename designer/server/src/test/scala/pl/touk/nussknacker/ui/{LoadableConfigBasedNussknackerConfigSpec.scala => config/processingtype/ProcessingTypeDataLoaderSpec.scala} (70%) delete mode 100644 utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigWithUnresolvedVersionExt.scala diff --git a/build.sbt b/build.sbt index 33c8fc01f76..01c8aca7bad 100644 --- a/build.sbt +++ b/build.sbt @@ -1,15 +1,15 @@ import com.typesafe.sbt.packager.SettingsHelper import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport.dockerUsername import pl.project13.scala.sbt.JmhPlugin -import pl.project13.scala.sbt.JmhPlugin._ -import sbt.Keys._ -import sbt._ +import pl.project13.scala.sbt.JmhPlugin.* +import sbt.* +import sbt.Keys.* import sbtassembly.AssemblyPlugin.autoImport.assembly import sbtassembly.MergeStrategy -import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations._ +import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations.* import scala.language.postfixOps -import scala.sys.process._ +import scala.sys.process.* import scala.util.Try import scala.xml.Elem import scala.xml.transform.{RewriteRule, RuleTransformer} @@ -1913,6 +1913,18 @@ lazy val listenerApi = (project in file("designer/listener-api")) ) .dependsOn(extensionsApi) +lazy val configLoaderApi = (project in file("designer/config-loader-api")) + .settings(commonSettings) + .settings( + name := "nussknacker-config-loader-api", + libraryDependencies ++= { + Seq( + "org.typelevel" %% "cats-effect" % catsEffectV + ) + } + ) + .dependsOn(extensionsApi) + lazy val deploymentManagerApi = (project in file("designer/deployment-manager-api")) .settings(commonSettings) .settings( @@ -1985,21 +1997,21 @@ lazy val designer = (project in file("designer/server")) assembly / assemblyMergeStrategy := designerMergeStrategy, libraryDependencies ++= { Seq( - "com.typesafe.akka" %% "akka-http" % akkaHttpV, - "com.typesafe.akka" %% "akka-slf4j" % akkaV, - "com.typesafe.akka" %% "akka-stream" % akkaV, - "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpV % Test, - "com.typesafe.akka" %% "akka-testkit" % akkaV % Test, - "de.heikoseeberger" %% "akka-http-circe" % akkaHttpCirceV, - "com.softwaremill.sttp.client3" %% "akka-http-backend" % sttpV, - "ch.qos.logback" % "logback-core" % logbackV, - "ch.qos.logback" % "logback-classic" % logbackV, - "ch.qos.logback.contrib" % "logback-json-classic" % logbackJsonV, - "ch.qos.logback.contrib" % "logback-jackson" % logbackJsonV, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonV, - "org.slf4j" % "log4j-over-slf4j" % slf4jV, - "com.carrotsearch" % "java-sizeof" % "0.0.5", - "org.typelevel" %% "case-insensitive" % "1.4.0", + "com.typesafe.akka" %% "akka-http" % akkaHttpV, + "com.typesafe.akka" %% "akka-slf4j" % akkaV, + "com.typesafe.akka" %% "akka-stream" % akkaV, + "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpV % Test, + "com.typesafe.akka" %% "akka-testkit" % akkaV % Test, + "de.heikoseeberger" %% "akka-http-circe" % akkaHttpCirceV, + "com.softwaremill.sttp.client3" %% "async-http-client-backend-cats" % sttpV, + "ch.qos.logback" % "logback-core" % logbackV, + "ch.qos.logback" % "logback-classic" % logbackV, + "ch.qos.logback.contrib" % "logback-json-classic" % logbackJsonV, + "ch.qos.logback.contrib" % "logback-jackson" % logbackJsonV, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonV, + "org.slf4j" % "log4j-over-slf4j" % slf4jV, + "com.carrotsearch" % "java-sizeof" % "0.0.5", + "org.typelevel" %% "case-insensitive" % "1.4.0", // It's needed by flinkDeploymentManager which has disabled includingScala "org.scala-lang" % "scala-compiler" % scalaVersion.value, @@ -2053,6 +2065,7 @@ lazy val designer = (project in file("designer/server")) deploymentManagerApi, restmodel, listenerApi, + configLoaderApi, defaultHelpers % Test, testUtils % Test, flinkTestUtils % Test, @@ -2193,6 +2206,7 @@ lazy val modules = List[ProjectReference]( httpUtils, restmodel, listenerApi, + configLoaderApi, deploymentManagerApi, designer, sqlComponents, diff --git a/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoader.scala b/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoader.scala new file mode 100644 index 00000000000..20adfc59eb2 --- /dev/null +++ b/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoader.scala @@ -0,0 +1,12 @@ +package pl.touk.nussknacker.ui.configloader + +import cats.effect.IO +import pl.touk.nussknacker.engine.ProcessingTypeConfig + +trait ProcessingTypeConfigsLoader { + + def loadProcessingTypeConfigs(): IO[ProcessingTypeConfigs] + +} + +case class ProcessingTypeConfigs(configByProcessingType: Map[String, ProcessingTypeConfig]) diff --git a/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoaderFactory.scala b/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoaderFactory.scala new file mode 100644 index 00000000000..5d7b2312687 --- /dev/null +++ b/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoaderFactory.scala @@ -0,0 +1,15 @@ +package pl.touk.nussknacker.ui.configloader + +import cats.effect.IO +import cats.effect.unsafe.IORuntime +import com.typesafe.config.Config +import sttp.client3.SttpBackend + +trait ProcessingTypeConfigsLoaderFactory { + + def create( + configLoaderConfig: Config, + sttpBackend: SttpBackend[IO, Any], + )(implicit ec: IORuntime): ProcessingTypeConfigsLoader + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala index 12b0aa71adb..25dc03d5742 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala @@ -1,7 +1,7 @@ package pl.touk.nussknacker.ui import cats.effect.{IO, Resource} -import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.config.{Config, ConfigFactory, ConfigValue, ConfigValueFactory} import org.apache.commons.io.FileUtils import pl.touk.nussknacker.engine.{DeploymentManagerProvider, ModelData} import pl.touk.nussknacker.ui.config.DesignerConfigLoader @@ -49,8 +49,14 @@ object LocalNussknackerWithSingleModel { modelData = Map(typeName -> (category, modelData)), deploymentManagerProvider = deploymentManagerProvider ) - val nussknackerConfig = new LoadableConfigBasedNussknackerConfig(IO.delay(DesignerConfigLoader.from(appConfig))) - val appFactory = new NussknackerAppFactory(nussknackerConfig, local) + val designerConfigLoader = DesignerConfigLoader.fromConfig( + // This map is ignored but must exist + appConfig.withValue("scenarioTypes", ConfigValueFactory.fromMap(Map.empty[String, ConfigValue].asJava)) + ) + val appFactory = new NussknackerAppFactory( + designerConfigLoader, + _ => local + ) appFactory.createApp() } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala index d99984f966a..f7b4c356e2c 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala @@ -1,13 +1,14 @@ package pl.touk.nussknacker.ui import cats.effect.{ExitCode, IO, IOApp} +import pl.touk.nussknacker.ui.config.{AlwaysLoadingFileBasedDesignerConfigLoader, DesignerConfigLoader} import pl.touk.nussknacker.ui.factory.NussknackerAppFactory object NussknackerApp extends IOApp { override def run(args: List[String]): IO[ExitCode] = { for { - appFactory <- IO(new NussknackerAppFactory(getClass.getClassLoader)) + appFactory <- IO(NussknackerAppFactory(AlwaysLoadingFileBasedDesignerConfigLoader(getClass.getClassLoader))) _ <- appFactory.createApp().use { _ => IO.never } } yield ExitCode.Success } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala deleted file mode 100644 index a05b7b175b8..00000000000 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala +++ /dev/null @@ -1,31 +0,0 @@ -package pl.touk.nussknacker.ui - -import cats.effect.IO -import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap -import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, ProcessingTypeConfig} -import pl.touk.nussknacker.engine.util.config.ConfigWithUnresolvedVersionExt._ -import pl.touk.nussknacker.ui.config.DesignerConfigLoader - -trait NussknackerConfig { - - def loadApplicationConfig(): IO[ConfigWithUnresolvedVersion] - - final def loadProcessingTypeConfigs(): IO[Map[String, ProcessingTypeConfig]] = { - loadApplicationConfig() - .map { config => - config - .readMap("scenarioTypes") - .getOrElse { throw new RuntimeException("No scenario types configuration provided") } - .mapValuesNow(ProcessingTypeConfig.read) - } - } - -} - -class LoadableConfigBasedNussknackerConfig(loadConfig: IO[ConfigWithUnresolvedVersion]) extends NussknackerConfig { - - override def loadApplicationConfig(): IO[ConfigWithUnresolvedVersion] = loadConfig -} - -class LoadableDesignerConfigBasedNussknackerConfig(classLoader: ClassLoader) - extends LoadableConfigBasedNussknackerConfig(DesignerConfigLoader.load(classLoader)) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DictApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DictApiEndpoints.scala index 36df4adceef..b177104b236 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DictApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DictApiEndpoints.scala @@ -15,9 +15,8 @@ import pl.touk.nussknacker.ui.api.description.DictApiEndpoints.DictError.{ } import pl.touk.nussknacker.ui.api.description.DictApiEndpoints.Dtos._ import sttp.model.StatusCode.{BadRequest, NotFound, Ok} -import sttp.tapir._ import sttp.tapir.json.circe._ -import sttp.tapir.Schema +import sttp.tapir._ import scala.language.implicitConversions diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/UserApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/UserApiEndpoints.scala index dc745e625d5..1095c09e4d6 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/UserApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/UserApiEndpoints.scala @@ -6,7 +6,7 @@ import pl.touk.nussknacker.restmodel.BaseEndpointDefinitions import pl.touk.nussknacker.restmodel.BaseEndpointDefinitions.SecuredEndpoint import pl.touk.nussknacker.security.AuthCredentials import pl.touk.nussknacker.ui.security.api.GlobalPermission.GlobalPermission -import pl.touk.nussknacker.ui.security.api.{AdminUser, CommonUser, ImpersonatedUser, LoggedUser, RealLoggedUser} +import pl.touk.nussknacker.ui.security.api._ import sttp.model.StatusCode.Ok import sttp.tapir.EndpointIO.Example import sttp.tapir.derevo.schema diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/scenarioActivity/InputOutput.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/scenarioActivity/InputOutput.scala index b475bab6015..317ebf02054 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/scenarioActivity/InputOutput.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/scenarioActivity/InputOutput.scala @@ -3,9 +3,9 @@ package pl.touk.nussknacker.ui.api.description.scenarioActivity import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.ui.api.description.scenarioActivity.Dtos.ScenarioActivityError import pl.touk.nussknacker.ui.api.description.scenarioActivity.Dtos.ScenarioActivityError.NoScenario -import sttp.model.StatusCode.{NotFound, NotImplemented} +import sttp.model.StatusCode.NotFound import sttp.tapir.EndpointIO.Example -import sttp.tapir.{EndpointOutput, emptyOutputAs, oneOf, oneOfVariantFromMatchType, plainBody} +import sttp.tapir.{EndpointOutput, oneOf, oneOfVariantFromMatchType, plainBody} object InputOutput { diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfig.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfig.scala new file mode 100644 index 00000000000..3cb1c86fd7e --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfig.scala @@ -0,0 +1,34 @@ +package pl.touk.nussknacker.ui.config + +import com.typesafe.config.{Config, ConfigFactory} +import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap +import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, ProcessingTypeConfig} +import pl.touk.nussknacker.ui.configloader.ProcessingTypeConfigs + +// TODO: We should extract a class for all configuration options that should be available to designer instead of returning raw hocon config. +// Thanks to that it will be easier to split processing type config from rest of configs and use this interface programmatically +final case class DesignerConfig private (rawConfig: ConfigWithUnresolvedVersion) { + + import net.ceedubs.ficus.Ficus._ + + def processingTypeConfigs: ProcessingTypeConfigs = + ProcessingTypeConfigs( + rawConfig + .readMap("scenarioTypes") + .getOrElse { + throw new RuntimeException("No scenario types configuration provided") + } + .mapValuesNow(ProcessingTypeConfig.read) + ) + + def configLoaderConfig: Config = rawConfig.resolved.getAs[Config]("configLoader").getOrElse(ConfigFactory.empty()) + +} + +object DesignerConfig { + + def from(config: Config): DesignerConfig = { + DesignerConfig(ConfigWithUnresolvedVersion(config)) + } + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala index b87cdb7f6eb..f0f00d3bb4e 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala @@ -3,8 +3,15 @@ package pl.touk.nussknacker.ui.config import cats.effect.IO import com.typesafe.config.{Config, ConfigFactory} import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.engine.util.UriUtils import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt +trait DesignerConfigLoader { + + def loadDesignerConfig(): IO[DesignerConfig] + +} + /** * This class handles two parts of ui config loading: * 1. Parsing of "base" config passed via nussknacker.config.locations system property (without resolution) @@ -14,27 +21,43 @@ import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt * Result of config loading still keep version with unresolved env variables for purpose of config loading on model side - see * InputConfigDuringExecution and ModelConfigLoader */ -object DesignerConfigLoader { +class AlwaysLoadingFileBasedDesignerConfigLoader private (classLoader: ClassLoader) extends DesignerConfigLoader { + + private val configLocationsProperty: String = "nussknacker.config.locations" private val defaultConfigResource = "defaultDesignerConfig.conf" - def load(classLoader: ClassLoader): IO[ConfigWithUnresolvedVersion] = { + override def loadDesignerConfig(): IO[DesignerConfig] = { + val locationsPropertyValueOpt = Option(System.getProperty(configLocationsProperty)) + val locations = locationsPropertyValueOpt.map(UriUtils.extractListOfLocations).getOrElse(List.empty) for { - baseConfig <- IO.blocking(ConfigFactoryExt.parseUnresolved(classLoader = classLoader)) - loadedConfig <- load(baseConfig, classLoader) - } yield loadedConfig + baseUnresolvedConfig <- IO.blocking(new ConfigFactoryExt(classLoader).parseUnresolved(locations)) + parsedDefaultUiConfig <- IO.blocking(ConfigFactory.parseResources(classLoader, defaultConfigResource)) + unresolvedConfigWithFallbackToDefaults = baseUnresolvedConfig.withFallback(parsedDefaultUiConfig) + } yield DesignerConfig(ConfigWithUnresolvedVersion(classLoader, unresolvedConfigWithFallbackToDefaults)) } - def load(baseUnresolvedConfig: Config, classLoader: ClassLoader): IO[ConfigWithUnresolvedVersion] = { - IO.blocking { - val parsedDefaultUiConfig = ConfigFactory.parseResources(classLoader, defaultConfigResource) - val unresolvedConfigWithFallbackToDefaults = baseUnresolvedConfig.withFallback(parsedDefaultUiConfig) - ConfigWithUnresolvedVersion(classLoader, unresolvedConfigWithFallbackToDefaults) - } - } +} - def from(config: Config): ConfigWithUnresolvedVersion = { - ConfigWithUnresolvedVersion(this.getClass.getClassLoader, config) - } +object AlwaysLoadingFileBasedDesignerConfigLoader { + + def apply(classLoader: ClassLoader): AlwaysLoadingFileBasedDesignerConfigLoader = + new AlwaysLoadingFileBasedDesignerConfigLoader(classLoader) + +} + +/** + * This implementation is more straightforward - it only parse config without any property checking and fallbacks + */ +class SimpleConfigLoadingDesignerConfigLoader(loadConfig: => Config) extends DesignerConfigLoader { + + override def loadDesignerConfig(): IO[DesignerConfig] = IO.delay(DesignerConfig.from(loadConfig)) + +} + +object DesignerConfigLoader { + + def fromConfig(loadConfig: => Config): SimpleConfigLoadingDesignerConfigLoader = + new SimpleConfigLoadingDesignerConfigLoader(loadConfig) } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/BaseEntityFactory.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/BaseEntityFactory.scala index 68b7f22e920..40baaff6cc4 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/BaseEntityFactory.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/BaseEntityFactory.scala @@ -9,8 +9,7 @@ import pl.touk.nussknacker.engine.api.deployment.{ } import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.newdeployment.DeploymentId -import slick.ast.BaseTypedType -import slick.jdbc.{JdbcProfile, JdbcType} +import slick.jdbc.JdbcProfile import java.util.UUID diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/ProcessEntityFactory.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/ProcessEntityFactory.scala index a16f394a22b..0edcf8e1b52 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/ProcessEntityFactory.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/ProcessEntityFactory.scala @@ -1,7 +1,6 @@ package pl.touk.nussknacker.ui.db.entity -import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName} -import pl.touk.nussknacker.engine.api.process.ProcessingType +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, ProcessingType} import slick.lifted.{ProvenShape, TableQuery => LTableQuery} import slick.sql.SqlProfile.ColumnOption.NotNull diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/timeseries/questdb/QuestDbFEStatisticsRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/timeseries/questdb/QuestDbFEStatisticsRepository.scala index f69597bb6de..ee78f21a61f 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/timeseries/questdb/QuestDbFEStatisticsRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/timeseries/questdb/QuestDbFEStatisticsRepository.scala @@ -10,7 +10,6 @@ import io.questdb.cairo.security.AllowAllSecurityContext import io.questdb.cairo.sql.RecordCursorFactory import io.questdb.cairo.wal.WalWriter import io.questdb.griffin.{SqlExecutionContext, SqlExecutionContextImpl} -import pl.touk.nussknacker.ui.db.timeseries.{FEStatisticsRepository, NoOpFEStatisticsRepository} import pl.touk.nussknacker.ui.db.timeseries.questdb.QuestDbExtensions.{ BuildCairoEngineExtension, CairoEngineExtension, @@ -22,6 +21,7 @@ import pl.touk.nussknacker.ui.db.timeseries.questdb.QuestDbFEStatisticsRepositor selectQuery, tableName } +import pl.touk.nussknacker.ui.db.timeseries.{FEStatisticsRepository, NoOpFEStatisticsRepository} import pl.touk.nussknacker.ui.statistics.RawFEStatistics import java.time.Clock diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala index e9be44d0bcf..816b4eb9b0d 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala @@ -2,62 +2,92 @@ package pl.touk.nussknacker.ui.factory import akka.actor.ActorSystem import akka.stream.Materializer +import cats.effect.unsafe.IORuntime import cats.effect.{IO, Resource} import com.typesafe.scalalogging.LazyLogging import io.dropwizard.metrics5.MetricRegistry import io.dropwizard.metrics5.jmx.JmxReporter import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader import pl.touk.nussknacker.engine.util.{JavaClassVersionChecker, SLF4JBridgeHandlerRegistrar} -import pl.touk.nussknacker.ui.{LoadableDesignerConfigBasedNussknackerConfig, NussknackerConfig} +import pl.touk.nussknacker.ui.config.{DesignerConfig, DesignerConfigLoader} +import pl.touk.nussknacker.ui.configloader.{ProcessingTypeConfigsLoader, ProcessingTypeConfigsLoaderFactory} import pl.touk.nussknacker.ui.db.DbRef import pl.touk.nussknacker.ui.db.timeseries.questdb.QuestDbFEStatisticsRepository -import pl.touk.nussknacker.ui.process.processingtype.loader._ +import pl.touk.nussknacker.ui.process.processingtype.loader.{ + ProcessingTypeDataLoader, + ProcessingTypesConfigBasedProcessingTypeDataLoader +} import pl.touk.nussknacker.ui.server.{AkkaHttpBasedRouteProvider, NussknackerHttpServer} +import pl.touk.nussknacker.ui.util.{ActorSystemBasedExecutionContextWithIORuntime, IOToFutureSttpBackendConverter} +import sttp.client3.SttpBackend +import sttp.client3.asynchttpclient.cats.AsyncHttpClientCatsBackend import java.time.Clock -object NussknackerAppFactory - -class NussknackerAppFactory(nussknackerConfig: NussknackerConfig, processingTypeDataLoader: ProcessingTypeDataLoader) - extends LazyLogging { - - def this(nussknackerConfig: NussknackerConfig) = { - this(nussknackerConfig, new ProcessingTypesConfigBasedProcessingTypeDataLoader(nussknackerConfig)) - } - - def this(classLoader: ClassLoader) = { - this(new LoadableDesignerConfigBasedNussknackerConfig(classLoader)) - } +class NussknackerAppFactory( + designerConfigLoader: DesignerConfigLoader, + createProcessingTypeDataLoader: ProcessingTypeConfigsLoader => ProcessingTypeDataLoader +) extends LazyLogging { def createApp(clock: Clock = Clock.systemUTC()): Resource[IO, Unit] = { for { - config <- Resource.eval(nussknackerConfig.loadApplicationConfig()) - system <- createActorSystem(config) - materializer = Materializer(system) + designerConfig <- Resource.eval(designerConfigLoader.loadDesignerConfig()) + system <- createActorSystem(designerConfig.rawConfig) + executionContextWithIORuntime = ActorSystemBasedExecutionContextWithIORuntime.createFrom(system) + ioSttpBackend <- AsyncHttpClientCatsBackend.resource[IO]() + processingTypeConfigsLoader = createProcessingTypeConfigsLoader( + designerConfig, + ioSttpBackend + )(executionContextWithIORuntime.ioRuntime) + processingTypeDataLoader = createProcessingTypeDataLoader(processingTypeConfigsLoader) + materializer = Materializer(system) _ <- Resource.eval(IO(JavaClassVersionChecker.check())) _ <- Resource.eval(IO(SLF4JBridgeHandlerRegistrar.register())) metricsRegistry <- createGeneralPurposeMetricsRegistry() - db <- DbRef.create(config.resolved) - feStatisticsRepository <- QuestDbFEStatisticsRepository.create(system, clock, config.resolved) + db <- DbRef.create(designerConfig.rawConfig.resolved) + feStatisticsRepository <- QuestDbFEStatisticsRepository.create(system, clock, designerConfig.rawConfig.resolved) server = new NussknackerHttpServer( new AkkaHttpBasedRouteProvider( db, metricsRegistry, + IOToFutureSttpBackendConverter.convert(ioSttpBackend)(executionContextWithIORuntime), processingTypeDataLoader, feStatisticsRepository, clock )( system, - materializer + materializer, + executionContextWithIORuntime ), system ) - _ <- server.start(config, metricsRegistry) + _ <- server.start(designerConfig, metricsRegistry) _ <- startJmxReporter(metricsRegistry) _ <- createStartAndStopLoggingEntries() } yield () } + private def createProcessingTypeConfigsLoader( + designerConfig: DesignerConfig, + sttpBackend: SttpBackend[IO, Any] + )(implicit ioRuntime: IORuntime): ProcessingTypeConfigsLoader = { + ScalaServiceLoader + .loadOne[ProcessingTypeConfigsLoaderFactory](getClass.getClassLoader) + .map { factory => + logger.debug( + s"Found custom ${classOf[ProcessingTypeConfigsLoaderFactory].getSimpleName}: ${factory.getClass.getName}. Using it for configuration loading" + ) + factory.create(designerConfig.configLoaderConfig, sttpBackend) + } + .getOrElse { + logger.debug( + s"No custom ${classOf[ProcessingTypeConfigsLoaderFactory].getSimpleName} found. Using the default implementation of loader" + ) + () => designerConfigLoader.loadDesignerConfig().map(_.processingTypeConfigs) + } + } + private def createActorSystem(config: ConfigWithUnresolvedVersion) = { Resource .make( @@ -87,3 +117,14 @@ class NussknackerAppFactory(nussknackerConfig: NussknackerConfig, processingType } } + +object NussknackerAppFactory { + + def apply(designerConfigLoader: DesignerConfigLoader): NussknackerAppFactory = { + new NussknackerAppFactory( + designerConfigLoader, + new ProcessingTypesConfigBasedProcessingTypeDataLoader(_) + ) + } + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala index 30500afd1a1..de70b5c0ba2 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala @@ -6,12 +6,12 @@ import pl.touk.nussknacker.engine._ import pl.touk.nussknacker.engine.api.process.ProcessingType import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader -import pl.touk.nussknacker.ui.NussknackerConfig +import pl.touk.nussknacker.ui.configloader.{ProcessingTypeConfigs, ProcessingTypeConfigsLoader} import pl.touk.nussknacker.ui.process.processingtype._ import pl.touk.nussknacker.ui.process.processingtype.loader.ProcessingTypeDataLoader.toValueWithRestriction import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataState -class ProcessingTypesConfigBasedProcessingTypeDataLoader(config: NussknackerConfig) +class ProcessingTypesConfigBasedProcessingTypeDataLoader(processingTypeConfigsLoader: ProcessingTypeConfigsLoader) extends ProcessingTypeDataLoader with LazyLogging { @@ -19,51 +19,57 @@ class ProcessingTypesConfigBasedProcessingTypeDataLoader(config: NussknackerConf getModelDependencies: ProcessingType => ModelDependencies, getDeploymentManagerDependencies: ProcessingType => DeploymentManagerDependencies, ): IO[ProcessingTypeDataState[ProcessingTypeData, CombinedProcessingTypeData]] = { - config + processingTypeConfigsLoader .loadProcessingTypeConfigs() - .map { processingTypesConfig => - // This step with splitting DeploymentManagerProvider loading for all processing types - // and after that creating ProcessingTypeData is done because of the deduplication of deployments - // See DeploymentManagerProvider.engineSetupIdentity - val providerWithNameInputData = processingTypesConfig.mapValuesNow { processingTypeConfig => - val provider = createDeploymentManagerProvider(processingTypeConfig) - val nameInputData = EngineNameInputData( - provider.defaultEngineSetupName, - provider.engineSetupIdentity(processingTypeConfig.deploymentConfig), - processingTypeConfig.engineSetupName - ) - (processingTypeConfig, provider, nameInputData) - } - val engineSetupNames = - ScenarioParametersDeterminer.determineEngineSetupNames(providerWithNameInputData.mapValuesNow(_._3)) - val processingTypesData = providerWithNameInputData - .map { case (processingType, (processingTypeConfig, deploymentManagerProvider, _)) => - logger.debug(s"Creating Processing Type: $processingType with config: $processingTypeConfig") - val modelDependencies = getModelDependencies(processingType) - val processingTypeData = ProcessingTypeData.createProcessingTypeData( - processingType, - ModelData(processingTypeConfig, modelDependencies), - deploymentManagerProvider, - getDeploymentManagerDependencies(processingType), - engineSetupNames(processingType), - processingTypeConfig.deploymentConfig, - processingTypeConfig.category, - modelDependencies.componentDefinitionExtractionMode - ) - processingType -> processingTypeData - } - - // Here all processing types are loaded and we are ready to perform additional configuration validations - // to assert the loaded configuration is correct (fail-fast approach). - val combinedData = CombinedProcessingTypeData.create(processingTypesData) + .map(createProcessingTypeData(_, getModelDependencies, getDeploymentManagerDependencies)) + } - ProcessingTypeDataState( - processingTypesData.mapValuesNow(toValueWithRestriction), - () => combinedData, - // We pass here new Object to enforce update of observers - new Object + private def createProcessingTypeData( + processingTypesConfig: ProcessingTypeConfigs, + getModelDependencies: ProcessingType => ModelDependencies, + getDeploymentManagerDependencies: ProcessingType => DeploymentManagerDependencies + ): ProcessingTypeDataState[ProcessingTypeData, CombinedProcessingTypeData] = { + // This step with splitting DeploymentManagerProvider loading for all processing types + // and after that creating ProcessingTypeData is done because of the deduplication of deployments + // See DeploymentManagerProvider.engineSetupIdentity + val providerWithNameInputData = processingTypesConfig.configByProcessingType.mapValuesNow { processingTypeConfig => + val provider = createDeploymentManagerProvider(processingTypeConfig) + val nameInputData = EngineNameInputData( + provider.defaultEngineSetupName, + provider.engineSetupIdentity(processingTypeConfig.deploymentConfig), + processingTypeConfig.engineSetupName + ) + (processingTypeConfig, provider, nameInputData) + } + val engineSetupNames = + ScenarioParametersDeterminer.determineEngineSetupNames(providerWithNameInputData.mapValuesNow(_._3)) + val processingTypesData = providerWithNameInputData + .map { case (processingType, (processingTypeConfig, deploymentManagerProvider, _)) => + logger.debug(s"Creating Processing Type: $processingType with config: $processingTypeConfig") + val modelDependencies = getModelDependencies(processingType) + val processingTypeData = ProcessingTypeData.createProcessingTypeData( + processingType, + ModelData(processingTypeConfig, modelDependencies), + deploymentManagerProvider, + getDeploymentManagerDependencies(processingType), + engineSetupNames(processingType), + processingTypeConfig.deploymentConfig, + processingTypeConfig.category, + modelDependencies.componentDefinitionExtractionMode ) + processingType -> processingTypeData } + + // Here all processing types are loaded and we are ready to perform additional configuration validations + // to assert the loaded configuration is correct (fail-fast approach). + val combinedData = CombinedProcessingTypeData.create(processingTypesData) + + ProcessingTypeDataState( + processingTypesData.mapValuesNow(toValueWithRestriction), + () => combinedData, + // We pass here new Object to enforce update of observers + new Object + ) } private def createDeploymentManagerProvider(typeConfig: ProcessingTypeConfig): DeploymentManagerProvider = { diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala index bb1356a2319..ce1ed9743ee 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala @@ -17,7 +17,7 @@ import pl.touk.nussknacker.engine.definition.test.ModelDataTestInfoProvider import pl.touk.nussknacker.engine.dict.ProcessDictSubstitutor import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader import pl.touk.nussknacker.engine.util.multiplicity.{Empty, Many, Multiplicity, One} -import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, DeploymentManagerDependencies, ModelDependencies} +import pl.touk.nussknacker.engine.{DeploymentManagerDependencies, ModelDependencies} import pl.touk.nussknacker.processCounts.influxdb.InfluxCountsReporterCreator import pl.touk.nussknacker.processCounts.{CountsReporter, CountsReporterCreator} import pl.touk.nussknacker.ui.api._ @@ -28,6 +28,7 @@ import pl.touk.nussknacker.ui.config.{ FeatureTogglesConfig, UsageStatisticsReportsConfig } +import pl.touk.nussknacker.ui.config.DesignerConfig import pl.touk.nussknacker.ui.db.DbRef import pl.touk.nussknacker.ui.db.timeseries.FEStatisticsRepository import pl.touk.nussknacker.ui.definition.component.{ComponentServiceProcessingTypeData, DefaultComponentService} @@ -92,12 +93,11 @@ import pl.touk.nussknacker.ui.validation.{ UIProcessValidator } import sttp.client3.SttpBackend -import sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend import java.time.Clock import java.util.concurrent.atomic.AtomicReference import java.util.function.Supplier -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.Future import scala.io.Source import scala.util.Try import scala.util.control.NonFatal @@ -105,27 +105,28 @@ import scala.util.control.NonFatal class AkkaHttpBasedRouteProvider( dbRef: DbRef, metricsRegistry: MetricRegistry, + sttpBackend: SttpBackend[Future, Any], processingTypeDataLoader: ProcessingTypeDataLoader, feStatisticsRepository: FEStatisticsRepository[Future], designerClock: Clock -)(implicit system: ActorSystem, materializer: Materializer) - extends RouteProvider[Route] +)( + implicit system: ActorSystem, + materializer: Materializer, + executionContextWithIORuntime: ExecutionContextWithIORuntime +) extends RouteProvider[Route] with Directives with LazyLogging { - override def createRoute(config: ConfigWithUnresolvedVersion): Resource[IO, Route] = { - implicit val executionContextWithIORuntime: ExecutionContextWithIORuntime = - ActorSystemBasedExecutionContextWithIORuntime.createFrom(system) + override def createRoute(designerConfig: DesignerConfig): Resource[IO, Route] = { import executionContextWithIORuntime.ioRuntime + val resolvedDesignerConfig = designerConfig.rawConfig.resolved + val environment = resolvedDesignerConfig.getString("environment") + val featureTogglesConfig = FeatureTogglesConfig.create(resolvedDesignerConfig) + logger.info(s"Designer config loaded: \nfeatureTogglesConfig: $featureTogglesConfig") for { - sttpBackend <- createSttpBackend() - resolvedConfig = config.resolved - environment = resolvedConfig.getString("environment") - featureTogglesConfig = FeatureTogglesConfig.create(resolvedConfig) - _ = logger.info(s"Designer config loaded: \nfeatureTogglesConfig: $featureTogglesConfig") countsReporter <- createCountsReporter(featureTogglesConfig, environment, sttpBackend) actionServiceSupplier = new DelayedInitActionServiceSupplier - additionalUIConfigProvider = createAdditionalUIConfigProvider(resolvedConfig, sttpBackend) + additionalUIConfigProvider = createAdditionalUIConfigProvider(resolvedDesignerConfig, sttpBackend) deploymentRepository = new DeploymentRepository(dbRef, Clock.systemDefaultZone()) scenarioActivityRepository = DbScenarioActivityRepository.create(dbRef, designerClock) dbioRunner = DBIOActionRunner(dbRef) @@ -150,7 +151,7 @@ class AkkaHttpBasedRouteProvider( val scheduler = new DeploymentsStatusesSynchronizationScheduler( system, deploymentsStatusesSynchronizer, - DeploymentsStatusesSynchronizationConfig.parse(resolvedConfig) + DeploymentsStatusesSynchronizationConfig.parse(resolvedDesignerConfig) ) scheduler.start() scheduler @@ -220,10 +221,10 @@ class AkkaHttpBasedRouteProvider( val processResolver = scenarioTestServiceDeps.mapValues(_._2) val scenarioResolver = scenarioTestServiceDeps.mapValues(_._3) - val notificationsConfig = resolvedConfig.as[NotificationConfig]("notifications") + val notificationsConfig = resolvedDesignerConfig.as[NotificationConfig]("notifications") val processChangeListener = ProcessChangeListenerLoader.loadListeners( getClass.getClassLoader, - resolvedConfig, + resolvedDesignerConfig, NussknackerServices(new PullProcessRepository(futureProcessRepository)) ) @@ -257,8 +258,9 @@ class AkkaHttpBasedRouteProvider( // correct classloader and that won't cause further delays during handling requests processingTypeDataProvider.reloadAll().unsafeRunSync() - val authenticationResources = AuthenticationResources(resolvedConfig, getClass.getClassLoader, sttpBackend) - val authManager = new AuthManager(authenticationResources) + val authenticationResources = + AuthenticationResources(resolvedDesignerConfig, getClass.getClassLoader, sttpBackend) + val authManager = new AuthManager(authenticationResources) Initialization.init( migrations, @@ -296,7 +298,7 @@ class AkkaHttpBasedRouteProvider( ) val configProcessToolbarService = new ConfigScenarioToolbarService( - CategoriesScenarioToolbarsConfigParser.parse(resolvedConfig) + CategoriesScenarioToolbarsConfigParser.parse(resolvedDesignerConfig) ) def prepareAlignedComponentsDefinitionProvider( @@ -305,7 +307,7 @@ class AkkaHttpBasedRouteProvider( AlignedComponentsDefinitionProvider(processingTypeData.designerModelData) val componentService = new DefaultComponentService( - ComponentLinksConfigExtractor.extract(resolvedConfig), + ComponentLinksConfigExtractor.extract(resolvedDesignerConfig), processingTypeDataProvider .mapValues { processingTypeData => val alignedModelDefinitionProvider = prepareAlignedComponentsDefinitionProvider(processingTypeData) @@ -328,7 +330,7 @@ class AkkaHttpBasedRouteProvider( ) val processAuthorizer = new AuthorizeProcess(futureProcessRepository) val appApiHttpService = new AppApiHttpService( - config = resolvedConfig, + config = resolvedDesignerConfig, authManager = authManager, processingTypeDataReloader = processingTypeDataProvider, modelBuildInfos = modelBuildInfo, @@ -340,7 +342,7 @@ class AkkaHttpBasedRouteProvider( val migrationApiAdapterService = new MigrationApiAdapterService() val migrationService = new MigrationService( - config = resolvedConfig, + config = resolvedDesignerConfig, processService = processService, processResolver = processResolver, processAuthorizer = processAuthorizer, @@ -416,7 +418,7 @@ class AkkaHttpBasedRouteProvider( scenarioService = processService, scenarioAuthorizer = processAuthorizer, new ScenarioAttachmentService( - AttachmentsConfig.create(resolvedConfig), + AttachmentsConfig.create(resolvedDesignerConfig), scenarioActivityRepository, dbioRunner, ), @@ -469,7 +471,7 @@ class AkkaHttpBasedRouteProvider( new DeploymentApiHttpService(authManager, activityService, deploymentService) } - initMetrics(metricsRegistry, resolvedConfig, futureProcessRepository) + initMetrics(metricsRegistry, resolvedDesignerConfig, futureProcessRepository) val apiResourcesWithAuthentication: List[RouteWithUser] = { val routes = List( @@ -505,7 +507,7 @@ class AkkaHttpBasedRouteProvider( prepareAlignedComponentsDefinitionProvider(processingTypeData), new ScenarioPropertiesConfigFinalizer(additionalUIConfigProvider, processingTypeData.name), fragmentRepository, - resolvedConfig.getAs[String]("fragmentPropertiesDocsUrl") + resolvedDesignerConfig.getAs[String]("fragmentPropertiesDocsUrl") ) ) } @@ -542,8 +544,9 @@ class AkkaHttpBasedRouteProvider( routes ++ optionalRoutes } - val usageStatisticsReportsConfig = resolvedConfig.as[UsageStatisticsReportsConfig]("usageStatisticsReports") - val fingerprintService = new FingerprintService(new FingerprintRepositoryImpl(dbRef)) + val usageStatisticsReportsConfig = + resolvedDesignerConfig.as[UsageStatisticsReportsConfig]("usageStatisticsReports") + val fingerprintService = new FingerprintService(new FingerprintRepositoryImpl(dbRef)) val usageStatisticsReportsSettingsService = UsageStatisticsReportsSettingsService( usageStatisticsReportsConfig, processService, @@ -608,7 +611,7 @@ class AkkaHttpBasedRouteProvider( val akkaHttpServerInterpreter = new NuAkkaHttpServerInterpreterForTapirPurposes() createAppRoute( - resolvedConfig = resolvedConfig, + resolvedConfig = resolvedDesignerConfig, authManager = authManager, tapirRelatedRoutes = akkaHttpServerInterpreter.toRoute(nuDesignerApi.allEndpoints) :: Nil, apiResourcesWithAuthentication = apiResourcesWithAuthentication, @@ -618,15 +621,6 @@ class AkkaHttpBasedRouteProvider( } } - private def createSttpBackend()(implicit executionContext: ExecutionContext) = { - Resource - .make( - acquire = IO(AsyncHttpClientFutureBackend.usingConfigBuilder(identity)) - )( - release = backend => IO.fromFuture(IO(backend.close())) - ) - } - private def initMetrics( metricsRegistry: MetricRegistry, config: Config, @@ -713,28 +707,27 @@ class AkkaHttpBasedRouteProvider( dbioActionRunner: DBIOActionRunner, sttpBackend: SttpBackend[Future, Any], featureTogglesConfig: FeatureTogglesConfig - )(implicit executionContext: ExecutionContext): Resource[IO, ReloadableProcessingTypeDataProvider] = { + ): Resource[IO, ReloadableProcessingTypeDataProvider] = { Resource .make( - acquire = IO( - new ReloadableProcessingTypeDataProvider( - processingTypeDataLoader.loadProcessingTypeData( - getModelDependencies( - additionalUIConfigProvider, - _, - featureTogglesConfig.componentDefinitionExtractionMode - ), - getDeploymentManagerDependencies( - additionalUIConfigProvider, - actionServiceProvider, - scenarioActivityRepository, - dbioActionRunner, - sttpBackend, - _ - ), - ) + acquire = IO { + val laodProcessingTypeDataIO = processingTypeDataLoader.loadProcessingTypeData( + getModelDependencies( + additionalUIConfigProvider, + _, + featureTogglesConfig.componentDefinitionExtractionMode + ), + getDeploymentManagerDependencies( + additionalUIConfigProvider, + actionServiceProvider, + scenarioActivityRepository, + dbioActionRunner, + sttpBackend, + _ + ), ) - ) + new ReloadableProcessingTypeDataProvider(laodProcessingTypeDataIO) + } )( release = _.close() ) @@ -747,7 +740,7 @@ class AkkaHttpBasedRouteProvider( dbioActionRunner: DBIOActionRunner, sttpBackend: SttpBackend[Future, Any], processingType: ProcessingType - )(implicit executionContext: ExecutionContext) = { + ) = { val additionalConfigsFromProvider = additionalUIConfigProvider.getAllForProcessingType(processingType) DeploymentManagerDependencies( DefaultProcessingTypeDeployedScenariosProvider(dbRef, processingType), @@ -781,11 +774,7 @@ class AkkaHttpBasedRouteProvider( ) } - private def createProcessingTypeDataReload() = {} - - private def createAdditionalUIConfigProvider(config: Config, sttpBackend: SttpBackend[Future, Any])( - implicit ec: ExecutionContext - ) = { + private def createAdditionalUIConfigProvider(config: Config, sttpBackend: SttpBackend[Future, Any]) = { val additionalUIConfigProviderFactory: AdditionalUIConfigProviderFactory = { Multiplicity( ScalaServiceLoader.load[AdditionalUIConfigProviderFactory](getClass.getClassLoader) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala index b15913b4f2c..a98e33fac83 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala @@ -10,6 +10,7 @@ import fr.davit.akka.http.metrics.core.{HttpMetricsRegistry, HttpMetricsSettings import fr.davit.akka.http.metrics.dropwizard.{DropwizardRegistry, DropwizardSettings} import io.dropwizard.metrics5.MetricRegistry import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.ui.config.DesignerConfig import pl.touk.nussknacker.ui.security.ssl.{HttpsConnectionContextFactory, SslConfigParser} import java.util.concurrent.atomic.AtomicReference @@ -23,10 +24,10 @@ class NussknackerHttpServer(routeProvider: RouteProvider[Route], system: ActorSy private implicit val systemImplicit: ActorSystem = system private implicit val executionContextImplicit: ExecutionContext = system.dispatcher - def start(config: ConfigWithUnresolvedVersion, metricRegistry: MetricRegistry): Resource[IO, Unit] = { + def start(designerConfig: DesignerConfig, metricRegistry: MetricRegistry): Resource[IO, Unit] = { for { - route <- routeProvider.createRoute(config) - _ <- createAkkaHttpBinding(config, route, metricRegistry) + route <- routeProvider.createRoute(designerConfig) + _ <- createAkkaHttpBinding(designerConfig.rawConfig, route, metricRegistry) } yield { RouteInterceptor.set(route) } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala index ce69787f60c..838760ce0e8 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala @@ -2,9 +2,10 @@ package pl.touk.nussknacker.ui.server import akka.http.scaladsl.server.Route import cats.effect.{IO, Resource} -import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.ui.config.DesignerConfig trait RouteProvider[R <: Route] { - def createRoute(config: ConfigWithUnresolvedVersion): Resource[IO, R] + def createRoute(config: DesignerConfig): Resource[IO, R] + } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/IOToFutureSttpBackendConverter.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/IOToFutureSttpBackendConverter.scala new file mode 100644 index 00000000000..6c1edb56bcd --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/IOToFutureSttpBackendConverter.scala @@ -0,0 +1,27 @@ +package pl.touk.nussknacker.ui.util + +import cats.arrow.FunctionK +import cats.effect.IO +import sttp.client3.SttpBackend +import sttp.client3.impl.cats.implicits.sttpBackendToCatsMappableSttpBackend +import sttp.monad.FutureMonad + +import scala.concurrent.Future + +object IOToFutureSttpBackendConverter { + + def convert( + ioBackend: SttpBackend[IO, Any] + )(implicit executionContextWithIORuntime: ExecutionContextWithIORuntime): SttpBackend[Future, Any] = { + import executionContextWithIORuntime.ioRuntime + ioBackend.mapK( + new FunctionK[IO, Future] { + override def apply[A](io: IO[A]): Future[A] = io.unsafeToFuture() + }, + new FunctionK[Future, IO] { + override def apply[A](future: Future[A]): IO[A] = IO.fromFuture(IO.pure(future)) + } + )(new FutureMonad) + } + +} diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala index 49ec79dc4b8..174f55d2179 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala @@ -8,10 +8,8 @@ import org.scalatest.{BeforeAndAfterAll, Suite} import pl.touk.nussknacker.test.DefaultUniquePortProvider import pl.touk.nussknacker.test.base.db.WithHsqlDbTesting import pl.touk.nussknacker.test.config.WithDesignerConfig -import pl.touk.nussknacker.ui.LoadableConfigBasedNussknackerConfig import pl.touk.nussknacker.ui.config.DesignerConfigLoader import pl.touk.nussknacker.ui.factory.NussknackerAppFactory -import pl.touk.nussknacker.ui.process.processingtype.loader._ trait NuItTest extends WithHsqlDbTesting with DefaultUniquePortProvider with WithClock with BeforeAndAfterAll { this: Suite with WithDesignerConfig => @@ -24,10 +22,8 @@ trait NuItTest extends WithHsqlDbTesting with DefaultUniquePortProvider with Wit override protected def beforeAll(): Unit = { super.beforeAll() - val nussknackerConfig = new LoadableConfigBasedNussknackerConfig( - IO.delay(DesignerConfigLoader.from(adjustNuTestConfig())) - ) - releaseAppResources = new NussknackerAppFactory(nussknackerConfig) + val designerConfigLoader = DesignerConfigLoader.fromConfig(adjustNuTestConfig()) + releaseAppResources = NussknackerAppFactory(designerConfigLoader) .createApp(clock = clock) .allocated .unsafeRunSync() diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala index cde22f36654..d47bcc52a6d 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala @@ -38,10 +38,10 @@ import pl.touk.nussknacker.test.mock.{MockDeploymentManager, MockManagerProvider import pl.touk.nussknacker.test.utils.domain.TestFactory._ import pl.touk.nussknacker.test.utils.domain.{ProcessTestData, TestFactory} import pl.touk.nussknacker.test.utils.scalas.AkkaHttpExtensions.toRequestEntity -import pl.touk.nussknacker.ui.LoadableConfigBasedNussknackerConfig import pl.touk.nussknacker.ui.api._ -import pl.touk.nussknacker.ui.config.FeatureTogglesConfig import pl.touk.nussknacker.ui.config.scenariotoolbar.CategoriesScenarioToolbarsConfigParser +import pl.touk.nussknacker.ui.config.FeatureTogglesConfig +import pl.touk.nussknacker.ui.config.DesignerConfig import pl.touk.nussknacker.ui.process.ProcessService.{CreateScenarioCommand, UpdateScenarioCommand} import pl.touk.nussknacker.ui.process._ import pl.touk.nussknacker.ui.process.deployment._ @@ -147,12 +147,13 @@ trait NuResourcesTest protected val featureTogglesConfig: FeatureTogglesConfig = FeatureTogglesConfig.create(testConfig) protected val typeToConfig: ProcessingTypeDataProvider[ProcessingTypeData, CombinedProcessingTypeData] = { - val processingTypeDataReader = new ProcessingTypesConfigBasedProcessingTypeDataLoader( - new LoadableConfigBasedNussknackerConfig(IO.pure(ConfigWithUnresolvedVersion(testConfig))) - ) + val designerConfig = DesignerConfig.from(testConfig) ProcessingTypeDataProvider( - processingTypeDataReader - .loadProcessingTypeData(_ => modelDependencies, _ => deploymentManagerDependencies) + new ProcessingTypesConfigBasedProcessingTypeDataLoader(() => IO.pure(designerConfig.processingTypeConfigs)) + .loadProcessingTypeData( + _ => modelDependencies, + _ => deploymentManagerDependencies + ) .unsafeRunSync() ) } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala index d90167c6e80..73ae8434f93 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala @@ -8,10 +8,10 @@ import com.typesafe.config.ConfigValueFactory.fromAnyRef import io.dropwizard.metrics5.MetricRegistry import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion -import pl.touk.nussknacker.test.{DefaultUniquePortProvider, WithTestHttpClientCreator} import pl.touk.nussknacker.test.config.ConfigWithScalaVersion import pl.touk.nussknacker.test.utils.scalas.CatsTestExtensions._ +import pl.touk.nussknacker.test.{DefaultUniquePortProvider, WithTestHttpClientCreator} +import pl.touk.nussknacker.ui.config.DesignerConfig import pl.touk.nussknacker.ui.security.ssl.HttpsConnectionContextFactory.prepareSSLContext import pl.touk.nussknacker.ui.security.ssl.KeyStoreConfig import pl.touk.nussknacker.ui.server.{NussknackerHttpServer, RouteProvider} @@ -32,7 +32,7 @@ class NussknackerHttpServerSpec server <- createHttpServer() client <- createHttpClient(Some(prepareSSLContext(keyStoreConfig))) _ <- server.start( - ConfigWithUnresolvedVersion( + DesignerConfig.from( ConfigFactory .empty() .withValue("http.interface", fromAnyRef("0.0.0.0")) @@ -64,7 +64,7 @@ class NussknackerHttpServerSpec private object DummyRouteProvider extends RouteProvider[Route] with Directives { - override def createRoute(config: ConfigWithUnresolvedVersion): Resource[IO, Route] = Resource.pure[IO, Route] { + override def createRoute(config: DesignerConfig): Resource[IO, Route] = Resource.pure[IO, Route] { path("test") { get { complete { diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/integration/ConfigurationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/config/ConfigurationTest.scala similarity index 69% rename from designer/server/src/test/scala/pl/touk/nussknacker/ui/integration/ConfigurationTest.scala rename to designer/server/src/test/scala/pl/touk/nussknacker/ui/config/ConfigurationTest.scala index 5631ea406fc..ac1e56a3885 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/integration/ConfigurationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/config/ConfigurationTest.scala @@ -1,17 +1,17 @@ -package pl.touk.nussknacker.ui.integration +package pl.touk.nussknacker.ui.config +import cats.effect.unsafe.implicits.global import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt import pl.touk.nussknacker.engine.{ModelData, ProcessingTypeConfig} import pl.touk.nussknacker.test.config.ConfigWithScalaVersion import pl.touk.nussknacker.test.utils.domain.TestFactory -import pl.touk.nussknacker.ui.config.DesignerConfigLoader -import cats.effect.unsafe.implicits.global + import java.net.URI import java.nio.file.Files import java.util.UUID +// TODO: We should spit DesignerConfigLoader tests and model ProcessingTypeConfig tests class ConfigurationTest extends AnyFunSuite with Matchers { // warning: can't be val - uses ConfigFactory.load which breaks "should preserve config overrides" test @@ -29,21 +29,23 @@ class ConfigurationTest extends AnyFunSuite with Matchers { } test("defaultConfig works") { - DesignerConfigLoader - .load(globalConfig, classLoader) + AlwaysLoadingFileBasedDesignerConfigLoader(classLoader) + .loadDesignerConfig() .unsafeRunSync() + .rawConfig .resolved .getString("db.driver") shouldBe "org.hsqldb.jdbc.JDBCDriver" } test("should be possible to config entries defined in default ui config from passed config") { val configUri = writeToTemp("foo: ${storageDir}") // storageDir is defined inside defaultDesignerConfig.conf + withNussknackerLocationsProperty(configUri.toString) { + val loadedConfig = AlwaysLoadingFileBasedDesignerConfigLoader(classLoader) + .loadDesignerConfig() + .unsafeRunSync() - val loadedConfig = DesignerConfigLoader - .load(ConfigFactoryExt.parseConfigFallbackChain(List(configUri), classLoader), classLoader) - .unsafeRunSync() - - loadedConfig.resolved.getString("foo") shouldEqual "./storage" + loadedConfig.rawConfig.resolved.getString("foo") shouldEqual "./storage" + } } test("defaultConfig is not accessible from model") { @@ -78,17 +80,32 @@ class ConfigurationTest extends AnyFunSuite with Matchers { |""".stripMargin val conf1 = writeToTemp(content) - val result = - try { - System.setProperty(randomPropertyName, "I win!") - DesignerConfigLoader - .load(ConfigFactoryExt.parseConfigFallbackChain(List(conf1), classLoader), classLoader) - .unsafeRunSync() - } finally { - System.getProperties.remove(randomPropertyName) - } - - result.resolved.getString(randomPropertyName) shouldBe "I win!" + withNussknackerLocationsProperty(conf1.toString) { + val result = + try { + System.setProperty(randomPropertyName, "I win!") + AlwaysLoadingFileBasedDesignerConfigLoader(classLoader) + .loadDesignerConfig() + .unsafeRunSync() + } finally { + System.getProperties.remove(randomPropertyName) + } + + result.rawConfig.resolved.getString(randomPropertyName) shouldBe "I win!" + } + } + + private def withNussknackerLocationsProperty(propertyValue: String)(f: => Unit): Unit = { + val locationsProperty = "nussknacker.config.locations" + val valueBeforeChange = Option(System.getProperty(locationsProperty)) + System.setProperty(locationsProperty, propertyValue) + try { + f + } finally { + valueBeforeChange + .map(System.setProperty(locationsProperty, _)) + .getOrElse(System.getProperties.remove(locationsProperty)) + } } // to be able to run this test: diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableConfigBasedNussknackerConfigSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/config/processingtype/ProcessingTypeDataLoaderSpec.scala similarity index 70% rename from designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableConfigBasedNussknackerConfigSpec.scala rename to designer/server/src/test/scala/pl/touk/nussknacker/ui/config/processingtype/ProcessingTypeDataLoaderSpec.scala index 51195a0c2ba..ec23803e342 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableConfigBasedNussknackerConfigSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/config/processingtype/ProcessingTypeDataLoaderSpec.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.ui +package pl.touk.nussknacker.ui.config.processingtype import cats.effect.unsafe.implicits.global import cats.effect.{IO, Ref} @@ -6,9 +6,10 @@ import com.typesafe import com.typesafe.config.{Config, ConfigFactory} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers.{convertToAnyShouldWrapper, include} -import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.ui.config.{DesignerConfig, DesignerConfigLoader} +import pl.touk.nussknacker.ui.configloader.ProcessingTypeConfigsLoader -class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { +class ProcessingTypeDataLoaderSpec extends AnyFunSuite { test("should throw when required configuration is missing") { val config = ConfigFactory @@ -31,7 +32,7 @@ class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { .resolve() intercept[typesafe.config.ConfigException] { - loadableConfigBasedNussknackerConfig(config) + staticConfigBasedProcessingTypeConfigsLoader(config) .loadProcessingTypeConfigs() .unsafeRunSync() }.getMessage should include("No configuration setting found for key 'deploymentConfig.type'") @@ -47,14 +48,14 @@ class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { .resolve() intercept[RuntimeException] { - loadableConfigBasedNussknackerConfig(config) + staticConfigBasedProcessingTypeConfigsLoader(config) .loadProcessingTypeConfigs() .unsafeRunSync() }.getMessage should include("No scenario types configuration provided") } test("should load the second config when reloaded") { - val nussknackerConfig = loadDifferentConfigPerInvocationNussknackerConfig( + val processingTypeConfigsLoader = loadDifferentConfigPerInvocationProcessingTypeConfigsLoader( config1 = ConfigFactory .parseString( """ @@ -103,39 +104,37 @@ class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { .resolve() ) - val processingTypes1 = nussknackerConfig + val processingTypes1 = processingTypeConfigsLoader .loadProcessingTypeConfigs() .unsafeRunSync() - processingTypes1.keys.toSet shouldBe Set("streaming") + processingTypes1.configByProcessingType.keys.toSet shouldBe Set("streaming") - val processingTypes2 = nussknackerConfig + val processingTypes2 = processingTypeConfigsLoader .loadProcessingTypeConfigs() .unsafeRunSync() - processingTypes2.keys.toSet shouldBe Set("streaming", "streaming2") + processingTypes2.configByProcessingType.keys.toSet shouldBe Set("streaming", "streaming2") } - private def loadableConfigBasedNussknackerConfig(config: Config): LoadableConfigBasedNussknackerConfig = { - loadableConfigBasedNussknackerConfig(IO.pure(ConfigWithUnresolvedVersion(config))) + private def staticConfigBasedProcessingTypeConfigsLoader(config: Config): ProcessingTypeConfigsLoader = { () => + DesignerConfigLoader.fromConfig(config).loadDesignerConfig().map(_.processingTypeConfigs) } - private def loadableConfigBasedNussknackerConfig( - loadConfig: IO[ConfigWithUnresolvedVersion] - ): LoadableConfigBasedNussknackerConfig = { - new LoadableConfigBasedNussknackerConfig(loadConfig) - } - - private def loadDifferentConfigPerInvocationNussknackerConfig(config1: Config, config2: Config, configs: Config*) = { + private def loadDifferentConfigPerInvocationProcessingTypeConfigsLoader( + config1: Config, + config2: Config, + configs: Config* + ): ProcessingTypeConfigsLoader = { val ref = Ref.unsafe[IO, Int](0) val allConfigs = config1 :: config2 :: configs.toList - val loadConfig = ref.getAndUpdate(_ + 1).flatMap { idx => + val loadDesignerConfig = ref.getAndUpdate(_ + 1).flatMap { idx => allConfigs.lift(idx) match { - case Some(config) => IO.pure(ConfigWithUnresolvedVersion(config)) + case Some(config) => IO.pure(DesignerConfig.from(config)) case None => IO.raiseError(throw new IllegalStateException(s"Cannot load the config more than [$idx]")) } } - loadableConfigBasedNussknackerConfig(loadConfig) + () => loadDesignerConfig.map(_.processingTypeConfigs) } } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala index 110f195b4da..f4a8e88d82d 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala @@ -55,7 +55,10 @@ class ProcessingTypeDataProviderSpec extends AnyFunSuite with Matchers { deploymentManagerProvider = new DeploymentManagerProviderStub ) loader - .loadProcessingTypeData(_ => modelDependencies, _ => TestFactory.deploymentManagerDependencies) + .loadProcessingTypeData( + _ => modelDependencies, + _ => TestFactory.deploymentManagerDependencies + ) .unsafeRunSync() } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala index d0502b2815c..fe079bdeec1 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala @@ -2,25 +2,25 @@ package pl.touk.nussknacker.ui.process.processingtype import cats.data.Validated.Invalid import cats.effect.IO +import cats.effect.unsafe.implicits.global import com.typesafe.config.ConfigFactory import com.typesafe.scalalogging.LazyLogging import org.scalatest.Inside.inside import org.scalatest.OptionValues import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, ModelDependencies} +import pl.touk.nussknacker.engine.ModelDependencies import pl.touk.nussknacker.engine.api.component.{ComponentProvider, DesignerWideComponentId, ProcessingMode} import pl.touk.nussknacker.engine.api.process.ProcessingType +import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode import pl.touk.nussknacker.engine.deployment.EngineSetupName import pl.touk.nussknacker.restmodel.scenariodetails.ScenarioParameters import pl.touk.nussknacker.security.Permission import pl.touk.nussknacker.test.ValidatedValuesDetailedMessage import pl.touk.nussknacker.test.utils.domain.TestFactory +import pl.touk.nussknacker.ui.config.DesignerConfig import pl.touk.nussknacker.ui.process.processingtype.loader.ProcessingTypesConfigBasedProcessingTypeDataLoader import pl.touk.nussknacker.ui.security.api.{LoggedUser, RealLoggedUser} -import cats.effect.unsafe.implicits.global -import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode -import pl.touk.nussknacker.ui.LoadableConfigBasedNussknackerConfig import java.nio.file.Path import scala.jdk.CollectionConverters._ @@ -281,25 +281,23 @@ class ScenarioParametersServiceTest val workPath = designerServerModuleDir.resolve("work") logDirectoryStructure(workPath) - val processingTypeDataReader = new ProcessingTypesConfigBasedProcessingTypeDataLoader( - new LoadableConfigBasedNussknackerConfig(IO.pure { - ConfigWithUnresolvedVersion(ConfigFactory.parseFile(devApplicationConfFile).withFallback(fallbackConfig)) - }) - ) - val processingTypeData = processingTypeDataReader - .loadProcessingTypeData( - processingType => - ModelDependencies( - Map.empty, - componentId => DesignerWideComponentId(componentId.toString), - Some(workPath), - shouldIncludeComponentProvider(processingType, _), - ComponentDefinitionExtractionMode.FinalDefinition - ), - _ => TestFactory.deploymentManagerDependencies, - ) - .unsafeRunSync() + val designerConfig = + DesignerConfig.from(ConfigFactory.parseFile(devApplicationConfFile).withFallback(fallbackConfig)) + val processingTypeData = + new ProcessingTypesConfigBasedProcessingTypeDataLoader(() => IO.pure(designerConfig.processingTypeConfigs)) + .loadProcessingTypeData( + processingType => + ModelDependencies( + Map.empty, + componentId => DesignerWideComponentId(componentId.toString), + Some(workPath), + shouldIncludeComponentProvider(processingType, _), + ComponentDefinitionExtractionMode.FinalDefinition + ), + _ => TestFactory.deploymentManagerDependencies, + ) + .unsafeRunSync() val parametersService = processingTypeData.getCombined().parametersService parametersService.scenarioParametersCombinationsWithWritePermission(TestFactory.adminUser()) shouldEqual List( diff --git a/engine/lite/runtime-app/src/main/scala/pl/touk/nussknacker/engine/lite/app/NuRuntimeApp.scala b/engine/lite/runtime-app/src/main/scala/pl/touk/nussknacker/engine/lite/app/NuRuntimeApp.scala index 887a6fbcbd0..2f33594193a 100644 --- a/engine/lite/runtime-app/src/main/scala/pl/touk/nussknacker/engine/lite/app/NuRuntimeApp.scala +++ b/engine/lite/runtime-app/src/main/scala/pl/touk/nussknacker/engine/lite/app/NuRuntimeApp.scala @@ -9,7 +9,7 @@ import com.typesafe.scalalogging.LazyLogging import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.marshall.ScenarioParser import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt -import pl.touk.nussknacker.engine.util.{JavaClassVersionChecker, ResourceLoader, SLF4JBridgeHandlerRegistrar} +import pl.touk.nussknacker.engine.util.{JavaClassVersionChecker, ResourceLoader, SLF4JBridgeHandlerRegistrar, UriUtils} import java.nio.file.Path import scala.concurrent.duration._ @@ -27,7 +27,13 @@ object NuRuntimeApp extends App with LazyLogging { val (scenarioFileLocation, deploymentConfigLocation) = parseArgs val scenario = parseScenario(scenarioFileLocation) val deploymentConfig = parseDeploymentConfig(deploymentConfigLocation) - val runtimeConfig = ConfigFactory.load(ConfigFactoryExt.parseUnresolved(classLoader = getClass.getClassLoader)) + + val runtimeConfig = { + val configLocationsProperty: String = "nussknacker.config.locations" + val locationsPropertyValueOpt = Option(System.getProperty(configLocationsProperty)) + val locations = locationsPropertyValueOpt.map(UriUtils.extractListOfLocations).getOrElse(List.empty) + ConfigFactory.load(new ConfigFactoryExt(getClass.getClassLoader).parseUnresolved(locations)) + } val httpConfig = runtimeConfig.as[HttpBindingConfig]("http") diff --git a/extensions-api/src/main/scala/pl/touk/nussknacker/engine/ConfigWithUnresolvedVersion.scala b/extensions-api/src/main/scala/pl/touk/nussknacker/engine/ConfigWithUnresolvedVersion.scala index fb63f036c08..16dad532540 100644 --- a/extensions-api/src/main/scala/pl/touk/nussknacker/engine/ConfigWithUnresolvedVersion.scala +++ b/extensions-api/src/main/scala/pl/touk/nussknacker/engine/ConfigWithUnresolvedVersion.scala @@ -2,12 +2,31 @@ package pl.touk.nussknacker.engine import com.typesafe.config.{Config, ConfigFactory, ConfigResolveOptions} +import scala.jdk.CollectionConverters._ + case class ConfigWithUnresolvedVersion private (withUnresolvedEnvVariables: Config, resolved: Config) { def getConfig(path: String): ConfigWithUnresolvedVersion = { ConfigWithUnresolvedVersion(withUnresolvedEnvVariables.getConfig(path), resolved.getConfig(path)) } + def readMap(path: String): Option[Map[String, ConfigWithUnresolvedVersion]] = { + if (resolved.hasPath(path)) { + val nestedConfig = getConfig(path) + Some( + nestedConfig.resolved + .root() + .entrySet() + .asScala + .map(_.getKey) + .map { key => key -> nestedConfig.getConfig(key) } + .toMap + ) + } else { + None + } + } + } object ConfigWithUnresolvedVersion { diff --git a/security/src/main/scala/pl/touk/nussknacker/ui/security/api/AuthenticationConfiguration.scala b/security/src/main/scala/pl/touk/nussknacker/ui/security/api/AuthenticationConfiguration.scala index 07f94a03962..4a0121a0aab 100644 --- a/security/src/main/scala/pl/touk/nussknacker/ui/security/api/AuthenticationConfiguration.scala +++ b/security/src/main/scala/pl/touk/nussknacker/ui/security/api/AuthenticationConfiguration.scala @@ -19,7 +19,7 @@ trait AuthenticationConfiguration { def isAdminImpersonationPossible: Boolean - private lazy val userConfig: Config = ConfigFactoryExt.parseUri(usersFile, getClass.getClassLoader) + private lazy val userConfig: Config = new ConfigFactoryExt(getClass.getClassLoader).parseUri(usersFile) protected lazy val usersOpt: Option[List[ConfigUser]] = userConfig.as[Option[List[ConfigUser]]](usersConfigurationPath) @@ -44,7 +44,7 @@ object AuthenticationConfiguration { val rulesConfigurationPath = "rules" private[security] def getRules(usersFile: URI): List[ConfigRule] = - ConfigFactoryExt.parseUri(usersFile, getClass.getClassLoader).as[List[ConfigRule]](rulesConfigurationPath) + new ConfigFactoryExt(getClass.getClassLoader).parseUri(usersFile).as[List[ConfigRule]](rulesConfigurationPath) final case class ConfigUser( identity: String, diff --git a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/UriUtils.scala b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/UriUtils.scala index a0ff3080a77..4cfb516b482 100644 --- a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/UriUtils.scala +++ b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/UriUtils.scala @@ -1,7 +1,9 @@ package pl.touk.nussknacker.engine.util -import java.net.{URLDecoder, URLEncoder} +import java.io.File +import java.net.{URI, URLDecoder, URLEncoder} import java.nio.charset.StandardCharsets +import scala.util.Try /** * Utility class for JavaScript compatible UTF-8 encoding and decoding. @@ -10,6 +12,13 @@ import java.nio.charset.StandardCharsets */ object UriUtils { + def extractListOfLocations(commaSeparatedListOfLocations: String): List[URI] = { + for { + singleElement <- commaSeparatedListOfLocations.split(",").toList + trimmed = singleElement.trim + } yield Try(URI.create(trimmed)).filter(_.getScheme.nonEmpty).getOrElse(new File(trimmed).toURI) + } + /** * Decodes the passed UTF-8 String using an algorithm that's compatible with * JavaScript's decodeURIComponent function. Returns diff --git a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExt.scala b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExt.scala index ec2dc6ed908..d2beeef99ff 100644 --- a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExt.scala +++ b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExt.scala @@ -2,13 +2,17 @@ package pl.touk.nussknacker.engine.util.config import com.typesafe.config.{Config, ConfigFactory} -import java.io.File import java.net.URI -import scala.util.Try -object ConfigFactoryExt extends URIExtensions { +class ConfigFactoryExt(classLoader: ClassLoader) extends URIExtensions { - def parseUri(uri: URI, classLoader: ClassLoader): Config = { + def parseUnresolved(locations: List[URI]): Config = + locations + .map(parseUri) + .reverse + .foldLeft(ConfigFactory.empty())(_.withFallback(_)) + + def parseUri(uri: URI): Config = { uri.getScheme match { // When we migrate to Java 9+ we can use SPI to load a URLStreamHandlerProvider // instance to handle the classpath scheme. @@ -17,24 +21,4 @@ object ConfigFactoryExt extends URIExtensions { } } - def parseConfigFallbackChain(resources: List[URI], classLoader: ClassLoader): Config = - resources - .map(ConfigFactoryExt.parseUri(_, classLoader)) - .reverse - .foldLeft(ConfigFactory.empty())(_.withFallback(_)) - - private val configLocationProperty: String = "nussknacker.config.locations" - - def parseUnresolved( - locationString: String = System.getProperty(configLocationProperty), - classLoader: ClassLoader - ): Config = { - val locations = for { - property <- Option(locationString).toList - singleElement <- property.split(",") - trimmed = singleElement.trim - } yield Try(URI.create(trimmed)).filter(_.getScheme.nonEmpty).getOrElse(new File(trimmed).toURI) - ConfigFactoryExt.parseConfigFallbackChain(locations, classLoader) - } - } diff --git a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigWithUnresolvedVersionExt.scala b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigWithUnresolvedVersionExt.scala deleted file mode 100644 index bbda0e21359..00000000000 --- a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigWithUnresolvedVersionExt.scala +++ /dev/null @@ -1,32 +0,0 @@ -package pl.touk.nussknacker.engine.util.config - -import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion - -import scala.jdk.CollectionConverters._ -import scala.language.implicitConversions - -class ConfigWithUnresolvedVersionExt(val config: ConfigWithUnresolvedVersion) { - - def readMap(path: String): Option[Map[String, ConfigWithUnresolvedVersion]] = { - if (config.resolved.hasPath(path)) { - val nestedConfig = config.getConfig(path) - Some( - nestedConfig.resolved - .root() - .entrySet() - .asScala - .map(_.getKey) - .map { key => key -> nestedConfig.getConfig(key) } - .toMap - ) - } else { - None - } - } - -} - -object ConfigWithUnresolvedVersionExt { - implicit def toConfigWithUnresolvedVersionExt(config: ConfigWithUnresolvedVersion): ConfigWithUnresolvedVersionExt = - new ConfigWithUnresolvedVersionExt(config) -} diff --git a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/loader/ScalaServiceLoader.scala b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/loader/ScalaServiceLoader.scala index 78ae2452575..2012bbf3c9a 100644 --- a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/loader/ScalaServiceLoader.scala +++ b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/loader/ScalaServiceLoader.scala @@ -42,6 +42,17 @@ object ScalaServiceLoader extends LazyLogging { } } + def loadOne[T: ClassTag](classLoader: ClassLoader): Option[T] = { + load[T](classLoader) match { + case Nil => + None + case one :: Nil => + Some(one) + case _ => + throw new IllegalStateException(s"More than one ${classTag[T].runtimeClass.getSimpleName} found") + } + } + def load[T: ClassTag](classLoader: ClassLoader): List[T] = { val interface: Class[T] = classTag[T].runtimeClass.asInstanceOf[Class[T]] val loadedClasses = ServiceLoader diff --git a/utils/utils-internal/src/test/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExtSpec.scala b/utils/utils-internal/src/test/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExtSpec.scala index a5d92ccaac5..bd71eb6a831 100644 --- a/utils/utils-internal/src/test/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExtSpec.scala +++ b/utils/utils-internal/src/test/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExtSpec.scala @@ -16,9 +16,8 @@ class ConfigFactoryExtSpec extends AnyFunSuite with Matchers { writeToTemp(Map("f1" -> "default", "f2" -> "not so default", "akka.http.server.request-timeout" -> "300s")) val conf2 = writeToTemp(Map("f1" -> "I win!")) - val result = ConfigFactoryExt.parseConfigFallbackChain( + val result = new ConfigFactoryExt(getClass.getClassLoader).parseUnresolved( List(conf1, conf2, URI.create("classpath:someConfig.conf")), - getClass.getClassLoader ) result.getString("f1") shouldBe "I win!"