From a5d530503ecde04c14ecc26cd59d7785e9321b10 Mon Sep 17 00:00:00 2001 From: Arek Burdach Date: Tue, 17 Dec 2024 23:35:02 +0100 Subject: [PATCH] [NU-1828] Processing types configs loader api (#7336) --- build.sbt | 54 +++++--- .../ProcessingTypeConfigsLoader.scala | 12 ++ .../ProcessingTypeConfigsLoaderFactory.scala | 15 +++ .../ui/LocalNussknackerWithSingleModel.scala | 12 +- .../touk/nussknacker/ui/NussknackerApp.scala | 3 +- .../nussknacker/ui/NussknackerConfig.scala | 31 ----- .../ui/api/description/DictApiEndpoints.scala | 3 +- .../ui/api/description/UserApiEndpoints.scala | 2 +- .../scenarioActivity/InputOutput.scala | 4 +- .../ui/config/DesignerConfig.scala | 34 ++++++ .../ui/config/DesignerConfigLoader.scala | 53 +++++--- .../ui/db/entity/BaseEntityFactory.scala | 3 +- .../ui/db/entity/ProcessEntityFactory.scala | 3 +- .../QuestDbFEStatisticsRepository.scala | 2 +- .../ui/factory/NussknackerAppFactory.scala | 83 +++++++++---- ...sConfigBasedProcessingTypeDataLoader.scala | 92 +++++++------- .../server/AkkaHttpBasedRouteProvider.scala | 115 ++++++++---------- .../ui/server/NussknackerHttpServer.scala | 7 +- .../nussknacker/ui/server/RouteProvider.scala | 5 +- .../util/IOToFutureSttpBackendConverter.scala | 27 ++++ .../nussknacker/test/base/it/NuItTest.scala | 8 +- .../test/base/it/NuResourcesTest.scala | 15 +-- .../ui/api/NussknackerHttpServerSpec.scala | 8 +- .../ConfigurationTest.scala | 61 ++++++---- .../ProcessingTypeDataLoaderSpec.scala} | 43 ++++--- .../ProcessingTypeDataProviderSpec.scala | 5 +- .../ScenarioParametersServiceTest.scala | 42 +++---- .../engine/lite/app/NuRuntimeApp.scala | 10 +- .../engine/ConfigWithUnresolvedVersion.scala | 19 +++ .../api/AuthenticationConfiguration.scala | 4 +- .../nussknacker/engine/util/UriUtils.scala | 11 +- .../engine/util/config/ConfigFactoryExt.scala | 32 ++--- .../ConfigWithUnresolvedVersionExt.scala | 32 ----- .../util/loader/ScalaServiceLoader.scala | 11 ++ .../util/config/ConfigFactoryExtSpec.scala | 3 +- 35 files changed, 505 insertions(+), 359 deletions(-) create mode 100644 designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoader.scala create mode 100644 designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoaderFactory.scala delete mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfig.scala create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/util/IOToFutureSttpBackendConverter.scala rename designer/server/src/test/scala/pl/touk/nussknacker/ui/{integration => config}/ConfigurationTest.scala (69%) rename designer/server/src/test/scala/pl/touk/nussknacker/ui/{LoadableConfigBasedNussknackerConfigSpec.scala => config/processingtype/ProcessingTypeDataLoaderSpec.scala} (70%) delete mode 100644 utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigWithUnresolvedVersionExt.scala diff --git a/build.sbt b/build.sbt index 33c8fc01f76..01c8aca7bad 100644 --- a/build.sbt +++ b/build.sbt @@ -1,15 +1,15 @@ import com.typesafe.sbt.packager.SettingsHelper import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport.dockerUsername import pl.project13.scala.sbt.JmhPlugin -import pl.project13.scala.sbt.JmhPlugin._ -import sbt.Keys._ -import sbt._ +import pl.project13.scala.sbt.JmhPlugin.* +import sbt.* +import sbt.Keys.* import sbtassembly.AssemblyPlugin.autoImport.assembly import sbtassembly.MergeStrategy -import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations._ +import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations.* import scala.language.postfixOps -import scala.sys.process._ +import scala.sys.process.* import scala.util.Try import scala.xml.Elem import scala.xml.transform.{RewriteRule, RuleTransformer} @@ -1913,6 +1913,18 @@ lazy val listenerApi = (project in file("designer/listener-api")) ) .dependsOn(extensionsApi) +lazy val configLoaderApi = (project in file("designer/config-loader-api")) + .settings(commonSettings) + .settings( + name := "nussknacker-config-loader-api", + libraryDependencies ++= { + Seq( + "org.typelevel" %% "cats-effect" % catsEffectV + ) + } + ) + .dependsOn(extensionsApi) + lazy val deploymentManagerApi = (project in file("designer/deployment-manager-api")) .settings(commonSettings) .settings( @@ -1985,21 +1997,21 @@ lazy val designer = (project in file("designer/server")) assembly / assemblyMergeStrategy := designerMergeStrategy, libraryDependencies ++= { Seq( - "com.typesafe.akka" %% "akka-http" % akkaHttpV, - "com.typesafe.akka" %% "akka-slf4j" % akkaV, - "com.typesafe.akka" %% "akka-stream" % akkaV, - "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpV % Test, - "com.typesafe.akka" %% "akka-testkit" % akkaV % Test, - "de.heikoseeberger" %% "akka-http-circe" % akkaHttpCirceV, - "com.softwaremill.sttp.client3" %% "akka-http-backend" % sttpV, - "ch.qos.logback" % "logback-core" % logbackV, - "ch.qos.logback" % "logback-classic" % logbackV, - "ch.qos.logback.contrib" % "logback-json-classic" % logbackJsonV, - "ch.qos.logback.contrib" % "logback-jackson" % logbackJsonV, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonV, - "org.slf4j" % "log4j-over-slf4j" % slf4jV, - "com.carrotsearch" % "java-sizeof" % "0.0.5", - "org.typelevel" %% "case-insensitive" % "1.4.0", + "com.typesafe.akka" %% "akka-http" % akkaHttpV, + "com.typesafe.akka" %% "akka-slf4j" % akkaV, + "com.typesafe.akka" %% "akka-stream" % akkaV, + "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpV % Test, + "com.typesafe.akka" %% "akka-testkit" % akkaV % Test, + "de.heikoseeberger" %% "akka-http-circe" % akkaHttpCirceV, + "com.softwaremill.sttp.client3" %% "async-http-client-backend-cats" % sttpV, + "ch.qos.logback" % "logback-core" % logbackV, + "ch.qos.logback" % "logback-classic" % logbackV, + "ch.qos.logback.contrib" % "logback-json-classic" % logbackJsonV, + "ch.qos.logback.contrib" % "logback-jackson" % logbackJsonV, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonV, + "org.slf4j" % "log4j-over-slf4j" % slf4jV, + "com.carrotsearch" % "java-sizeof" % "0.0.5", + "org.typelevel" %% "case-insensitive" % "1.4.0", // It's needed by flinkDeploymentManager which has disabled includingScala "org.scala-lang" % "scala-compiler" % scalaVersion.value, @@ -2053,6 +2065,7 @@ lazy val designer = (project in file("designer/server")) deploymentManagerApi, restmodel, listenerApi, + configLoaderApi, defaultHelpers % Test, testUtils % Test, flinkTestUtils % Test, @@ -2193,6 +2206,7 @@ lazy val modules = List[ProjectReference]( httpUtils, restmodel, listenerApi, + configLoaderApi, deploymentManagerApi, designer, sqlComponents, diff --git a/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoader.scala b/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoader.scala new file mode 100644 index 00000000000..20adfc59eb2 --- /dev/null +++ b/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoader.scala @@ -0,0 +1,12 @@ +package pl.touk.nussknacker.ui.configloader + +import cats.effect.IO +import pl.touk.nussknacker.engine.ProcessingTypeConfig + +trait ProcessingTypeConfigsLoader { + + def loadProcessingTypeConfigs(): IO[ProcessingTypeConfigs] + +} + +case class ProcessingTypeConfigs(configByProcessingType: Map[String, ProcessingTypeConfig]) diff --git a/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoaderFactory.scala b/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoaderFactory.scala new file mode 100644 index 00000000000..5d7b2312687 --- /dev/null +++ b/designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoaderFactory.scala @@ -0,0 +1,15 @@ +package pl.touk.nussknacker.ui.configloader + +import cats.effect.IO +import cats.effect.unsafe.IORuntime +import com.typesafe.config.Config +import sttp.client3.SttpBackend + +trait ProcessingTypeConfigsLoaderFactory { + + def create( + configLoaderConfig: Config, + sttpBackend: SttpBackend[IO, Any], + )(implicit ec: IORuntime): ProcessingTypeConfigsLoader + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala index 12b0aa71adb..25dc03d5742 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala @@ -1,7 +1,7 @@ package pl.touk.nussknacker.ui import cats.effect.{IO, Resource} -import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.config.{Config, ConfigFactory, ConfigValue, ConfigValueFactory} import org.apache.commons.io.FileUtils import pl.touk.nussknacker.engine.{DeploymentManagerProvider, ModelData} import pl.touk.nussknacker.ui.config.DesignerConfigLoader @@ -49,8 +49,14 @@ object LocalNussknackerWithSingleModel { modelData = Map(typeName -> (category, modelData)), deploymentManagerProvider = deploymentManagerProvider ) - val nussknackerConfig = new LoadableConfigBasedNussknackerConfig(IO.delay(DesignerConfigLoader.from(appConfig))) - val appFactory = new NussknackerAppFactory(nussknackerConfig, local) + val designerConfigLoader = DesignerConfigLoader.fromConfig( + // This map is ignored but must exist + appConfig.withValue("scenarioTypes", ConfigValueFactory.fromMap(Map.empty[String, ConfigValue].asJava)) + ) + val appFactory = new NussknackerAppFactory( + designerConfigLoader, + _ => local + ) appFactory.createApp() } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala index d99984f966a..f7b4c356e2c 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala @@ -1,13 +1,14 @@ package pl.touk.nussknacker.ui import cats.effect.{ExitCode, IO, IOApp} +import pl.touk.nussknacker.ui.config.{AlwaysLoadingFileBasedDesignerConfigLoader, DesignerConfigLoader} import pl.touk.nussknacker.ui.factory.NussknackerAppFactory object NussknackerApp extends IOApp { override def run(args: List[String]): IO[ExitCode] = { for { - appFactory <- IO(new NussknackerAppFactory(getClass.getClassLoader)) + appFactory <- IO(NussknackerAppFactory(AlwaysLoadingFileBasedDesignerConfigLoader(getClass.getClassLoader))) _ <- appFactory.createApp().use { _ => IO.never } } yield ExitCode.Success } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala deleted file mode 100644 index a05b7b175b8..00000000000 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala +++ /dev/null @@ -1,31 +0,0 @@ -package pl.touk.nussknacker.ui - -import cats.effect.IO -import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap -import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, ProcessingTypeConfig} -import pl.touk.nussknacker.engine.util.config.ConfigWithUnresolvedVersionExt._ -import pl.touk.nussknacker.ui.config.DesignerConfigLoader - -trait NussknackerConfig { - - def loadApplicationConfig(): IO[ConfigWithUnresolvedVersion] - - final def loadProcessingTypeConfigs(): IO[Map[String, ProcessingTypeConfig]] = { - loadApplicationConfig() - .map { config => - config - .readMap("scenarioTypes") - .getOrElse { throw new RuntimeException("No scenario types configuration provided") } - .mapValuesNow(ProcessingTypeConfig.read) - } - } - -} - -class LoadableConfigBasedNussknackerConfig(loadConfig: IO[ConfigWithUnresolvedVersion]) extends NussknackerConfig { - - override def loadApplicationConfig(): IO[ConfigWithUnresolvedVersion] = loadConfig -} - -class LoadableDesignerConfigBasedNussknackerConfig(classLoader: ClassLoader) - extends LoadableConfigBasedNussknackerConfig(DesignerConfigLoader.load(classLoader)) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DictApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DictApiEndpoints.scala index 36df4adceef..b177104b236 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DictApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DictApiEndpoints.scala @@ -15,9 +15,8 @@ import pl.touk.nussknacker.ui.api.description.DictApiEndpoints.DictError.{ } import pl.touk.nussknacker.ui.api.description.DictApiEndpoints.Dtos._ import sttp.model.StatusCode.{BadRequest, NotFound, Ok} -import sttp.tapir._ import sttp.tapir.json.circe._ -import sttp.tapir.Schema +import sttp.tapir._ import scala.language.implicitConversions diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/UserApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/UserApiEndpoints.scala index dc745e625d5..1095c09e4d6 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/UserApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/UserApiEndpoints.scala @@ -6,7 +6,7 @@ import pl.touk.nussknacker.restmodel.BaseEndpointDefinitions import pl.touk.nussknacker.restmodel.BaseEndpointDefinitions.SecuredEndpoint import pl.touk.nussknacker.security.AuthCredentials import pl.touk.nussknacker.ui.security.api.GlobalPermission.GlobalPermission -import pl.touk.nussknacker.ui.security.api.{AdminUser, CommonUser, ImpersonatedUser, LoggedUser, RealLoggedUser} +import pl.touk.nussknacker.ui.security.api._ import sttp.model.StatusCode.Ok import sttp.tapir.EndpointIO.Example import sttp.tapir.derevo.schema diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/scenarioActivity/InputOutput.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/scenarioActivity/InputOutput.scala index b475bab6015..317ebf02054 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/scenarioActivity/InputOutput.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/scenarioActivity/InputOutput.scala @@ -3,9 +3,9 @@ package pl.touk.nussknacker.ui.api.description.scenarioActivity import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.ui.api.description.scenarioActivity.Dtos.ScenarioActivityError import pl.touk.nussknacker.ui.api.description.scenarioActivity.Dtos.ScenarioActivityError.NoScenario -import sttp.model.StatusCode.{NotFound, NotImplemented} +import sttp.model.StatusCode.NotFound import sttp.tapir.EndpointIO.Example -import sttp.tapir.{EndpointOutput, emptyOutputAs, oneOf, oneOfVariantFromMatchType, plainBody} +import sttp.tapir.{EndpointOutput, oneOf, oneOfVariantFromMatchType, plainBody} object InputOutput { diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfig.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfig.scala new file mode 100644 index 00000000000..3cb1c86fd7e --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfig.scala @@ -0,0 +1,34 @@ +package pl.touk.nussknacker.ui.config + +import com.typesafe.config.{Config, ConfigFactory} +import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap +import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, ProcessingTypeConfig} +import pl.touk.nussknacker.ui.configloader.ProcessingTypeConfigs + +// TODO: We should extract a class for all configuration options that should be available to designer instead of returning raw hocon config. +// Thanks to that it will be easier to split processing type config from rest of configs and use this interface programmatically +final case class DesignerConfig private (rawConfig: ConfigWithUnresolvedVersion) { + + import net.ceedubs.ficus.Ficus._ + + def processingTypeConfigs: ProcessingTypeConfigs = + ProcessingTypeConfigs( + rawConfig + .readMap("scenarioTypes") + .getOrElse { + throw new RuntimeException("No scenario types configuration provided") + } + .mapValuesNow(ProcessingTypeConfig.read) + ) + + def configLoaderConfig: Config = rawConfig.resolved.getAs[Config]("configLoader").getOrElse(ConfigFactory.empty()) + +} + +object DesignerConfig { + + def from(config: Config): DesignerConfig = { + DesignerConfig(ConfigWithUnresolvedVersion(config)) + } + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala index b87cdb7f6eb..f0f00d3bb4e 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala @@ -3,8 +3,15 @@ package pl.touk.nussknacker.ui.config import cats.effect.IO import com.typesafe.config.{Config, ConfigFactory} import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.engine.util.UriUtils import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt +trait DesignerConfigLoader { + + def loadDesignerConfig(): IO[DesignerConfig] + +} + /** * This class handles two parts of ui config loading: * 1. Parsing of "base" config passed via nussknacker.config.locations system property (without resolution) @@ -14,27 +21,43 @@ import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt * Result of config loading still keep version with unresolved env variables for purpose of config loading on model side - see * InputConfigDuringExecution and ModelConfigLoader */ -object DesignerConfigLoader { +class AlwaysLoadingFileBasedDesignerConfigLoader private (classLoader: ClassLoader) extends DesignerConfigLoader { + + private val configLocationsProperty: String = "nussknacker.config.locations" private val defaultConfigResource = "defaultDesignerConfig.conf" - def load(classLoader: ClassLoader): IO[ConfigWithUnresolvedVersion] = { + override def loadDesignerConfig(): IO[DesignerConfig] = { + val locationsPropertyValueOpt = Option(System.getProperty(configLocationsProperty)) + val locations = locationsPropertyValueOpt.map(UriUtils.extractListOfLocations).getOrElse(List.empty) for { - baseConfig <- IO.blocking(ConfigFactoryExt.parseUnresolved(classLoader = classLoader)) - loadedConfig <- load(baseConfig, classLoader) - } yield loadedConfig + baseUnresolvedConfig <- IO.blocking(new ConfigFactoryExt(classLoader).parseUnresolved(locations)) + parsedDefaultUiConfig <- IO.blocking(ConfigFactory.parseResources(classLoader, defaultConfigResource)) + unresolvedConfigWithFallbackToDefaults = baseUnresolvedConfig.withFallback(parsedDefaultUiConfig) + } yield DesignerConfig(ConfigWithUnresolvedVersion(classLoader, unresolvedConfigWithFallbackToDefaults)) } - def load(baseUnresolvedConfig: Config, classLoader: ClassLoader): IO[ConfigWithUnresolvedVersion] = { - IO.blocking { - val parsedDefaultUiConfig = ConfigFactory.parseResources(classLoader, defaultConfigResource) - val unresolvedConfigWithFallbackToDefaults = baseUnresolvedConfig.withFallback(parsedDefaultUiConfig) - ConfigWithUnresolvedVersion(classLoader, unresolvedConfigWithFallbackToDefaults) - } - } +} - def from(config: Config): ConfigWithUnresolvedVersion = { - ConfigWithUnresolvedVersion(this.getClass.getClassLoader, config) - } +object AlwaysLoadingFileBasedDesignerConfigLoader { + + def apply(classLoader: ClassLoader): AlwaysLoadingFileBasedDesignerConfigLoader = + new AlwaysLoadingFileBasedDesignerConfigLoader(classLoader) + +} + +/** + * This implementation is more straightforward - it only parse config without any property checking and fallbacks + */ +class SimpleConfigLoadingDesignerConfigLoader(loadConfig: => Config) extends DesignerConfigLoader { + + override def loadDesignerConfig(): IO[DesignerConfig] = IO.delay(DesignerConfig.from(loadConfig)) + +} + +object DesignerConfigLoader { + + def fromConfig(loadConfig: => Config): SimpleConfigLoadingDesignerConfigLoader = + new SimpleConfigLoadingDesignerConfigLoader(loadConfig) } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/BaseEntityFactory.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/BaseEntityFactory.scala index 68b7f22e920..40baaff6cc4 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/BaseEntityFactory.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/BaseEntityFactory.scala @@ -9,8 +9,7 @@ import pl.touk.nussknacker.engine.api.deployment.{ } import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.newdeployment.DeploymentId -import slick.ast.BaseTypedType -import slick.jdbc.{JdbcProfile, JdbcType} +import slick.jdbc.JdbcProfile import java.util.UUID diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/ProcessEntityFactory.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/ProcessEntityFactory.scala index a16f394a22b..0edcf8e1b52 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/ProcessEntityFactory.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/ProcessEntityFactory.scala @@ -1,7 +1,6 @@ package pl.touk.nussknacker.ui.db.entity -import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName} -import pl.touk.nussknacker.engine.api.process.ProcessingType +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, ProcessingType} import slick.lifted.{ProvenShape, TableQuery => LTableQuery} import slick.sql.SqlProfile.ColumnOption.NotNull diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/timeseries/questdb/QuestDbFEStatisticsRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/timeseries/questdb/QuestDbFEStatisticsRepository.scala index f69597bb6de..ee78f21a61f 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/timeseries/questdb/QuestDbFEStatisticsRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/timeseries/questdb/QuestDbFEStatisticsRepository.scala @@ -10,7 +10,6 @@ import io.questdb.cairo.security.AllowAllSecurityContext import io.questdb.cairo.sql.RecordCursorFactory import io.questdb.cairo.wal.WalWriter import io.questdb.griffin.{SqlExecutionContext, SqlExecutionContextImpl} -import pl.touk.nussknacker.ui.db.timeseries.{FEStatisticsRepository, NoOpFEStatisticsRepository} import pl.touk.nussknacker.ui.db.timeseries.questdb.QuestDbExtensions.{ BuildCairoEngineExtension, CairoEngineExtension, @@ -22,6 +21,7 @@ import pl.touk.nussknacker.ui.db.timeseries.questdb.QuestDbFEStatisticsRepositor selectQuery, tableName } +import pl.touk.nussknacker.ui.db.timeseries.{FEStatisticsRepository, NoOpFEStatisticsRepository} import pl.touk.nussknacker.ui.statistics.RawFEStatistics import java.time.Clock diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala index e9be44d0bcf..816b4eb9b0d 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala @@ -2,62 +2,92 @@ package pl.touk.nussknacker.ui.factory import akka.actor.ActorSystem import akka.stream.Materializer +import cats.effect.unsafe.IORuntime import cats.effect.{IO, Resource} import com.typesafe.scalalogging.LazyLogging import io.dropwizard.metrics5.MetricRegistry import io.dropwizard.metrics5.jmx.JmxReporter import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader import pl.touk.nussknacker.engine.util.{JavaClassVersionChecker, SLF4JBridgeHandlerRegistrar} -import pl.touk.nussknacker.ui.{LoadableDesignerConfigBasedNussknackerConfig, NussknackerConfig} +import pl.touk.nussknacker.ui.config.{DesignerConfig, DesignerConfigLoader} +import pl.touk.nussknacker.ui.configloader.{ProcessingTypeConfigsLoader, ProcessingTypeConfigsLoaderFactory} import pl.touk.nussknacker.ui.db.DbRef import pl.touk.nussknacker.ui.db.timeseries.questdb.QuestDbFEStatisticsRepository -import pl.touk.nussknacker.ui.process.processingtype.loader._ +import pl.touk.nussknacker.ui.process.processingtype.loader.{ + ProcessingTypeDataLoader, + ProcessingTypesConfigBasedProcessingTypeDataLoader +} import pl.touk.nussknacker.ui.server.{AkkaHttpBasedRouteProvider, NussknackerHttpServer} +import pl.touk.nussknacker.ui.util.{ActorSystemBasedExecutionContextWithIORuntime, IOToFutureSttpBackendConverter} +import sttp.client3.SttpBackend +import sttp.client3.asynchttpclient.cats.AsyncHttpClientCatsBackend import java.time.Clock -object NussknackerAppFactory - -class NussknackerAppFactory(nussknackerConfig: NussknackerConfig, processingTypeDataLoader: ProcessingTypeDataLoader) - extends LazyLogging { - - def this(nussknackerConfig: NussknackerConfig) = { - this(nussknackerConfig, new ProcessingTypesConfigBasedProcessingTypeDataLoader(nussknackerConfig)) - } - - def this(classLoader: ClassLoader) = { - this(new LoadableDesignerConfigBasedNussknackerConfig(classLoader)) - } +class NussknackerAppFactory( + designerConfigLoader: DesignerConfigLoader, + createProcessingTypeDataLoader: ProcessingTypeConfigsLoader => ProcessingTypeDataLoader +) extends LazyLogging { def createApp(clock: Clock = Clock.systemUTC()): Resource[IO, Unit] = { for { - config <- Resource.eval(nussknackerConfig.loadApplicationConfig()) - system <- createActorSystem(config) - materializer = Materializer(system) + designerConfig <- Resource.eval(designerConfigLoader.loadDesignerConfig()) + system <- createActorSystem(designerConfig.rawConfig) + executionContextWithIORuntime = ActorSystemBasedExecutionContextWithIORuntime.createFrom(system) + ioSttpBackend <- AsyncHttpClientCatsBackend.resource[IO]() + processingTypeConfigsLoader = createProcessingTypeConfigsLoader( + designerConfig, + ioSttpBackend + )(executionContextWithIORuntime.ioRuntime) + processingTypeDataLoader = createProcessingTypeDataLoader(processingTypeConfigsLoader) + materializer = Materializer(system) _ <- Resource.eval(IO(JavaClassVersionChecker.check())) _ <- Resource.eval(IO(SLF4JBridgeHandlerRegistrar.register())) metricsRegistry <- createGeneralPurposeMetricsRegistry() - db <- DbRef.create(config.resolved) - feStatisticsRepository <- QuestDbFEStatisticsRepository.create(system, clock, config.resolved) + db <- DbRef.create(designerConfig.rawConfig.resolved) + feStatisticsRepository <- QuestDbFEStatisticsRepository.create(system, clock, designerConfig.rawConfig.resolved) server = new NussknackerHttpServer( new AkkaHttpBasedRouteProvider( db, metricsRegistry, + IOToFutureSttpBackendConverter.convert(ioSttpBackend)(executionContextWithIORuntime), processingTypeDataLoader, feStatisticsRepository, clock )( system, - materializer + materializer, + executionContextWithIORuntime ), system ) - _ <- server.start(config, metricsRegistry) + _ <- server.start(designerConfig, metricsRegistry) _ <- startJmxReporter(metricsRegistry) _ <- createStartAndStopLoggingEntries() } yield () } + private def createProcessingTypeConfigsLoader( + designerConfig: DesignerConfig, + sttpBackend: SttpBackend[IO, Any] + )(implicit ioRuntime: IORuntime): ProcessingTypeConfigsLoader = { + ScalaServiceLoader + .loadOne[ProcessingTypeConfigsLoaderFactory](getClass.getClassLoader) + .map { factory => + logger.debug( + s"Found custom ${classOf[ProcessingTypeConfigsLoaderFactory].getSimpleName}: ${factory.getClass.getName}. Using it for configuration loading" + ) + factory.create(designerConfig.configLoaderConfig, sttpBackend) + } + .getOrElse { + logger.debug( + s"No custom ${classOf[ProcessingTypeConfigsLoaderFactory].getSimpleName} found. Using the default implementation of loader" + ) + () => designerConfigLoader.loadDesignerConfig().map(_.processingTypeConfigs) + } + } + private def createActorSystem(config: ConfigWithUnresolvedVersion) = { Resource .make( @@ -87,3 +117,14 @@ class NussknackerAppFactory(nussknackerConfig: NussknackerConfig, processingType } } + +object NussknackerAppFactory { + + def apply(designerConfigLoader: DesignerConfigLoader): NussknackerAppFactory = { + new NussknackerAppFactory( + designerConfigLoader, + new ProcessingTypesConfigBasedProcessingTypeDataLoader(_) + ) + } + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala index 30500afd1a1..de70b5c0ba2 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala @@ -6,12 +6,12 @@ import pl.touk.nussknacker.engine._ import pl.touk.nussknacker.engine.api.process.ProcessingType import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader -import pl.touk.nussknacker.ui.NussknackerConfig +import pl.touk.nussknacker.ui.configloader.{ProcessingTypeConfigs, ProcessingTypeConfigsLoader} import pl.touk.nussknacker.ui.process.processingtype._ import pl.touk.nussknacker.ui.process.processingtype.loader.ProcessingTypeDataLoader.toValueWithRestriction import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataState -class ProcessingTypesConfigBasedProcessingTypeDataLoader(config: NussknackerConfig) +class ProcessingTypesConfigBasedProcessingTypeDataLoader(processingTypeConfigsLoader: ProcessingTypeConfigsLoader) extends ProcessingTypeDataLoader with LazyLogging { @@ -19,51 +19,57 @@ class ProcessingTypesConfigBasedProcessingTypeDataLoader(config: NussknackerConf getModelDependencies: ProcessingType => ModelDependencies, getDeploymentManagerDependencies: ProcessingType => DeploymentManagerDependencies, ): IO[ProcessingTypeDataState[ProcessingTypeData, CombinedProcessingTypeData]] = { - config + processingTypeConfigsLoader .loadProcessingTypeConfigs() - .map { processingTypesConfig => - // This step with splitting DeploymentManagerProvider loading for all processing types - // and after that creating ProcessingTypeData is done because of the deduplication of deployments - // See DeploymentManagerProvider.engineSetupIdentity - val providerWithNameInputData = processingTypesConfig.mapValuesNow { processingTypeConfig => - val provider = createDeploymentManagerProvider(processingTypeConfig) - val nameInputData = EngineNameInputData( - provider.defaultEngineSetupName, - provider.engineSetupIdentity(processingTypeConfig.deploymentConfig), - processingTypeConfig.engineSetupName - ) - (processingTypeConfig, provider, nameInputData) - } - val engineSetupNames = - ScenarioParametersDeterminer.determineEngineSetupNames(providerWithNameInputData.mapValuesNow(_._3)) - val processingTypesData = providerWithNameInputData - .map { case (processingType, (processingTypeConfig, deploymentManagerProvider, _)) => - logger.debug(s"Creating Processing Type: $processingType with config: $processingTypeConfig") - val modelDependencies = getModelDependencies(processingType) - val processingTypeData = ProcessingTypeData.createProcessingTypeData( - processingType, - ModelData(processingTypeConfig, modelDependencies), - deploymentManagerProvider, - getDeploymentManagerDependencies(processingType), - engineSetupNames(processingType), - processingTypeConfig.deploymentConfig, - processingTypeConfig.category, - modelDependencies.componentDefinitionExtractionMode - ) - processingType -> processingTypeData - } - - // Here all processing types are loaded and we are ready to perform additional configuration validations - // to assert the loaded configuration is correct (fail-fast approach). - val combinedData = CombinedProcessingTypeData.create(processingTypesData) + .map(createProcessingTypeData(_, getModelDependencies, getDeploymentManagerDependencies)) + } - ProcessingTypeDataState( - processingTypesData.mapValuesNow(toValueWithRestriction), - () => combinedData, - // We pass here new Object to enforce update of observers - new Object + private def createProcessingTypeData( + processingTypesConfig: ProcessingTypeConfigs, + getModelDependencies: ProcessingType => ModelDependencies, + getDeploymentManagerDependencies: ProcessingType => DeploymentManagerDependencies + ): ProcessingTypeDataState[ProcessingTypeData, CombinedProcessingTypeData] = { + // This step with splitting DeploymentManagerProvider loading for all processing types + // and after that creating ProcessingTypeData is done because of the deduplication of deployments + // See DeploymentManagerProvider.engineSetupIdentity + val providerWithNameInputData = processingTypesConfig.configByProcessingType.mapValuesNow { processingTypeConfig => + val provider = createDeploymentManagerProvider(processingTypeConfig) + val nameInputData = EngineNameInputData( + provider.defaultEngineSetupName, + provider.engineSetupIdentity(processingTypeConfig.deploymentConfig), + processingTypeConfig.engineSetupName + ) + (processingTypeConfig, provider, nameInputData) + } + val engineSetupNames = + ScenarioParametersDeterminer.determineEngineSetupNames(providerWithNameInputData.mapValuesNow(_._3)) + val processingTypesData = providerWithNameInputData + .map { case (processingType, (processingTypeConfig, deploymentManagerProvider, _)) => + logger.debug(s"Creating Processing Type: $processingType with config: $processingTypeConfig") + val modelDependencies = getModelDependencies(processingType) + val processingTypeData = ProcessingTypeData.createProcessingTypeData( + processingType, + ModelData(processingTypeConfig, modelDependencies), + deploymentManagerProvider, + getDeploymentManagerDependencies(processingType), + engineSetupNames(processingType), + processingTypeConfig.deploymentConfig, + processingTypeConfig.category, + modelDependencies.componentDefinitionExtractionMode ) + processingType -> processingTypeData } + + // Here all processing types are loaded and we are ready to perform additional configuration validations + // to assert the loaded configuration is correct (fail-fast approach). + val combinedData = CombinedProcessingTypeData.create(processingTypesData) + + ProcessingTypeDataState( + processingTypesData.mapValuesNow(toValueWithRestriction), + () => combinedData, + // We pass here new Object to enforce update of observers + new Object + ) } private def createDeploymentManagerProvider(typeConfig: ProcessingTypeConfig): DeploymentManagerProvider = { diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala index bb1356a2319..ce1ed9743ee 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala @@ -17,7 +17,7 @@ import pl.touk.nussknacker.engine.definition.test.ModelDataTestInfoProvider import pl.touk.nussknacker.engine.dict.ProcessDictSubstitutor import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader import pl.touk.nussknacker.engine.util.multiplicity.{Empty, Many, Multiplicity, One} -import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, DeploymentManagerDependencies, ModelDependencies} +import pl.touk.nussknacker.engine.{DeploymentManagerDependencies, ModelDependencies} import pl.touk.nussknacker.processCounts.influxdb.InfluxCountsReporterCreator import pl.touk.nussknacker.processCounts.{CountsReporter, CountsReporterCreator} import pl.touk.nussknacker.ui.api._ @@ -28,6 +28,7 @@ import pl.touk.nussknacker.ui.config.{ FeatureTogglesConfig, UsageStatisticsReportsConfig } +import pl.touk.nussknacker.ui.config.DesignerConfig import pl.touk.nussknacker.ui.db.DbRef import pl.touk.nussknacker.ui.db.timeseries.FEStatisticsRepository import pl.touk.nussknacker.ui.definition.component.{ComponentServiceProcessingTypeData, DefaultComponentService} @@ -92,12 +93,11 @@ import pl.touk.nussknacker.ui.validation.{ UIProcessValidator } import sttp.client3.SttpBackend -import sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend import java.time.Clock import java.util.concurrent.atomic.AtomicReference import java.util.function.Supplier -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.Future import scala.io.Source import scala.util.Try import scala.util.control.NonFatal @@ -105,27 +105,28 @@ import scala.util.control.NonFatal class AkkaHttpBasedRouteProvider( dbRef: DbRef, metricsRegistry: MetricRegistry, + sttpBackend: SttpBackend[Future, Any], processingTypeDataLoader: ProcessingTypeDataLoader, feStatisticsRepository: FEStatisticsRepository[Future], designerClock: Clock -)(implicit system: ActorSystem, materializer: Materializer) - extends RouteProvider[Route] +)( + implicit system: ActorSystem, + materializer: Materializer, + executionContextWithIORuntime: ExecutionContextWithIORuntime +) extends RouteProvider[Route] with Directives with LazyLogging { - override def createRoute(config: ConfigWithUnresolvedVersion): Resource[IO, Route] = { - implicit val executionContextWithIORuntime: ExecutionContextWithIORuntime = - ActorSystemBasedExecutionContextWithIORuntime.createFrom(system) + override def createRoute(designerConfig: DesignerConfig): Resource[IO, Route] = { import executionContextWithIORuntime.ioRuntime + val resolvedDesignerConfig = designerConfig.rawConfig.resolved + val environment = resolvedDesignerConfig.getString("environment") + val featureTogglesConfig = FeatureTogglesConfig.create(resolvedDesignerConfig) + logger.info(s"Designer config loaded: \nfeatureTogglesConfig: $featureTogglesConfig") for { - sttpBackend <- createSttpBackend() - resolvedConfig = config.resolved - environment = resolvedConfig.getString("environment") - featureTogglesConfig = FeatureTogglesConfig.create(resolvedConfig) - _ = logger.info(s"Designer config loaded: \nfeatureTogglesConfig: $featureTogglesConfig") countsReporter <- createCountsReporter(featureTogglesConfig, environment, sttpBackend) actionServiceSupplier = new DelayedInitActionServiceSupplier - additionalUIConfigProvider = createAdditionalUIConfigProvider(resolvedConfig, sttpBackend) + additionalUIConfigProvider = createAdditionalUIConfigProvider(resolvedDesignerConfig, sttpBackend) deploymentRepository = new DeploymentRepository(dbRef, Clock.systemDefaultZone()) scenarioActivityRepository = DbScenarioActivityRepository.create(dbRef, designerClock) dbioRunner = DBIOActionRunner(dbRef) @@ -150,7 +151,7 @@ class AkkaHttpBasedRouteProvider( val scheduler = new DeploymentsStatusesSynchronizationScheduler( system, deploymentsStatusesSynchronizer, - DeploymentsStatusesSynchronizationConfig.parse(resolvedConfig) + DeploymentsStatusesSynchronizationConfig.parse(resolvedDesignerConfig) ) scheduler.start() scheduler @@ -220,10 +221,10 @@ class AkkaHttpBasedRouteProvider( val processResolver = scenarioTestServiceDeps.mapValues(_._2) val scenarioResolver = scenarioTestServiceDeps.mapValues(_._3) - val notificationsConfig = resolvedConfig.as[NotificationConfig]("notifications") + val notificationsConfig = resolvedDesignerConfig.as[NotificationConfig]("notifications") val processChangeListener = ProcessChangeListenerLoader.loadListeners( getClass.getClassLoader, - resolvedConfig, + resolvedDesignerConfig, NussknackerServices(new PullProcessRepository(futureProcessRepository)) ) @@ -257,8 +258,9 @@ class AkkaHttpBasedRouteProvider( // correct classloader and that won't cause further delays during handling requests processingTypeDataProvider.reloadAll().unsafeRunSync() - val authenticationResources = AuthenticationResources(resolvedConfig, getClass.getClassLoader, sttpBackend) - val authManager = new AuthManager(authenticationResources) + val authenticationResources = + AuthenticationResources(resolvedDesignerConfig, getClass.getClassLoader, sttpBackend) + val authManager = new AuthManager(authenticationResources) Initialization.init( migrations, @@ -296,7 +298,7 @@ class AkkaHttpBasedRouteProvider( ) val configProcessToolbarService = new ConfigScenarioToolbarService( - CategoriesScenarioToolbarsConfigParser.parse(resolvedConfig) + CategoriesScenarioToolbarsConfigParser.parse(resolvedDesignerConfig) ) def prepareAlignedComponentsDefinitionProvider( @@ -305,7 +307,7 @@ class AkkaHttpBasedRouteProvider( AlignedComponentsDefinitionProvider(processingTypeData.designerModelData) val componentService = new DefaultComponentService( - ComponentLinksConfigExtractor.extract(resolvedConfig), + ComponentLinksConfigExtractor.extract(resolvedDesignerConfig), processingTypeDataProvider .mapValues { processingTypeData => val alignedModelDefinitionProvider = prepareAlignedComponentsDefinitionProvider(processingTypeData) @@ -328,7 +330,7 @@ class AkkaHttpBasedRouteProvider( ) val processAuthorizer = new AuthorizeProcess(futureProcessRepository) val appApiHttpService = new AppApiHttpService( - config = resolvedConfig, + config = resolvedDesignerConfig, authManager = authManager, processingTypeDataReloader = processingTypeDataProvider, modelBuildInfos = modelBuildInfo, @@ -340,7 +342,7 @@ class AkkaHttpBasedRouteProvider( val migrationApiAdapterService = new MigrationApiAdapterService() val migrationService = new MigrationService( - config = resolvedConfig, + config = resolvedDesignerConfig, processService = processService, processResolver = processResolver, processAuthorizer = processAuthorizer, @@ -416,7 +418,7 @@ class AkkaHttpBasedRouteProvider( scenarioService = processService, scenarioAuthorizer = processAuthorizer, new ScenarioAttachmentService( - AttachmentsConfig.create(resolvedConfig), + AttachmentsConfig.create(resolvedDesignerConfig), scenarioActivityRepository, dbioRunner, ), @@ -469,7 +471,7 @@ class AkkaHttpBasedRouteProvider( new DeploymentApiHttpService(authManager, activityService, deploymentService) } - initMetrics(metricsRegistry, resolvedConfig, futureProcessRepository) + initMetrics(metricsRegistry, resolvedDesignerConfig, futureProcessRepository) val apiResourcesWithAuthentication: List[RouteWithUser] = { val routes = List( @@ -505,7 +507,7 @@ class AkkaHttpBasedRouteProvider( prepareAlignedComponentsDefinitionProvider(processingTypeData), new ScenarioPropertiesConfigFinalizer(additionalUIConfigProvider, processingTypeData.name), fragmentRepository, - resolvedConfig.getAs[String]("fragmentPropertiesDocsUrl") + resolvedDesignerConfig.getAs[String]("fragmentPropertiesDocsUrl") ) ) } @@ -542,8 +544,9 @@ class AkkaHttpBasedRouteProvider( routes ++ optionalRoutes } - val usageStatisticsReportsConfig = resolvedConfig.as[UsageStatisticsReportsConfig]("usageStatisticsReports") - val fingerprintService = new FingerprintService(new FingerprintRepositoryImpl(dbRef)) + val usageStatisticsReportsConfig = + resolvedDesignerConfig.as[UsageStatisticsReportsConfig]("usageStatisticsReports") + val fingerprintService = new FingerprintService(new FingerprintRepositoryImpl(dbRef)) val usageStatisticsReportsSettingsService = UsageStatisticsReportsSettingsService( usageStatisticsReportsConfig, processService, @@ -608,7 +611,7 @@ class AkkaHttpBasedRouteProvider( val akkaHttpServerInterpreter = new NuAkkaHttpServerInterpreterForTapirPurposes() createAppRoute( - resolvedConfig = resolvedConfig, + resolvedConfig = resolvedDesignerConfig, authManager = authManager, tapirRelatedRoutes = akkaHttpServerInterpreter.toRoute(nuDesignerApi.allEndpoints) :: Nil, apiResourcesWithAuthentication = apiResourcesWithAuthentication, @@ -618,15 +621,6 @@ class AkkaHttpBasedRouteProvider( } } - private def createSttpBackend()(implicit executionContext: ExecutionContext) = { - Resource - .make( - acquire = IO(AsyncHttpClientFutureBackend.usingConfigBuilder(identity)) - )( - release = backend => IO.fromFuture(IO(backend.close())) - ) - } - private def initMetrics( metricsRegistry: MetricRegistry, config: Config, @@ -713,28 +707,27 @@ class AkkaHttpBasedRouteProvider( dbioActionRunner: DBIOActionRunner, sttpBackend: SttpBackend[Future, Any], featureTogglesConfig: FeatureTogglesConfig - )(implicit executionContext: ExecutionContext): Resource[IO, ReloadableProcessingTypeDataProvider] = { + ): Resource[IO, ReloadableProcessingTypeDataProvider] = { Resource .make( - acquire = IO( - new ReloadableProcessingTypeDataProvider( - processingTypeDataLoader.loadProcessingTypeData( - getModelDependencies( - additionalUIConfigProvider, - _, - featureTogglesConfig.componentDefinitionExtractionMode - ), - getDeploymentManagerDependencies( - additionalUIConfigProvider, - actionServiceProvider, - scenarioActivityRepository, - dbioActionRunner, - sttpBackend, - _ - ), - ) + acquire = IO { + val laodProcessingTypeDataIO = processingTypeDataLoader.loadProcessingTypeData( + getModelDependencies( + additionalUIConfigProvider, + _, + featureTogglesConfig.componentDefinitionExtractionMode + ), + getDeploymentManagerDependencies( + additionalUIConfigProvider, + actionServiceProvider, + scenarioActivityRepository, + dbioActionRunner, + sttpBackend, + _ + ), ) - ) + new ReloadableProcessingTypeDataProvider(laodProcessingTypeDataIO) + } )( release = _.close() ) @@ -747,7 +740,7 @@ class AkkaHttpBasedRouteProvider( dbioActionRunner: DBIOActionRunner, sttpBackend: SttpBackend[Future, Any], processingType: ProcessingType - )(implicit executionContext: ExecutionContext) = { + ) = { val additionalConfigsFromProvider = additionalUIConfigProvider.getAllForProcessingType(processingType) DeploymentManagerDependencies( DefaultProcessingTypeDeployedScenariosProvider(dbRef, processingType), @@ -781,11 +774,7 @@ class AkkaHttpBasedRouteProvider( ) } - private def createProcessingTypeDataReload() = {} - - private def createAdditionalUIConfigProvider(config: Config, sttpBackend: SttpBackend[Future, Any])( - implicit ec: ExecutionContext - ) = { + private def createAdditionalUIConfigProvider(config: Config, sttpBackend: SttpBackend[Future, Any]) = { val additionalUIConfigProviderFactory: AdditionalUIConfigProviderFactory = { Multiplicity( ScalaServiceLoader.load[AdditionalUIConfigProviderFactory](getClass.getClassLoader) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala index b15913b4f2c..a98e33fac83 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala @@ -10,6 +10,7 @@ import fr.davit.akka.http.metrics.core.{HttpMetricsRegistry, HttpMetricsSettings import fr.davit.akka.http.metrics.dropwizard.{DropwizardRegistry, DropwizardSettings} import io.dropwizard.metrics5.MetricRegistry import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.ui.config.DesignerConfig import pl.touk.nussknacker.ui.security.ssl.{HttpsConnectionContextFactory, SslConfigParser} import java.util.concurrent.atomic.AtomicReference @@ -23,10 +24,10 @@ class NussknackerHttpServer(routeProvider: RouteProvider[Route], system: ActorSy private implicit val systemImplicit: ActorSystem = system private implicit val executionContextImplicit: ExecutionContext = system.dispatcher - def start(config: ConfigWithUnresolvedVersion, metricRegistry: MetricRegistry): Resource[IO, Unit] = { + def start(designerConfig: DesignerConfig, metricRegistry: MetricRegistry): Resource[IO, Unit] = { for { - route <- routeProvider.createRoute(config) - _ <- createAkkaHttpBinding(config, route, metricRegistry) + route <- routeProvider.createRoute(designerConfig) + _ <- createAkkaHttpBinding(designerConfig.rawConfig, route, metricRegistry) } yield { RouteInterceptor.set(route) } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala index ce69787f60c..838760ce0e8 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala @@ -2,9 +2,10 @@ package pl.touk.nussknacker.ui.server import akka.http.scaladsl.server.Route import cats.effect.{IO, Resource} -import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.ui.config.DesignerConfig trait RouteProvider[R <: Route] { - def createRoute(config: ConfigWithUnresolvedVersion): Resource[IO, R] + def createRoute(config: DesignerConfig): Resource[IO, R] + } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/IOToFutureSttpBackendConverter.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/IOToFutureSttpBackendConverter.scala new file mode 100644 index 00000000000..6c1edb56bcd --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/IOToFutureSttpBackendConverter.scala @@ -0,0 +1,27 @@ +package pl.touk.nussknacker.ui.util + +import cats.arrow.FunctionK +import cats.effect.IO +import sttp.client3.SttpBackend +import sttp.client3.impl.cats.implicits.sttpBackendToCatsMappableSttpBackend +import sttp.monad.FutureMonad + +import scala.concurrent.Future + +object IOToFutureSttpBackendConverter { + + def convert( + ioBackend: SttpBackend[IO, Any] + )(implicit executionContextWithIORuntime: ExecutionContextWithIORuntime): SttpBackend[Future, Any] = { + import executionContextWithIORuntime.ioRuntime + ioBackend.mapK( + new FunctionK[IO, Future] { + override def apply[A](io: IO[A]): Future[A] = io.unsafeToFuture() + }, + new FunctionK[Future, IO] { + override def apply[A](future: Future[A]): IO[A] = IO.fromFuture(IO.pure(future)) + } + )(new FutureMonad) + } + +} diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala index 49ec79dc4b8..174f55d2179 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala @@ -8,10 +8,8 @@ import org.scalatest.{BeforeAndAfterAll, Suite} import pl.touk.nussknacker.test.DefaultUniquePortProvider import pl.touk.nussknacker.test.base.db.WithHsqlDbTesting import pl.touk.nussknacker.test.config.WithDesignerConfig -import pl.touk.nussknacker.ui.LoadableConfigBasedNussknackerConfig import pl.touk.nussknacker.ui.config.DesignerConfigLoader import pl.touk.nussknacker.ui.factory.NussknackerAppFactory -import pl.touk.nussknacker.ui.process.processingtype.loader._ trait NuItTest extends WithHsqlDbTesting with DefaultUniquePortProvider with WithClock with BeforeAndAfterAll { this: Suite with WithDesignerConfig => @@ -24,10 +22,8 @@ trait NuItTest extends WithHsqlDbTesting with DefaultUniquePortProvider with Wit override protected def beforeAll(): Unit = { super.beforeAll() - val nussknackerConfig = new LoadableConfigBasedNussknackerConfig( - IO.delay(DesignerConfigLoader.from(adjustNuTestConfig())) - ) - releaseAppResources = new NussknackerAppFactory(nussknackerConfig) + val designerConfigLoader = DesignerConfigLoader.fromConfig(adjustNuTestConfig()) + releaseAppResources = NussknackerAppFactory(designerConfigLoader) .createApp(clock = clock) .allocated .unsafeRunSync() diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala index cde22f36654..d47bcc52a6d 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala @@ -38,10 +38,10 @@ import pl.touk.nussknacker.test.mock.{MockDeploymentManager, MockManagerProvider import pl.touk.nussknacker.test.utils.domain.TestFactory._ import pl.touk.nussknacker.test.utils.domain.{ProcessTestData, TestFactory} import pl.touk.nussknacker.test.utils.scalas.AkkaHttpExtensions.toRequestEntity -import pl.touk.nussknacker.ui.LoadableConfigBasedNussknackerConfig import pl.touk.nussknacker.ui.api._ -import pl.touk.nussknacker.ui.config.FeatureTogglesConfig import pl.touk.nussknacker.ui.config.scenariotoolbar.CategoriesScenarioToolbarsConfigParser +import pl.touk.nussknacker.ui.config.FeatureTogglesConfig +import pl.touk.nussknacker.ui.config.DesignerConfig import pl.touk.nussknacker.ui.process.ProcessService.{CreateScenarioCommand, UpdateScenarioCommand} import pl.touk.nussknacker.ui.process._ import pl.touk.nussknacker.ui.process.deployment._ @@ -147,12 +147,13 @@ trait NuResourcesTest protected val featureTogglesConfig: FeatureTogglesConfig = FeatureTogglesConfig.create(testConfig) protected val typeToConfig: ProcessingTypeDataProvider[ProcessingTypeData, CombinedProcessingTypeData] = { - val processingTypeDataReader = new ProcessingTypesConfigBasedProcessingTypeDataLoader( - new LoadableConfigBasedNussknackerConfig(IO.pure(ConfigWithUnresolvedVersion(testConfig))) - ) + val designerConfig = DesignerConfig.from(testConfig) ProcessingTypeDataProvider( - processingTypeDataReader - .loadProcessingTypeData(_ => modelDependencies, _ => deploymentManagerDependencies) + new ProcessingTypesConfigBasedProcessingTypeDataLoader(() => IO.pure(designerConfig.processingTypeConfigs)) + .loadProcessingTypeData( + _ => modelDependencies, + _ => deploymentManagerDependencies + ) .unsafeRunSync() ) } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala index d90167c6e80..73ae8434f93 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala @@ -8,10 +8,10 @@ import com.typesafe.config.ConfigValueFactory.fromAnyRef import io.dropwizard.metrics5.MetricRegistry import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion -import pl.touk.nussknacker.test.{DefaultUniquePortProvider, WithTestHttpClientCreator} import pl.touk.nussknacker.test.config.ConfigWithScalaVersion import pl.touk.nussknacker.test.utils.scalas.CatsTestExtensions._ +import pl.touk.nussknacker.test.{DefaultUniquePortProvider, WithTestHttpClientCreator} +import pl.touk.nussknacker.ui.config.DesignerConfig import pl.touk.nussknacker.ui.security.ssl.HttpsConnectionContextFactory.prepareSSLContext import pl.touk.nussknacker.ui.security.ssl.KeyStoreConfig import pl.touk.nussknacker.ui.server.{NussknackerHttpServer, RouteProvider} @@ -32,7 +32,7 @@ class NussknackerHttpServerSpec server <- createHttpServer() client <- createHttpClient(Some(prepareSSLContext(keyStoreConfig))) _ <- server.start( - ConfigWithUnresolvedVersion( + DesignerConfig.from( ConfigFactory .empty() .withValue("http.interface", fromAnyRef("0.0.0.0")) @@ -64,7 +64,7 @@ class NussknackerHttpServerSpec private object DummyRouteProvider extends RouteProvider[Route] with Directives { - override def createRoute(config: ConfigWithUnresolvedVersion): Resource[IO, Route] = Resource.pure[IO, Route] { + override def createRoute(config: DesignerConfig): Resource[IO, Route] = Resource.pure[IO, Route] { path("test") { get { complete { diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/integration/ConfigurationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/config/ConfigurationTest.scala similarity index 69% rename from designer/server/src/test/scala/pl/touk/nussknacker/ui/integration/ConfigurationTest.scala rename to designer/server/src/test/scala/pl/touk/nussknacker/ui/config/ConfigurationTest.scala index 5631ea406fc..ac1e56a3885 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/integration/ConfigurationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/config/ConfigurationTest.scala @@ -1,17 +1,17 @@ -package pl.touk.nussknacker.ui.integration +package pl.touk.nussknacker.ui.config +import cats.effect.unsafe.implicits.global import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt import pl.touk.nussknacker.engine.{ModelData, ProcessingTypeConfig} import pl.touk.nussknacker.test.config.ConfigWithScalaVersion import pl.touk.nussknacker.test.utils.domain.TestFactory -import pl.touk.nussknacker.ui.config.DesignerConfigLoader -import cats.effect.unsafe.implicits.global + import java.net.URI import java.nio.file.Files import java.util.UUID +// TODO: We should spit DesignerConfigLoader tests and model ProcessingTypeConfig tests class ConfigurationTest extends AnyFunSuite with Matchers { // warning: can't be val - uses ConfigFactory.load which breaks "should preserve config overrides" test @@ -29,21 +29,23 @@ class ConfigurationTest extends AnyFunSuite with Matchers { } test("defaultConfig works") { - DesignerConfigLoader - .load(globalConfig, classLoader) + AlwaysLoadingFileBasedDesignerConfigLoader(classLoader) + .loadDesignerConfig() .unsafeRunSync() + .rawConfig .resolved .getString("db.driver") shouldBe "org.hsqldb.jdbc.JDBCDriver" } test("should be possible to config entries defined in default ui config from passed config") { val configUri = writeToTemp("foo: ${storageDir}") // storageDir is defined inside defaultDesignerConfig.conf + withNussknackerLocationsProperty(configUri.toString) { + val loadedConfig = AlwaysLoadingFileBasedDesignerConfigLoader(classLoader) + .loadDesignerConfig() + .unsafeRunSync() - val loadedConfig = DesignerConfigLoader - .load(ConfigFactoryExt.parseConfigFallbackChain(List(configUri), classLoader), classLoader) - .unsafeRunSync() - - loadedConfig.resolved.getString("foo") shouldEqual "./storage" + loadedConfig.rawConfig.resolved.getString("foo") shouldEqual "./storage" + } } test("defaultConfig is not accessible from model") { @@ -78,17 +80,32 @@ class ConfigurationTest extends AnyFunSuite with Matchers { |""".stripMargin val conf1 = writeToTemp(content) - val result = - try { - System.setProperty(randomPropertyName, "I win!") - DesignerConfigLoader - .load(ConfigFactoryExt.parseConfigFallbackChain(List(conf1), classLoader), classLoader) - .unsafeRunSync() - } finally { - System.getProperties.remove(randomPropertyName) - } - - result.resolved.getString(randomPropertyName) shouldBe "I win!" + withNussknackerLocationsProperty(conf1.toString) { + val result = + try { + System.setProperty(randomPropertyName, "I win!") + AlwaysLoadingFileBasedDesignerConfigLoader(classLoader) + .loadDesignerConfig() + .unsafeRunSync() + } finally { + System.getProperties.remove(randomPropertyName) + } + + result.rawConfig.resolved.getString(randomPropertyName) shouldBe "I win!" + } + } + + private def withNussknackerLocationsProperty(propertyValue: String)(f: => Unit): Unit = { + val locationsProperty = "nussknacker.config.locations" + val valueBeforeChange = Option(System.getProperty(locationsProperty)) + System.setProperty(locationsProperty, propertyValue) + try { + f + } finally { + valueBeforeChange + .map(System.setProperty(locationsProperty, _)) + .getOrElse(System.getProperties.remove(locationsProperty)) + } } // to be able to run this test: diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableConfigBasedNussknackerConfigSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/config/processingtype/ProcessingTypeDataLoaderSpec.scala similarity index 70% rename from designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableConfigBasedNussknackerConfigSpec.scala rename to designer/server/src/test/scala/pl/touk/nussknacker/ui/config/processingtype/ProcessingTypeDataLoaderSpec.scala index 51195a0c2ba..ec23803e342 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableConfigBasedNussknackerConfigSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/config/processingtype/ProcessingTypeDataLoaderSpec.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.ui +package pl.touk.nussknacker.ui.config.processingtype import cats.effect.unsafe.implicits.global import cats.effect.{IO, Ref} @@ -6,9 +6,10 @@ import com.typesafe import com.typesafe.config.{Config, ConfigFactory} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers.{convertToAnyShouldWrapper, include} -import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.ui.config.{DesignerConfig, DesignerConfigLoader} +import pl.touk.nussknacker.ui.configloader.ProcessingTypeConfigsLoader -class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { +class ProcessingTypeDataLoaderSpec extends AnyFunSuite { test("should throw when required configuration is missing") { val config = ConfigFactory @@ -31,7 +32,7 @@ class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { .resolve() intercept[typesafe.config.ConfigException] { - loadableConfigBasedNussknackerConfig(config) + staticConfigBasedProcessingTypeConfigsLoader(config) .loadProcessingTypeConfigs() .unsafeRunSync() }.getMessage should include("No configuration setting found for key 'deploymentConfig.type'") @@ -47,14 +48,14 @@ class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { .resolve() intercept[RuntimeException] { - loadableConfigBasedNussknackerConfig(config) + staticConfigBasedProcessingTypeConfigsLoader(config) .loadProcessingTypeConfigs() .unsafeRunSync() }.getMessage should include("No scenario types configuration provided") } test("should load the second config when reloaded") { - val nussknackerConfig = loadDifferentConfigPerInvocationNussknackerConfig( + val processingTypeConfigsLoader = loadDifferentConfigPerInvocationProcessingTypeConfigsLoader( config1 = ConfigFactory .parseString( """ @@ -103,39 +104,37 @@ class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { .resolve() ) - val processingTypes1 = nussknackerConfig + val processingTypes1 = processingTypeConfigsLoader .loadProcessingTypeConfigs() .unsafeRunSync() - processingTypes1.keys.toSet shouldBe Set("streaming") + processingTypes1.configByProcessingType.keys.toSet shouldBe Set("streaming") - val processingTypes2 = nussknackerConfig + val processingTypes2 = processingTypeConfigsLoader .loadProcessingTypeConfigs() .unsafeRunSync() - processingTypes2.keys.toSet shouldBe Set("streaming", "streaming2") + processingTypes2.configByProcessingType.keys.toSet shouldBe Set("streaming", "streaming2") } - private def loadableConfigBasedNussknackerConfig(config: Config): LoadableConfigBasedNussknackerConfig = { - loadableConfigBasedNussknackerConfig(IO.pure(ConfigWithUnresolvedVersion(config))) + private def staticConfigBasedProcessingTypeConfigsLoader(config: Config): ProcessingTypeConfigsLoader = { () => + DesignerConfigLoader.fromConfig(config).loadDesignerConfig().map(_.processingTypeConfigs) } - private def loadableConfigBasedNussknackerConfig( - loadConfig: IO[ConfigWithUnresolvedVersion] - ): LoadableConfigBasedNussknackerConfig = { - new LoadableConfigBasedNussknackerConfig(loadConfig) - } - - private def loadDifferentConfigPerInvocationNussknackerConfig(config1: Config, config2: Config, configs: Config*) = { + private def loadDifferentConfigPerInvocationProcessingTypeConfigsLoader( + config1: Config, + config2: Config, + configs: Config* + ): ProcessingTypeConfigsLoader = { val ref = Ref.unsafe[IO, Int](0) val allConfigs = config1 :: config2 :: configs.toList - val loadConfig = ref.getAndUpdate(_ + 1).flatMap { idx => + val loadDesignerConfig = ref.getAndUpdate(_ + 1).flatMap { idx => allConfigs.lift(idx) match { - case Some(config) => IO.pure(ConfigWithUnresolvedVersion(config)) + case Some(config) => IO.pure(DesignerConfig.from(config)) case None => IO.raiseError(throw new IllegalStateException(s"Cannot load the config more than [$idx]")) } } - loadableConfigBasedNussknackerConfig(loadConfig) + () => loadDesignerConfig.map(_.processingTypeConfigs) } } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala index 110f195b4da..f4a8e88d82d 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala @@ -55,7 +55,10 @@ class ProcessingTypeDataProviderSpec extends AnyFunSuite with Matchers { deploymentManagerProvider = new DeploymentManagerProviderStub ) loader - .loadProcessingTypeData(_ => modelDependencies, _ => TestFactory.deploymentManagerDependencies) + .loadProcessingTypeData( + _ => modelDependencies, + _ => TestFactory.deploymentManagerDependencies + ) .unsafeRunSync() } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala index d0502b2815c..fe079bdeec1 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala @@ -2,25 +2,25 @@ package pl.touk.nussknacker.ui.process.processingtype import cats.data.Validated.Invalid import cats.effect.IO +import cats.effect.unsafe.implicits.global import com.typesafe.config.ConfigFactory import com.typesafe.scalalogging.LazyLogging import org.scalatest.Inside.inside import org.scalatest.OptionValues import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, ModelDependencies} +import pl.touk.nussknacker.engine.ModelDependencies import pl.touk.nussknacker.engine.api.component.{ComponentProvider, DesignerWideComponentId, ProcessingMode} import pl.touk.nussknacker.engine.api.process.ProcessingType +import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode import pl.touk.nussknacker.engine.deployment.EngineSetupName import pl.touk.nussknacker.restmodel.scenariodetails.ScenarioParameters import pl.touk.nussknacker.security.Permission import pl.touk.nussknacker.test.ValidatedValuesDetailedMessage import pl.touk.nussknacker.test.utils.domain.TestFactory +import pl.touk.nussknacker.ui.config.DesignerConfig import pl.touk.nussknacker.ui.process.processingtype.loader.ProcessingTypesConfigBasedProcessingTypeDataLoader import pl.touk.nussknacker.ui.security.api.{LoggedUser, RealLoggedUser} -import cats.effect.unsafe.implicits.global -import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode -import pl.touk.nussknacker.ui.LoadableConfigBasedNussknackerConfig import java.nio.file.Path import scala.jdk.CollectionConverters._ @@ -281,25 +281,23 @@ class ScenarioParametersServiceTest val workPath = designerServerModuleDir.resolve("work") logDirectoryStructure(workPath) - val processingTypeDataReader = new ProcessingTypesConfigBasedProcessingTypeDataLoader( - new LoadableConfigBasedNussknackerConfig(IO.pure { - ConfigWithUnresolvedVersion(ConfigFactory.parseFile(devApplicationConfFile).withFallback(fallbackConfig)) - }) - ) - val processingTypeData = processingTypeDataReader - .loadProcessingTypeData( - processingType => - ModelDependencies( - Map.empty, - componentId => DesignerWideComponentId(componentId.toString), - Some(workPath), - shouldIncludeComponentProvider(processingType, _), - ComponentDefinitionExtractionMode.FinalDefinition - ), - _ => TestFactory.deploymentManagerDependencies, - ) - .unsafeRunSync() + val designerConfig = + DesignerConfig.from(ConfigFactory.parseFile(devApplicationConfFile).withFallback(fallbackConfig)) + val processingTypeData = + new ProcessingTypesConfigBasedProcessingTypeDataLoader(() => IO.pure(designerConfig.processingTypeConfigs)) + .loadProcessingTypeData( + processingType => + ModelDependencies( + Map.empty, + componentId => DesignerWideComponentId(componentId.toString), + Some(workPath), + shouldIncludeComponentProvider(processingType, _), + ComponentDefinitionExtractionMode.FinalDefinition + ), + _ => TestFactory.deploymentManagerDependencies, + ) + .unsafeRunSync() val parametersService = processingTypeData.getCombined().parametersService parametersService.scenarioParametersCombinationsWithWritePermission(TestFactory.adminUser()) shouldEqual List( diff --git a/engine/lite/runtime-app/src/main/scala/pl/touk/nussknacker/engine/lite/app/NuRuntimeApp.scala b/engine/lite/runtime-app/src/main/scala/pl/touk/nussknacker/engine/lite/app/NuRuntimeApp.scala index 887a6fbcbd0..2f33594193a 100644 --- a/engine/lite/runtime-app/src/main/scala/pl/touk/nussknacker/engine/lite/app/NuRuntimeApp.scala +++ b/engine/lite/runtime-app/src/main/scala/pl/touk/nussknacker/engine/lite/app/NuRuntimeApp.scala @@ -9,7 +9,7 @@ import com.typesafe.scalalogging.LazyLogging import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.marshall.ScenarioParser import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt -import pl.touk.nussknacker.engine.util.{JavaClassVersionChecker, ResourceLoader, SLF4JBridgeHandlerRegistrar} +import pl.touk.nussknacker.engine.util.{JavaClassVersionChecker, ResourceLoader, SLF4JBridgeHandlerRegistrar, UriUtils} import java.nio.file.Path import scala.concurrent.duration._ @@ -27,7 +27,13 @@ object NuRuntimeApp extends App with LazyLogging { val (scenarioFileLocation, deploymentConfigLocation) = parseArgs val scenario = parseScenario(scenarioFileLocation) val deploymentConfig = parseDeploymentConfig(deploymentConfigLocation) - val runtimeConfig = ConfigFactory.load(ConfigFactoryExt.parseUnresolved(classLoader = getClass.getClassLoader)) + + val runtimeConfig = { + val configLocationsProperty: String = "nussknacker.config.locations" + val locationsPropertyValueOpt = Option(System.getProperty(configLocationsProperty)) + val locations = locationsPropertyValueOpt.map(UriUtils.extractListOfLocations).getOrElse(List.empty) + ConfigFactory.load(new ConfigFactoryExt(getClass.getClassLoader).parseUnresolved(locations)) + } val httpConfig = runtimeConfig.as[HttpBindingConfig]("http") diff --git a/extensions-api/src/main/scala/pl/touk/nussknacker/engine/ConfigWithUnresolvedVersion.scala b/extensions-api/src/main/scala/pl/touk/nussknacker/engine/ConfigWithUnresolvedVersion.scala index fb63f036c08..16dad532540 100644 --- a/extensions-api/src/main/scala/pl/touk/nussknacker/engine/ConfigWithUnresolvedVersion.scala +++ b/extensions-api/src/main/scala/pl/touk/nussknacker/engine/ConfigWithUnresolvedVersion.scala @@ -2,12 +2,31 @@ package pl.touk.nussknacker.engine import com.typesafe.config.{Config, ConfigFactory, ConfigResolveOptions} +import scala.jdk.CollectionConverters._ + case class ConfigWithUnresolvedVersion private (withUnresolvedEnvVariables: Config, resolved: Config) { def getConfig(path: String): ConfigWithUnresolvedVersion = { ConfigWithUnresolvedVersion(withUnresolvedEnvVariables.getConfig(path), resolved.getConfig(path)) } + def readMap(path: String): Option[Map[String, ConfigWithUnresolvedVersion]] = { + if (resolved.hasPath(path)) { + val nestedConfig = getConfig(path) + Some( + nestedConfig.resolved + .root() + .entrySet() + .asScala + .map(_.getKey) + .map { key => key -> nestedConfig.getConfig(key) } + .toMap + ) + } else { + None + } + } + } object ConfigWithUnresolvedVersion { diff --git a/security/src/main/scala/pl/touk/nussknacker/ui/security/api/AuthenticationConfiguration.scala b/security/src/main/scala/pl/touk/nussknacker/ui/security/api/AuthenticationConfiguration.scala index 07f94a03962..4a0121a0aab 100644 --- a/security/src/main/scala/pl/touk/nussknacker/ui/security/api/AuthenticationConfiguration.scala +++ b/security/src/main/scala/pl/touk/nussknacker/ui/security/api/AuthenticationConfiguration.scala @@ -19,7 +19,7 @@ trait AuthenticationConfiguration { def isAdminImpersonationPossible: Boolean - private lazy val userConfig: Config = ConfigFactoryExt.parseUri(usersFile, getClass.getClassLoader) + private lazy val userConfig: Config = new ConfigFactoryExt(getClass.getClassLoader).parseUri(usersFile) protected lazy val usersOpt: Option[List[ConfigUser]] = userConfig.as[Option[List[ConfigUser]]](usersConfigurationPath) @@ -44,7 +44,7 @@ object AuthenticationConfiguration { val rulesConfigurationPath = "rules" private[security] def getRules(usersFile: URI): List[ConfigRule] = - ConfigFactoryExt.parseUri(usersFile, getClass.getClassLoader).as[List[ConfigRule]](rulesConfigurationPath) + new ConfigFactoryExt(getClass.getClassLoader).parseUri(usersFile).as[List[ConfigRule]](rulesConfigurationPath) final case class ConfigUser( identity: String, diff --git a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/UriUtils.scala b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/UriUtils.scala index a0ff3080a77..4cfb516b482 100644 --- a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/UriUtils.scala +++ b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/UriUtils.scala @@ -1,7 +1,9 @@ package pl.touk.nussknacker.engine.util -import java.net.{URLDecoder, URLEncoder} +import java.io.File +import java.net.{URI, URLDecoder, URLEncoder} import java.nio.charset.StandardCharsets +import scala.util.Try /** * Utility class for JavaScript compatible UTF-8 encoding and decoding. @@ -10,6 +12,13 @@ import java.nio.charset.StandardCharsets */ object UriUtils { + def extractListOfLocations(commaSeparatedListOfLocations: String): List[URI] = { + for { + singleElement <- commaSeparatedListOfLocations.split(",").toList + trimmed = singleElement.trim + } yield Try(URI.create(trimmed)).filter(_.getScheme.nonEmpty).getOrElse(new File(trimmed).toURI) + } + /** * Decodes the passed UTF-8 String using an algorithm that's compatible with * JavaScript's decodeURIComponent function. Returns diff --git a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExt.scala b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExt.scala index ec2dc6ed908..d2beeef99ff 100644 --- a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExt.scala +++ b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExt.scala @@ -2,13 +2,17 @@ package pl.touk.nussknacker.engine.util.config import com.typesafe.config.{Config, ConfigFactory} -import java.io.File import java.net.URI -import scala.util.Try -object ConfigFactoryExt extends URIExtensions { +class ConfigFactoryExt(classLoader: ClassLoader) extends URIExtensions { - def parseUri(uri: URI, classLoader: ClassLoader): Config = { + def parseUnresolved(locations: List[URI]): Config = + locations + .map(parseUri) + .reverse + .foldLeft(ConfigFactory.empty())(_.withFallback(_)) + + def parseUri(uri: URI): Config = { uri.getScheme match { // When we migrate to Java 9+ we can use SPI to load a URLStreamHandlerProvider // instance to handle the classpath scheme. @@ -17,24 +21,4 @@ object ConfigFactoryExt extends URIExtensions { } } - def parseConfigFallbackChain(resources: List[URI], classLoader: ClassLoader): Config = - resources - .map(ConfigFactoryExt.parseUri(_, classLoader)) - .reverse - .foldLeft(ConfigFactory.empty())(_.withFallback(_)) - - private val configLocationProperty: String = "nussknacker.config.locations" - - def parseUnresolved( - locationString: String = System.getProperty(configLocationProperty), - classLoader: ClassLoader - ): Config = { - val locations = for { - property <- Option(locationString).toList - singleElement <- property.split(",") - trimmed = singleElement.trim - } yield Try(URI.create(trimmed)).filter(_.getScheme.nonEmpty).getOrElse(new File(trimmed).toURI) - ConfigFactoryExt.parseConfigFallbackChain(locations, classLoader) - } - } diff --git a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigWithUnresolvedVersionExt.scala b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigWithUnresolvedVersionExt.scala deleted file mode 100644 index bbda0e21359..00000000000 --- a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigWithUnresolvedVersionExt.scala +++ /dev/null @@ -1,32 +0,0 @@ -package pl.touk.nussknacker.engine.util.config - -import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion - -import scala.jdk.CollectionConverters._ -import scala.language.implicitConversions - -class ConfigWithUnresolvedVersionExt(val config: ConfigWithUnresolvedVersion) { - - def readMap(path: String): Option[Map[String, ConfigWithUnresolvedVersion]] = { - if (config.resolved.hasPath(path)) { - val nestedConfig = config.getConfig(path) - Some( - nestedConfig.resolved - .root() - .entrySet() - .asScala - .map(_.getKey) - .map { key => key -> nestedConfig.getConfig(key) } - .toMap - ) - } else { - None - } - } - -} - -object ConfigWithUnresolvedVersionExt { - implicit def toConfigWithUnresolvedVersionExt(config: ConfigWithUnresolvedVersion): ConfigWithUnresolvedVersionExt = - new ConfigWithUnresolvedVersionExt(config) -} diff --git a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/loader/ScalaServiceLoader.scala b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/loader/ScalaServiceLoader.scala index 78ae2452575..2012bbf3c9a 100644 --- a/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/loader/ScalaServiceLoader.scala +++ b/utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/loader/ScalaServiceLoader.scala @@ -42,6 +42,17 @@ object ScalaServiceLoader extends LazyLogging { } } + def loadOne[T: ClassTag](classLoader: ClassLoader): Option[T] = { + load[T](classLoader) match { + case Nil => + None + case one :: Nil => + Some(one) + case _ => + throw new IllegalStateException(s"More than one ${classTag[T].runtimeClass.getSimpleName} found") + } + } + def load[T: ClassTag](classLoader: ClassLoader): List[T] = { val interface: Class[T] = classTag[T].runtimeClass.asInstanceOf[Class[T]] val loadedClasses = ServiceLoader diff --git a/utils/utils-internal/src/test/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExtSpec.scala b/utils/utils-internal/src/test/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExtSpec.scala index a5d92ccaac5..bd71eb6a831 100644 --- a/utils/utils-internal/src/test/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExtSpec.scala +++ b/utils/utils-internal/src/test/scala/pl/touk/nussknacker/engine/util/config/ConfigFactoryExtSpec.scala @@ -16,9 +16,8 @@ class ConfigFactoryExtSpec extends AnyFunSuite with Matchers { writeToTemp(Map("f1" -> "default", "f2" -> "not so default", "akka.http.server.request-timeout" -> "300s")) val conf2 = writeToTemp(Map("f1" -> "I win!")) - val result = ConfigFactoryExt.parseConfigFallbackChain( + val result = new ConfigFactoryExt(getClass.getClassLoader).parseUnresolved( List(conf1, conf2, URI.create("classpath:someConfig.conf")), - getClass.getClassLoader ) result.getString("f1") shouldBe "I win!"