From f0356be3a892fc75a6034cf1eb2fe6f0b1356cb5 Mon Sep 17 00:00:00 2001 From: Arek Burdach Date: Thu, 5 Dec 2024 18:00:27 +0100 Subject: [PATCH 01/10] [NU-1828] SPI for processing types configs loading --- .../ui/LocalNussknackerWithSingleModel.scala | 7 +- .../touk/nussknacker/ui/NussknackerApp.scala | 8 +- .../nussknacker/ui/NussknackerConfig.scala | 31 ------ ...r.scala => DesignerRootConfigLoader.scala} | 18 ++- .../ui/factory/NussknackerAppFactory.scala | 42 ++++--- .../LoadableDesignerRootConfig.scala | 18 +++ .../LoadableProcessingTypeConfigs.scala | 56 ++++++++++ .../LocalProcessingTypeDataLoader.scala | 2 + .../loader/ProcessingTypeDataLoader.scala | 2 + ...sConfigBasedProcessingTypeDataLoader.scala | 104 ++++++++++-------- .../server/AkkaHttpBasedRouteProvider.scala | 77 ++++++------- .../ui/server/NussknackerHttpServer.scala | 7 +- .../nussknacker/ui/server/RouteProvider.scala | 5 +- .../nussknacker/test/base/it/NuItTest.scala | 11 +- .../test/base/it/NuResourcesTest.scala | 16 +-- ...a => LoadableDesignerRootConfigSpec.scala} | 47 ++++---- .../ui/api/NussknackerHttpServerSpec.scala | 7 +- .../ui/integration/ConfigurationTest.scala | 8 +- .../ProcessingTypeDataProviderSpec.scala | 7 +- .../ScenarioParametersServiceTest.scala | 21 ++-- 20 files changed, 290 insertions(+), 204 deletions(-) delete mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala rename designer/server/src/main/scala/pl/touk/nussknacker/ui/config/{DesignerConfigLoader.scala => DesignerRootConfigLoader.scala} (70%) create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/loadableconfig/LoadableDesignerRootConfig.scala create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/loadableconfig/LoadableProcessingTypeConfigs.scala rename designer/server/src/test/scala/pl/touk/nussknacker/ui/{LoadableConfigBasedNussknackerConfigSpec.scala => LoadableDesignerRootConfigSpec.scala} (67%) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala index 12b0aa71adb..21338bb9037 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala @@ -4,8 +4,9 @@ import cats.effect.{IO, Resource} import com.typesafe.config.{Config, ConfigFactory} import org.apache.commons.io.FileUtils import pl.touk.nussknacker.engine.{DeploymentManagerProvider, ModelData} -import pl.touk.nussknacker.ui.config.DesignerConfigLoader +import pl.touk.nussknacker.ui.config.DesignerRootConfig import pl.touk.nussknacker.ui.factory.NussknackerAppFactory +import pl.touk.nussknacker.ui.loadableconfig.LoadableDesignerRootConfig import pl.touk.nussknacker.ui.process.processingtype.loader.LocalProcessingTypeDataLoader import java.io.File @@ -49,8 +50,8 @@ object LocalNussknackerWithSingleModel { modelData = Map(typeName -> (category, modelData)), deploymentManagerProvider = deploymentManagerProvider ) - val nussknackerConfig = new LoadableConfigBasedNussknackerConfig(IO.delay(DesignerConfigLoader.from(appConfig))) - val appFactory = new NussknackerAppFactory(nussknackerConfig, local) + val loadableDesignerConfig = LoadableDesignerRootConfig(IO.delay(DesignerRootConfig.from(appConfig))) + val appFactory = new NussknackerAppFactory(loadableDesignerConfig, local) appFactory.createApp() } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala index d99984f966a..137c495dfa8 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala @@ -1,14 +1,18 @@ package pl.touk.nussknacker.ui import cats.effect.{ExitCode, IO, IOApp} +import pl.touk.nussknacker.ui.config.DesignerRootConfigLoader import pl.touk.nussknacker.ui.factory.NussknackerAppFactory +import pl.touk.nussknacker.ui.loadableconfig.LoadableDesignerRootConfig object NussknackerApp extends IOApp { override def run(args: List[String]): IO[ExitCode] = { for { - appFactory <- IO(new NussknackerAppFactory(getClass.getClassLoader)) - _ <- appFactory.createApp().use { _ => IO.never } + appFactory <- IO( + NussknackerAppFactory(LoadableDesignerRootConfig(DesignerRootConfigLoader.load(getClass.getClassLoader))) + ) + _ <- appFactory.createApp().use { _ => IO.never } } yield ExitCode.Success } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala deleted file mode 100644 index a05b7b175b8..00000000000 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerConfig.scala +++ /dev/null @@ -1,31 +0,0 @@ -package pl.touk.nussknacker.ui - -import cats.effect.IO -import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap -import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, ProcessingTypeConfig} -import pl.touk.nussknacker.engine.util.config.ConfigWithUnresolvedVersionExt._ -import pl.touk.nussknacker.ui.config.DesignerConfigLoader - -trait NussknackerConfig { - - def loadApplicationConfig(): IO[ConfigWithUnresolvedVersion] - - final def loadProcessingTypeConfigs(): IO[Map[String, ProcessingTypeConfig]] = { - loadApplicationConfig() - .map { config => - config - .readMap("scenarioTypes") - .getOrElse { throw new RuntimeException("No scenario types configuration provided") } - .mapValuesNow(ProcessingTypeConfig.read) - } - } - -} - -class LoadableConfigBasedNussknackerConfig(loadConfig: IO[ConfigWithUnresolvedVersion]) extends NussknackerConfig { - - override def loadApplicationConfig(): IO[ConfigWithUnresolvedVersion] = loadConfig -} - -class LoadableDesignerConfigBasedNussknackerConfig(classLoader: ClassLoader) - extends LoadableConfigBasedNussknackerConfig(DesignerConfigLoader.load(classLoader)) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerRootConfigLoader.scala similarity index 70% rename from designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala rename to designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerRootConfigLoader.scala index b87cdb7f6eb..7eed8d3ec81 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerRootConfigLoader.scala @@ -14,15 +14,15 @@ import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt * Result of config loading still keep version with unresolved env variables for purpose of config loading on model side - see * InputConfigDuringExecution and ModelConfigLoader */ -object DesignerConfigLoader { +object DesignerRootConfigLoader { private val defaultConfigResource = "defaultDesignerConfig.conf" - def load(classLoader: ClassLoader): IO[ConfigWithUnresolvedVersion] = { + def load(classLoader: ClassLoader): IO[DesignerRootConfig] = { for { baseConfig <- IO.blocking(ConfigFactoryExt.parseUnresolved(classLoader = classLoader)) loadedConfig <- load(baseConfig, classLoader) - } yield loadedConfig + } yield DesignerRootConfig(loadedConfig) } def load(baseUnresolvedConfig: Config, classLoader: ClassLoader): IO[ConfigWithUnresolvedVersion] = { @@ -33,8 +33,16 @@ object DesignerConfigLoader { } } - def from(config: Config): ConfigWithUnresolvedVersion = { - ConfigWithUnresolvedVersion(this.getClass.getClassLoader, config) +} + +// TODO: We should extract a class for all configuration options that should be available to designer instead of returning raw hocon config. +// Thanks to that it will be easier to split processing type config from rest of configs and use this interface programmatically +final case class DesignerRootConfig(rawConfig: ConfigWithUnresolvedVersion) + +object DesignerRootConfig { + + def from(config: Config): DesignerRootConfig = { + DesignerRootConfig(ConfigWithUnresolvedVersion(config)) } } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala index e9be44d0bcf..75da524598b 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/factory/NussknackerAppFactory.scala @@ -8,37 +8,33 @@ import io.dropwizard.metrics5.MetricRegistry import io.dropwizard.metrics5.jmx.JmxReporter import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion import pl.touk.nussknacker.engine.util.{JavaClassVersionChecker, SLF4JBridgeHandlerRegistrar} -import pl.touk.nussknacker.ui.{LoadableDesignerConfigBasedNussknackerConfig, NussknackerConfig} import pl.touk.nussknacker.ui.db.DbRef import pl.touk.nussknacker.ui.db.timeseries.questdb.QuestDbFEStatisticsRepository +import pl.touk.nussknacker.ui.loadableconfig.{ + EachTimeLoadingRootConfigLoadableProcessingTypeConfigs, + LoadableDesignerRootConfig, + LoadableProcessingTypeConfigs +} import pl.touk.nussknacker.ui.process.processingtype.loader._ import pl.touk.nussknacker.ui.server.{AkkaHttpBasedRouteProvider, NussknackerHttpServer} import java.time.Clock -object NussknackerAppFactory - -class NussknackerAppFactory(nussknackerConfig: NussknackerConfig, processingTypeDataLoader: ProcessingTypeDataLoader) - extends LazyLogging { - - def this(nussknackerConfig: NussknackerConfig) = { - this(nussknackerConfig, new ProcessingTypesConfigBasedProcessingTypeDataLoader(nussknackerConfig)) - } - - def this(classLoader: ClassLoader) = { - this(new LoadableDesignerConfigBasedNussknackerConfig(classLoader)) - } +class NussknackerAppFactory( + loadableDesignerRootConfig: LoadableDesignerRootConfig, + processingTypeDataLoader: ProcessingTypeDataLoader +) extends LazyLogging { def createApp(clock: Clock = Clock.systemUTC()): Resource[IO, Unit] = { for { - config <- Resource.eval(nussknackerConfig.loadApplicationConfig()) - system <- createActorSystem(config) + rootConfig <- Resource.eval(loadableDesignerRootConfig.loadDesignerRootConfig()) + system <- createActorSystem(rootConfig.rawConfig) materializer = Materializer(system) _ <- Resource.eval(IO(JavaClassVersionChecker.check())) _ <- Resource.eval(IO(SLF4JBridgeHandlerRegistrar.register())) metricsRegistry <- createGeneralPurposeMetricsRegistry() - db <- DbRef.create(config.resolved) - feStatisticsRepository <- QuestDbFEStatisticsRepository.create(system, clock, config.resolved) + db <- DbRef.create(rootConfig.rawConfig.resolved) + feStatisticsRepository <- QuestDbFEStatisticsRepository.create(system, clock, rootConfig.rawConfig.resolved) server = new NussknackerHttpServer( new AkkaHttpBasedRouteProvider( db, @@ -52,7 +48,7 @@ class NussknackerAppFactory(nussknackerConfig: NussknackerConfig, processingType ), system ) - _ <- server.start(config, metricsRegistry) + _ <- server.start(rootConfig, metricsRegistry) _ <- startJmxReporter(metricsRegistry) _ <- createStartAndStopLoggingEntries() } yield () @@ -87,3 +83,13 @@ class NussknackerAppFactory(nussknackerConfig: NussknackerConfig, processingType } } + +object NussknackerAppFactory { + + def apply(loadableDesignerRootConfig: LoadableDesignerRootConfig): NussknackerAppFactory = { + val loadableProcessingTypeConfig = LoadableProcessingTypeConfigs.default(loadableDesignerRootConfig) + val processingTypeDataLoader = new ProcessingTypesConfigBasedProcessingTypeDataLoader(loadableProcessingTypeConfig) + new NussknackerAppFactory(loadableDesignerRootConfig, processingTypeDataLoader) + } + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/loadableconfig/LoadableDesignerRootConfig.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/loadableconfig/LoadableDesignerRootConfig.scala new file mode 100644 index 00000000000..7be9bc290f2 --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/loadableconfig/LoadableDesignerRootConfig.scala @@ -0,0 +1,18 @@ +package pl.touk.nussknacker.ui.loadableconfig + +import cats.effect.IO +import pl.touk.nussknacker.ui.config.DesignerRootConfig + +trait LoadableDesignerRootConfig { + + def loadDesignerRootConfig(): IO[DesignerRootConfig] + +} + +object LoadableDesignerRootConfig { + + def apply(loadConfig: IO[DesignerRootConfig]): LoadableDesignerRootConfig = { () => + loadConfig + } + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/loadableconfig/LoadableProcessingTypeConfigs.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/loadableconfig/LoadableProcessingTypeConfigs.scala new file mode 100644 index 00000000000..2c22ed52d7a --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/loadableconfig/LoadableProcessingTypeConfigs.scala @@ -0,0 +1,56 @@ +package pl.touk.nussknacker.ui.loadableconfig + +import cats.effect.IO +import com.typesafe.scalalogging.LazyLogging +import pl.touk.nussknacker.engine.ProcessingTypeConfig +import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap +import pl.touk.nussknacker.engine.util.config.ConfigWithUnresolvedVersionExt._ +import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader +import pl.touk.nussknacker.ui.config.DesignerRootConfig + +trait LoadableProcessingTypeConfigs { + + // rootConfigLoadedAtStart is used for external project purpose - don't remove it + def loadProcessingTypeConfigs(rootConfigLoadedAtStart: DesignerRootConfig): IO[Map[String, ProcessingTypeConfig]] + +} + +class EachTimeLoadingRootConfigLoadableProcessingTypeConfigs(loadableDesignerRootConfig: LoadableDesignerRootConfig) + extends LoadableProcessingTypeConfigs { + + def loadProcessingTypeConfigs( + rootConfigLoadedAtStart: DesignerRootConfig + ): IO[Map[String, ProcessingTypeConfig]] = + loadableDesignerRootConfig + .loadDesignerRootConfig() + .map(LoadableProcessingTypeConfigs.extractProcessingTypeConfigs) + +} + +object LoadableProcessingTypeConfigs extends LazyLogging { + + def extractProcessingTypeConfigs(rootConfig: DesignerRootConfig): Map[String, ProcessingTypeConfig] = { + rootConfig.rawConfig + .readMap("scenarioTypes") + .getOrElse { + throw new RuntimeException("No scenario types configuration provided") + } + .mapValuesNow(ProcessingTypeConfig.read) + } + + def default(loadableDesignerRootConfig: LoadableDesignerRootConfig): LoadableProcessingTypeConfigs = { + ScalaServiceLoader.load[LoadableProcessingTypeConfigs](getClass.getClassLoader) match { + case one :: Nil => + logger.debug( + s"Found custom ${classOf[LoadableProcessingTypeConfigs].getSimpleName}: ${one.getClass.getName}. Using it for configuration loading" + ) + one + case Nil => + logger.debug(s"No custom ${classOf[LoadableProcessingTypeConfigs].getSimpleName} found. Using the default one") + new EachTimeLoadingRootConfigLoadableProcessingTypeConfigs(loadableDesignerRootConfig) + case _ => + throw new IllegalStateException(s"More than one ${classOf[LoadableProcessingTypeConfigs].getSimpleName} found") + } + } + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/LocalProcessingTypeDataLoader.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/LocalProcessingTypeDataLoader.scala index 2fae13dc5c2..02ef05d4016 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/LocalProcessingTypeDataLoader.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/LocalProcessingTypeDataLoader.scala @@ -5,6 +5,7 @@ import com.typesafe.config.ConfigFactory import pl.touk.nussknacker.engine._ import pl.touk.nussknacker.engine.api.process.ProcessingType import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap +import pl.touk.nussknacker.ui.config.DesignerRootConfig import pl.touk.nussknacker.ui.process.processingtype.loader.ProcessingTypeDataLoader.toValueWithRestriction import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataState import pl.touk.nussknacker.ui.process.processingtype.{CombinedProcessingTypeData, ProcessingTypeData} @@ -15,6 +16,7 @@ class LocalProcessingTypeDataLoader( ) extends ProcessingTypeDataLoader { override def loadProcessingTypeData( + rootConfigLoadedAtStart: DesignerRootConfig, getModelDependencies: ProcessingType => ModelDependencies, getDeploymentManagerDependencies: ProcessingType => DeploymentManagerDependencies ): IO[ProcessingTypeDataState[ProcessingTypeData, CombinedProcessingTypeData]] = IO { diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypeDataLoader.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypeDataLoader.scala index b2b526fc02f..dbb92129d3d 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypeDataLoader.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypeDataLoader.scala @@ -3,6 +3,7 @@ package pl.touk.nussknacker.ui.process.processingtype.loader import cats.effect.IO import pl.touk.nussknacker.engine.api.process.ProcessingType import pl.touk.nussknacker.engine.{DeploymentManagerDependencies, ModelDependencies} +import pl.touk.nussknacker.ui.config.DesignerRootConfig import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataState import pl.touk.nussknacker.ui.process.processingtype.{ CombinedProcessingTypeData, @@ -13,6 +14,7 @@ import pl.touk.nussknacker.ui.process.processingtype.{ trait ProcessingTypeDataLoader { def loadProcessingTypeData( + rootConfigLoadedAtStart: DesignerRootConfig, getModelDependencies: ProcessingType => ModelDependencies, getDeploymentManagerDependencies: ProcessingType => DeploymentManagerDependencies, ): IO[ProcessingTypeDataState[ProcessingTypeData, CombinedProcessingTypeData]] diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala index 30500afd1a1..b6df9b2b79d 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/loader/ProcessingTypesConfigBasedProcessingTypeDataLoader.scala @@ -6,64 +6,78 @@ import pl.touk.nussknacker.engine._ import pl.touk.nussknacker.engine.api.process.ProcessingType import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader -import pl.touk.nussknacker.ui.NussknackerConfig +import pl.touk.nussknacker.ui.config.DesignerRootConfig +import pl.touk.nussknacker.ui.loadableconfig.LoadableProcessingTypeConfigs import pl.touk.nussknacker.ui.process.processingtype._ import pl.touk.nussknacker.ui.process.processingtype.loader.ProcessingTypeDataLoader.toValueWithRestriction import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataState -class ProcessingTypesConfigBasedProcessingTypeDataLoader(config: NussknackerConfig) - extends ProcessingTypeDataLoader - with LazyLogging { +class ProcessingTypesConfigBasedProcessingTypeDataLoader(loadableProcessingTypeConfigs: LoadableProcessingTypeConfigs) + extends ProcessingTypeDataLoader { override def loadProcessingTypeData( + designerRootConfig: DesignerRootConfig, getModelDependencies: ProcessingType => ModelDependencies, getDeploymentManagerDependencies: ProcessingType => DeploymentManagerDependencies, ): IO[ProcessingTypeDataState[ProcessingTypeData, CombinedProcessingTypeData]] = { - config - .loadProcessingTypeConfigs() - .map { processingTypesConfig => - // This step with splitting DeploymentManagerProvider loading for all processing types - // and after that creating ProcessingTypeData is done because of the deduplication of deployments - // See DeploymentManagerProvider.engineSetupIdentity - val providerWithNameInputData = processingTypesConfig.mapValuesNow { processingTypeConfig => - val provider = createDeploymentManagerProvider(processingTypeConfig) - val nameInputData = EngineNameInputData( - provider.defaultEngineSetupName, - provider.engineSetupIdentity(processingTypeConfig.deploymentConfig), - processingTypeConfig.engineSetupName - ) - (processingTypeConfig, provider, nameInputData) - } - val engineSetupNames = - ScenarioParametersDeterminer.determineEngineSetupNames(providerWithNameInputData.mapValuesNow(_._3)) - val processingTypesData = providerWithNameInputData - .map { case (processingType, (processingTypeConfig, deploymentManagerProvider, _)) => - logger.debug(s"Creating Processing Type: $processingType with config: $processingTypeConfig") - val modelDependencies = getModelDependencies(processingType) - val processingTypeData = ProcessingTypeData.createProcessingTypeData( - processingType, - ModelData(processingTypeConfig, modelDependencies), - deploymentManagerProvider, - getDeploymentManagerDependencies(processingType), - engineSetupNames(processingType), - processingTypeConfig.deploymentConfig, - processingTypeConfig.category, - modelDependencies.componentDefinitionExtractionMode - ) - processingType -> processingTypeData - } + loadableProcessingTypeConfigs + .loadProcessingTypeConfigs(designerRootConfig) + .map( + ProcessingTypesConfigBasedProcessingTypeDataLoader + .loadProcessingTypeData(_, getModelDependencies, getDeploymentManagerDependencies) + ) + } + +} - // Here all processing types are loaded and we are ready to perform additional configuration validations - // to assert the loaded configuration is correct (fail-fast approach). - val combinedData = CombinedProcessingTypeData.create(processingTypesData) +object ProcessingTypesConfigBasedProcessingTypeDataLoader extends LazyLogging { - ProcessingTypeDataState( - processingTypesData.mapValuesNow(toValueWithRestriction), - () => combinedData, - // We pass here new Object to enforce update of observers - new Object + def loadProcessingTypeData( + processingTypesConfig: Map[String, ProcessingTypeConfig], + getModelDependencies: ProcessingType => ModelDependencies, + getDeploymentManagerDependencies: ProcessingType => DeploymentManagerDependencies + ): ProcessingTypeDataState[ProcessingTypeData, CombinedProcessingTypeData] = { + // This step with splitting DeploymentManagerProvider loading for all processing types + // and after that creating ProcessingTypeData is done because of the deduplication of deployments + // See DeploymentManagerProvider.engineSetupIdentity + val providerWithNameInputData = processingTypesConfig.mapValuesNow { processingTypeConfig => + val provider = createDeploymentManagerProvider(processingTypeConfig) + val nameInputData = EngineNameInputData( + provider.defaultEngineSetupName, + provider.engineSetupIdentity(processingTypeConfig.deploymentConfig), + processingTypeConfig.engineSetupName + ) + (processingTypeConfig, provider, nameInputData) + } + val engineSetupNames = + ScenarioParametersDeterminer.determineEngineSetupNames(providerWithNameInputData.mapValuesNow(_._3)) + val processingTypesData = providerWithNameInputData + .map { case (processingType, (processingTypeConfig, deploymentManagerProvider, _)) => + logger.debug(s"Creating Processing Type: $processingType with config: $processingTypeConfig") + val modelDependencies = getModelDependencies(processingType) + val processingTypeData = ProcessingTypeData.createProcessingTypeData( + processingType, + ModelData(processingTypeConfig, modelDependencies), + deploymentManagerProvider, + getDeploymentManagerDependencies(processingType), + engineSetupNames(processingType), + processingTypeConfig.deploymentConfig, + processingTypeConfig.category, + modelDependencies.componentDefinitionExtractionMode ) + processingType -> processingTypeData } + + // Here all processing types are loaded and we are ready to perform additional configuration validations + // to assert the loaded configuration is correct (fail-fast approach). + val combinedData = CombinedProcessingTypeData.create(processingTypesData) + + ProcessingTypeDataState( + processingTypesData.mapValuesNow(toValueWithRestriction), + () => combinedData, + // We pass here new Object to enforce update of observers + new Object + ) } private def createDeploymentManagerProvider(typeConfig: ProcessingTypeConfig): DeploymentManagerProvider = { diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala index bb1356a2319..c51be4d9002 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala @@ -17,7 +17,7 @@ import pl.touk.nussknacker.engine.definition.test.ModelDataTestInfoProvider import pl.touk.nussknacker.engine.dict.ProcessDictSubstitutor import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader import pl.touk.nussknacker.engine.util.multiplicity.{Empty, Many, Multiplicity, One} -import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, DeploymentManagerDependencies, ModelDependencies} +import pl.touk.nussknacker.engine.{DeploymentManagerDependencies, ModelDependencies} import pl.touk.nussknacker.processCounts.influxdb.InfluxCountsReporterCreator import pl.touk.nussknacker.processCounts.{CountsReporter, CountsReporterCreator} import pl.touk.nussknacker.ui.api._ @@ -25,6 +25,7 @@ import pl.touk.nussknacker.ui.config.scenariotoolbar.CategoriesScenarioToolbarsC import pl.touk.nussknacker.ui.config.{ AttachmentsConfig, ComponentLinksConfigExtractor, + DesignerRootConfig, FeatureTogglesConfig, UsageStatisticsReportsConfig } @@ -113,23 +114,24 @@ class AkkaHttpBasedRouteProvider( with Directives with LazyLogging { - override def createRoute(config: ConfigWithUnresolvedVersion): Resource[IO, Route] = { + override def createRoute(rootConfig: DesignerRootConfig): Resource[IO, Route] = { implicit val executionContextWithIORuntime: ExecutionContextWithIORuntime = ActorSystemBasedExecutionContextWithIORuntime.createFrom(system) import executionContextWithIORuntime.ioRuntime for { sttpBackend <- createSttpBackend() - resolvedConfig = config.resolved - environment = resolvedConfig.getString("environment") - featureTogglesConfig = FeatureTogglesConfig.create(resolvedConfig) + resolvedRootConfig = rootConfig.rawConfig.resolved + environment = resolvedRootConfig.getString("environment") + featureTogglesConfig = FeatureTogglesConfig.create(resolvedRootConfig) _ = logger.info(s"Designer config loaded: \nfeatureTogglesConfig: $featureTogglesConfig") countsReporter <- createCountsReporter(featureTogglesConfig, environment, sttpBackend) actionServiceSupplier = new DelayedInitActionServiceSupplier - additionalUIConfigProvider = createAdditionalUIConfigProvider(resolvedConfig, sttpBackend) + additionalUIConfigProvider = createAdditionalUIConfigProvider(resolvedRootConfig, sttpBackend) deploymentRepository = new DeploymentRepository(dbRef, Clock.systemDefaultZone()) scenarioActivityRepository = DbScenarioActivityRepository.create(dbRef, designerClock) dbioRunner = DBIOActionRunner(dbRef) processingTypeDataProvider <- prepareProcessingTypeDataReload( + rootConfig, additionalUIConfigProvider, actionServiceSupplier, scenarioActivityRepository, @@ -150,7 +152,7 @@ class AkkaHttpBasedRouteProvider( val scheduler = new DeploymentsStatusesSynchronizationScheduler( system, deploymentsStatusesSynchronizer, - DeploymentsStatusesSynchronizationConfig.parse(resolvedConfig) + DeploymentsStatusesSynchronizationConfig.parse(resolvedRootConfig) ) scheduler.start() scheduler @@ -220,10 +222,10 @@ class AkkaHttpBasedRouteProvider( val processResolver = scenarioTestServiceDeps.mapValues(_._2) val scenarioResolver = scenarioTestServiceDeps.mapValues(_._3) - val notificationsConfig = resolvedConfig.as[NotificationConfig]("notifications") + val notificationsConfig = resolvedRootConfig.as[NotificationConfig]("notifications") val processChangeListener = ProcessChangeListenerLoader.loadListeners( getClass.getClassLoader, - resolvedConfig, + resolvedRootConfig, NussknackerServices(new PullProcessRepository(futureProcessRepository)) ) @@ -257,7 +259,7 @@ class AkkaHttpBasedRouteProvider( // correct classloader and that won't cause further delays during handling requests processingTypeDataProvider.reloadAll().unsafeRunSync() - val authenticationResources = AuthenticationResources(resolvedConfig, getClass.getClassLoader, sttpBackend) + val authenticationResources = AuthenticationResources(resolvedRootConfig, getClass.getClassLoader, sttpBackend) val authManager = new AuthManager(authenticationResources) Initialization.init( @@ -296,7 +298,7 @@ class AkkaHttpBasedRouteProvider( ) val configProcessToolbarService = new ConfigScenarioToolbarService( - CategoriesScenarioToolbarsConfigParser.parse(resolvedConfig) + CategoriesScenarioToolbarsConfigParser.parse(resolvedRootConfig) ) def prepareAlignedComponentsDefinitionProvider( @@ -305,7 +307,7 @@ class AkkaHttpBasedRouteProvider( AlignedComponentsDefinitionProvider(processingTypeData.designerModelData) val componentService = new DefaultComponentService( - ComponentLinksConfigExtractor.extract(resolvedConfig), + ComponentLinksConfigExtractor.extract(resolvedRootConfig), processingTypeDataProvider .mapValues { processingTypeData => val alignedModelDefinitionProvider = prepareAlignedComponentsDefinitionProvider(processingTypeData) @@ -328,7 +330,7 @@ class AkkaHttpBasedRouteProvider( ) val processAuthorizer = new AuthorizeProcess(futureProcessRepository) val appApiHttpService = new AppApiHttpService( - config = resolvedConfig, + config = resolvedRootConfig, authManager = authManager, processingTypeDataReloader = processingTypeDataProvider, modelBuildInfos = modelBuildInfo, @@ -340,7 +342,7 @@ class AkkaHttpBasedRouteProvider( val migrationApiAdapterService = new MigrationApiAdapterService() val migrationService = new MigrationService( - config = resolvedConfig, + config = resolvedRootConfig, processService = processService, processResolver = processResolver, processAuthorizer = processAuthorizer, @@ -416,7 +418,7 @@ class AkkaHttpBasedRouteProvider( scenarioService = processService, scenarioAuthorizer = processAuthorizer, new ScenarioAttachmentService( - AttachmentsConfig.create(resolvedConfig), + AttachmentsConfig.create(resolvedRootConfig), scenarioActivityRepository, dbioRunner, ), @@ -469,7 +471,7 @@ class AkkaHttpBasedRouteProvider( new DeploymentApiHttpService(authManager, activityService, deploymentService) } - initMetrics(metricsRegistry, resolvedConfig, futureProcessRepository) + initMetrics(metricsRegistry, resolvedRootConfig, futureProcessRepository) val apiResourcesWithAuthentication: List[RouteWithUser] = { val routes = List( @@ -505,7 +507,7 @@ class AkkaHttpBasedRouteProvider( prepareAlignedComponentsDefinitionProvider(processingTypeData), new ScenarioPropertiesConfigFinalizer(additionalUIConfigProvider, processingTypeData.name), fragmentRepository, - resolvedConfig.getAs[String]("fragmentPropertiesDocsUrl") + resolvedRootConfig.getAs[String]("fragmentPropertiesDocsUrl") ) ) } @@ -542,7 +544,7 @@ class AkkaHttpBasedRouteProvider( routes ++ optionalRoutes } - val usageStatisticsReportsConfig = resolvedConfig.as[UsageStatisticsReportsConfig]("usageStatisticsReports") + val usageStatisticsReportsConfig = resolvedRootConfig.as[UsageStatisticsReportsConfig]("usageStatisticsReports") val fingerprintService = new FingerprintService(new FingerprintRepositoryImpl(dbRef)) val usageStatisticsReportsSettingsService = UsageStatisticsReportsSettingsService( usageStatisticsReportsConfig, @@ -608,7 +610,7 @@ class AkkaHttpBasedRouteProvider( val akkaHttpServerInterpreter = new NuAkkaHttpServerInterpreterForTapirPurposes() createAppRoute( - resolvedConfig = resolvedConfig, + resolvedConfig = resolvedRootConfig, authManager = authManager, tapirRelatedRoutes = akkaHttpServerInterpreter.toRoute(nuDesignerApi.allEndpoints) :: Nil, apiResourcesWithAuthentication = apiResourcesWithAuthentication, @@ -707,6 +709,7 @@ class AkkaHttpBasedRouteProvider( } private def prepareProcessingTypeDataReload( + rootConfigLoadedAtStart: DesignerRootConfig, additionalUIConfigProvider: AdditionalUIConfigProvider, actionServiceProvider: Supplier[ActionService], scenarioActivityRepository: ScenarioActivityRepository, @@ -716,25 +719,25 @@ class AkkaHttpBasedRouteProvider( )(implicit executionContext: ExecutionContext): Resource[IO, ReloadableProcessingTypeDataProvider] = { Resource .make( - acquire = IO( - new ReloadableProcessingTypeDataProvider( - processingTypeDataLoader.loadProcessingTypeData( - getModelDependencies( - additionalUIConfigProvider, - _, - featureTogglesConfig.componentDefinitionExtractionMode - ), - getDeploymentManagerDependencies( - additionalUIConfigProvider, - actionServiceProvider, - scenarioActivityRepository, - dbioActionRunner, - sttpBackend, - _ - ), - ) + acquire = IO.pure { + val laodProcessingTypeDataIO = processingTypeDataLoader.loadProcessingTypeData( + rootConfigLoadedAtStart, + getModelDependencies( + additionalUIConfigProvider, + _, + featureTogglesConfig.componentDefinitionExtractionMode + ), + getDeploymentManagerDependencies( + additionalUIConfigProvider, + actionServiceProvider, + scenarioActivityRepository, + dbioActionRunner, + sttpBackend, + _ + ), ) - ) + new ReloadableProcessingTypeDataProvider(laodProcessingTypeDataIO) + } )( release = _.close() ) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala index b15913b4f2c..fef8d77fa1e 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/NussknackerHttpServer.scala @@ -10,6 +10,7 @@ import fr.davit.akka.http.metrics.core.{HttpMetricsRegistry, HttpMetricsSettings import fr.davit.akka.http.metrics.dropwizard.{DropwizardRegistry, DropwizardSettings} import io.dropwizard.metrics5.MetricRegistry import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.ui.config.DesignerRootConfig import pl.touk.nussknacker.ui.security.ssl.{HttpsConnectionContextFactory, SslConfigParser} import java.util.concurrent.atomic.AtomicReference @@ -23,10 +24,10 @@ class NussknackerHttpServer(routeProvider: RouteProvider[Route], system: ActorSy private implicit val systemImplicit: ActorSystem = system private implicit val executionContextImplicit: ExecutionContext = system.dispatcher - def start(config: ConfigWithUnresolvedVersion, metricRegistry: MetricRegistry): Resource[IO, Unit] = { + def start(rootConfig: DesignerRootConfig, metricRegistry: MetricRegistry): Resource[IO, Unit] = { for { - route <- routeProvider.createRoute(config) - _ <- createAkkaHttpBinding(config, route, metricRegistry) + route <- routeProvider.createRoute(rootConfig) + _ <- createAkkaHttpBinding(rootConfig.rawConfig, route, metricRegistry) } yield { RouteInterceptor.set(route) } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala index ce69787f60c..98bdc56e6d5 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/RouteProvider.scala @@ -2,9 +2,10 @@ package pl.touk.nussknacker.ui.server import akka.http.scaladsl.server.Route import cats.effect.{IO, Resource} -import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.ui.config.DesignerRootConfig trait RouteProvider[R <: Route] { - def createRoute(config: ConfigWithUnresolvedVersion): Resource[IO, R] + def createRoute(config: DesignerRootConfig): Resource[IO, R] + } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala index 49ec79dc4b8..561fdcdef42 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuItTest.scala @@ -8,10 +8,9 @@ import org.scalatest.{BeforeAndAfterAll, Suite} import pl.touk.nussknacker.test.DefaultUniquePortProvider import pl.touk.nussknacker.test.base.db.WithHsqlDbTesting import pl.touk.nussknacker.test.config.WithDesignerConfig -import pl.touk.nussknacker.ui.LoadableConfigBasedNussknackerConfig -import pl.touk.nussknacker.ui.config.DesignerConfigLoader +import pl.touk.nussknacker.ui.config.DesignerRootConfig import pl.touk.nussknacker.ui.factory.NussknackerAppFactory -import pl.touk.nussknacker.ui.process.processingtype.loader._ +import pl.touk.nussknacker.ui.loadableconfig.LoadableDesignerRootConfig trait NuItTest extends WithHsqlDbTesting with DefaultUniquePortProvider with WithClock with BeforeAndAfterAll { this: Suite with WithDesignerConfig => @@ -24,10 +23,10 @@ trait NuItTest extends WithHsqlDbTesting with DefaultUniquePortProvider with Wit override protected def beforeAll(): Unit = { super.beforeAll() - val nussknackerConfig = new LoadableConfigBasedNussknackerConfig( - IO.delay(DesignerConfigLoader.from(adjustNuTestConfig())) + val loadableDesignerConfig = LoadableDesignerRootConfig( + IO.delay(DesignerRootConfig.from(adjustNuTestConfig())) ) - releaseAppResources = new NussknackerAppFactory(nussknackerConfig) + releaseAppResources = NussknackerAppFactory(loadableDesignerConfig) .createApp(clock = clock) .allocated .unsafeRunSync() diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala index cde22f36654..dcab0e68103 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala @@ -4,8 +4,6 @@ import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCode, StatusCod import akka.http.scaladsl.server.{Directives, Route} import akka.http.scaladsl.testkit.ScalatestRouteTest import akka.http.scaladsl.unmarshalling.FromEntityUnmarshaller -import cats.effect.IO -import cats.effect.unsafe.implicits.global import com.typesafe.config.Config import com.typesafe.scalalogging.LazyLogging import db.util.DBIOActionInstances.DB @@ -38,10 +36,10 @@ import pl.touk.nussknacker.test.mock.{MockDeploymentManager, MockManagerProvider import pl.touk.nussknacker.test.utils.domain.TestFactory._ import pl.touk.nussknacker.test.utils.domain.{ProcessTestData, TestFactory} import pl.touk.nussknacker.test.utils.scalas.AkkaHttpExtensions.toRequestEntity -import pl.touk.nussknacker.ui.LoadableConfigBasedNussknackerConfig import pl.touk.nussknacker.ui.api._ -import pl.touk.nussknacker.ui.config.FeatureTogglesConfig import pl.touk.nussknacker.ui.config.scenariotoolbar.CategoriesScenarioToolbarsConfigParser +import pl.touk.nussknacker.ui.config.{DesignerRootConfig, FeatureTogglesConfig} +import pl.touk.nussknacker.ui.loadableconfig.LoadableProcessingTypeConfigs import pl.touk.nussknacker.ui.process.ProcessService.{CreateScenarioCommand, UpdateScenarioCommand} import pl.touk.nussknacker.ui.process._ import pl.touk.nussknacker.ui.process.deployment._ @@ -147,13 +145,11 @@ trait NuResourcesTest protected val featureTogglesConfig: FeatureTogglesConfig = FeatureTogglesConfig.create(testConfig) protected val typeToConfig: ProcessingTypeDataProvider[ProcessingTypeData, CombinedProcessingTypeData] = { - val processingTypeDataReader = new ProcessingTypesConfigBasedProcessingTypeDataLoader( - new LoadableConfigBasedNussknackerConfig(IO.pure(ConfigWithUnresolvedVersion(testConfig))) - ) + val rootConfig = DesignerRootConfig.from(testConfig) + val processingTypeConfigs = LoadableProcessingTypeConfigs.extractProcessingTypeConfigs(rootConfig) ProcessingTypeDataProvider( - processingTypeDataReader - .loadProcessingTypeData(_ => modelDependencies, _ => deploymentManagerDependencies) - .unsafeRunSync() + ProcessingTypesConfigBasedProcessingTypeDataLoader + .loadProcessingTypeData(processingTypeConfigs, _ => modelDependencies, _ => deploymentManagerDependencies) ) } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableConfigBasedNussknackerConfigSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableDesignerRootConfigSpec.scala similarity index 67% rename from designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableConfigBasedNussknackerConfigSpec.scala rename to designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableDesignerRootConfigSpec.scala index 51195a0c2ba..4117e13d43c 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableConfigBasedNussknackerConfigSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/LoadableDesignerRootConfigSpec.scala @@ -6,9 +6,14 @@ import com.typesafe import com.typesafe.config.{Config, ConfigFactory} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers.{convertToAnyShouldWrapper, include} -import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion +import pl.touk.nussknacker.ui.config.DesignerRootConfig +import pl.touk.nussknacker.ui.loadableconfig.{ + EachTimeLoadingRootConfigLoadableProcessingTypeConfigs, + LoadableDesignerRootConfig, + LoadableProcessingTypeConfigs +} -class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { +class LoadableDesignerRootConfigSpec extends AnyFunSuite { test("should throw when required configuration is missing") { val config = ConfigFactory @@ -31,8 +36,8 @@ class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { .resolve() intercept[typesafe.config.ConfigException] { - loadableConfigBasedNussknackerConfig(config) - .loadProcessingTypeConfigs() + staticConfigBasedLoadableProcessingTypeConfigs(config) + .loadProcessingTypeConfigs(DesignerRootConfig.from(ConfigFactory.empty())) .unsafeRunSync() }.getMessage should include("No configuration setting found for key 'deploymentConfig.type'") } @@ -47,14 +52,14 @@ class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { .resolve() intercept[RuntimeException] { - loadableConfigBasedNussknackerConfig(config) - .loadProcessingTypeConfigs() + staticConfigBasedLoadableProcessingTypeConfigs(config) + .loadProcessingTypeConfigs(DesignerRootConfig.from(ConfigFactory.empty())) .unsafeRunSync() }.getMessage should include("No scenario types configuration provided") } test("should load the second config when reloaded") { - val nussknackerConfig = loadDifferentConfigPerInvocationNussknackerConfig( + val loadableProcessingTypeConfigs = loadDifferentConfigPerInvocationLoadableProcessingTypeConfigs( config1 = ConfigFactory .parseString( """ @@ -103,39 +108,37 @@ class LoadableConfigBasedNussknackerConfigSpec extends AnyFunSuite { .resolve() ) - val processingTypes1 = nussknackerConfig - .loadProcessingTypeConfigs() + val processingTypes1 = loadableProcessingTypeConfigs + .loadProcessingTypeConfigs(DesignerRootConfig.from(ConfigFactory.empty())) .unsafeRunSync() processingTypes1.keys.toSet shouldBe Set("streaming") - val processingTypes2 = nussknackerConfig - .loadProcessingTypeConfigs() + val processingTypes2 = loadableProcessingTypeConfigs + .loadProcessingTypeConfigs(DesignerRootConfig.from(ConfigFactory.empty())) .unsafeRunSync() processingTypes2.keys.toSet shouldBe Set("streaming", "streaming2") } - private def loadableConfigBasedNussknackerConfig(config: Config): LoadableConfigBasedNussknackerConfig = { - loadableConfigBasedNussknackerConfig(IO.pure(ConfigWithUnresolvedVersion(config))) - } - - private def loadableConfigBasedNussknackerConfig( - loadConfig: IO[ConfigWithUnresolvedVersion] - ): LoadableConfigBasedNussknackerConfig = { - new LoadableConfigBasedNussknackerConfig(loadConfig) + private def staticConfigBasedLoadableProcessingTypeConfigs(config: Config): LoadableProcessingTypeConfigs = { + new EachTimeLoadingRootConfigLoadableProcessingTypeConfigs( + LoadableDesignerRootConfig(IO.pure(DesignerRootConfig.from(config))) + ) } - private def loadDifferentConfigPerInvocationNussknackerConfig(config1: Config, config2: Config, configs: Config*) = { + private def loadDifferentConfigPerInvocationLoadableProcessingTypeConfigs(config1: Config, config2: Config, configs: Config*): LoadableProcessingTypeConfigs = { val ref = Ref.unsafe[IO, Int](0) val allConfigs = config1 :: config2 :: configs.toList val loadConfig = ref.getAndUpdate(_ + 1).flatMap { idx => allConfigs.lift(idx) match { - case Some(config) => IO.pure(ConfigWithUnresolvedVersion(config)) + case Some(config) => IO.pure(DesignerRootConfig.from(config)) case None => IO.raiseError(throw new IllegalStateException(s"Cannot load the config more than [$idx]")) } } - loadableConfigBasedNussknackerConfig(loadConfig) + new EachTimeLoadingRootConfigLoadableProcessingTypeConfigs( + LoadableDesignerRootConfig(loadConfig) + ) } } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala index d90167c6e80..a6c60ba3767 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/NussknackerHttpServerSpec.scala @@ -9,9 +9,10 @@ import io.dropwizard.metrics5.MetricRegistry import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion -import pl.touk.nussknacker.test.{DefaultUniquePortProvider, WithTestHttpClientCreator} import pl.touk.nussknacker.test.config.ConfigWithScalaVersion import pl.touk.nussknacker.test.utils.scalas.CatsTestExtensions._ +import pl.touk.nussknacker.test.{DefaultUniquePortProvider, WithTestHttpClientCreator} +import pl.touk.nussknacker.ui.config.DesignerRootConfig import pl.touk.nussknacker.ui.security.ssl.HttpsConnectionContextFactory.prepareSSLContext import pl.touk.nussknacker.ui.security.ssl.KeyStoreConfig import pl.touk.nussknacker.ui.server.{NussknackerHttpServer, RouteProvider} @@ -32,7 +33,7 @@ class NussknackerHttpServerSpec server <- createHttpServer() client <- createHttpClient(Some(prepareSSLContext(keyStoreConfig))) _ <- server.start( - ConfigWithUnresolvedVersion( + DesignerRootConfig.from( ConfigFactory .empty() .withValue("http.interface", fromAnyRef("0.0.0.0")) @@ -64,7 +65,7 @@ class NussknackerHttpServerSpec private object DummyRouteProvider extends RouteProvider[Route] with Directives { - override def createRoute(config: ConfigWithUnresolvedVersion): Resource[IO, Route] = Resource.pure[IO, Route] { + override def createRoute(config: DesignerRootConfig): Resource[IO, Route] = Resource.pure[IO, Route] { path("test") { get { complete { diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/integration/ConfigurationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/integration/ConfigurationTest.scala index 5631ea406fc..9f515bd026f 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/integration/ConfigurationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/integration/ConfigurationTest.scala @@ -6,7 +6,7 @@ import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt import pl.touk.nussknacker.engine.{ModelData, ProcessingTypeConfig} import pl.touk.nussknacker.test.config.ConfigWithScalaVersion import pl.touk.nussknacker.test.utils.domain.TestFactory -import pl.touk.nussknacker.ui.config.DesignerConfigLoader +import pl.touk.nussknacker.ui.config.DesignerRootConfigLoader import cats.effect.unsafe.implicits.global import java.net.URI import java.nio.file.Files @@ -29,7 +29,7 @@ class ConfigurationTest extends AnyFunSuite with Matchers { } test("defaultConfig works") { - DesignerConfigLoader + DesignerRootConfigLoader .load(globalConfig, classLoader) .unsafeRunSync() .resolved @@ -39,7 +39,7 @@ class ConfigurationTest extends AnyFunSuite with Matchers { test("should be possible to config entries defined in default ui config from passed config") { val configUri = writeToTemp("foo: ${storageDir}") // storageDir is defined inside defaultDesignerConfig.conf - val loadedConfig = DesignerConfigLoader + val loadedConfig = DesignerRootConfigLoader .load(ConfigFactoryExt.parseConfigFallbackChain(List(configUri), classLoader), classLoader) .unsafeRunSync() @@ -81,7 +81,7 @@ class ConfigurationTest extends AnyFunSuite with Matchers { val result = try { System.setProperty(randomPropertyName, "I win!") - DesignerConfigLoader + DesignerRootConfigLoader .load(ConfigFactoryExt.parseConfigFallbackChain(List(conf1), classLoader), classLoader) .unsafeRunSync() } finally { diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala index 110f195b4da..dbe5be9c18c 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ProcessingTypeDataProviderSpec.scala @@ -10,6 +10,7 @@ import pl.touk.nussknacker.engine.testing.{DeploymentManagerProviderStub, LocalM import pl.touk.nussknacker.security.Permission import pl.touk.nussknacker.test.utils.domain.TestFactory import pl.touk.nussknacker.ui.UnauthorizedError +import pl.touk.nussknacker.ui.config.DesignerRootConfig import pl.touk.nussknacker.ui.process.processingtype.loader.LocalProcessingTypeDataLoader import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataProvider import pl.touk.nussknacker.ui.security.api.RealLoggedUser @@ -55,7 +56,11 @@ class ProcessingTypeDataProviderSpec extends AnyFunSuite with Matchers { deploymentManagerProvider = new DeploymentManagerProviderStub ) loader - .loadProcessingTypeData(_ => modelDependencies, _ => TestFactory.deploymentManagerDependencies) + .loadProcessingTypeData( + DesignerRootConfig.from(ConfigFactory.empty()), + _ => modelDependencies, + _ => TestFactory.deploymentManagerDependencies + ) .unsafeRunSync() } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala index d0502b2815c..f21597a49c7 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/processingtype/ScenarioParametersServiceTest.scala @@ -1,26 +1,25 @@ package pl.touk.nussknacker.ui.process.processingtype import cats.data.Validated.Invalid -import cats.effect.IO import com.typesafe.config.ConfigFactory import com.typesafe.scalalogging.LazyLogging import org.scalatest.Inside.inside import org.scalatest.OptionValues import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, ModelDependencies} +import pl.touk.nussknacker.engine.ModelDependencies import pl.touk.nussknacker.engine.api.component.{ComponentProvider, DesignerWideComponentId, ProcessingMode} import pl.touk.nussknacker.engine.api.process.ProcessingType +import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode import pl.touk.nussknacker.engine.deployment.EngineSetupName import pl.touk.nussknacker.restmodel.scenariodetails.ScenarioParameters import pl.touk.nussknacker.security.Permission import pl.touk.nussknacker.test.ValidatedValuesDetailedMessage import pl.touk.nussknacker.test.utils.domain.TestFactory +import pl.touk.nussknacker.ui.config.DesignerRootConfig +import pl.touk.nussknacker.ui.loadableconfig.LoadableProcessingTypeConfigs import pl.touk.nussknacker.ui.process.processingtype.loader.ProcessingTypesConfigBasedProcessingTypeDataLoader import pl.touk.nussknacker.ui.security.api.{LoggedUser, RealLoggedUser} -import cats.effect.unsafe.implicits.global -import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode -import pl.touk.nussknacker.ui.LoadableConfigBasedNussknackerConfig import java.nio.file.Path import scala.jdk.CollectionConverters._ @@ -281,14 +280,13 @@ class ScenarioParametersServiceTest val workPath = designerServerModuleDir.resolve("work") logDirectoryStructure(workPath) - val processingTypeDataReader = new ProcessingTypesConfigBasedProcessingTypeDataLoader( - new LoadableConfigBasedNussknackerConfig(IO.pure { - ConfigWithUnresolvedVersion(ConfigFactory.parseFile(devApplicationConfFile).withFallback(fallbackConfig)) - }) - ) - val processingTypeData = processingTypeDataReader + val processingTypeConfigs = LoadableProcessingTypeConfigs.extractProcessingTypeConfigs( + DesignerRootConfig.from(ConfigFactory.parseFile(devApplicationConfFile).withFallback(fallbackConfig)) + ) + val processingTypeData = ProcessingTypesConfigBasedProcessingTypeDataLoader .loadProcessingTypeData( + processingTypeConfigs, processingType => ModelDependencies( Map.empty, @@ -299,7 +297,6 @@ class ScenarioParametersServiceTest ), _ => TestFactory.deploymentManagerDependencies, ) - .unsafeRunSync() val parametersService = processingTypeData.getCombined().parametersService parametersService.scenarioParametersCombinationsWithWritePermission(TestFactory.adminUser()) shouldEqual List( From 95e9b3e3753770233319dd0ef64497a64f4bba3f Mon Sep 17 00:00:00 2001 From: Arek Burdach Date: Thu, 5 Dec 2024 19:23:49 +0100 Subject: [PATCH 02/10] api module extracted --- .run/NussknackerApp-dist-config.run.xml | 4 +- .run/NussknackerApp-postgres.run.xml | 4 +- .run/NussknackerApp.run.xml | 4 +- build.sbt | 24 ++++++-- .../loadableconfig/DesignerRootConfig.scala | 16 ++++++ .../LoadableProcessingTypeConfigs.scala | 26 +++++++++ .../ui/LocalNussknackerWithSingleModel.scala | 4 +- .../touk/nussknacker/ui/NussknackerApp.scala | 4 +- ...tConfigLoadableProcessingTypeConfigs.scala | 18 ++++++ ...LoadableProcessingTypeConfigsFactory.scala | 25 +++++++++ .../{ => root}/DesignerRootConfigLoader.scala | 14 +---- .../root}/LoadableDesignerRootConfig.scala | 4 +- .../ui/factory/NussknackerAppFactory.scala | 9 +-- .../LoadableProcessingTypeConfigs.scala | 56 ------------------- .../LocalProcessingTypeDataLoader.scala | 2 +- .../loader/ProcessingTypeDataLoader.scala | 2 +- ...sConfigBasedProcessingTypeDataLoader.scala | 3 +- .../server/AkkaHttpBasedRouteProvider.scala | 2 +- .../ui/server/NussknackerHttpServer.scala | 2 +- .../nussknacker/ui/server/RouteProvider.scala | 2 +- .../nussknacker/test/base/it/NuItTest.scala | 4 +- .../test/base/it/NuResourcesTest.scala | 4 +- .../ui/api/NussknackerHttpServerSpec.scala | 3 +- ...igLoadableProcessingTypeConfigsSpec.scala} | 18 +++--- .../ui/integration/ConfigurationTest.scala | 2 +- .../ProcessingTypeDataProviderSpec.scala | 2 +- .../ScenarioParametersServiceTest.scala | 3 +- .../engine/ConfigWithUnresolvedVersion.scala | 19 +++++++ .../ConfigWithUnresolvedVersionExt.scala | 32 ----------- 29 files changed, 163 insertions(+), 149 deletions(-) create mode 100644 designer/loadable-config-api/src/main/scala/pl/touk/nussknacker/ui/loadableconfig/DesignerRootConfig.scala create mode 100644 designer/loadable-config-api/src/main/scala/pl/touk/nussknacker/ui/loadableconfig/LoadableProcessingTypeConfigs.scala create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/config/processingtype/EachTimeLoadingRootConfigLoadableProcessingTypeConfigs.scala create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/config/processingtype/LoadableProcessingTypeConfigsFactory.scala rename designer/server/src/main/scala/pl/touk/nussknacker/ui/config/{ => root}/DesignerRootConfigLoader.scala (75%) rename designer/server/src/main/scala/pl/touk/nussknacker/ui/{loadableconfig => config/root}/LoadableDesignerRootConfig.scala (71%) delete mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/loadableconfig/LoadableProcessingTypeConfigs.scala rename designer/server/src/test/scala/pl/touk/nussknacker/ui/{LoadableDesignerRootConfigSpec.scala => config/processingtype/EachTimeLoadingRootConfigLoadableProcessingTypeConfigsSpec.scala} (91%) delete mode 100644 utils/utils-internal/src/main/scala/pl/touk/nussknacker/engine/util/config/ConfigWithUnresolvedVersionExt.scala diff --git a/.run/NussknackerApp-dist-config.run.xml b/.run/NussknackerApp-dist-config.run.xml index 3ef7ee7f1ee..3f27dd337b3 100644 --- a/.run/NussknackerApp-dist-config.run.xml +++ b/.run/NussknackerApp-dist-config.run.xml @@ -26,7 +26,7 @@