[NU-1828] Processing types configs loader api

build.sbt

@@ -1,15 +1,15 @@
 import com.typesafe.sbt.packager.SettingsHelper
 import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport.dockerUsername
 import pl.project13.scala.sbt.JmhPlugin
-import pl.project13.scala.sbt.JmhPlugin._
-import sbt.Keys._
-import sbt._
+import pl.project13.scala.sbt.JmhPlugin.*
+import sbt.*
+import sbt.Keys.*
 import sbtassembly.AssemblyPlugin.autoImport.assembly
 import sbtassembly.MergeStrategy
-import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations._
+import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations.*
 
 import scala.language.postfixOps
-import scala.sys.process._
+import scala.sys.process.*
 import scala.util.Try
 import scala.xml.Elem
 import scala.xml.transform.{RewriteRule, RuleTransformer}
@@ -1913,6 +1913,18 @@ lazy val listenerApi = (project in file("designer/listener-api"))
   )
   .dependsOn(extensionsApi)
 
+lazy val configLoaderApi = (project in file("designer/config-loader-api"))
+  .settings(commonSettings)
+  .settings(
+    name := "nussknacker-config-loader-api",
+    libraryDependencies ++= {
+      Seq(
+        "org.typelevel" %% "cats-effect" % catsEffectV
+      )
+    }
+  )
+  .dependsOn(extensionsApi)
+
 lazy val deploymentManagerApi = (project in file("designer/deployment-manager-api"))
   .settings(commonSettings)
   .settings(
@@ -1985,21 +1997,21 @@ lazy val designer = (project in file("designer/server"))
     assembly / assemblyMergeStrategy := designerMergeStrategy,
     libraryDependencies ++= {
       Seq(
-        "com.typesafe.akka"             %% "akka-http"            % akkaHttpV,
-        "com.typesafe.akka"             %% "akka-slf4j"           % akkaV,
-        "com.typesafe.akka"             %% "akka-stream"          % akkaV,
-        "com.typesafe.akka"             %% "akka-http-testkit"    % akkaHttpV % Test,
-        "com.typesafe.akka"             %% "akka-testkit"         % akkaV     % Test,
-        "de.heikoseeberger"             %% "akka-http-circe"      % akkaHttpCirceV,
-        "com.softwaremill.sttp.client3" %% "akka-http-backend"    % sttpV,
-        "ch.qos.logback"                 % "logback-core"         % logbackV,
-        "ch.qos.logback"                 % "logback-classic"      % logbackV,
-        "ch.qos.logback.contrib"         % "logback-json-classic" % logbackJsonV,
-        "ch.qos.logback.contrib"         % "logback-jackson"      % logbackJsonV,
-        "com.fasterxml.jackson.core"     % "jackson-databind"     % jacksonV,
-        "org.slf4j"                      % "log4j-over-slf4j"     % slf4jV,
-        "com.carrotsearch"               % "java-sizeof"          % "0.0.5",
-        "org.typelevel"                 %% "case-insensitive"     % "1.4.0",
+        "com.typesafe.akka"             %% "akka-http"                      % akkaHttpV,
+        "com.typesafe.akka"             %% "akka-slf4j"                     % akkaV,
+        "com.typesafe.akka"             %% "akka-stream"                    % akkaV,
+        "com.typesafe.akka"             %% "akka-http-testkit"              % akkaHttpV % Test,
+        "com.typesafe.akka"             %% "akka-testkit"                   % akkaV     % Test,
+        "de.heikoseeberger"             %% "akka-http-circe"                % akkaHttpCirceV,
+        "com.softwaremill.sttp.client3" %% "async-http-client-backend-cats" % sttpV,
+        "ch.qos.logback"                 % "logback-core"                   % logbackV,
+        "ch.qos.logback"                 % "logback-classic"                % logbackV,
+        "ch.qos.logback.contrib"         % "logback-json-classic"           % logbackJsonV,
+        "ch.qos.logback.contrib"         % "logback-jackson"                % logbackJsonV,
+        "com.fasterxml.jackson.core"     % "jackson-databind"               % jacksonV,
+        "org.slf4j"                      % "log4j-over-slf4j"               % slf4jV,
+        "com.carrotsearch"               % "java-sizeof"                    % "0.0.5",
+        "org.typelevel"                 %% "case-insensitive"               % "1.4.0",
 
         // It's needed by flinkDeploymentManager which has disabled includingScala
         "org.scala-lang"                 % "scala-compiler"                  % scalaVersion.value,
@@ -2053,6 +2065,7 @@ lazy val designer = (project in file("designer/server"))
     deploymentManagerApi,
     restmodel,
     listenerApi,
+    configLoaderApi,
     defaultHelpers                    % Test,
     testUtils                         % Test,
     flinkTestUtils                    % Test,
@@ -2193,6 +2206,7 @@ lazy val modules = List[ProjectReference](
   httpUtils,
   restmodel,
   listenerApi,
+  configLoaderApi,
   deploymentManagerApi,
   designer,
   sqlComponents,

designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoader.scala

@@ -0,0 +1,10 @@
+package pl.touk.nussknacker.ui.configloader
+
+import cats.effect.IO
+import pl.touk.nussknacker.engine.ProcessingTypeConfig
+
+trait ProcessingTypeConfigsLoader {
+
+  def loadProcessingTypeConfigs(): IO[Map[String, ProcessingTypeConfig]]
+
+}

designer/config-loader-api/src/main/scala/pl/touk/nussknacker/ui/configloader/ProcessingTypeConfigsLoaderFactory.scala

@@ -0,0 +1,16 @@
+package pl.touk.nussknacker.ui.configloader
+
+import cats.effect.IO
+import com.typesafe.config.Config
+import sttp.client3.SttpBackend
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait ProcessingTypeConfigsLoaderFactory {
+
+  def create(
+      configLoaderConfig: Config,
+      sttpBackend: SttpBackend[IO, Any],
+  )(implicit ec: ExecutionContext): ProcessingTypeConfigsLoader
+
+}

designer/server/src/main/scala/pl/touk/nussknacker/ui/LocalNussknackerWithSingleModel.scala

@@ -4,13 +4,16 @@ import cats.effect.{IO, Resource}
 import com.typesafe.config.{Config, ConfigFactory}
 import org.apache.commons.io.FileUtils
 import pl.touk.nussknacker.engine.{DeploymentManagerProvider, ModelData}
-import pl.touk.nussknacker.ui.config.DesignerConfigLoader
 import pl.touk.nussknacker.ui.factory.NussknackerAppFactory
+import pl.touk.nussknacker.ui.config.DesignerConfigLoader
+import pl.touk.nussknacker.ui.configloader.{ProcessingTypeConfigsLoader, ProcessingTypeConfigsLoaderFactory}
 import pl.touk.nussknacker.ui.process.processingtype.loader.LocalProcessingTypeDataLoader
+import sttp.client3.SttpBackend
 
 import java.io.File
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
+import scala.concurrent.{ExecutionContext, Future}
 import scala.jdk.CollectionConverters._
 
 //This is helper, which allows for starting UI with given model without having to build jar etc.
@@ -49,8 +52,19 @@ object LocalNussknackerWithSingleModel {
       modelData = Map(typeName -> (category, modelData)),
       deploymentManagerProvider = deploymentManagerProvider
     )
-    val nussknackerConfig = new LoadableConfigBasedNussknackerConfig(IO.delay(DesignerConfigLoader.from(appConfig)))
-    val appFactory        = new NussknackerAppFactory(nussknackerConfig, local)
+    val designerConfigLoader = DesignerConfigLoader.fromConfig(appConfig)
+    val processingTypeConfigsLoaderFactory = new ProcessingTypeConfigsLoaderFactory {
+      override def create(configLoaderConfig: Config, sttpBackend: SttpBackend[IO, Any])(
+          implicit ec: ExecutionContext
+      ): ProcessingTypeConfigsLoader = { () =>
+        IO.pure(Map.empty)
+      }
+    }
+    val appFactory = new NussknackerAppFactory(
+      designerConfigLoader,
+      Some(processingTypeConfigsLoaderFactory),
+      _ => local
+    )
     appFactory.createApp()
   }
 

designer/server/src/main/scala/pl/touk/nussknacker/ui/NussknackerApp.scala

@@ -1,13 +1,14 @@
 package pl.touk.nussknacker.ui
 
 import cats.effect.{ExitCode, IO, IOApp}
+import pl.touk.nussknacker.ui.config.DesignerConfigLoader
 import pl.touk.nussknacker.ui.factory.NussknackerAppFactory
 
 object NussknackerApp extends IOApp {
 
   override def run(args: List[String]): IO[ExitCode] = {
     for {
-      appFactory <- IO(new NussknackerAppFactory(getClass.getClassLoader))
+      appFactory <- IO(NussknackerAppFactory(DesignerConfigLoader(getClass.getClassLoader)))
       _          <- appFactory.createApp().use { _ => IO.never }
     } yield ExitCode.Success
   }

designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DictApiEndpoints.scala

@@ -15,9 +15,8 @@ import pl.touk.nussknacker.ui.api.description.DictApiEndpoints.DictError.{
 }
 import pl.touk.nussknacker.ui.api.description.DictApiEndpoints.Dtos._
 import sttp.model.StatusCode.{BadRequest, NotFound, Ok}
-import sttp.tapir._
 import sttp.tapir.json.circe._
-import sttp.tapir.Schema
+import sttp.tapir._
 
 import scala.language.implicitConversions
 

designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/UserApiEndpoints.scala

@@ -6,7 +6,7 @@ import pl.touk.nussknacker.restmodel.BaseEndpointDefinitions
 import pl.touk.nussknacker.restmodel.BaseEndpointDefinitions.SecuredEndpoint
 import pl.touk.nussknacker.security.AuthCredentials
 import pl.touk.nussknacker.ui.security.api.GlobalPermission.GlobalPermission
-import pl.touk.nussknacker.ui.security.api.{AdminUser, CommonUser, ImpersonatedUser, LoggedUser, RealLoggedUser}
+import pl.touk.nussknacker.ui.security.api._
 import sttp.model.StatusCode.Ok
 import sttp.tapir.EndpointIO.Example
 import sttp.tapir.derevo.schema

designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/scenarioActivity/InputOutput.scala

@@ -3,9 +3,9 @@ package pl.touk.nussknacker.ui.api.description.scenarioActivity
 import pl.touk.nussknacker.engine.api.process.ProcessName
 import pl.touk.nussknacker.ui.api.description.scenarioActivity.Dtos.ScenarioActivityError
 import pl.touk.nussknacker.ui.api.description.scenarioActivity.Dtos.ScenarioActivityError.NoScenario
-import sttp.model.StatusCode.{NotFound, NotImplemented}
+import sttp.model.StatusCode.NotFound
 import sttp.tapir.EndpointIO.Example
-import sttp.tapir.{EndpointOutput, emptyOutputAs, oneOf, oneOfVariantFromMatchType, plainBody}
+import sttp.tapir.{EndpointOutput, oneOf, oneOfVariantFromMatchType, plainBody}
 
 object InputOutput {
 

designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfig.scala

@@ -0,0 +1,28 @@
+package pl.touk.nussknacker.ui.config
+
+import com.typesafe.config.Config
+import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap
+import pl.touk.nussknacker.engine.{ConfigWithUnresolvedVersion, ProcessingTypeConfig}
+
+// TODO: We should extract a class for all configuration options that should be available to designer instead of returning raw hocon config.
+//       Thanks to that it will be easier to split processing type config from rest of configs and use this interface programmatically
+final case class DesignerConfig private (rawConfig: ConfigWithUnresolvedVersion) {
+
+  def processingTypeConfigs: Map[String, ProcessingTypeConfig] = {
+    rawConfig
+      .readMap("scenarioTypes")
+      .getOrElse {
+        throw new RuntimeException("No scenario types configuration provided")
+      }
+      .mapValuesNow(ProcessingTypeConfig.read)
+  }
+
+}
+
+object DesignerConfig {
+
+  def from(config: Config): DesignerConfig = {
+    DesignerConfig(ConfigWithUnresolvedVersion(config))
+  }
+
+}

designer/server/src/main/scala/pl/touk/nussknacker/ui/config/DesignerConfigLoader.scala

@@ -3,8 +3,15 @@ package pl.touk.nussknacker.ui.config
 import cats.effect.IO
 import com.typesafe.config.{Config, ConfigFactory}
 import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion
+import pl.touk.nussknacker.engine.util.UriUtils
 import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt
 
+trait DesignerConfigLoader {
+
+  def loadDesignerConfig(): IO[DesignerConfig]
+
+}
+
 /**
   * This class handles two parts of ui config loading:
   * 1. Parsing of "base" config passed via nussknacker.config.locations system property (without resolution)
@@ -14,27 +21,39 @@ import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt
   * Result of config loading still keep version with unresolved env variables for purpose of config loading on model side - see
   * InputConfigDuringExecution and ModelConfigLoader
   */
-object DesignerConfigLoader {
+class AlwaysLoadingFileBasedDesignerConfigLoader(classLoader: ClassLoader) extends DesignerConfigLoader {
+
+  private val configLocationsProperty: String = "nussknacker.config.locations"
 
   private val defaultConfigResource = "defaultDesignerConfig.conf"
 
-  def load(classLoader: ClassLoader): IO[ConfigWithUnresolvedVersion] = {
+  override def loadDesignerConfig(): IO[DesignerConfig] = {
+    val locationsPropertyValueOpt = Option(System.getProperty(configLocationsProperty))
+    val locations                 = locationsPropertyValueOpt.map(UriUtils.extractListOfLocations).getOrElse(List.empty)
     for {
-      baseConfig   <- IO.blocking(ConfigFactoryExt.parseUnresolved(classLoader = classLoader))
-      loadedConfig <- load(baseConfig, classLoader)
-    } yield loadedConfig
+      baseUnresolvedConfig  <- IO.blocking(new ConfigFactoryExt(classLoader).parseUnresolved(locations))
+      parsedDefaultUiConfig <- IO.blocking(ConfigFactory.parseResources(classLoader, defaultConfigResource))
+      unresolvedConfigWithFallbackToDefaults = baseUnresolvedConfig.withFallback(parsedDefaultUiConfig)
+    } yield DesignerConfig(ConfigWithUnresolvedVersion(classLoader, unresolvedConfigWithFallbackToDefaults))
   }
 
-  def load(baseUnresolvedConfig: Config, classLoader: ClassLoader): IO[ConfigWithUnresolvedVersion] = {
-    IO.blocking {
-      val parsedDefaultUiConfig                  = ConfigFactory.parseResources(classLoader, defaultConfigResource)
-      val unresolvedConfigWithFallbackToDefaults = baseUnresolvedConfig.withFallback(parsedDefaultUiConfig)
-      ConfigWithUnresolvedVersion(classLoader, unresolvedConfigWithFallbackToDefaults)
-    }
-  }
+}
 
-  def from(config: Config): ConfigWithUnresolvedVersion = {
-    ConfigWithUnresolvedVersion(this.getClass.getClassLoader, config)
-  }
+/**
+ * This implementation is more straightforward - it only parse config without any property checking and fallbacks
+ */
+class SimpleConfigLoadingDesignerConfigLoader(loadConfig: => Config) extends DesignerConfigLoader {
+
+  override def loadDesignerConfig(): IO[DesignerConfig] = IO.delay(DesignerConfig.from(loadConfig))
+
+}
+
+object DesignerConfigLoader {
+
+  def apply(classLoader: ClassLoader): AlwaysLoadingFileBasedDesignerConfigLoader =
+    new AlwaysLoadingFileBasedDesignerConfigLoader(classLoader)
+
+  def fromConfig(loadConfig: => Config): SimpleConfigLoadingDesignerConfigLoader =
+    new SimpleConfigLoadingDesignerConfigLoader(loadConfig)
 
 }

designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/BaseEntityFactory.scala

@@ -9,8 +9,7 @@ import pl.touk.nussknacker.engine.api.deployment.{
 }
 import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId}
 import pl.touk.nussknacker.engine.newdeployment.DeploymentId
-import slick.ast.BaseTypedType
-import slick.jdbc.{JdbcProfile, JdbcType}
+import slick.jdbc.JdbcProfile
 
 import java.util.UUID
 

designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/ProcessEntityFactory.scala

@@ -1,7 +1,6 @@
 package pl.touk.nussknacker.ui.db.entity
 
-import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName}
-import pl.touk.nussknacker.engine.api.process.ProcessingType
+import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, ProcessingType}
 import slick.lifted.{ProvenShape, TableQuery => LTableQuery}
 import slick.sql.SqlProfile.ColumnOption.NotNull
 

designer/server/src/main/scala/pl/touk/nussknacker/ui/db/timeseries/questdb/QuestDbFEStatisticsRepository.scala

@@ -10,7 +10,6 @@ import io.questdb.cairo.security.AllowAllSecurityContext
 import io.questdb.cairo.sql.RecordCursorFactory
 import io.questdb.cairo.wal.WalWriter
 import io.questdb.griffin.{SqlExecutionContext, SqlExecutionContextImpl}
-import pl.touk.nussknacker.ui.db.timeseries.{FEStatisticsRepository, NoOpFEStatisticsRepository}
 import pl.touk.nussknacker.ui.db.timeseries.questdb.QuestDbExtensions.{
   BuildCairoEngineExtension,
   CairoEngineExtension,
@@ -22,6 +21,7 @@ import pl.touk.nussknacker.ui.db.timeseries.questdb.QuestDbFEStatisticsRepositor
   selectQuery,
   tableName
 }
+import pl.touk.nussknacker.ui.db.timeseries.{FEStatisticsRepository, NoOpFEStatisticsRepository}
 import pl.touk.nussknacker.ui.statistics.RawFEStatistics
 
 import java.time.Clock