Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[NU-1828] Processing types configs loader api #7336

Merged
merged 10 commits into from
Dec 17, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions build.sbt
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
import com.typesafe.sbt.packager.SettingsHelper
import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport.dockerUsername
import pl.project13.scala.sbt.JmhPlugin
import pl.project13.scala.sbt.JmhPlugin._
import sbt.Keys._
import sbt._
import pl.project13.scala.sbt.JmhPlugin.*
import sbt.*
import sbt.Keys.*
import sbtassembly.AssemblyPlugin.autoImport.assembly
import sbtassembly.MergeStrategy
import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations._
import sbtrelease.ReleasePlugin.autoImport.ReleaseTransformations.*

import scala.language.postfixOps
import scala.sys.process._
import scala.sys.process.*
import scala.util.Try
import scala.xml.Elem
import scala.xml.transform.{RewriteRule, RuleTransformer}
Expand Down Expand Up @@ -1913,6 +1913,18 @@ lazy val listenerApi = (project in file("designer/listener-api"))
)
.dependsOn(extensionsApi)

lazy val configLoaderApi = (project in file("designer/config-loader-api"))
.settings(commonSettings)
.settings(
name := "nussknacker-config-loader-api",
libraryDependencies ++= {
Seq(
"org.typelevel" %% "cats-effect" % catsEffectV
)
}
)
.dependsOn(extensionsApi)

lazy val deploymentManagerApi = (project in file("designer/deployment-manager-api"))
.settings(commonSettings)
.settings(
Expand Down Expand Up @@ -2053,6 +2065,7 @@ lazy val designer = (project in file("designer/server"))
deploymentManagerApi,
restmodel,
listenerApi,
configLoaderApi,
defaultHelpers % Test,
testUtils % Test,
flinkTestUtils % Test,
Expand Down Expand Up @@ -2193,6 +2206,7 @@ lazy val modules = List[ProjectReference](
httpUtils,
restmodel,
listenerApi,
configLoaderApi,
deploymentManagerApi,
designer,
sqlComponents,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package pl.touk.nussknacker.ui.configloader

import com.typesafe.config.Config
import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion

// TODO: We should extract a class for all configuration options that should be available to designer instead of returning raw hocon config.
// Thanks to that it will be easier to split processing type config from rest of configs and use this interface programmatically
final case class DesignerRootConfig(rawConfig: ConfigWithUnresolvedVersion)
arkadius marked this conversation as resolved.
Show resolved Hide resolved

object DesignerRootConfig {

def from(config: Config): DesignerRootConfig = {
DesignerRootConfig(ConfigWithUnresolvedVersion(config))
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package pl.touk.nussknacker.ui.configloader

import com.typesafe.scalalogging.LazyLogging
import pl.touk.nussknacker.engine.ProcessingTypeConfig
import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap
import cats.effect.IO

trait ProcessingTypeConfigsLoader {

def loadProcessingTypeConfigs(): IO[Map[String, ProcessingTypeConfig]]

}

object ProcessingTypeConfigsLoader extends LazyLogging {

def extractProcessingTypeConfigs(rootConfig: DesignerRootConfig): Map[String, ProcessingTypeConfig] = {
coutoPL marked this conversation as resolved.
Show resolved Hide resolved
rootConfig.rawConfig
.readMap("scenarioTypes")
.getOrElse {
throw new RuntimeException("No scenario types configuration provided")
}
.mapValuesNow(ProcessingTypeConfig.read)
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package pl.touk.nussknacker.ui.configloader

import sttp.client3.SttpBackend

import scala.concurrent.{ExecutionContext, Future}

trait ProcessingTypeConfigsLoaderFactory {

def create(
designerRootConfigLoadedAtStart: DesignerRootConfig,
sttpBackend: SttpBackend[Future, Any],
coutoPL marked this conversation as resolved.
Show resolved Hide resolved
ec: ExecutionContext
coutoPL marked this conversation as resolved.
Show resolved Hide resolved
): ProcessingTypeConfigsLoader

}
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import cats.effect.{IO, Resource}
import com.typesafe.config.{Config, ConfigFactory}
import org.apache.commons.io.FileUtils
import pl.touk.nussknacker.engine.{DeploymentManagerProvider, ModelData}
import pl.touk.nussknacker.ui.config.DesignerConfigLoader
import pl.touk.nussknacker.ui.factory.NussknackerAppFactory
import pl.touk.nussknacker.ui.config.root.DesignerRootConfigLoader
import pl.touk.nussknacker.ui.process.processingtype.loader.LocalProcessingTypeDataLoader

import java.io.File
Expand Down Expand Up @@ -49,8 +49,12 @@ object LocalNussknackerWithSingleModel {
modelData = Map(typeName -> (category, modelData)),
deploymentManagerProvider = deploymentManagerProvider
)
val nussknackerConfig = new LoadableConfigBasedNussknackerConfig(IO.delay(DesignerConfigLoader.from(appConfig)))
val appFactory = new NussknackerAppFactory(nussknackerConfig, local)
val designerConfigLoader = DesignerRootConfigLoader.fromConfig(appConfig)
val appFactory = new NussknackerAppFactory(
coutoPL marked this conversation as resolved.
Show resolved Hide resolved
designerConfigLoader,
(_, _, _) => () => IO.pure(Map.empty),
_ => local
)
appFactory.createApp()
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
package pl.touk.nussknacker.ui

import cats.effect.{ExitCode, IO, IOApp}
import pl.touk.nussknacker.ui.config.root.DesignerRootConfigLoader
import pl.touk.nussknacker.ui.factory.NussknackerAppFactory

object NussknackerApp extends IOApp {

override def run(args: List[String]): IO[ExitCode] = {
for {
appFactory <- IO(new NussknackerAppFactory(getClass.getClassLoader))
appFactory <- IO(NussknackerAppFactory(DesignerRootConfigLoader(getClass.getClassLoader)))
_ <- appFactory.createApp().use { _ => IO.never }
} yield ExitCode.Success
}
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@ import pl.touk.nussknacker.ui.api.description.DictApiEndpoints.DictError.{
}
import pl.touk.nussknacker.ui.api.description.DictApiEndpoints.Dtos._
import sttp.model.StatusCode.{BadRequest, NotFound, Ok}
import sttp.tapir._
import sttp.tapir.{Schema, _}
coutoPL marked this conversation as resolved.
Show resolved Hide resolved
import sttp.tapir.json.circe._
import sttp.tapir.Schema

import scala.language.implicitConversions

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import pl.touk.nussknacker.restmodel.BaseEndpointDefinitions
import pl.touk.nussknacker.restmodel.BaseEndpointDefinitions.SecuredEndpoint
import pl.touk.nussknacker.security.AuthCredentials
import pl.touk.nussknacker.ui.security.api.GlobalPermission.GlobalPermission
import pl.touk.nussknacker.ui.security.api.{AdminUser, CommonUser, ImpersonatedUser, LoggedUser, RealLoggedUser}
import pl.touk.nussknacker.ui.security.api._
import sttp.model.StatusCode.Ok
import sttp.tapir.EndpointIO.Example
import sttp.tapir.derevo.schema
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package pl.touk.nussknacker.ui.api.description.scenarioActivity
import pl.touk.nussknacker.engine.api.process.ProcessName
import pl.touk.nussknacker.ui.api.description.scenarioActivity.Dtos.ScenarioActivityError
import pl.touk.nussknacker.ui.api.description.scenarioActivity.Dtos.ScenarioActivityError.NoScenario
import sttp.model.StatusCode.{NotFound, NotImplemented}
import sttp.model.StatusCode.NotFound
import sttp.tapir.EndpointIO.Example
import sttp.tapir.{EndpointOutput, emptyOutputAs, oneOf, oneOfVariantFromMatchType, plainBody}
import sttp.tapir.{EndpointOutput, oneOf, oneOfVariantFromMatchType, plainBody}

object InputOutput {

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package pl.touk.nussknacker.ui.config.processingtype

import cats.effect.IO
import pl.touk.nussknacker.engine.ProcessingTypeConfig
import pl.touk.nussknacker.ui.config.root.DesignerRootConfigLoader
import pl.touk.nussknacker.ui.configloader.ProcessingTypeConfigsLoader

class EachTimeLoadingRootConfigProcessingTypeConfigsLoader(designerRootConfigLoader: DesignerRootConfigLoader)
coutoPL marked this conversation as resolved.
Show resolved Hide resolved
extends ProcessingTypeConfigsLoader {

def loadProcessingTypeConfigs(): IO[Map[String, ProcessingTypeConfig]] =
coutoPL marked this conversation as resolved.
Show resolved Hide resolved
designerRootConfigLoader
.loadDesignerRootConfig()
.map(ProcessingTypeConfigsLoader.extractProcessingTypeConfigs)

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package pl.touk.nussknacker.ui.config.processingtype

import com.typesafe.scalalogging.LazyLogging
import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader
import pl.touk.nussknacker.ui.config.root.DesignerRootConfigLoader
import pl.touk.nussknacker.ui.configloader.{
DesignerRootConfig,
ProcessingTypeConfigsLoader,
ProcessingTypeConfigsLoaderFactory
}
import sttp.client3.SttpBackend

import scala.concurrent.{ExecutionContext, Future}

object ProcessingTypeConfigsLoaderFactoryServiceLoader extends LazyLogging {

def loadService(designerRootConfigLoader: DesignerRootConfigLoader): ProcessingTypeConfigsLoaderFactory = {
coutoPL marked this conversation as resolved.
Show resolved Hide resolved
ScalaServiceLoader.load[ProcessingTypeConfigsLoaderFactory](getClass.getClassLoader) match {
case one :: Nil =>
logger.debug(
s"Found custom ${classOf[ProcessingTypeConfigsLoaderFactory].getSimpleName}: ${one.getClass.getName}. Using it for configuration loading"
)
one
case Nil =>
logger.debug(
s"No custom ${classOf[ProcessingTypeConfigsLoaderFactory].getSimpleName} found. Using the default one"
)
(_: DesignerRootConfig, _: SttpBackend[Future, Any], _: ExecutionContext) =>
new EachTimeLoadingRootConfigProcessingTypeConfigsLoader(designerRootConfigLoader)
case _ =>
throw new IllegalStateException(s"More than one ${classOf[ProcessingTypeConfigsLoader].getSimpleName} found")
}
}

}
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
package pl.touk.nussknacker.ui.config
package pl.touk.nussknacker.ui.config.root

coutoPL marked this conversation as resolved.
Show resolved Hide resolved
import cats.effect.IO
import com.typesafe.config.{Config, ConfigFactory}
import pl.touk.nussknacker.engine.ConfigWithUnresolvedVersion
import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt
import pl.touk.nussknacker.ui.configloader.DesignerRootConfig

trait DesignerRootConfigLoader {

def loadDesignerRootConfig(): IO[DesignerRootConfig]

}

/**
* This class handles two parts of ui config loading:
Expand All @@ -14,27 +21,31 @@ import pl.touk.nussknacker.engine.util.config.ConfigFactoryExt
* Result of config loading still keep version with unresolved env variables for purpose of config loading on model side - see
* InputConfigDuringExecution and ModelConfigLoader
*/
object DesignerConfigLoader {
private[root] class DesignerRootConfigLoaderImpl(classLoader: ClassLoader) extends DesignerRootConfigLoader {

coutoPL marked this conversation as resolved.
Show resolved Hide resolved
private val defaultConfigResource = "defaultDesignerConfig.conf"

def load(classLoader: ClassLoader): IO[ConfigWithUnresolvedVersion] = {
override def loadDesignerRootConfig(): IO[DesignerRootConfig] = {
for {
baseConfig <- IO.blocking(ConfigFactoryExt.parseUnresolved(classLoader = classLoader))
loadedConfig <- load(baseConfig, classLoader)
} yield loadedConfig
loadedConfig <- load(baseConfig)
} yield DesignerRootConfig(loadedConfig)
}

def load(baseUnresolvedConfig: Config, classLoader: ClassLoader): IO[ConfigWithUnresolvedVersion] = {
def load(baseUnresolvedConfig: Config): IO[ConfigWithUnresolvedVersion] = {
IO.blocking {
coutoPL marked this conversation as resolved.
Show resolved Hide resolved
val parsedDefaultUiConfig = ConfigFactory.parseResources(classLoader, defaultConfigResource)
val unresolvedConfigWithFallbackToDefaults = baseUnresolvedConfig.withFallback(parsedDefaultUiConfig)
ConfigWithUnresolvedVersion(classLoader, unresolvedConfigWithFallbackToDefaults)
}
}

def from(config: Config): ConfigWithUnresolvedVersion = {
ConfigWithUnresolvedVersion(this.getClass.getClassLoader, config)
}
}

object DesignerRootConfigLoader {

def apply(classLoader: ClassLoader): DesignerRootConfigLoader = new DesignerRootConfigLoaderImpl(classLoader)

def fromConfig(config: => Config): DesignerRootConfigLoader = () => IO.delay(DesignerRootConfig.from(config))

}
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@ import pl.touk.nussknacker.engine.api.deployment.{
}
import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId}
import pl.touk.nussknacker.engine.newdeployment.DeploymentId
import slick.ast.BaseTypedType
import slick.jdbc.{JdbcProfile, JdbcType}
import slick.jdbc.JdbcProfile

import java.util.UUID

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package pl.touk.nussknacker.ui.db.entity

import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName}
import pl.touk.nussknacker.engine.api.process.ProcessingType
import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, ProcessingType}
import slick.lifted.{ProvenShape, TableQuery => LTableQuery}
import slick.sql.SqlProfile.ColumnOption.NotNull

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import io.questdb.cairo.security.AllowAllSecurityContext
import io.questdb.cairo.sql.RecordCursorFactory
import io.questdb.cairo.wal.WalWriter
import io.questdb.griffin.{SqlExecutionContext, SqlExecutionContextImpl}
import pl.touk.nussknacker.ui.db.timeseries.{FEStatisticsRepository, NoOpFEStatisticsRepository}
import pl.touk.nussknacker.ui.db.timeseries.questdb.QuestDbExtensions.{
BuildCairoEngineExtension,
CairoEngineExtension,
Expand All @@ -22,6 +21,7 @@ import pl.touk.nussknacker.ui.db.timeseries.questdb.QuestDbFEStatisticsRepositor
selectQuery,
tableName
}
import pl.touk.nussknacker.ui.db.timeseries.{FEStatisticsRepository, NoOpFEStatisticsRepository}
import pl.touk.nussknacker.ui.statistics.RawFEStatistics

import java.time.Clock
Expand Down
Loading
Loading