From 03286b21c14fcd8643fe9aa2d1c54319b844e527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Wed, 18 Dec 2024 17:14:41 +0100 Subject: [PATCH 01/11] stashed and rebased --- build.sbt | 38 ++- .../img/toolbarButtons/run-off-schedule.svg | 7 - .../DeploymentManagerDependencies.scala | 4 +- .../periodic/PeriodicProcessesManager.scala | 150 ++++++++++ .../PeriodicProcessesManagerProvider.scala | 19 ++ .../model/DeploymentWithRuntimeParams.scala | 11 + .../periodic/model/PeriodicProcess.scala | 11 +- .../model/PeriodicProcessDeployment.scala | 42 +++ .../periodic/model/SchedulesState.scala | 44 +-- .../web/static/assets/states/scheduled.svg | 0 .../static/assets/states/wait-reschedule.svg | 0 ...dicDeploymentManagerTablesDefinition.scala | 190 +++++++++++++ ...060__PeriodicDeploymentManagerTables.scala | 8 + ...060__PeriodicDeploymentManagerTables.scala | 8 + .../PeriodicProcessDeploymentsTable.scala | 15 +- .../ui/db/entity/PeriodicProcessesTable.scala | 140 ++++++++++ ...ositoryBasedPeriodicProcessesManager.scala | 106 +++++++ ...asedPeriodicProcessesManagerProvider.scala | 21 ++ .../PeriodicProcessesRepository.scala | 249 +++++++++-------- .../repository/ScenarioActionRepository.scala | 1 + .../server/AkkaHttpBasedRouteProvider.scala | 29 +- ...tionsAndCommentsToScenarioActivities.scala | 1 - .../nussknacker/test/base/db/DbTesting.scala | 2 + .../test/mock/MockDeploymentManager.scala | 6 + .../test/utils/domain/TestFactory.scala | 5 + ...eriodicProcessServiceIntegrationTest.scala | 234 +++++++++------- .../common}/periodic/DeploymentActor.scala | 24 +- .../periodic/PeriodicBatchConfig.scala | 2 +- .../periodic/PeriodicDeploymentHandler.scala | 26 ++ .../periodic/PeriodicDeploymentManager.scala | 55 ++-- .../PeriodicDeploymentManagerProvider.scala} | 31 ++- .../periodic/PeriodicProcessException.scala | 2 +- .../periodic/PeriodicProcessService.scala | 192 +++++++------ ...eriodicProcessStateDefinitionManager.scala | 4 +- .../periodic/PeriodicStateStatus.scala | 2 +- .../periodic/RescheduleFinishedActor.scala | 6 +- .../periodic/SchedulePropertyExtractor.scala | 4 +- .../SchedulePropertyExtractorFactory.scala | 2 +- .../periodic/SingleScheduleProperty.scala | 36 ++- .../engine/common}/periodic/Utils.scala | 2 +- .../cron/CronParameterValidator.scala | 4 +- .../AdditionalDeploymentDataProvider.scala | 11 +- .../service/PeriodicProcessListener.scala | 22 +- .../service/ProcessConfigEnricher.scala | 13 +- ...ssknacker.engine.DeploymentManagerProvider | 2 +- ...ne.api.definition.CustomParameterValidator | 2 +- .../assets/custom-actions/batch-instant.svg | 0 .../management/periodic/JarManager.scala | 23 -- .../periodic/db/DbInitializer.scala | 56 ---- .../periodic/db/PeriodicProcessesTable.scala | 261 ------------------ ...a => FlinkPeriodicDeploymentHandler.scala} | 103 ++++--- ...inkPeriodicDeploymentManagerProvider.scala | 37 +++ .../model/DeploymentWithJarData.scala | 25 -- .../model/PeriodicProcessDeployment.scala | 52 ---- .../management/periodic/JarManagerStub.scala | 38 --- .../CronSchedulePropertyExtractorTest.scala | 5 +- .../CronSchedulePropertyTest.scala | 6 +- .../{ => flink}/DeploymentActorTest.scala | 26 +- .../{ => flink}/DeploymentManagerStub.scala | 4 +- .../{ => flink}/FlinkClientStub.scala | 2 +- .../flink/PeriodicDeploymentHandlerStub.scala | 38 +++ .../PeriodicDeploymentHandlerTest.scala} | 51 ++-- .../PeriodicDeploymentManagerTest.scala | 134 ++++----- .../PeriodicProcessDeploymentGen.scala | 14 +- .../{ => flink}/PeriodicProcessGen.scala | 15 +- .../PeriodicProcessServiceTest.scala | 178 ++++++------ ...dicProcessStateDefinitionManagerTest.scala | 13 +- .../RescheduleFinishedActorTest.scala | 3 +- .../periodic/{ => flink}/UtilsSpec.scala | 5 +- .../db/InMemPeriodicProcessesManager.scala} | 232 +++++++++++----- ...amingDeploymentManagerProviderHelper.scala | 15 +- .../management/FlinkRestManagerSpec.scala | 10 +- ...esponseEmbeddedDeploymentManagerTest.scala | 15 +- .../BaseK8sDeploymentManagerTest.scala | 4 +- .../K8sDeploymentManagerOnMocksTest.scala | 6 +- 75 files changed, 1833 insertions(+), 1321 deletions(-) delete mode 100644 designer/client/src/assets/img/toolbarButtons/run-off-schedule.svg create mode 100644 designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala create mode 100644 designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManagerProvider.scala create mode 100644 designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala rename {engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment}/periodic/model/PeriodicProcess.scala (54%) create mode 100644 designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala rename {engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment}/periodic/model/SchedulesState.scala (67%) rename {engine/flink/management/periodic => designer/server}/src/main/resources/web/static/assets/states/scheduled.svg (100%) rename {engine/flink/management/periodic => designer/server}/src/main/resources/web/static/assets/states/wait-reschedule.svg (100%) create mode 100644 designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala create mode 100644 designer/server/src/main/scala/db/migration/hsql/V1_060__PeriodicDeploymentManagerTables.scala create mode 100644 designer/server/src/main/scala/db/migration/postgres/V1_060__PeriodicDeploymentManagerTables.scala rename {engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db => designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity}/PeriodicProcessDeploymentsTable.scala (82%) create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala rename {engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db => designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository}/PeriodicProcessesRepository.scala (70%) rename {engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management => designer/server/src/test/scala/pl/touk/nussknacker/ui/process}/periodic/PeriodicProcessServiceIntegrationTest.scala (78%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/DeploymentActor.scala (71%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/PeriodicBatchConfig.scala (96%) create mode 100644 engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/PeriodicDeploymentManager.scala (88%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala} (74%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/PeriodicProcessException.scala (74%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/PeriodicProcessService.scala (84%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/PeriodicProcessStateDefinitionManager.scala (92%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/PeriodicStateStatus.scala (98%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/RescheduleFinishedActor.scala (84%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/SchedulePropertyExtractor.scala (95%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/SchedulePropertyExtractorFactory.scala (70%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/SingleScheduleProperty.scala (64%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/Utils.scala (95%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/cron/CronParameterValidator.scala (92%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/service/AdditionalDeploymentDataProvider.scala (55%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/service/PeriodicProcessListener.scala (64%) rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common}/periodic/service/ProcessConfigEnricher.scala (83%) create mode 100644 engine/flink/management/periodic/src/main/resources/web/static/assets/custom-actions/batch-instant.svg delete mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala delete mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/DbInitializer.scala delete mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesTable.scala rename engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/{FlinkJarManager.scala => FlinkPeriodicDeploymentHandler.scala} (50%) create mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala delete mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/DeploymentWithJarData.scala delete mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala delete mode 100644 engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/CronSchedulePropertyExtractorTest.scala (88%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/CronSchedulePropertyTest.scala (88%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/DeploymentActorTest.scala (66%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/DeploymentManagerStub.scala (92%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/FlinkClientStub.scala (96%) create mode 100644 engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{JarManagerTest.scala => flink/PeriodicDeploymentHandlerTest.scala} (56%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/PeriodicDeploymentManagerTest.scala (75%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/PeriodicProcessDeploymentGen.scala (53%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/PeriodicProcessGen.scala (58%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/PeriodicProcessServiceTest.scala (72%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/PeriodicProcessStateDefinitionManagerTest.scala (84%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/RescheduleFinishedActorTest.scala (91%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{ => flink}/UtilsSpec.scala (94%) rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{db/InMemPeriodicProcessesRepository.scala => flink/db/InMemPeriodicProcessesManager.scala} (57%) diff --git a/build.sbt b/build.sbt index 01c8aca7bad..22e45792737 100644 --- a/build.sbt +++ b/build.sbt @@ -644,21 +644,16 @@ lazy val flinkPeriodicDeploymentManager = (project in flink("management/periodic name := "nussknacker-flink-periodic-manager", libraryDependencies ++= { Seq( - "org.typelevel" %% "cats-core" % catsV % Provided, - "com.typesafe.slick" %% "slick" % slickV % Provided, - "com.typesafe.slick" %% "slick-hikaricp" % slickV % "provided, test", - "com.github.tminglei" %% "slick-pg" % slickPgV, - "org.hsqldb" % "hsqldb" % hsqldbV % Test, - "org.flywaydb" % "flyway-core" % flywayV % Provided, - "com.cronutils" % "cron-utils" % cronParserV, - "com.typesafe.akka" %% "akka-actor" % akkaV, - "com.typesafe.akka" %% "akka-testkit" % akkaV % Test, - "com.dimafeng" %% "testcontainers-scala-scalatest" % testContainersScalaV % Test, - "com.dimafeng" %% "testcontainers-scala-postgresql" % testContainersScalaV % Test, + "com.typesafe.slick" %% "slick-hikaricp" % slickV % "provided, test", + "org.hsqldb" % "hsqldb" % hsqldbV % Test, + "com.typesafe.akka" %% "akka-testkit" % akkaV % Test, + "com.dimafeng" %% "testcontainers-scala-scalatest" % testContainersScalaV % Test, + "com.dimafeng" %% "testcontainers-scala-postgresql" % testContainersScalaV % Test, ) } ) .dependsOn( + commonPeriodicDeploymentManager, flinkDeploymentManager, deploymentManagerApi % Provided, scenarioCompiler % Provided, @@ -1788,6 +1783,25 @@ lazy val commonComponentsTests = (project in engine("common/components-tests")) flinkComponentsTestkit % Test ) +lazy val commonPeriodicDeploymentManager = (project in engine("common/periodic-deployment-manager")) + .settings(commonSettings) + .settings(publishAssemblySettings: _*) + .settings( + name := "nussknacker-common-periodic-deployment-manager", + libraryDependencies ++= { + Seq( + "com.typesafe.akka" %% "akka-actor" % akkaV, + "org.typelevel" %% "cats-core" % catsV % Provided, + "com.cronutils" % "cron-utils" % cronParserV, + "com.softwaremill.retry" %% "retry" % retryV, + "com.github.tminglei" %% "slick-pg" % slickPgV, + ) + } + ) + .dependsOn( + deploymentManagerApi % Provided, + ) + lazy val flinkBaseComponents = (project in flink("components/base")) .settings(commonSettings) .settings(assemblyNoScala("flinkBase.jar"): _*) @@ -2080,7 +2094,7 @@ lazy val designer = (project in file("designer/server")) liteEmbeddedDeploymentManager % Provided, liteK8sDeploymentManager % Provided, developmentTestsDeploymentManager % Provided, - flinkPeriodicDeploymentManager % Provided, + flinkPeriodicDeploymentManager % "provided, test->test", schemedKafkaComponentsUtils % Provided, ) diff --git a/designer/client/src/assets/img/toolbarButtons/run-off-schedule.svg b/designer/client/src/assets/img/toolbarButtons/run-off-schedule.svg deleted file mode 100644 index c9b3eb0110d..00000000000 --- a/designer/client/src/assets/img/toolbarButtons/run-off-schedule.svg +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala index bf8fed6e669..4f7242b16be 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala @@ -1,12 +1,13 @@ package pl.touk.nussknacker.engine import akka.actor.ActorSystem +import pl.touk.nussknacker.engine.api.component.{ComponentAdditionalConfig, DesignerWideComponentId} +import pl.touk.nussknacker.engine.api.deployment.periodic.{PeriodicProcessesManager, PeriodicProcessesManagerProvider} import pl.touk.nussknacker.engine.api.deployment.{ ProcessingTypeActionService, ProcessingTypeDeployedScenariosProvider, ScenarioActivityManager } -import pl.touk.nussknacker.engine.api.component.{ComponentAdditionalConfig, DesignerWideComponentId} import sttp.client3.SttpBackend import scala.concurrent.{ExecutionContext, Future} @@ -15,6 +16,7 @@ case class DeploymentManagerDependencies( deployedScenariosProvider: ProcessingTypeDeployedScenariosProvider, actionService: ProcessingTypeActionService, scenarioActivityManager: ScenarioActivityManager, + periodicProcessesManagerProvider: PeriodicProcessesManagerProvider, executionContext: ExecutionContext, actorSystem: ActorSystem, sttpBackend: SttpBackend[Future, Any], diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala new file mode 100644 index 00000000000..457072ae5a0 --- /dev/null +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala @@ -0,0 +1,150 @@ +package pl.touk.nussknacker.engine.api.deployment.periodic + +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager.ScheduleProperty +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess + +import java.time.LocalDateTime +import scala.concurrent.Future + +trait PeriodicProcessesManager { + + def create( + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: ScheduleProperty, + processActionId: ProcessActionId, + ): Future[PeriodicProcess] + + def markInactive(processId: PeriodicProcessId): Future[Unit] + + def schedule( + id: PeriodicProcessId, + scheduleName: ScheduleName, + runAt: LocalDateTime, + deployMaxRetries: Int, + ): Future[PeriodicProcessDeployment] + + def findProcessData(id: PeriodicProcessDeploymentId): Future[PeriodicProcessDeployment] + + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] + + def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] + + def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] + + def markFinished(id: PeriodicProcessDeploymentId): Future[Unit] + + def markFailed(id: PeriodicProcessDeploymentId): Future[Unit] + + def markFailedOnDeployWithStatus( + id: PeriodicProcessDeploymentId, + status: PeriodicProcessDeploymentStatus, + deployRetries: Int, + retryAt: Option[LocalDateTime] + ): Future[Unit] + + def getSchedulesState( + scenarioName: ProcessName + ): Future[SchedulesState] + + def getLatestDeploymentsForActiveSchedules( + processName: ProcessName, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] + + def getLatestDeploymentsForLatestInactiveSchedules( + processName: ProcessName, + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] + + def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + ): Future[SchedulesState] + + def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId, + ): Future[Option[CanonicalProcess]] + +} + +object PeriodicProcessesManager { + + sealed trait ScheduleProperty + + sealed trait SingleScheduleProperty extends ScheduleProperty + + case class MultipleScheduleProperty(schedules: Map[String, SingleScheduleProperty]) extends ScheduleProperty + + case class CronScheduleProperty(labelOrCronExpr: String) extends SingleScheduleProperty + +} + +object NoOpPeriodicProcessesManager extends PeriodicProcessesManager { + + override def create( + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: ScheduleProperty, + processActionId: ProcessActionId, + ): Future[PeriodicProcess] = notImplemented + + override def markInactive(processId: PeriodicProcessId): Future[Unit] = notImplemented + + override def schedule( + id: PeriodicProcessId, + scheduleName: ScheduleName, + runAt: LocalDateTime, + deployMaxRetries: Int + ): Future[PeriodicProcessDeployment] = notImplemented + + override def findProcessData( + id: PeriodicProcessDeploymentId, + ): Future[PeriodicProcessDeployment] = notImplemented + + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = notImplemented + + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = notImplemented + + override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = notImplemented + + override def markFinished(id: PeriodicProcessDeploymentId): Future[Unit] = notImplemented + + override def markFailed(id: PeriodicProcessDeploymentId): Future[Unit] = notImplemented + + override def markFailedOnDeployWithStatus( + id: PeriodicProcessDeploymentId, + status: PeriodicProcessDeploymentStatus, + deployRetries: Int, + retryAt: Option[LocalDateTime] + ): Future[Unit] = notImplemented + + override def getSchedulesState(scenarioName: ProcessName): Future[SchedulesState] = notImplemented + + override def getLatestDeploymentsForActiveSchedules( + processName: ProcessName, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = notImplemented + + override def getLatestDeploymentsForLatestInactiveSchedules( + processName: ProcessName, + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = notImplemented + + override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + ): Future[SchedulesState] = notImplemented + + override def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId + ): Future[Option[CanonicalProcess]] = notImplemented + + private def notImplemented: Future[Nothing] = + Future.failed(new NotImplementedError()) + +} diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManagerProvider.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManagerProvider.scala new file mode 100644 index 00000000000..c756a754d7f --- /dev/null +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManagerProvider.scala @@ -0,0 +1,19 @@ +package pl.touk.nussknacker.engine.api.deployment.periodic + +trait PeriodicProcessesManagerProvider { + + def provide( + deploymentManagerName: String, + processingType: String, + ): PeriodicProcessesManager + +} + +object NoOpPeriodicProcessesManagerProvider extends PeriodicProcessesManagerProvider { + + override def provide( + deploymentManagerName: String, + processingType: String + ): PeriodicProcessesManager = NoOpPeriodicProcessesManager + +} diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala new file mode 100644 index 00000000000..84811ae0c9a --- /dev/null +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala @@ -0,0 +1,11 @@ +package pl.touk.nussknacker.engine.api.deployment.periodic.model + +import pl.touk.nussknacker.engine.api.ProcessVersion + +case class DeploymentWithRuntimeParams( + processVersion: ProcessVersion, + inputConfigDuringExecutionJson: String, + runtimeParams: RuntimeParams, +) + +final case class RuntimeParams(params: Map[String, String]) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala similarity index 54% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala rename to designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala index e89deab2320..ab1c666a008 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala @@ -1,17 +1,16 @@ -package pl.touk.nussknacker.engine.management.periodic.model +package pl.touk.nussknacker.engine.api.deployment.periodic.model import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId -import pl.touk.nussknacker.engine.management.periodic.ScheduleProperty -import slick.lifted.MappedTo +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager.ScheduleProperty import java.time.LocalDateTime -case class PeriodicProcessId(value: Long) extends MappedTo[Long] +case class PeriodicProcessId(value: Long) -case class PeriodicProcess[DeploymentData <: DeploymentWithJarData]( +case class PeriodicProcess( id: PeriodicProcessId, - deploymentData: DeploymentData, + deploymentData: DeploymentWithRuntimeParams, scheduleProperty: ScheduleProperty, active: Boolean, createdAt: LocalDateTime, diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala new file mode 100644 index 00000000000..e67b46142dc --- /dev/null +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala @@ -0,0 +1,42 @@ +package pl.touk.nussknacker.engine.api.deployment.periodic.model + +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus + +import java.time.LocalDateTime + +// TODO: We should separate schedules concept from deployments - fully switch to ScheduleData and ScheduleDeploymentData +case class PeriodicProcessDeployment( + id: PeriodicProcessDeploymentId, + periodicProcess: PeriodicProcess, + createdAt: LocalDateTime, + runAt: LocalDateTime, + scheduleName: ScheduleName, + retriesLeft: Int, + nextRetryAt: Option[LocalDateTime], + state: PeriodicProcessDeploymentState +) { + + def display: String = + s"${periodicProcess.processVersion} with scheduleName=${scheduleName.display} and deploymentId=$id" + +} + +case class PeriodicProcessDeploymentState( + deployedAt: Option[LocalDateTime], + completedAt: Option[LocalDateTime], + status: PeriodicProcessDeploymentStatus +) + +case class PeriodicProcessDeploymentId(value: Long) { + override def toString: String = value.toString +} + +object PeriodicProcessDeploymentStatus extends Enumeration { + type PeriodicProcessDeploymentStatus = Value + + val Scheduled, Deployed, Finished, Failed, RetryingDeploy, FailedOnDeploy = Value +} + +case class ScheduleName(value: Option[String]) { + def display: String = value.getOrElse("[default]") +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala similarity index 67% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala rename to designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala index ca697fe27d9..ecfddb07ab0 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala @@ -1,8 +1,6 @@ -package pl.touk.nussknacker.engine.management.periodic.model +package pl.touk.nussknacker.engine.api.deployment.periodic.model import pl.touk.nussknacker.engine.api.process.ProcessName -import pl.touk.nussknacker.engine.management.periodic.db.{PeriodicProcessDeploymentEntity, PeriodicProcessesRepository} -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithoutCanonicalProcess import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap import java.time.LocalDateTime @@ -36,10 +34,7 @@ case class SchedulesState(schedules: Map[ScheduleId, ScheduleData]) { // For most operations it will contain only one latest deployment but for purpose of statuses of historical deployments // it has list instead of one element. // This structure should contain SingleScheduleProperty as well. See note above -case class ScheduleData( - process: PeriodicProcess[WithoutCanonicalProcess], - latestDeployments: List[ScheduleDeploymentData] -) +case class ScheduleData(process: PeriodicProcess, latestDeployments: List[ScheduleDeploymentData]) // To identify schedule we need scheduleName - None for SingleScheduleProperty and Some(key) for MultipleScheduleProperty keys // Also we need PeriodicProcessId to distinguish between active schedules and some inactive from the past for the same PeriodicProcessId @@ -57,42 +52,17 @@ case class ScheduleDeploymentData( ) { def toFullDeploymentData( - process: PeriodicProcess[WithoutCanonicalProcess], + process: PeriodicProcess, scheduleName: ScheduleName - ): PeriodicProcessDeployment[WithoutCanonicalProcess] = + ): PeriodicProcessDeployment = PeriodicProcessDeployment(id, process, createdAt, runAt, scheduleName, retriesLeft, nextRetryAt, state) def display = s"deploymentId=$id" } -object ScheduleDeploymentData { - - def apply(deployment: PeriodicProcessDeploymentEntity): ScheduleDeploymentData = { - ScheduleDeploymentData( - deployment.id, - deployment.createdAt, - deployment.runAt, - deployment.deployedAt, - deployment.retriesLeft, - deployment.nextRetryAt, - PeriodicProcessesRepository.createPeriodicDeploymentState(deployment) - ) - } - -} - // These below are temporary structures, see notice next to SchedulesState case class PeriodicProcessScheduleData( - process: PeriodicProcess[WithoutCanonicalProcess], - deployments: List[PeriodicProcessDeployment[WithoutCanonicalProcess]] -) { - def existsDeployment(predicate: PeriodicProcessDeployment[WithoutCanonicalProcess] => Boolean): Boolean = - deployments.exists(predicate) - - def display: String = { - val deploymentsForSchedules = deployments.map(_.display) - s"processName=${process.processVersion.processName}, deploymentsForSchedules=$deploymentsForSchedules" - } - -} + process: PeriodicProcess, + deployments: List[PeriodicProcessDeployment] +) diff --git a/engine/flink/management/periodic/src/main/resources/web/static/assets/states/scheduled.svg b/designer/server/src/main/resources/web/static/assets/states/scheduled.svg similarity index 100% rename from engine/flink/management/periodic/src/main/resources/web/static/assets/states/scheduled.svg rename to designer/server/src/main/resources/web/static/assets/states/scheduled.svg diff --git a/engine/flink/management/periodic/src/main/resources/web/static/assets/states/wait-reschedule.svg b/designer/server/src/main/resources/web/static/assets/states/wait-reschedule.svg similarity index 100% rename from engine/flink/management/periodic/src/main/resources/web/static/assets/states/wait-reschedule.svg rename to designer/server/src/main/resources/web/static/assets/states/wait-reschedule.svg diff --git a/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala b/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala new file mode 100644 index 00000000000..fd1b42362f4 --- /dev/null +++ b/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala @@ -0,0 +1,190 @@ +package db.migration + +import com.typesafe.scalalogging.LazyLogging +import db.migration.V1_060__PeriodicDeploymentManagerTablesDefinition.Definitions +import pl.touk.nussknacker.ui.db.migration.SlickMigration +import slick.jdbc.JdbcProfile +import slick.lifted.ProvenShape +import slick.sql.SqlProfile.ColumnOption.NotNull + +import java.time.LocalDateTime +import java.util.UUID +import scala.concurrent.ExecutionContext.Implicits.global + +trait V1_060__PeriodicDeploymentManagerTablesDefinition extends SlickMigration with LazyLogging { + + import profile.api._ + + private val definitions = new Definitions(profile) + + override def migrateActions: DBIOAction[Any, NoStream, Effect.All] = { + logger.info("Starting migration V1_056__CreateScenarioActivitiesDefinition") + for { + _ <- definitions.periodicProcessesTable.schema.create + _ <- definitions.periodicProcessDeploymentsTable.schema.create + } yield () + } + +} + +object V1_060__PeriodicDeploymentManagerTablesDefinition { + + class Definitions(val profile: JdbcProfile) { + import profile.api._ + + val periodicProcessDeploymentsTable = TableQuery[PeriodicProcessDeploymentsTable] + + class PeriodicProcessDeploymentsTable(tag: Tag) + extends Table[PeriodicProcessDeploymentEntity](tag, "periodic_process_deployments") { + + def id: Rep[Long] = column[Long]("id", O.PrimaryKey, O.AutoInc) + + def periodicProcessId: Rep[Long] = column[Long]("periodic_process_id", NotNull) + + def createdAt: Rep[LocalDateTime] = column[LocalDateTime]("created_at", NotNull) + + def runAt: Rep[LocalDateTime] = column[LocalDateTime]("run_at", NotNull) + + def scheduleName: Rep[Option[String]] = column[Option[String]]("schedule_name") + + def deployedAt: Rep[Option[LocalDateTime]] = column[Option[LocalDateTime]]("deployed_at") + + def completedAt: Rep[Option[LocalDateTime]] = column[Option[LocalDateTime]]("completed_at") + + def retriesLeft: Rep[Int] = column[Int]("retries_left") + + def nextRetryAt: Rep[Option[LocalDateTime]] = column[Option[LocalDateTime]]("next_retry_at") + + def status: Rep[String] = column[String]("status", NotNull) + + override def * : ProvenShape[PeriodicProcessDeploymentEntity] = ( + id, + periodicProcessId, + createdAt, + runAt, + scheduleName, + deployedAt, + completedAt, + retriesLeft, + nextRetryAt, + status + ) <> + ((PeriodicProcessDeploymentEntity.apply _).tupled, PeriodicProcessDeploymentEntity.unapply) + + } + + case class PeriodicProcessDeploymentEntity( + id: Long, + periodicProcessId: Long, + createdAt: LocalDateTime, + runAt: LocalDateTime, + scheduleName: Option[String], + deployedAt: Option[LocalDateTime], + completedAt: Option[LocalDateTime], + retriesLeft: Int, + nextRetryAt: Option[LocalDateTime], + status: String + ) + + val periodicProcessesTable = TableQuery[PeriodicProcessesTable] + + class PeriodicProcessesTable(tag: Tag) extends Table[PeriodicProcessEntity](tag, "periodic_processes") { + + def id: Rep[Long] = column[Long]("id", O.PrimaryKey, O.AutoInc) + + def processName: Rep[String] = column[String]("process_name", NotNull) + + def processVersionId: Rep[Long] = column[Long]("process_version_id", NotNull) + + def processingType: Rep[String] = column[String]("processing_type", NotNull) + + def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) + + // This is a legacy column left after migrating periodic processes to core + // The periodic deployment manager is now decoupled from Flink, and its runtime params are stored in runtime_params column + def jarFileName: Rep[Option[String]] = column[Option[String]]("jar_file_name") + + def runtimeParams: Rep[String] = column[String]("runtime_params") + + def scheduleProperty: Rep[String] = column[String]("schedule_property", NotNull) + + def active: Rep[Boolean] = column[Boolean]("active", NotNull) + + def createdAt: Rep[LocalDateTime] = column[LocalDateTime]("created_at", NotNull) + + def processActionId: Rep[Option[UUID]] = column[Option[UUID]]("process_action_id") + + override def * : ProvenShape[PeriodicProcessEntity] = ( + id, + processName, + processVersionId, + processingType, + inputConfigDuringExecutionJson, + jarFileName, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) <> ( + tuple => + PeriodicProcessEntity( + id = tuple._1, + processName = tuple._2, + processVersionId = tuple._3, + processingType = tuple._4, + inputConfigDuringExecutionJson = tuple._5, + runtimeParams = tuple._7, + scheduleProperty = tuple._8, + active = tuple._9, + createdAt = tuple._10, + processActionId = tuple._11, + ), + (e: PeriodicProcessEntity) => + PeriodicProcessEntity.unapply(e).map { + case ( + id, + processName, + versionId, + processingType, + inputConfigDuringExecutionJson, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) => + ( + id, + processName, + versionId, + processingType, + inputConfigDuringExecutionJson, + None, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) + } + ) + + } + + case class PeriodicProcessEntity( + id: Long, + processName: String, + processVersionId: Long, + processingType: String, + inputConfigDuringExecutionJson: String, + runtimeParams: String, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[UUID] + ) + + } + +} diff --git a/designer/server/src/main/scala/db/migration/hsql/V1_060__PeriodicDeploymentManagerTables.scala b/designer/server/src/main/scala/db/migration/hsql/V1_060__PeriodicDeploymentManagerTables.scala new file mode 100644 index 00000000000..3e485f8f4da --- /dev/null +++ b/designer/server/src/main/scala/db/migration/hsql/V1_060__PeriodicDeploymentManagerTables.scala @@ -0,0 +1,8 @@ +package db.migration.hsql + +import db.migration.V1_060__PeriodicDeploymentManagerTablesDefinition +import slick.jdbc.{HsqldbProfile, JdbcProfile} + +class V1_060__PeriodicDeploymentManagerTables extends V1_060__PeriodicDeploymentManagerTablesDefinition { + override protected lazy val profile: JdbcProfile = HsqldbProfile +} diff --git a/designer/server/src/main/scala/db/migration/postgres/V1_060__PeriodicDeploymentManagerTables.scala b/designer/server/src/main/scala/db/migration/postgres/V1_060__PeriodicDeploymentManagerTables.scala new file mode 100644 index 00000000000..b5f19970a7d --- /dev/null +++ b/designer/server/src/main/scala/db/migration/postgres/V1_060__PeriodicDeploymentManagerTables.scala @@ -0,0 +1,8 @@ +package db.migration.postgres + +import db.migration.V1_060__PeriodicDeploymentManagerTablesDefinition +import slick.jdbc.{JdbcProfile, PostgresProfile} + +class V1_060__PeriodicDeploymentManagerTables extends V1_060__PeriodicDeploymentManagerTablesDefinition { + override protected lazy val profile: JdbcProfile = PostgresProfile +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessDeploymentsTable.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessDeploymentsTable.scala similarity index 82% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessDeploymentsTable.scala rename to designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessDeploymentsTable.scala index f9b1bacb69b..f75cbff1395 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessDeploymentsTable.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessDeploymentsTable.scala @@ -1,11 +1,7 @@ -package pl.touk.nussknacker.engine.management.periodic.db +package pl.touk.nussknacker.ui.db.entity -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model.{ - PeriodicProcessDeploymentId, - PeriodicProcessDeploymentStatus, - PeriodicProcessId -} +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{PeriodicProcessDeploymentId, PeriodicProcessDeploymentStatus, PeriodicProcessId} import slick.jdbc.{JdbcProfile, JdbcType} import slick.lifted.ProvenShape import slick.sql.SqlProfile.ColumnOption.NotNull @@ -14,10 +10,11 @@ import java.time.LocalDateTime trait PeriodicProcessDeploymentsTableFactory extends PeriodicProcessesTableFactory { - protected val profile: JdbcProfile - import profile.api._ + implicit val periodicProcessDeploymentIdMapping: BaseColumnType[PeriodicProcessDeploymentId] = + MappedColumnType.base[PeriodicProcessDeploymentId, Long](_.value, PeriodicProcessDeploymentId.apply) + implicit val periodicProcessDeploymentStatusColumnTyped: JdbcType[PeriodicProcessDeploymentStatus] = MappedColumnType.base[PeriodicProcessDeploymentStatus, String](_.toString, PeriodicProcessDeploymentStatus.withName) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala new file mode 100644 index 00000000000..0d3b85155c4 --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala @@ -0,0 +1,140 @@ +package pl.touk.nussknacker.ui.db.entity + +import io.circe.Decoder +import io.circe.syntax.EncoderOps +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{PeriodicProcessId, RuntimeParams} +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} +import slick.jdbc.JdbcProfile +import slick.lifted.ProvenShape +import slick.sql.SqlProfile.ColumnOption.NotNull + +import java.time.LocalDateTime +import java.util.UUID + +trait PeriodicProcessesTableFactory extends BaseEntityFactory{ + + protected val profile: JdbcProfile + + import profile.api._ + + implicit val periodicProcessIdMapping: BaseColumnType[PeriodicProcessId] = + MappedColumnType.base[PeriodicProcessId, Long](_.value, PeriodicProcessId.apply) + + private implicit val ProcessActionIdTypedType: BaseColumnType[ProcessActionId] = + MappedColumnType.base[ProcessActionId, UUID]( + _.value, + ProcessActionId(_) + ) + + implicit val runtimeParamsTypedType: BaseColumnType[RuntimeParams] = + MappedColumnType.base[RuntimeParams, String]( + _.params.asJson.noSpaces, + jsonStr => + io.circe.parser.parse(jsonStr).flatMap(Decoder[Map[String, String]].decodeJson) match { + case Right(params) => RuntimeParams(params) + case Left(error) => throw error + } + ) + + class PeriodicProcessesTable(tag: Tag) extends Table[PeriodicProcessEntity](tag, "periodic_processes") { + + def id: Rep[PeriodicProcessId] = column[PeriodicProcessId]("id", O.PrimaryKey, O.AutoInc) + + def processName: Rep[ProcessName] = column[ProcessName]("process_name", NotNull) + + def processVersionId: Rep[VersionId] = column[VersionId]("process_version_id", NotNull) + + def processingType: Rep[String] = column[String]("processing_type", NotNull) + + def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) + + // This is a legacy column left after migrating periodic processes to core + // The periodic deployment manager is now decoupled from Flink, and its runtime params are stored in runtime_params column + def jarFileName: Rep[Option[String]] = column[Option[String]]("jar_file_name") + + def runtimeParams: Rep[RuntimeParams] = column[RuntimeParams]("runtime_params") + + def scheduleProperty: Rep[String] = column[String]("schedule_property", NotNull) + + def active: Rep[Boolean] = column[Boolean]("active", NotNull) + + def createdAt: Rep[LocalDateTime] = column[LocalDateTime]("created_at", NotNull) + + def processActionId: Rep[Option[ProcessActionId]] = column[Option[ProcessActionId]]("process_action_id") + + override def * : ProvenShape[PeriodicProcessEntity] = ( + id, + processName, + processVersionId, + processingType, + inputConfigDuringExecutionJson, + jarFileName, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) <> ( + tuple => + PeriodicProcessEntity( + id = tuple._1, + processName = tuple._2, + processVersionId = tuple._3, + processingType = tuple._4, + inputConfigDuringExecutionJson = tuple._5, + runtimeParams = + RuntimeParams(tuple._6.map(f => Map("jarFileName" -> f)).getOrElse(Map.empty) ++ tuple._7.params), + scheduleProperty = tuple._8, + active = tuple._9, + createdAt = tuple._10, + processActionId = tuple._11, + ), + (e: PeriodicProcessEntity) => + PeriodicProcessEntity.unapply(e).map { + case ( + id, + processName, + versionId, + processingType, + inputConfigDuringExecutionJson, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) => + ( + id, + processName, + versionId, + processingType, + inputConfigDuringExecutionJson, + None, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) + } + ) + + } + + object PeriodicProcesses extends TableQuery(new PeriodicProcessesTable(_)) + +} + +final case class PeriodicProcessEntity( + id: PeriodicProcessId, + processName: ProcessName, + processVersionId: VersionId, + processingType: String, + inputConfigDuringExecutionJson: String, + runtimeParams: RuntimeParams, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[ProcessActionId] +) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala new file mode 100644 index 00000000000..957b11dc6ba --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala @@ -0,0 +1,106 @@ +package pl.touk.nussknacker.ui.process.periodic + +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.ui.process.repository.PeriodicProcessesRepository + +import java.time.LocalDateTime +import scala.concurrent.Future + +class RepositoryBasedPeriodicProcessesManager( + deploymentManagerName: String, + processingType: String, + periodicProcessesRepository: PeriodicProcessesRepository, +) extends PeriodicProcessesManager { + + import periodicProcessesRepository._ + + override def create( + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: PeriodicProcessesManager.ScheduleProperty, + processActionId: ProcessActionId, + ): Future[PeriodicProcess] = + periodicProcessesRepository + .create(deploymentWithRuntimeParams, scheduleProperty, processActionId, processingType) + .run + + override def markInactive(processId: PeriodicProcessId): Future[Unit] = + periodicProcessesRepository.markInactive(processId).run + + override def schedule( + id: PeriodicProcessId, + scheduleName: ScheduleName, + runAt: LocalDateTime, + deployMaxRetries: Int + ): Future[PeriodicProcessDeployment] = + periodicProcessesRepository.schedule(id, scheduleName, runAt, deployMaxRetries).run + + override def findProcessData( + id: PeriodicProcessDeploymentId, + ): Future[PeriodicProcessDeployment] = + periodicProcessesRepository.findProcessData(id).run + + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = + periodicProcessesRepository.findToBeDeployed(processingType).run + + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = + periodicProcessesRepository.findToBeRetried(processingType).run + + override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = + periodicProcessesRepository.markDeployed(id).run + + override def markFinished(id: PeriodicProcessDeploymentId): Future[Unit] = + periodicProcessesRepository.markFinished(id).run + + override def markFailed(id: PeriodicProcessDeploymentId): Future[Unit] = + periodicProcessesRepository.markFailed(id).run + + override def markFailedOnDeployWithStatus( + id: PeriodicProcessDeploymentId, + status: PeriodicProcessDeploymentStatus, + deployRetries: Int, + retryAt: Option[LocalDateTime] + ): Future[Unit] = periodicProcessesRepository.markFailedOnDeployWithStatus(id, status, deployRetries, retryAt).run + + override def getSchedulesState(scenarioName: ProcessName): Future[SchedulesState] = + periodicProcessesRepository.getSchedulesState(scenarioName).run + + override def getLatestDeploymentsForActiveSchedules( + processName: ProcessName, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = + periodicProcessesRepository + .getLatestDeploymentsForActiveSchedules(processName, deploymentsPerScheduleMaxCount, processingType) + .run + + override def getLatestDeploymentsForLatestInactiveSchedules( + processName: ProcessName, + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = + periodicProcessesRepository + .getLatestDeploymentsForLatestInactiveSchedules( + processName, + inactiveProcessesMaxCount, + deploymentsPerScheduleMaxCount, + processingType, + ) + .run + + override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + ): Future[SchedulesState] = periodicProcessesRepository + .findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus(expectedDeploymentStatuses, processingType) + .run + + override def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId + ): Future[Option[CanonicalProcess]] = + periodicProcessesRepository.fetchCanonicalProcess(processName, versionId).run + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala new file mode 100644 index 00000000000..f2b576e35a0 --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala @@ -0,0 +1,21 @@ +package pl.touk.nussknacker.ui.process.periodic + +import pl.touk.nussknacker.engine.api.deployment.periodic.{PeriodicProcessesManager, PeriodicProcessesManagerProvider} +import pl.touk.nussknacker.ui.process.repository.PeriodicProcessesRepository + +class RepositoryBasedPeriodicProcessesManagerProvider( + periodicProcessesRepository: PeriodicProcessesRepository, +) extends PeriodicProcessesManagerProvider { + + override def provide( + deploymentManagerName: String, + processingType: String + ): PeriodicProcessesManager = { + new RepositoryBasedPeriodicProcessesManager( + deploymentManagerName, + processingType, + periodicProcessesRepository + ) + } + +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala similarity index 70% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala rename to designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala index 0b2e2c48c8d..6cb2531a9c6 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala @@ -1,19 +1,21 @@ -package pl.touk.nussknacker.engine.management.periodic.db +package pl.touk.nussknacker.ui.process.repository +import _root_.db.util.DBIOActionInstances import cats.Monad import com.github.tminglei.slickpg.ExPostgresProfile import com.typesafe.scalalogging.LazyLogging import io.circe.parser.decode +import io.circe.syntax.EncoderOps import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId -import pl.touk.nussknacker.engine.api.process.ProcessName -import pl.touk.nussknacker.engine.management.periodic._ -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.{ - WithCanonicalProcess, - WithoutCanonicalProcess -} -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model._ +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} +import pl.touk.nussknacker.engine.common.periodic._ +import pl.touk.nussknacker.ui.db.entity._ import slick.dbio.{DBIOAction, Effect, NoStream} import slick.jdbc.PostgresProfile.api._ import slick.jdbc.{JdbcBackend, JdbcProfile} @@ -25,10 +27,10 @@ import scala.language.higherKinds object PeriodicProcessesRepository { def createPeriodicProcessDeployment( - processEntity: PeriodicProcessEntityWithJson, - processDeploymentEntity: PeriodicProcessDeploymentEntity - ): PeriodicProcessDeployment[WithCanonicalProcess] = { - val process = createPeriodicProcessWithJson(processEntity) + processEntity: PeriodicProcessEntity, + processDeploymentEntity: PeriodicProcessDeploymentEntity, + ): PeriodicProcessDeployment = { + val process = createPeriodicProcess(processEntity) PeriodicProcessDeployment( processDeploymentEntity.id, process, @@ -51,36 +53,15 @@ object PeriodicProcessesRepository { ) } - def createPeriodicProcessWithJson( - processEntity: PeriodicProcessEntityWithJson - ): PeriodicProcess[WithCanonicalProcess] = { + def createPeriodicProcess(processEntity: PeriodicProcessEntity): PeriodicProcess = { val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, - model.DeploymentWithJarData.WithCanonicalProcess( + DeploymentWithRuntimeParams( processVersion = processVersion, inputConfigDuringExecutionJson = processEntity.inputConfigDuringExecutionJson, - jarFileName = processEntity.jarFileName, - process = processEntity.processJson, - ), - scheduleProperty, - processEntity.active, - processEntity.createdAt, - processEntity.processActionId - ) - } - - def createPeriodicProcessWithoutJson( - processEntity: PeriodicProcessEntity - ): PeriodicProcess[WithoutCanonicalProcess] = { - val processVersion = createProcessVersion(processEntity) - val scheduleProperty = prepareScheduleProperty(processEntity) - PeriodicProcess( - processEntity.id, - model.DeploymentWithJarData.WithoutCanonicalProcess( - processVersion = processVersion, - jarFileName = processEntity.jarFileName, + runtimeParams = processEntity.runtimeParams, ), scheduleProperty, processEntity.active, @@ -92,7 +73,7 @@ object PeriodicProcessesRepository { private def prepareScheduleProperty(processEntity: PeriodicProcessEntity) = { val scheduleProperty = decode[ScheduleProperty](processEntity.scheduleProperty) .fold(e => throw new IllegalArgumentException(e), identity) - scheduleProperty + toApi(scheduleProperty) } private def createProcessVersion(processEntity: PeriodicProcessEntity): ProcessVersion = { @@ -121,33 +102,35 @@ trait PeriodicProcessesRepository { ): Action[SchedulesState] def create( - deploymentWithJarData: DeploymentWithJarData.WithCanonicalProcess, - scheduleProperty: ScheduleProperty, - processActionId: ProcessActionId - ): Action[PeriodicProcess[WithCanonicalProcess]] + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: PeriodicProcessesManager.ScheduleProperty, + processActionId: ProcessActionId, + processingType: String, + ): Action[PeriodicProcess] def getLatestDeploymentsForActiveSchedules( processName: ProcessName, - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[SchedulesState] def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[SchedulesState] - def findToBeDeployed: Action[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] + def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment]] - def findToBeRetried: Action[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] + def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment]] def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( - expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus] + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + processingType: String, ): Action[SchedulesState] - def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment[WithCanonicalProcess]] - - def findProcessData(processName: ProcessName): Action[Seq[PeriodicProcess[WithCanonicalProcess]]] + def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment] def markDeployed(id: PeriodicProcessDeploymentId): Action[Unit] @@ -167,7 +150,12 @@ trait PeriodicProcessesRepository { scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Action[PeriodicProcessDeployment[WithCanonicalProcess]] + ): Action[PeriodicProcessDeployment] + + def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId, + ): Action[Option[WithCanonicalProcess]] } @@ -175,14 +163,14 @@ class SlickPeriodicProcessesRepository( db: JdbcBackend.DatabaseDef, override val profile: JdbcProfile, clock: Clock, - processingType: String )(implicit ec: ExecutionContext) extends PeriodicProcessesRepository with PeriodicProcessesTableFactory with PeriodicProcessDeploymentsTableFactory + with ProcessVersionEntityFactory + with ProcessEntityFactory with LazyLogging { - import io.circe.syntax._ import pl.touk.nussknacker.engine.util.Implicits._ type Action[T] = DBIOActionInstances.DB[T] @@ -195,7 +183,7 @@ class SlickPeriodicProcessesRepository( scenarioName: ProcessName, afterOpt: Option[LocalDateTime], ): Action[SchedulesState] = { - PeriodicProcessesWithoutJson + PeriodicProcesses .filter(_.processName === scenarioName) .join(PeriodicProcessDeployments) .on(_.id === _.periodicProcessId) @@ -205,63 +193,66 @@ class SlickPeriodicProcessesRepository( } override def create( - deploymentWithJarData: DeploymentWithJarData.WithCanonicalProcess, - scheduleProperty: ScheduleProperty, - processActionId: ProcessActionId - ): Action[PeriodicProcess[WithCanonicalProcess]] = { - val processEntity = PeriodicProcessEntityWithJson( + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: PeriodicProcessesManager.ScheduleProperty, + processActionId: ProcessActionId, + processingType: String, + ): Action[PeriodicProcess] = { + val processEntity = PeriodicProcessEntity( id = PeriodicProcessId(-1), - processName = deploymentWithJarData.processVersion.processName, - processVersionId = deploymentWithJarData.processVersion.versionId, + processName = deploymentWithRuntimeParams.processVersion.processName, + processVersionId = deploymentWithRuntimeParams.processVersion.versionId, processingType = processingType, - processJson = deploymentWithJarData.process, - inputConfigDuringExecutionJson = deploymentWithJarData.inputConfigDuringExecutionJson, - jarFileName = deploymentWithJarData.jarFileName, - scheduleProperty = scheduleProperty.asJson.noSpaces, + inputConfigDuringExecutionJson = deploymentWithRuntimeParams.inputConfigDuringExecutionJson, + runtimeParams = deploymentWithRuntimeParams.runtimeParams, + scheduleProperty = fromApi(scheduleProperty).asJson.noSpaces, active = true, createdAt = now(), Some(processActionId) ) - ((PeriodicProcessesWithJson returning PeriodicProcessesWithJson into ((_, id) => id)) += processEntity) - .map(PeriodicProcessesRepository.createPeriodicProcessWithJson) + ((PeriodicProcesses returning PeriodicProcesses into ((_, id) => id)) += processEntity) + .map(PeriodicProcessesRepository.createPeriodicProcess) } private def now(): LocalDateTime = LocalDateTime.now(clock) - override def findToBeDeployed: Action[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] = - activePeriodicProcessWithDeploymentQuery + override def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment]] = findProcesses( + activePeriodicProcessWithDeploymentQuery(processingType) .filter { case (_, d) => d.runAt <= now() && d.status === (PeriodicProcessDeploymentStatus.Scheduled: PeriodicProcessDeploymentStatus) } - .result - .map(_.map((PeriodicProcessesRepository.createPeriodicProcessDeployment _).tupled)) + ) - override def findToBeRetried: Action[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] = - activePeriodicProcessWithDeploymentQuery + override def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment]] = findProcesses( + activePeriodicProcessWithDeploymentQuery(processingType) .filter { case (_, d) => d.nextRetryAt <= now() && d.status === (PeriodicProcessDeploymentStatus.RetryingDeploy: PeriodicProcessDeploymentStatus) } - .result - .map(_.map((PeriodicProcessesRepository.createPeriodicProcessDeployment _).tupled)) - - override def findProcessData( - id: PeriodicProcessDeploymentId - ): Action[PeriodicProcessDeployment[WithCanonicalProcess]] = { - (PeriodicProcessesWithJson join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) - .filter { case (_, deployment) => deployment.id === id } - .result - .head - .map((PeriodicProcessesRepository.createPeriodicProcessDeployment _).tupled) + ) + + private def findProcesses( + query: Query[ + (PeriodicProcessesTable, PeriodicProcessDeploymentsTable), + (PeriodicProcessEntity, PeriodicProcessDeploymentEntity), + Seq + ] + ) = { + query.result + .map(_.map { case (periodicProcess, periodicDeployment) => + PeriodicProcessesRepository.createPeriodicProcessDeployment( + periodicProcess, + periodicDeployment, + ) + }) } - override def findProcessData(processName: ProcessName): Action[Seq[PeriodicProcess[WithCanonicalProcess]]] = { - PeriodicProcessesWithJson - .filter(p => p.active === true && p.processName === processName) - .result - .map(_.map(PeriodicProcessesRepository.createPeriodicProcessWithJson)) - } + override def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment] = + findProcesses( + (PeriodicProcesses join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) + .filter { case (_, deployment) => deployment.id === id } + ).map(_.head) override def markDeployed(id: PeriodicProcessDeploymentId): Action[Unit] = { val q = for { @@ -304,9 +295,10 @@ class SlickPeriodicProcessesRepository( } override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( - expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus] + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + processingType: String, ): Action[SchedulesState] = { - val processesHavingDeploymentsWithMatchingStatus = PeriodicProcessesWithoutJson.filter(p => + val processesHavingDeploymentsWithMatchingStatus = PeriodicProcesses.filter(p => p.active && PeriodicProcessDeployments .filter(d => d.periodicProcessId === p.id && d.status.inSet(expectedDeploymentStatuses)) @@ -314,33 +306,37 @@ class SlickPeriodicProcessesRepository( ) getLatestDeploymentsForEachSchedule( processesHavingDeploymentsWithMatchingStatus, - deploymentsPerScheduleMaxCount = 1 + deploymentsPerScheduleMaxCount = 1, + processingType = processingType, ) } override def getLatestDeploymentsForActiveSchedules( processName: ProcessName, - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[SchedulesState] = { - val activeProcessesQuery = PeriodicProcessesWithoutJson.filter(p => p.processName === processName && p.active) - getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount) + val activeProcessesQuery = PeriodicProcesses.filter(p => p.processName === processName && p.active) + getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount, processingType) } override def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[SchedulesState] = { - val filteredProcessesQuery = PeriodicProcessesWithoutJson + val filteredProcessesQuery = PeriodicProcesses .filter(p => p.processName === processName && !p.active) .sortBy(_.createdAt.desc) .take(inactiveProcessesMaxCount) - getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount) + getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount, processingType) } private def getLatestDeploymentsForEachSchedule( periodicProcessesQuery: Query[PeriodicProcessWithoutJson, PeriodicProcessEntityWithoutJson, Seq], - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[SchedulesState] = { val filteredPeriodicProcessQuery = periodicProcessesQuery.filter(p => p.processingType === processingType) val latestDeploymentsForSchedules = profile match { @@ -429,7 +425,7 @@ class SlickPeriodicProcessesRepository( scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Action[PeriodicProcessDeployment[WithCanonicalProcess]] = { + ): Action[PeriodicProcessDeployment] = { val deploymentEntity = PeriodicProcessDeploymentEntity( id = PeriodicProcessDeploymentId(-1), periodicProcessId = id, @@ -449,14 +445,25 @@ class SlickPeriodicProcessesRepository( override def markInactive(processId: PeriodicProcessId): Action[Unit] = { val q = for { - p <- PeriodicProcessesWithoutJson if p.id === processId + p <- PeriodicProcesses if p.id === processId } yield p.active val update = q.update(false) update.map(_ => ()) } - private def activePeriodicProcessWithDeploymentQuery = { - (PeriodicProcessesWithJson.filter(p => p.active === true && p.processingType === processingType) + def fetchCanonicalProcess(processName: ProcessName, versionId: VersionId): Action[Option[CanonicalProcess]] = { + processesTable + .filter(_.name === processName) + .join(processVersionsTable) + .on((process, version) => process.id === version.processId) + .filter { case (_, version) => version.id === versionId } + .result + .headOption + .map(_.flatMap(_._2.json)) + } + + private def activePeriodicProcessWithDeploymentQuery(processingType: String) = { + (PeriodicProcesses.filter(p => p.active === true && p.processingType === processingType) join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) } @@ -466,8 +473,8 @@ class SlickPeriodicProcessesRepository( .map { case (process, deployment) => val scheduleId = ScheduleId(process.id, ScheduleName(deployment.scheduleName)) val scheduleDataWithoutDeployment = - (scheduleId, PeriodicProcessesRepository.createPeriodicProcessWithoutJson(process)) - val scheduleDeployment = ScheduleDeploymentData(deployment) + (scheduleId, PeriodicProcessesRepository.createPeriodicProcess(process)) + val scheduleDeployment = scheduleDeploymentData(deployment) (scheduleDataWithoutDeployment, scheduleDeployment) } .toList @@ -480,26 +487,16 @@ class SlickPeriodicProcessesRepository( ) } -} - -//Copied from designer/server. -object DBIOActionInstances { - - type DB[A] = DBIOAction[A, NoStream, Effect.All] - - implicit def dbMonad(implicit ec: ExecutionContext): Monad[DB] = new Monad[DB] { - - override def pure[A](x: A) = DBIO.successful(x) - - override def flatMap[A, B](fa: DB[A])(f: (A) => DB[B]) = fa.flatMap(f) - - // this is *not* tail recursive - override def tailRecM[A, B](a: A)(f: (A) => DB[Either[A, B]]): DB[B] = - f(a).flatMap { - case Right(r) => pure(r) - case Left(l) => tailRecM(l)(f) - } - + private def scheduleDeploymentData(deployment: PeriodicProcessDeploymentEntity): ScheduleDeploymentData = { + ScheduleDeploymentData( + deployment.id, + deployment.createdAt, + deployment.runAt, + deployment.deployedAt, + deployment.retriesLeft, + deployment.nextRetryAt, + PeriodicProcessesRepository.createPeriodicDeploymentState(deployment) + ) } } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala index 68aecc26a42..5c9e6f83ef5 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala @@ -7,6 +7,7 @@ import pl.touk.nussknacker.engine.api.Comment import pl.touk.nussknacker.engine.api.deployment.ProcessActionState.ProcessActionState import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, ProcessingType, VersionId} +import pl.touk.nussknacker.engine.common.periodic.InstantBatchCustomAction import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap import pl.touk.nussknacker.ui.app.BuildInfo import pl.touk.nussknacker.ui.db.entity.{ diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala index ce1ed9743ee..55148cc2091 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala @@ -65,6 +65,10 @@ import pl.touk.nussknacker.ui.process.newdeployment.synchronize.{ DeploymentsStatusesSynchronizer } import pl.touk.nussknacker.ui.process.newdeployment.{DeploymentRepository, DeploymentService} +import pl.touk.nussknacker.ui.process.periodic.{ + RepositoryBasedPeriodicProcessesManager, + RepositoryBasedPeriodicProcessesManagerProvider +} import pl.touk.nussknacker.ui.process.processingtype.ProcessingTypeData import pl.touk.nussknacker.ui.process.processingtype.loader.ProcessingTypeDataLoader import pl.touk.nussknacker.ui.process.processingtype.provider.ReloadableProcessingTypeDataProvider @@ -125,15 +129,17 @@ class AkkaHttpBasedRouteProvider( logger.info(s"Designer config loaded: \nfeatureTogglesConfig: $featureTogglesConfig") for { countsReporter <- createCountsReporter(featureTogglesConfig, environment, sttpBackend) - actionServiceSupplier = new DelayedInitActionServiceSupplier - additionalUIConfigProvider = createAdditionalUIConfigProvider(resolvedDesignerConfig, sttpBackend) - deploymentRepository = new DeploymentRepository(dbRef, Clock.systemDefaultZone()) - scenarioActivityRepository = DbScenarioActivityRepository.create(dbRef, designerClock) - dbioRunner = DBIOActionRunner(dbRef) + actionServiceSupplier = new DelayedInitActionServiceSupplier + additionalUIConfigProvider = createAdditionalUIConfigProvider(resolvedDesignerConfig, sttpBackend) + deploymentRepository = new DeploymentRepository(dbRef, Clock.systemDefaultZone()) + scenarioActivityRepository = DbScenarioActivityRepository.create(dbRef, designerClock) + periodicProcessesRepository = new SlickPeriodicProcessesRepository(dbRef.db, dbRef.profile, designerClock) + dbioRunner = DBIOActionRunner(dbRef) processingTypeDataProvider <- prepareProcessingTypeDataReload( additionalUIConfigProvider, actionServiceSupplier, scenarioActivityRepository, + periodicProcessesRepository, dbioRunner, sttpBackend, featureTogglesConfig, @@ -704,6 +710,7 @@ class AkkaHttpBasedRouteProvider( additionalUIConfigProvider: AdditionalUIConfigProvider, actionServiceProvider: Supplier[ActionService], scenarioActivityRepository: ScenarioActivityRepository, + periodicProcessesRepository: PeriodicProcessesRepository, dbioActionRunner: DBIOActionRunner, sttpBackend: SttpBackend[Future, Any], featureTogglesConfig: FeatureTogglesConfig @@ -721,10 +728,12 @@ class AkkaHttpBasedRouteProvider( additionalUIConfigProvider, actionServiceProvider, scenarioActivityRepository, - dbioActionRunner, - sttpBackend, - _ - ), + periodicProcessesRepository, + dbioActionRunner, + sttpBackend, + _ + ), + ) new ReloadableProcessingTypeDataProvider(laodProcessingTypeDataIO) } @@ -737,6 +746,7 @@ class AkkaHttpBasedRouteProvider( additionalUIConfigProvider: AdditionalUIConfigProvider, actionServiceProvider: Supplier[ActionService], scenarioActivityRepository: ScenarioActivityRepository, + periodicProcessesRepository: PeriodicProcessesRepository, dbioActionRunner: DBIOActionRunner, sttpBackend: SttpBackend[Future, Any], processingType: ProcessingType @@ -752,6 +762,7 @@ class AkkaHttpBasedRouteProvider( scenarioActivityRepository, dbioActionRunner, ), + new RepositoryBasedPeriodicProcessesManagerProvider(periodicProcessesRepository), system.dispatcher, system, sttpBackend, diff --git a/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala b/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala index 38aaa699fb2..6fc518f7cd2 100644 --- a/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala +++ b/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala @@ -8,7 +8,6 @@ import db.migration.V1_057__MigrateActionsAndCommentsToScenarioActivitiesDefinit import io.circe.syntax.EncoderOps import org.scalatest.freespec.AnyFreeSpecLike import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.api.deployment.ScenarioComment.WithContent import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.api.{MetaData, ProcessAdditionalFields, RequestResponseMetaData} diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala index fbc1af4f450..adf9fd3b3cf 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala @@ -93,6 +93,8 @@ trait DbTesting extends BeforeAndAfterEach with BeforeAndAfterAll { session.prepareStatement("""delete from "environments"""").execute() session.prepareStatement("""delete from "processes"""").execute() session.prepareStatement("""delete from "fingerprints"""").execute() + session.prepareStatement("""delete from "periodic_processes"""").execute() + session.prepareStatement("""delete from "periodic_process_deployments"""").execute() } } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala index dcd25483e6d..605749a8ee9 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala @@ -13,6 +13,10 @@ import pl.touk.nussknacker.engine.api.definition.{ StringParameterEditor } import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.{ + NoOpPeriodicProcessesManagerProvider, + PeriodicProcessesManagerProvider +} import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.api.{ProcessVersion, StreamMetaData} @@ -44,6 +48,7 @@ class MockDeploymentManager( new ProcessingTypeDeployedScenariosProviderStub(List.empty), actionService: ProcessingTypeActionService = new ProcessingTypeActionServiceStub, scenarioActivityManager: ScenarioActivityManager = NoOpScenarioActivityManager, + periodicProcessesManagerProvider: PeriodicProcessesManagerProvider = NoOpPeriodicProcessesManagerProvider, ) extends FlinkDeploymentManager( ModelData( ProcessingTypeConfig.read(ConfigWithScalaVersion.StreamingProcessTypeConfig), @@ -53,6 +58,7 @@ class MockDeploymentManager( deployedScenariosProvider, actionService, scenarioActivityManager, + periodicProcessesManagerProvider, ExecutionContext.global, ActorSystem("MockDeploymentManager"), SttpBackendStub.asynchronousFuture diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala index c95746e8536..3a8779b0516 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala @@ -8,6 +8,10 @@ import com.typesafe.config.ConfigFactory import db.util.DBIOActionInstances._ import pl.touk.nussknacker.engine.api.component.{ComponentAdditionalConfig, DesignerWideComponentId, ProcessingMode} import pl.touk.nussknacker.engine.api.definition.FixedExpressionValue +import pl.touk.nussknacker.engine.api.deployment.periodic.{ + NoOpPeriodicProcessesManager, + NoOpPeriodicProcessesManagerProvider +} import pl.touk.nussknacker.engine.api.deployment.{ NoOpScenarioActivityManager, ProcessingTypeActionServiceStub, @@ -139,6 +143,7 @@ object TestFactory { new ProcessingTypeDeployedScenariosProviderStub(List.empty), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, actorSystem.dispatcher, actorSystem, SttpBackendStub.asynchronousFuture diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala similarity index 78% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala rename to designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala index e7807319126..9232b146cfa 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -1,11 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.ui.process.periodic +import _root_.db.util.DBIOActionInstances.DB import com.cronutils.builder.CronBuilder import com.cronutils.model.CronType import com.cronutils.model.definition.CronDefinitionBuilder import com.cronutils.model.field.expression.FieldExpressionFactory.{on, questionMark} -import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} -import com.typesafe.config.{Config, ConfigFactory} import com.typesafe.scalalogging.LazyLogging import org.scalatest.LoneElement._ import org.scalatest.OptionValues @@ -13,27 +12,37 @@ import org.scalatest.concurrent.ScalaFutures import org.scalatest.exceptions.TestFailedException import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import org.testcontainers.utility.DockerImageName +import org.scalatest.time.{Millis, Seconds, Span} import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus -import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, ProcessName} +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, ProcessName, VersionId} import pl.touk.nussknacker.engine.api.{MetaData, ProcessVersion, StreamMetaData} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.PeriodicProcessStatus -import pl.touk.nussknacker.engine.management.periodic.db.{DbInitializer, SlickPeriodicProcessesRepository} -import pl.touk.nussknacker.engine.management.periodic.model._ -import pl.touk.nussknacker.engine.management.periodic.service._ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.PeriodicProcessStatus +import pl.touk.nussknacker.engine.common.periodic._ +import pl.touk.nussknacker.engine.common.periodic.service._ +import pl.touk.nussknacker.engine.management.periodic.flink.{DeploymentManagerStub, PeriodicDeploymentHandlerStub} import pl.touk.nussknacker.test.PatientScalaFutures -import slick.jdbc -import slick.jdbc.{JdbcBackend, JdbcProfile} +import pl.touk.nussknacker.test.base.db.WithPostgresDbTesting +import pl.touk.nussknacker.test.base.it.WithClock +import pl.touk.nussknacker.test.utils.domain.TestFactory.newWriteProcessRepository +import pl.touk.nussknacker.test.utils.scalas.DBIOActionValues +import pl.touk.nussknacker.ui.db.DbRef +import pl.touk.nussknacker.ui.process.repository.ProcessRepository.CreateProcessAction +import pl.touk.nussknacker.ui.process.repository.{ + DBIOActionRunner, + DBProcessRepository, + SlickPeriodicProcessesRepository +} +import pl.touk.nussknacker.ui.security.api.AdminUser import java.time._ import java.time.temporal.ChronoUnit import java.util.UUID import scala.collection.mutable.ArrayBuffer import scala.concurrent.Future -import scala.jdk.CollectionConverters._ //Integration test with both in-memory hsql and postgres from test containers class PeriodicProcessServiceIntegrationTest @@ -42,23 +51,21 @@ class PeriodicProcessServiceIntegrationTest with OptionValues with ScalaFutures with PatientScalaFutures - with ForAllTestContainer - with LazyLogging { - - override val container: PostgreSQLContainer = PostgreSQLContainer(DockerImageName.parse("postgres:11.2")) + with WithPostgresDbTesting + with LazyLogging + with WithClock + with DBIOActionValues { import scala.concurrent.ExecutionContext.Implicits.global + override protected def dbioRunner: DBIOActionRunner = new DBIOActionRunner(testDbRef) + implicit val freshnessPolicy: DataFreshnessPolicy = DataFreshnessPolicy.Fresh private val processingType = "testProcessingType" private val processName = ProcessName("test") - private val processIdWithName = ProcessIdWithName(ProcessId(1), processName) - - private val sampleProcess = CanonicalProcess(MetaData(processName.value, StreamMetaData()), Nil) - private val startTime = Instant.parse("2021-04-06T13:18:00Z") // we truncate to millis, as HSQL stores with that precision... @@ -74,58 +81,37 @@ class PeriodicProcessServiceIntegrationTest executionConfig: PeriodicExecutionConfig = PeriodicExecutionConfig(), maxFetchedPeriodicScenarioActivities: Option[Int] = None, )(testCode: Fixture => Any): Unit = { - val postgresConfig = ConfigFactory.parseMap( - Map( - "user" -> container.username, - "password" -> container.password, - "url" -> container.jdbcUrl, - "driver" -> "org.postgresql.Driver", - "schema" -> UUID.randomUUID().toString - ).asJava - ) - - val hsqlConfig = ConfigFactory.parseMap( - Map( - "url" -> s"jdbc:hsqldb:mem:periodic-${UUID.randomUUID().toString};sql.syntax_ora=true", - "user" -> "SA", - "password" -> "" - ).asJava - ) - - def runTestCodeWithDbConfig(config: Config) = { - val (db: jdbc.JdbcBackend.DatabaseDef, dbProfile: JdbcProfile) = DbInitializer.init(config) - try { - testCode( - new Fixture(db, dbProfile, deploymentRetryConfig, executionConfig, maxFetchedPeriodicScenarioActivities) + def runTestCodeWithDbConfig = { + testCode( + new Fixture(testDbRef, deploymentRetryConfig, executionConfig, maxFetchedPeriodicScenarioActivities) ) - } finally { - db.close() - } } - logger.debug("Running test with hsql") - runTestCodeWithDbConfig(hsqlConfig) - logger.debug("Running test with postgres") - runTestCodeWithDbConfig(postgresConfig) + logger.debug("Running test with database") + runTestCodeWithDbConfig } class Fixture( - db: JdbcBackend.DatabaseDef, - dbProfile: JdbcProfile, + dbRef: DbRef, deploymentRetryConfig: DeploymentRetryConfig, executionConfig: PeriodicExecutionConfig, maxFetchedPeriodicScenarioActivities: Option[Int], ) { val delegateDeploymentManagerStub = new DeploymentManagerStub - val jarManagerStub = new JarManagerStub + val periodicDeploymentHandlerStub = new PeriodicDeploymentHandlerStub val events = new ArrayBuffer[PeriodicProcessEvent]() var failListener = false - def periodicProcessService(currentTime: Instant, processingType: String = processingType) = + def periodicProcessService( + currentTime: Instant, + deploymentManagerName: String = "testPeriodicDeploymentManager", + processingType: String = processingType + ) = new PeriodicProcessService( delegateDeploymentManager = delegateDeploymentManagerStub, - jarManager = jarManagerStub, - scheduledProcessesRepository = - new SlickPeriodicProcessesRepository(db, dbProfile, fixedClock(currentTime), processingType), + periodicDeploymentHandler = periodicDeploymentHandlerStub, + periodicProcessesManager = new RepositoryBasedPeriodicProcessesManagerProvider( + new SlickPeriodicProcessesRepository(dbRef.db, dbRef.profile, fixedClock(currentTime)) + ).provide(deploymentManagerName, processingType), periodicProcessListener = new PeriodicProcessListener { override def onPeriodicProcessEvent: PartialFunction[PeriodicProcessEvent, Unit] = { @@ -141,9 +127,27 @@ class PeriodicProcessServiceIntegrationTest processConfigEnricher = ProcessConfigEnricher.identity, clock = fixedClock(currentTime), new ProcessingTypeActionServiceStub, - Map.empty + Map.empty, ) + def writeProcessRepository: DBProcessRepository = newWriteProcessRepository(dbRef, clock) + + def prepareProcess(processName: ProcessName): DB[ProcessIdWithName] = { + val canonicalProcess = CanonicalProcess(MetaData(processName.value, StreamMetaData()), Nil) + val action = CreateProcessAction( + processName = processName, + category = "Category1", + canonicalProcess = canonicalProcess, + processingType = "streaming", + isFragment = false, + forwardedUserName = None + ) + writeProcessRepository + .saveNewProcess(action)(AdminUser("artificialTestAdmin", "artificialTestAdmin")) + .map(_.value.processId) + .map(ProcessIdWithName(_, processName)) + } + } it should "handle basic flow" in withFixture() { f => @@ -159,36 +163,34 @@ class PeriodicProcessServiceIntegrationTest def otherProcessingTypeService = f.periodicProcessService(currentTime, processingType = "other") val otherProcessName = ProcessName("other") + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( cronEveryHour, - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue service .schedule( cronEvery30Minutes, ProcessVersion.empty.copy(processName = every30MinutesProcessName), - sampleProcess, - randomProcessActionId + randomProcessActionId, ) .futureValue service .schedule( cronEvery4Hours, ProcessVersion.empty.copy(processName = every4HoursProcessName), - sampleProcess, - randomProcessActionId + randomProcessActionId, ) .futureValue otherProcessingTypeService .schedule( cronEveryHour, ProcessVersion.empty.copy(processName = otherProcessName), - sampleProcess, - randomProcessActionId + randomProcessActionId, ) .futureValue @@ -251,7 +253,7 @@ class PeriodicProcessServiceIntegrationTest val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] activities shouldBe List( ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), + scenarioId = ScenarioId(processIdWithName.id.value), scenarioActivityId = firstActivity.scenarioActivityId, user = ScenarioUser(None, UserName("Nussknacker"), None, None), date = firstActivity.date, @@ -271,15 +273,17 @@ class PeriodicProcessServiceIntegrationTest ) { f => val timeToTriggerCheck = startTime.plus(2, ChronoUnit.HOURS) var currentTime = startTime - f.jarManagerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) + f.periodicDeploymentHandlerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) def service = f.periodicProcessService(currentTime) + + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( cronEveryHour, - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue @@ -300,7 +304,7 @@ class PeriodicProcessServiceIntegrationTest val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] activities shouldBe List( ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), + scenarioId = ScenarioId(processIdWithName.id.value), scenarioActivityId = firstActivity.scenarioActivityId, user = ScenarioUser(None, UserName("Nussknacker"), None, None), date = firstActivity.date, @@ -325,6 +329,9 @@ class PeriodicProcessServiceIntegrationTest val scheduleMinute5 = "scheduleMinute5" val scheduleMinute10 = "scheduleMinute10" + + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( MultipleScheduleProperty( @@ -333,9 +340,8 @@ class PeriodicProcessServiceIntegrationTest scheduleMinute10 -> CronScheduleProperty("0 10 * * * ?") ) ), - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue @@ -349,8 +355,7 @@ class PeriodicProcessServiceIntegrationTest ) ), ProcessVersion.empty.copy(processName = ProcessName("other")), - sampleProcess, - randomProcessActionId + randomProcessActionId, ) .futureValue @@ -390,6 +395,9 @@ class PeriodicProcessServiceIntegrationTest val firstSchedule = "schedule1" val secondSchedule = "schedule2" + + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( MultipleScheduleProperty( @@ -398,9 +406,8 @@ class PeriodicProcessServiceIntegrationTest secondSchedule -> CronScheduleProperty("0 5 * * * ?") ) ), - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue @@ -430,7 +437,7 @@ class PeriodicProcessServiceIntegrationTest } firstActivity shouldBe ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), + scenarioId = ScenarioId(processIdWithName.id.value), scenarioActivityId = firstActivity.scenarioActivityId, user = ScenarioUser(None, UserName("Nussknacker"), None, None), date = firstActivity.date, @@ -521,6 +528,9 @@ class PeriodicProcessServiceIntegrationTest val schedule1 = "schedule1" val schedule2 = "schedule2" + + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( MultipleScheduleProperty( @@ -529,9 +539,8 @@ class PeriodicProcessServiceIntegrationTest schedule2 -> CronScheduleProperty(convertDateToCron(localTime(timeToTriggerSchedule2))) ) ), - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue @@ -603,6 +612,37 @@ class PeriodicProcessServiceIntegrationTest inactiveStates.latestDeploymentForSchedule(schedule1).state.status shouldBe PeriodicProcessDeploymentStatus.Finished inactiveStates.latestDeploymentForSchedule(schedule2).state.status shouldBe PeriodicProcessDeploymentStatus.Finished + val activities = service.getScenarioActivitiesSpecificToPeriodicProcess(processIdWithName).futureValue + val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] + val secondActivity = activities(1).asInstanceOf[ScenarioActivity.PerformedScheduledExecution] + activities shouldBe List( + ScenarioActivity.PerformedScheduledExecution( + scenarioId = ScenarioId(processIdWithName.id.value), + scenarioActivityId = firstActivity.scenarioActivityId, + user = ScenarioUser(None, UserName("Nussknacker"), None, None), + date = firstActivity.date, + scenarioVersionId = Some(ScenarioVersionId(1)), + dateFinished = firstActivity.dateFinished, + scheduleName = "schedule1", + scheduledExecutionStatus = ScheduledExecutionStatus.Finished, + createdAt = firstActivity.createdAt, + retriesLeft = None, + nextRetryAt = None + ), + ScenarioActivity.PerformedScheduledExecution( + scenarioId = ScenarioId(processIdWithName.id.value), + scenarioActivityId = secondActivity.scenarioActivityId, + user = ScenarioUser(None, UserName("Nussknacker"), None, None), + date = secondActivity.date, + scenarioVersionId = Some(ScenarioVersionId(1)), + dateFinished = secondActivity.dateFinished, + scheduleName = "schedule2", + scheduledExecutionStatus = ScheduledExecutionStatus.Finished, + createdAt = secondActivity.createdAt, + retriesLeft = None, + nextRetryAt = None + ), + ) } it should "handle failed event handler" in withFixture() { f => @@ -612,19 +652,20 @@ class PeriodicProcessServiceIntegrationTest def service = f.periodicProcessService(currentTime) - def tryWithFailedListener[T](action: () => Future[T]): T = { + def tryWithFailedListener[T](action: () => Future[T]): Unit = { f.failListener = true - intercept[TestFailedException](action().futureValue).getCause shouldBe a[PeriodicProcessException] + val exception = intercept[TestFailedException](action().futureValue) + exception.getCause shouldBe a[PeriodicProcessException] f.failListener = false - action().futureValue } + val processIdWithName = f.prepareProcess(processName).dbioActionValues + tryWithFailedListener { () => service.schedule( cronEveryHour, - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) } @@ -646,15 +687,16 @@ class PeriodicProcessServiceIntegrationTest val timeToTriggerCheck = startTime.plus(1, ChronoUnit.HOURS) var currentTime = startTime - f.jarManagerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) + f.periodicDeploymentHandlerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) def service = f.periodicProcessService(currentTime) + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( cronEveryHour, - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue currentTime = timeToTriggerCheck @@ -675,7 +717,7 @@ class PeriodicProcessServiceIntegrationTest val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] activities shouldBe List( ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), + scenarioId = ScenarioId(processIdWithName.id.value), scenarioActivityId = firstActivity.scenarioActivityId, user = ScenarioUser(None, UserName("Nussknacker"), None, None), date = firstActivity.date, diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActor.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala similarity index 71% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActor.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala index 1beb82e32ac..ba383b46eab 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActor.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala @@ -1,15 +1,13 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import akka.actor.{Actor, Props, Timers} import com.typesafe.scalalogging.LazyLogging -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.DeploymentActor.{ +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment +import pl.touk.nussknacker.engine.common.periodic.DeploymentActor.{ CheckToBeDeployed, DeploymentCompleted, WaitingForDeployment } -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment import scala.concurrent.Future import scala.concurrent.duration._ @@ -21,24 +19,24 @@ object DeploymentActor { props(service.findToBeDeployed, service.deploy, interval) } - private[periodic] def props( - findToBeDeployed: => Future[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]], - deploy: PeriodicProcessDeployment[WithCanonicalProcess] => Future[Unit], + private[engine] def props( + findToBeDeployed: => Future[Seq[PeriodicProcessDeployment]], + deploy: PeriodicProcessDeployment => Future[Unit], interval: FiniteDuration ) = { Props(new DeploymentActor(findToBeDeployed, deploy, interval)) } - private[periodic] case object CheckToBeDeployed + private[engine] case object CheckToBeDeployed - private case class WaitingForDeployment(ids: List[PeriodicProcessDeployment[WithCanonicalProcess]]) + private case class WaitingForDeployment(ids: List[PeriodicProcessDeployment]) private case object DeploymentCompleted } class DeploymentActor( - findToBeDeployed: => Future[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]], - deploy: PeriodicProcessDeployment[WithCanonicalProcess] => Future[Unit], + findToBeDeployed: => Future[Seq[PeriodicProcessDeployment]], + deploy: PeriodicProcessDeployment => Future[Unit], interval: FiniteDuration ) extends Actor with Timers @@ -74,7 +72,7 @@ class DeploymentActor( } } - private def receiveOngoingDeployment(runDetails: PeriodicProcessDeployment[WithCanonicalProcess]): Receive = { + private def receiveOngoingDeployment(runDetails: PeriodicProcessDeployment): Receive = { case CheckToBeDeployed => logger.debug(s"Still waiting for ${runDetails.display} to be deployed") case DeploymentCompleted => diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicBatchConfig.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicBatchConfig.scala similarity index 96% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicBatchConfig.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicBatchConfig.scala index 61b8f9bb1ac..138a166ace1 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicBatchConfig.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicBatchConfig.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import com.typesafe.config.Config diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala new file mode 100644 index 00000000000..81a0b8bd3f2 --- /dev/null +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala @@ -0,0 +1,26 @@ +package pl.touk.nussknacker.engine.common.periodic + +import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{DeploymentWithRuntimeParams, RuntimeParams} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} + +import scala.concurrent.Future + +trait PeriodicDeploymentHandler { + + def prepareDeploymentWithRuntimeParams( + processVersion: ProcessVersion, + ): Future[DeploymentWithRuntimeParams] + + def deployWithRuntimeParams( + deploymentWithJarData: DeploymentWithRuntimeParams, + deploymentData: DeploymentData, + canonicalProcess: CanonicalProcess, + ): Future[Option[ExternalDeploymentId]] + + def cleanAfterDeployment( + runtimeParams: RuntimeParams + ): Future[Unit] + +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala similarity index 88% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala index 9ad4fdf8342..162b98de32e 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala @@ -1,29 +1,21 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import cats.data.OptionT import com.typesafe.config.Config import com.typesafe.scalalogging.LazyLogging +import pl.touk.nussknacker.engine.DeploymentManagerDependencies import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager import pl.touk.nussknacker.engine.api.process.{ProcessIdWithName, ProcessName, VersionId} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, ExternalDeploymentId, RunOffScheduleResult} -import pl.touk.nussknacker.engine.management.FlinkConfig -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.PeriodicProcessStatus -import pl.touk.nussknacker.engine.management.periodic.Utils.{createActorWithRetry, runSafely} -import pl.touk.nussknacker.engine.management.periodic.db.{ - DbInitializer, - PeriodicProcessesRepository, - SlickPeriodicProcessesRepository -} -import pl.touk.nussknacker.engine.management.periodic.flink.FlinkJarManager -import pl.touk.nussknacker.engine.management.periodic.service.{ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.PeriodicProcessStatus +import pl.touk.nussknacker.engine.common.periodic.Utils.{createActorWithRetry, runSafely} +import pl.touk.nussknacker.engine.common.periodic.service.{ AdditionalDeploymentDataProvider, PeriodicProcessListenerFactory, ProcessConfigEnricherFactory } -import pl.touk.nussknacker.engine.{BaseModelData, DeploymentManagerDependencies} -import slick.jdbc -import slick.jdbc.JdbcProfile +import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, ExternalDeploymentId} import java.time.{Clock, Instant} import scala.concurrent.{ExecutionContext, Future} @@ -32,30 +24,25 @@ object PeriodicDeploymentManager { def apply( delegate: DeploymentManager, + periodicDeploymentHandler: PeriodicDeploymentHandler, schedulePropertyExtractorFactory: SchedulePropertyExtractorFactory, processConfigEnricherFactory: ProcessConfigEnricherFactory, periodicBatchConfig: PeriodicBatchConfig, - flinkConfig: FlinkConfig, originalConfig: Config, - modelData: BaseModelData, listenerFactory: PeriodicProcessListenerFactory, additionalDeploymentDataProvider: AdditionalDeploymentDataProvider, - dependencies: DeploymentManagerDependencies + dependencies: DeploymentManagerDependencies, + periodicProcessesManager: PeriodicProcessesManager, ): PeriodicDeploymentManager = { import dependencies._ - val clock = Clock.systemDefaultZone() - - val (db: jdbc.JdbcBackend.DatabaseDef, dbProfile: JdbcProfile) = DbInitializer.init(periodicBatchConfig.db) - val scheduledProcessesRepository = - new SlickPeriodicProcessesRepository(db, dbProfile, clock, periodicBatchConfig.processingType) - val jarManager = FlinkJarManager(flinkConfig, periodicBatchConfig, modelData) + val clock = Clock.systemDefaultZone() val listener = listenerFactory.create(originalConfig) val processConfigEnricher = processConfigEnricherFactory(originalConfig) val service = new PeriodicProcessService( delegate, - jarManager, - scheduledProcessesRepository, + periodicDeploymentHandler, + periodicProcessesManager, listener, additionalDeploymentDataProvider, periodicBatchConfig.deploymentRetry, @@ -64,7 +51,7 @@ object PeriodicDeploymentManager { processConfigEnricher, clock, dependencies.actionService, - dependencies.configsFromProvider + dependencies.configsFromProvider, ) // These actors have to be created with retries because they can initially fail to create due to taken names, @@ -80,30 +67,39 @@ object PeriodicDeploymentManager { dependencies.actorSystem ) + val customActionsProvider = customActionsProviderFactory.create( + periodicProcessesManager, + service, + periodicBatchConfig.processingType + ) + val toClose = () => { runSafely(listener.close()) // deploymentActor and rescheduleFinishedActor just call methods from PeriodicProcessService on interval, // they don't have any internal state, so stopping them non-gracefully is safe runSafely(dependencies.actorSystem.stop(deploymentActor)) runSafely(dependencies.actorSystem.stop(rescheduleFinishedActor)) - runSafely(db.close()) } new PeriodicDeploymentManager( delegate, service, scheduledProcessesRepository, schedulePropertyExtractorFactory(originalConfig), + customActionsProvider, + periodicBatchConfig.processingType, toClose ) } } -class PeriodicDeploymentManager private[periodic] ( +class PeriodicDeploymentManager private[engine] ( val delegate: DeploymentManager, service: PeriodicProcessService, repository: PeriodicProcessesRepository, schedulePropertyExtractor: SchedulePropertyExtractor, + customActionsProvider: PeriodicCustomActionsProvider, + processingType: String, toClose: () => Unit )(implicit val ec: ExecutionContext) extends DeploymentManager @@ -145,7 +141,6 @@ class PeriodicDeploymentManager private[periodic] ( .schedule( scheduleProperty, processVersion, - canonicalProcess, deploymentData.deploymentId.toActionIdOpt.getOrElse( throw new IllegalArgumentException(s"deploymentData.deploymentId should be valid ProcessActionId") ), diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala similarity index 74% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala index bd64c7e8bd8..3f5e2bfa414 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import cats.data.ValidatedNel import com.typesafe.config.Config @@ -6,11 +6,9 @@ import com.typesafe.scalalogging.LazyLogging import pl.touk.nussknacker.engine.api.component.ScenarioPropertyConfig import pl.touk.nussknacker.engine.api.definition.{MandatoryParameterValidator, StringParameterEditor} import pl.touk.nussknacker.engine.api.deployment.DeploymentManager +import pl.touk.nussknacker.engine.common.periodic.cron.CronParameterValidator +import pl.touk.nussknacker.engine.common.periodic.service._ import pl.touk.nussknacker.engine.deployment.EngineSetupName -import pl.touk.nussknacker.engine.management.periodic.cron.CronParameterValidator -import pl.touk.nussknacker.engine.management.{FlinkConfig, FlinkStreamingDeploymentManagerProvider} -import pl.touk.nussknacker.engine.management.periodic.service._ -import pl.touk.nussknacker.engine.util.config.ConfigEnrichments.RichConfig import pl.touk.nussknacker.engine.{ BaseModelData, DeploymentManagerDependencies, @@ -20,9 +18,11 @@ import pl.touk.nussknacker.engine.{ import scala.concurrent.duration.FiniteDuration -class FlinkPeriodicDeploymentManagerProvider extends DeploymentManagerProvider with LazyLogging { - - private val delegate = new FlinkStreamingDeploymentManagerProvider() +abstract class PeriodicDeploymentManagerProvider( + override val name: String, + delegate: DeploymentManagerProvider, +) extends DeploymentManagerProvider + with LazyLogging { private val cronConfig = CronSchedulePropertyExtractor.CronPropertyDefaultName -> ScenarioPropertyConfig( defaultValue = None, @@ -32,7 +32,11 @@ class FlinkPeriodicDeploymentManagerProvider extends DeploymentManagerProvider w hintText = Some("Quartz cron syntax. You can specify multiple schedulers separated by '|'.") ) - override def name: String = "flinkPeriodic" + protected def createPeriodicDeploymentHandler( + modelData: BaseModelData, + dependencies: DeploymentManagerDependencies, + config: Config, + ): PeriodicDeploymentHandler override def createDeploymentManager( modelData: BaseModelData, @@ -40,25 +44,24 @@ class FlinkPeriodicDeploymentManagerProvider extends DeploymentManagerProvider w config: Config, scenarioStateCacheTTL: Option[FiniteDuration] ): ValidatedNel[String, DeploymentManager] = { - logger.info("Creating FlinkPeriodic scenario manager") + logger.info("Creating periodic scenario manager") delegate.createDeploymentManager(modelData, dependencies, config, scenarioStateCacheTTL).map { delegateDeploymentManager => import net.ceedubs.ficus.Ficus._ import net.ceedubs.ficus.readers.ArbitraryTypeReader._ val periodicBatchConfig = config.as[PeriodicBatchConfig]("deploymentManager") - val flinkConfig = config.rootAs[FlinkConfig] PeriodicDeploymentManager( delegate = delegateDeploymentManager, + periodicDeploymentHandler = createPeriodicDeploymentHandler(modelData, dependencies, config), schedulePropertyExtractorFactory = _ => CronSchedulePropertyExtractor(), processConfigEnricherFactory = ProcessConfigEnricherFactory.noOp, periodicBatchConfig = periodicBatchConfig, - flinkConfig = flinkConfig, originalConfig = config, - modelData = modelData, EmptyPeriodicProcessListenerFactory, DefaultAdditionalDeploymentDataProvider, - dependencies + dependencies, + dependencies.periodicProcessesManagerProvider.provide(name, periodicBatchConfig.processingType) ) } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessException.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessException.scala similarity index 74% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessException.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessException.scala index 72a267b3317..9890980319f 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessException.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessException.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic class PeriodicProcessException(message: String, parent: Throwable) extends RuntimeException(message, parent) { def this(message: String) = this(message, null) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala similarity index 84% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala index 6e48530d7c6..bfbd375cb38 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import cats.implicits._ import com.typesafe.scalalogging.LazyLogging @@ -10,28 +10,23 @@ import pl.touk.nussknacker.engine.api.component.{ } import pl.touk.nussknacker.engine.api.deployment.StateStatus.StatusName import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus import pl.touk.nussknacker.engine.api.process.{ProcessIdWithName, ProcessName} -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{AdditionalModelConfigs, DeploymentData, DeploymentId} -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.{ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.{ DeploymentStatus, EngineStatusesToReschedule, FinishedScheduledExecutionMetadata, MaxDeploymentsStatus, PeriodicProcessStatus } -import pl.touk.nussknacker.engine.management.periodic.PeriodicStateStatus.{ScheduledStatus, WaitingForScheduleStatus} -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.{ - WithCanonicalProcess, - WithoutCanonicalProcess -} -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model._ -import pl.touk.nussknacker.engine.management.periodic.service._ -import pl.touk.nussknacker.engine.management.periodic.util.DeterministicUUIDFromLong +import pl.touk.nussknacker.engine.common.periodic.PeriodicStateStatus.{ScheduledStatus, WaitingForScheduleStatus} +import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} +import pl.touk.nussknacker.engine.common.periodic.service._ +import pl.touk.nussknacker.engine.deployment.{AdditionalModelConfigs, DeploymentData, DeploymentId} import pl.touk.nussknacker.engine.util.AdditionalComponentConfigsForRuntimeExtractor import java.time.chrono.ChronoLocalDateTime @@ -42,8 +37,8 @@ import scala.util.control.NonFatal class PeriodicProcessService( delegateDeploymentManager: DeploymentManager, - jarManager: JarManager, - scheduledProcessesRepository: PeriodicProcessesRepository, + periodicDeploymentHandler: PeriodicDeploymentHandler, + periodicProcessesManager: PeriodicProcessesManager, periodicProcessListener: PeriodicProcessListener, additionalDeploymentDataProvider: AdditionalDeploymentDataProvider, deploymentRetryConfig: DeploymentRetryConfig, @@ -52,19 +47,18 @@ class PeriodicProcessService( processConfigEnricher: ProcessConfigEnricher, clock: Clock, actionService: ProcessingTypeActionService, - configsFromProvider: Map[DesignerWideComponentId, ComponentAdditionalConfig] + configsFromProvider: Map[DesignerWideComponentId, ComponentAdditionalConfig], )(implicit ec: ExecutionContext) extends LazyLogging { import cats.syntax.all._ - import scheduledProcessesRepository._ - private type RepositoryAction[T] = scheduledProcessesRepository.Action[T] - private type Callback = () => Future[Unit] - private type NeedsReschedule = Boolean - private implicit class WithCallbacksSeq(result: RepositoryAction[List[Callback]]) { + private type Callback = () => Future[Unit] + private type NeedsReschedule = Boolean + + private implicit class WithCallbacksSeq(result: Future[List[Callback]]) { def runWithCallbacks: Future[Unit] = - result.run.flatMap(callbacks => Future.sequence(callbacks.map(_()))).map(_ => ()) + result.flatMap(callbacks => Future.sequence(callbacks.map(_()))).map(_ => ()) } private val emptyCallback: Callback = () => Future.successful(()) @@ -75,9 +69,7 @@ class PeriodicProcessService( processIdWithName: ProcessIdWithName, after: Option[Instant], ): Future[List[ScenarioActivity]] = for { - schedulesState <- scheduledProcessesRepository - .getSchedulesState(processIdWithName.name, after.map(localDateTimeAtSystemDefaultZone)) - .run + schedulesState <- periodicProcessesManager.getSchedulesState(processIdWithName.name) groupedByProcess = schedulesState.groupedByPeriodicProcess deployments = groupedByProcess.flatMap(_.deployments) deploymentsWithStatuses = deployments.flatMap(d => scheduledExecutionStatusAndDateFinished(d).map((d, _))) @@ -109,15 +101,12 @@ class PeriodicProcessService( def schedule( schedule: ScheduleProperty, processVersion: ProcessVersion, - canonicalProcess: CanonicalProcess, processActionId: ProcessActionId, beforeSchedule: => Future[Unit] = Future.unit ): Future[Unit] = { prepareInitialScheduleDates(schedule) match { case Right(scheduleDates) => - beforeSchedule.flatMap(_ => - scheduleWithInitialDates(schedule, processVersion, canonicalProcess, scheduleDates, processActionId) - ) + beforeSchedule.flatMap(_ => scheduleWithInitialDates(schedule, processVersion, scheduleDates, processActionId)) case Left(error) => Future.failed(error) } @@ -146,58 +135,62 @@ class PeriodicProcessService( private def scheduleWithInitialDates( scheduleProperty: ScheduleProperty, processVersion: ProcessVersion, - canonicalProcess: CanonicalProcess, scheduleDates: List[(ScheduleName, Option[LocalDateTime])], - processActionId: ProcessActionId + processActionId: ProcessActionId, ): Future[Unit] = { logger.info("Scheduling periodic scenario: {} on {}", processVersion, scheduleDates) for { - deploymentWithJarData <- jarManager.prepareDeploymentWithJar(processVersion, canonicalProcess) + deploymentWithJarData <- periodicDeploymentHandler.prepareDeploymentWithRuntimeParams( + processVersion, + ) enrichedProcessConfig <- processConfigEnricher.onInitialSchedule( ProcessConfigEnricher.InitialScheduleData( - deploymentWithJarData.process, deploymentWithJarData.inputConfigDuringExecutionJson ) ) enrichedDeploymentWithJarData = deploymentWithJarData.copy(inputConfigDuringExecutionJson = enrichedProcessConfig.inputConfigDuringExecutionJson ) - _ <- initialSchedule(scheduleProperty, scheduleDates, enrichedDeploymentWithJarData, processActionId) + _ <- initialSchedule( + scheduleProperty, + scheduleDates, + enrichedDeploymentWithJarData, + processActionId, + ) } yield () } private def initialSchedule( scheduleMap: ScheduleProperty, scheduleDates: List[(ScheduleName, Option[LocalDateTime])], - deploymentWithJarData: DeploymentWithJarData.WithCanonicalProcess, - processActionId: ProcessActionId + deploymentWithJarData: DeploymentWithRuntimeParams, + processActionId: ProcessActionId, ): Future[Unit] = { - scheduledProcessesRepository - .create(deploymentWithJarData, scheduleMap, processActionId) + periodicProcessesManager + .create(deploymentWithJarData, toApi(scheduleMap), processActionId) .flatMap { process => scheduleDates.collect { case (name, Some(date)) => - scheduledProcessesRepository + periodicProcessesManager .schedule(process.id, name, date, deploymentRetryConfig.deployMaxRetries) .flatMap { data => handleEvent(ScheduledEvent(data, firstSchedule = true)) } case (name, None) => logger.warn(s"Schedule $name does not have date to schedule") - monad.pure(()) + Future.successful(()) }.sequence } - .run .map(_ => ()) } - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] = { + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { for { - toBeDeployed <- scheduledProcessesRepository.findToBeDeployed.run.flatMap { toDeployList => + toBeDeployed <- periodicProcessesManager.findToBeDeployed.flatMap { toDeployList => Future.sequence(toDeployList.map(checkIfNotRunning)).map(_.flatten) } // We retry scenarios that failed on deployment. Failure recovery of running scenarios should be handled by Flink's restart strategy - toBeRetried <- scheduledProcessesRepository.findToBeRetried.run + toBeRetried <- periodicProcessesManager.findToBeRetried // We don't block scheduled deployments by retries } yield toBeDeployed.sortBy(d => (d.runAt, d.createdAt)) ++ toBeRetried.sortBy(d => (d.nextRetryAt, d.createdAt)) } @@ -205,8 +198,8 @@ class PeriodicProcessService( // Currently we don't allow simultaneous runs of one scenario - only sequential, so if other schedule kicks in, it'll have to wait // TODO: we show allow to deploy scenarios with different scheduleName to be deployed simultaneous private def checkIfNotRunning( - toDeploy: PeriodicProcessDeployment[WithCanonicalProcess] - ): Future[Option[PeriodicProcessDeployment[WithCanonicalProcess]]] = { + toDeploy: PeriodicProcessDeployment + ): Future[Option[PeriodicProcessDeployment]] = { delegateDeploymentManager .getProcessStates(toDeploy.periodicProcess.processVersion.processName)(DataFreshnessPolicy.Fresh) .map( @@ -228,7 +221,7 @@ class PeriodicProcessService( schedules.groupedByPeriodicProcess .collect { case processScheduleData - if processScheduleData.existsDeployment(d => needRescheduleDeploymentIds.contains(d.id)) => + if processScheduleData.deployments.exists(d => needRescheduleDeploymentIds.contains(d.id)) => reschedule(processScheduleData, needRescheduleDeploymentIds) } .sequence @@ -236,11 +229,10 @@ class PeriodicProcessService( } for { - schedules <- scheduledProcessesRepository + schedules <- periodicProcessesManager .findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( - Set(PeriodicProcessDeploymentStatus.Deployed, PeriodicProcessDeploymentStatus.FailedOnDeploy) + Set(PeriodicProcessDeploymentStatus.Deployed, PeriodicProcessDeploymentStatus.FailedOnDeploy), ) - .run // we handle each job separately, if we fail at some point, we will continue on next handleFinished run _ <- Future.sequence(schedules.groupByProcessName.toList.map(handleSingleProcess _ tupled)) } yield () @@ -269,7 +261,7 @@ class PeriodicProcessService( ) needRescheduleDeployments <- Future .sequence(scheduleDeploymentsWithStatus.map { case (deploymentData, statusOpt) => - synchronizeDeploymentState(deploymentData, statusOpt).run.map { needReschedule => + synchronizeDeploymentState(deploymentData, statusOpt).map { needReschedule => Option(deploymentData.id).filter(_ => needReschedule) } }) @@ -284,9 +276,9 @@ class PeriodicProcessService( private def synchronizeDeploymentState( deployment: ScheduleDeploymentData, processState: Option[StatusDetails] - ): RepositoryAction[NeedsReschedule] = { - implicit class RichRepositoryAction[Unit](a: RepositoryAction[Unit]) { - def needsReschedule(value: Boolean): RepositoryAction[NeedsReschedule] = a.map(_ => value) + ): Future[NeedsReschedule] = { + implicit class RichFuture[Unit](a: Future[Unit]) { + def needsReschedule(value: Boolean): Future[NeedsReschedule] = a.map(_ => value) } processState.map(_.status) match { case Some(status) @@ -308,21 +300,21 @@ class PeriodicProcessService( // so freshly deployed deployments aren't considered markFinished(deployment, processState).needsReschedule(value = true) case _ => - scheduledProcessesRepository.monad.pure(()).needsReschedule(value = false) + Future.successful(()).needsReschedule(value = false) } } private def reschedule( processScheduleData: PeriodicProcessScheduleData, needRescheduleDeploymentIds: Set[PeriodicProcessDeploymentId] - ): RepositoryAction[Callback] = { + ): Future[Callback] = { import processScheduleData._ val scheduleActions = deployments.map { deployment => if (needRescheduleDeploymentIds.contains(deployment.id)) - deployment.nextRunAt(clock) match { + nextRunAt(deployment, clock) match { case Right(Some(futureDate)) => logger.info(s"Rescheduling ${deployment.display} to $futureDate") - val action = scheduledProcessesRepository + val action = periodicProcessesManager .schedule(process.id, deployment.scheduleName, futureDate, deploymentRetryConfig.deployMaxRetries) .flatMap { data => handleEvent(ScheduledEvent(data, firstSchedule = false)) @@ -338,7 +330,7 @@ class PeriodicProcessService( else Option(deployment) .filter(_.state.status == PeriodicProcessDeploymentStatus.Scheduled) - .map(_ => scheduledProcessesRepository.monad.pure(())) + .map(_ => Future.successful(())) } @@ -352,18 +344,26 @@ class PeriodicProcessService( scheduleActions.flatten.sequence.as(emptyCallback) } - private def markFinished(deployment: ScheduleDeploymentData, state: Option[StatusDetails]): RepositoryAction[Unit] = { + private def nextRunAt(deployment: PeriodicProcessDeployment, clock: Clock): Either[String, Option[LocalDateTime]] = + (fromApi(deployment.periodicProcess.scheduleProperty), deployment.scheduleName.value) match { + case (MultipleScheduleProperty(schedules), Some(name)) => + schedules.get(name).toRight(s"Failed to find schedule: $deployment.scheduleName").flatMap(_.nextRunAt(clock)) + case (e: SingleScheduleProperty, None) => e.nextRunAt(clock) + case (schedule, name) => Left(s"Schedule name: $name mismatch with schedule: $schedule") + } + + private def markFinished(deployment: ScheduleDeploymentData, state: Option[StatusDetails]): Future[Unit] = { logger.info(s"Marking ${deployment.display} with status: ${deployment.state.status} as finished") for { - _ <- scheduledProcessesRepository.markFinished(deployment.id) - currentState <- scheduledProcessesRepository.findProcessData(deployment.id) + _ <- periodicProcessesManager.markFinished(deployment.id) + currentState <- periodicProcessesManager.findProcessData(deployment.id) } yield handleEvent(FinishedEvent(currentState, state)) } private def handleFailedDeployment( - deployment: PeriodicProcessDeployment[_], + deployment: PeriodicProcessDeployment, state: Option[StatusDetails] - ): RepositoryAction[Unit] = { + ): Future[Unit] = { def calculateNextRetryAt = now().plus(deploymentRetryConfig.deployRetryPenalize.toMillis, ChronoUnit.MILLIS) val retriesLeft = @@ -382,19 +382,19 @@ class PeriodicProcessService( ) for { - _ <- scheduledProcessesRepository.markFailedOnDeployWithStatus(deployment.id, status, retriesLeft, nextRetryAt) - currentState <- scheduledProcessesRepository.findProcessData(deployment.id) + _ <- periodicProcessesManager.markFailedOnDeployWithStatus(deployment.id, status, retriesLeft, nextRetryAt) + currentState <- periodicProcessesManager.findProcessData(deployment.id) } yield handleEvent(FailedOnDeployEvent(currentState, state)) } private def markFailedAction( deployment: ScheduleDeploymentData, state: Option[StatusDetails] - ): RepositoryAction[Unit] = { + ): Future[Unit] = { logger.info(s"Marking ${deployment.display} as failed.") for { - _ <- scheduledProcessesRepository.markFailed(deployment.id) - currentState <- scheduledProcessesRepository.findProcessData(deployment.id) + _ <- periodicProcessesManager.markFailed(deployment.id) + currentState <- periodicProcessesManager.findProcessData(deployment.id) } yield handleEvent(FailedOnRunEvent(currentState, state)) } @@ -405,26 +405,26 @@ class PeriodicProcessService( _ <- activeSchedules.groupedByPeriodicProcess.map(p => deactivateAction(p.process)).sequence.runWithCallbacks } yield runningDeploymentsForSchedules.map(deployment => DeploymentId(deployment.toString)) - private def deactivateAction(process: PeriodicProcess[WithoutCanonicalProcess]): RepositoryAction[Callback] = { + private def deactivateAction(process: PeriodicProcess): Future[Callback] = { logger.info(s"Deactivate periodic process id: ${process.id.value}") for { - _ <- scheduledProcessesRepository.markInactive(process.id) + _ <- periodicProcessesManager.markInactive(process.id) // we want to delete jars only after we successfully mark process as inactive. It's better to leave jar garbage than // have process without jar - } yield () => jarManager.deleteJar(process.deploymentData.jarFileName) + } yield () => periodicDeploymentHandler.cleanAfterDeployment(process.deploymentData.runtimeParams) } private def markProcessActionExecutionFinished( processActionIdOption: Option[ProcessActionId] - ): RepositoryAction[Callback] = - scheduledProcessesRepository.monad.pure { () => + ): Future[Callback] = + Future.successful { () => processActionIdOption .map(actionService.markActionExecutionFinished) .sequence .map(_ => ()) } - def deploy(deployment: PeriodicProcessDeployment[WithCanonicalProcess]): Future[Unit] = { + def deploy(deployment: PeriodicProcessDeployment): Future[Unit] = { // TODO: set status before deployment? val id = deployment.id val deploymentData = DeploymentData( @@ -442,38 +442,48 @@ class PeriodicProcessService( _ <- Future.successful( logger.info("Deploying scenario {} for deployment id {}", deploymentWithJarData.processVersion, id) ) + processName = deploymentWithJarData.processVersion.processName + versionId = deploymentWithJarData.processVersion.versionId + canonicalProcessOrError <- periodicProcessesManager.fetchCanonicalProcess(processName, versionId) + canonicalProcess = canonicalProcessOrError.getOrElse { + throw new PeriodicProcessException( + s"Could not fetch CanonicalProcess for processName=$processName, versionId=$versionId" + ) + } enrichedProcessConfig <- processConfigEnricher.onDeploy( ProcessConfigEnricher.DeployData( - deploymentWithJarData.process, deploymentWithJarData.inputConfigDuringExecutionJson, - deployment + deployment, ) ) enrichedDeploymentWithJarData = deploymentWithJarData.copy(inputConfigDuringExecutionJson = enrichedProcessConfig.inputConfigDuringExecutionJson ) - externalDeploymentId <- jarManager.deployWithJar(enrichedDeploymentWithJarData, deploymentData) + externalDeploymentId <- periodicDeploymentHandler.deployWithRuntimeParams( + enrichedDeploymentWithJarData, + deploymentData, + canonicalProcess, + ) } yield externalDeploymentId deploymentAction .flatMap { externalDeploymentId => logger.info("Scenario has been deployed {} for deployment id {}", deploymentWithJarData.processVersion, id) // TODO: add externalDeploymentId?? - scheduledProcessesRepository + periodicProcessesManager .markDeployed(id) - .flatMap(_ => scheduledProcessesRepository.findProcessData(id)) + .flatMap(_ => periodicProcessesManager.findProcessData(id)) .flatMap(afterChange => handleEvent(DeployedEvent(afterChange, externalDeploymentId))) - .run } // We can recover since deployment actor watches only future completion. .recoverWith { case exception => logger.error(s"Scenario deployment ${deployment.display} failed", exception) - handleFailedDeployment(deployment, None).run + handleFailedDeployment(deployment, None) } } // TODO: allow access to DB in listener? - private def handleEvent(event: PeriodicProcessEvent): scheduledProcessesRepository.Action[Unit] = { - scheduledProcessesRepository.monad.pure { + private def handleEvent(event: PeriodicProcessEvent): Future[Unit] = { + Future.successful { try { periodicProcessListener.onPeriodicProcessEvent.applyOrElse(event, (_: PeriodicProcessEvent) => ()) } catch { @@ -532,20 +542,22 @@ class PeriodicProcessService( processName: ProcessName, deploymentsPerScheduleMaxCount: Int = 1 ): Future[SchedulesState] = - scheduledProcessesRepository.getLatestDeploymentsForActiveSchedules(processName, deploymentsPerScheduleMaxCount).run + periodicProcessesManager.getLatestDeploymentsForActiveSchedules( + processName, + deploymentsPerScheduleMaxCount, + ) def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int ): Future[SchedulesState] = - scheduledProcessesRepository + periodicProcessesManager .getLatestDeploymentsForLatestInactiveSchedules( processName, inactiveProcessesMaxCount, - deploymentsPerScheduleMaxCount + deploymentsPerScheduleMaxCount, ) - .run implicit class RuntimeStatusesExt(runtimeStatuses: List[StatusDetails]) { @@ -556,7 +568,7 @@ class PeriodicProcessService( } private def scheduledExecutionStatusAndDateFinished( - entity: PeriodicProcessDeployment[WithoutCanonicalProcess], + entity: PeriodicProcessDeployment, ): Option[FinishedScheduledExecutionMetadata] = { for { status <- entity.state.status match { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManager.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessStateDefinitionManager.scala similarity index 92% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManager.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessStateDefinitionManager.scala index 008aabcca05..709735a125f 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManager.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessStateDefinitionManager.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import pl.touk.nussknacker.engine.api.deployment.ProcessStateDefinitionManager.defaultVisibleActions import pl.touk.nussknacker.engine.api.deployment.{ @@ -7,7 +7,7 @@ import pl.touk.nussknacker.engine.api.deployment.{ ScenarioActionName, StateStatus } -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.{DeploymentStatus, PeriodicProcessStatus} +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.{DeploymentStatus, PeriodicProcessStatus} class PeriodicProcessStateDefinitionManager(delegate: ProcessStateDefinitionManager) extends OverridingProcessStateDefinitionManager( diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicStateStatus.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicStateStatus.scala similarity index 98% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicStateStatus.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicStateStatus.scala index a90c0cee3fb..a1a0e68dd13 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicStateStatus.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicStateStatus.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import pl.touk.nussknacker.engine.api.deployment.ProcessStateDefinitionManager.ProcessStatus import pl.touk.nussknacker.engine.api.deployment.StateStatus.StatusName diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActor.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/RescheduleFinishedActor.scala similarity index 84% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActor.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/RescheduleFinishedActor.scala index d53b95c95c0..f4ed347af3e 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActor.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/RescheduleFinishedActor.scala @@ -1,8 +1,8 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import akka.actor.{Actor, Props, Timers} import com.typesafe.scalalogging.LazyLogging -import pl.touk.nussknacker.engine.management.periodic.RescheduleFinishedActor.{CheckStates, CheckStatesCompleted} +import pl.touk.nussknacker.engine.common.periodic.RescheduleFinishedActor.{CheckStates, CheckStatesCompleted} import scala.concurrent.Future import scala.concurrent.duration._ @@ -14,7 +14,7 @@ object RescheduleFinishedActor { props(service.handleFinished, interval) } - private[periodic] def props(handleFinished: => Future[Unit], interval: FiniteDuration): Props = { + private[engine] def props(handleFinished: => Future[Unit], interval: FiniteDuration): Props = { Props(new RescheduleFinishedActor(handleFinished, interval)) } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractor.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractor.scala similarity index 95% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractor.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractor.scala index 989d625cb51..d692045356c 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractor.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractor.scala @@ -1,10 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import cats.instances.list._ import cats.syntax.traverse._ import com.typesafe.scalalogging.LazyLogging import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.CronSchedulePropertyExtractor.CronPropertyDefaultName +import pl.touk.nussknacker.engine.common.periodic.CronSchedulePropertyExtractor.CronPropertyDefaultName import java.time.Clock diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractorFactory.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractorFactory.scala similarity index 70% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractorFactory.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractorFactory.scala index 209f9adb675..6acc84c23f4 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractorFactory.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractorFactory.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import com.typesafe.config.Config diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SingleScheduleProperty.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SingleScheduleProperty.scala similarity index 64% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SingleScheduleProperty.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SingleScheduleProperty.scala index 97b7aa80af9..91e23eeb792 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SingleScheduleProperty.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SingleScheduleProperty.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import com.cronutils.model.definition.CronDefinitionBuilder import com.cronutils.model.time.ExecutionTime @@ -6,12 +6,44 @@ import com.cronutils.model.{Cron, CronType} import com.cronutils.parser.CronParser import io.circe.generic.JsonCodec import io.circe.generic.extras.{Configuration, ConfiguredJsonCodec} +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager import java.time.{Clock, LocalDateTime, ZoneId, ZonedDateTime} +import scala.collection.compat._ import scala.util.Try object ScheduleProperty { implicit val configuration: Configuration = Configuration.default.withDefaults.withDiscriminator("type") + + def toApi(scheduleProperty: ScheduleProperty): PeriodicProcessesManager.ScheduleProperty = scheduleProperty match { + case MultipleScheduleProperty(schedules) => + PeriodicProcessesManager.MultipleScheduleProperty(schedules.view.mapValues(singleToApi).toMap) + case p: CronScheduleProperty => + singleToApi(p) + } + + private def singleToApi( + scheduleProperty: SingleScheduleProperty + ): PeriodicProcessesManager.SingleScheduleProperty = scheduleProperty match { + case CronScheduleProperty(labelOrCronExpr) => + PeriodicProcessesManager.CronScheduleProperty(labelOrCronExpr) + } + + def fromApi( + scheduleProperty: PeriodicProcessesManager.ScheduleProperty + ): ScheduleProperty = scheduleProperty match { + case PeriodicProcessesManager.MultipleScheduleProperty(schedules) => + MultipleScheduleProperty(schedules.view.mapValues(singleFromApi).toMap) + case p: PeriodicProcessesManager.CronScheduleProperty => + singleFromApi(p) + } + + private def singleFromApi( + singleScheduleProperty: PeriodicProcessesManager.SingleScheduleProperty + ): SingleScheduleProperty = singleScheduleProperty match { + case PeriodicProcessesManager.CronScheduleProperty(labelOrCronExpr) => CronScheduleProperty(labelOrCronExpr) + } + } @ConfiguredJsonCodec sealed trait ScheduleProperty @@ -34,7 +66,7 @@ object SingleScheduleProperty { @JsonCodec case class CronScheduleProperty(labelOrCronExpr: String) extends SingleScheduleProperty { import cats.implicits._ - import pl.touk.nussknacker.engine.management.periodic.CronScheduleProperty._ + import pl.touk.nussknacker.engine.common.periodic.CronScheduleProperty._ private lazy val cronsOrError: Either[String, List[Cron]] = { val (errors, crons) = labelOrCronExpr diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/Utils.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/Utils.scala similarity index 95% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/Utils.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/Utils.scala index b3fe622b411..35bd201f79b 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/Utils.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/Utils.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import akka.actor.{ActorRef, ActorSystem, Props} import com.typesafe.scalalogging.LazyLogging diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/cron/CronParameterValidator.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/cron/CronParameterValidator.scala similarity index 92% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/cron/CronParameterValidator.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/cron/CronParameterValidator.scala index 9eddcdbde26..c701c175cdc 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/cron/CronParameterValidator.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/cron/CronParameterValidator.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic.cron +package pl.touk.nussknacker.engine.common.periodic.cron import cats.data.Validated import cats.data.Validated.{invalid, valid} @@ -12,7 +12,7 @@ import pl.touk.nussknacker.engine.api.definition.{ } import pl.touk.nussknacker.engine.api.parameter.ParameterName import pl.touk.nussknacker.engine.graph.expression.Expression -import pl.touk.nussknacker.engine.management.periodic.SchedulePropertyExtractor +import pl.touk.nussknacker.engine.common.periodic.SchedulePropertyExtractor object CronParameterValidator extends CronParameterValidator { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/AdditionalDeploymentDataProvider.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala similarity index 55% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/AdditionalDeploymentDataProvider.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala index 3131908a091..2fcdfcaab12 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/AdditionalDeploymentDataProvider.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala @@ -1,22 +1,19 @@ -package pl.touk.nussknacker.engine.management.periodic.service +package pl.touk.nussknacker.engine.common.periodic.service +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment import java.time.format.DateTimeFormatter trait AdditionalDeploymentDataProvider { - def prepareAdditionalData(runDetails: PeriodicProcessDeployment[WithCanonicalProcess]): Map[String, String] + def prepareAdditionalData(runDetails: PeriodicProcessDeployment): Map[String, String] } object DefaultAdditionalDeploymentDataProvider extends AdditionalDeploymentDataProvider { - override def prepareAdditionalData( - runDetails: PeriodicProcessDeployment[WithCanonicalProcess] - ): Map[String, String] = { + override def prepareAdditionalData(runDetails: PeriodicProcessDeployment): Map[String, String] = { Map( "deploymentId" -> runDetails.id.value.toString, "runAt" -> runDetails.runAt.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/PeriodicProcessListener.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala similarity index 64% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/PeriodicProcessListener.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala index d3068aac568..80ef76f6564 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/PeriodicProcessListener.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala @@ -1,11 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic.service +package pl.touk.nussknacker.engine.common.periodic.service import com.typesafe.config.Config import pl.touk.nussknacker.engine.api.deployment.StatusDetails +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.ExternalDeploymentId -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment /* Listener is at-least-once. If there are problems e.g. with DB, invocation can be repeated for same event. @@ -22,31 +21,28 @@ trait PeriodicProcessListenerFactory { } sealed trait PeriodicProcessEvent { - val deployment: PeriodicProcessDeployment[WithCanonicalProcess] + val deployment: PeriodicProcessDeployment } case class DeployedEvent( - deployment: PeriodicProcessDeployment[WithCanonicalProcess], + deployment: PeriodicProcessDeployment, externalDeploymentId: Option[ExternalDeploymentId] ) extends PeriodicProcessEvent -case class FinishedEvent( - deployment: PeriodicProcessDeployment[WithCanonicalProcess], - processState: Option[StatusDetails] -) extends PeriodicProcessEvent +case class FinishedEvent(deployment: PeriodicProcessDeployment, processState: Option[StatusDetails]) + extends PeriodicProcessEvent case class FailedOnDeployEvent( - deployment: PeriodicProcessDeployment[WithCanonicalProcess], + deployment: PeriodicProcessDeployment, processState: Option[StatusDetails] ) extends PeriodicProcessEvent case class FailedOnRunEvent( - deployment: PeriodicProcessDeployment[WithCanonicalProcess], + deployment: PeriodicProcessDeployment, processState: Option[StatusDetails] ) extends PeriodicProcessEvent -case class ScheduledEvent(deployment: PeriodicProcessDeployment[WithCanonicalProcess], firstSchedule: Boolean) - extends PeriodicProcessEvent +case class ScheduledEvent(deployment: PeriodicProcessDeployment, firstSchedule: Boolean) extends PeriodicProcessEvent object EmptyListener extends EmptyListener diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/ProcessConfigEnricher.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala similarity index 83% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/ProcessConfigEnricher.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala index e428dbd5ba4..5f439a0601b 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/ProcessConfigEnricher.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala @@ -1,10 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic.service +package pl.touk.nussknacker.engine.common.periodic.service import com.typesafe.config.{Config, ConfigFactory} +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment -import pl.touk.nussknacker.engine.management.periodic.service.ProcessConfigEnricher.{ +import pl.touk.nussknacker.engine.common.periodic.service.ProcessConfigEnricher.{ DeployData, EnrichedProcessConfig, InitialScheduleData @@ -32,7 +32,6 @@ trait ProcessConfigEnricher { object ProcessConfigEnricher { trait ProcessConfigEnricherInputData { - def canonicalProcess: CanonicalProcess def inputConfigDuringExecutionJson: String def inputConfigDuringExecution: Config = { @@ -41,13 +40,11 @@ object ProcessConfigEnricher { } - case class InitialScheduleData(canonicalProcess: CanonicalProcess, inputConfigDuringExecutionJson: String) - extends ProcessConfigEnricherInputData + case class InitialScheduleData(inputConfigDuringExecutionJson: String) extends ProcessConfigEnricherInputData case class DeployData( - canonicalProcess: CanonicalProcess, inputConfigDuringExecutionJson: String, - deployment: PeriodicProcessDeployment[WithCanonicalProcess] + deployment: PeriodicProcessDeployment ) extends ProcessConfigEnricherInputData case class EnrichedProcessConfig(inputConfigDuringExecutionJson: String) diff --git a/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.DeploymentManagerProvider b/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.DeploymentManagerProvider index e3b1553f5e4..76e20a0b331 100644 --- a/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.DeploymentManagerProvider +++ b/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.DeploymentManagerProvider @@ -1 +1 @@ -pl.touk.nussknacker.engine.management.periodic.FlinkPeriodicDeploymentManagerProvider +pl.touk.nussknacker.engine.management.periodic.flink.FlinkPeriodicDeploymentManagerProvider diff --git a/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.api.definition.CustomParameterValidator b/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.api.definition.CustomParameterValidator index 202f7c5d586..82cb6147dba 100644 --- a/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.api.definition.CustomParameterValidator +++ b/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.api.definition.CustomParameterValidator @@ -1 +1 @@ -pl.touk.nussknacker.engine.management.periodic.cron.CronParameterValidator \ No newline at end of file +pl.touk.nussknacker.engine.management.periodic.flink.cron.CronParameterValidator diff --git a/engine/flink/management/periodic/src/main/resources/web/static/assets/custom-actions/batch-instant.svg b/engine/flink/management/periodic/src/main/resources/web/static/assets/custom-actions/batch-instant.svg new file mode 100644 index 00000000000..e69de29bb2d diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala deleted file mode 100644 index bf5cab7a4f7..00000000000 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala +++ /dev/null @@ -1,23 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic - -import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData - -import scala.concurrent.Future - -private[periodic] trait JarManager { - - def prepareDeploymentWithJar( - processVersion: ProcessVersion, - canonicalProcess: CanonicalProcess - ): Future[DeploymentWithJarData.WithCanonicalProcess] - - def deployWithJar( - deploymentWithJarData: DeploymentWithJarData.WithCanonicalProcess, - deploymentData: DeploymentData, - ): Future[Option[ExternalDeploymentId]] - - def deleteJar(jarFileName: String): Future[Unit] -} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/DbInitializer.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/DbInitializer.scala deleted file mode 100644 index bb65cef5794..00000000000 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/DbInitializer.scala +++ /dev/null @@ -1,56 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic.db - -import com.github.tminglei.slickpg.ExPostgresProfile -import com.typesafe.config.Config -import com.typesafe.scalalogging.LazyLogging -import net.ceedubs.ficus.readers.ValueReader -import org.flywaydb.core.Flyway -import org.flywaydb.core.api.configuration.FluentConfiguration -import org.flywaydb.core.internal.database.postgresql.PostgreSQLDatabaseType -import slick.jdbc.{HsqldbProfile, JdbcBackend, JdbcProfile, PostgresProfile} - -object DbInitializer extends LazyLogging { - - def init(configDb: Config): (JdbcBackend.DatabaseDef, JdbcProfile) = { - import net.ceedubs.ficus.Ficus._ - val url = configDb.as[String]("url") - val profile = chooseDbProfile(url) - logger.info("Applying db migrations") - - // we want to set property on FluentConfiguration only if there is property in config - implicit class OptionalConfig(config: FluentConfiguration) { - def setOptional[A](name: String, setAction: (FluentConfiguration, A) => FluentConfiguration)( - implicit reader: ValueReader[Option[A]] - ): FluentConfiguration = { - configDb.getAs[A](name).fold(config)(setAction(config, _)) - } - } - - Flyway - .configure() - .locations( - (profile match { - case _: HsqldbProfile => Array("db/batch_periodic/migration/hsql", "db/batch_periodic/migration/common") - case _: PostgresProfile => Array("db/batch_periodic/migration/postgres", "db/batch_periodic/migration/common") - case _ => - throw new IllegalArgumentException(s"Unsupported database url: $url. Use either PostgreSQL or HSQLDB.") - }): _* - ) - .dataSource(url, configDb.as[String]("user"), configDb.as[String]("password")) - .setOptional[String]("schema", _.schemas(_)) - .setOptional[String]("table", _.table(_)) - .baselineOnMigrate(true) - .load() - .migrate() - - (JdbcBackend.Database.forConfig(path = "", configDb), profile) - } - - private def chooseDbProfile(dbUrl: String): JdbcProfile = { - dbUrl match { - case url if (new PostgreSQLDatabaseType).handlesJDBCUrl(url) => ExPostgresProfile - case _ => HsqldbProfile - } - } - -} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesTable.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesTable.scala deleted file mode 100644 index 1dfaef719eb..00000000000 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesTable.scala +++ /dev/null @@ -1,261 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic.db - -import io.circe.syntax.EncoderOps -import pl.touk.nussknacker.engine.api.deployment.ProcessActionId -import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessId -import pl.touk.nussknacker.engine.marshall.ProcessMarshaller -import slick.ast.TypedType -import slick.jdbc.JdbcProfile -import slick.lifted.MappedToBase.mappedToIsomorphism -import slick.lifted.ProvenShape -import slick.sql.SqlProfile.ColumnOption.NotNull - -import java.time.LocalDateTime -import java.util.UUID - -trait PeriodicProcessesTableFactory { - - protected val profile: JdbcProfile - - import profile.api._ - - implicit val processNameMapping: BaseColumnType[ProcessName] = - MappedColumnType.base[ProcessName, String](_.value, ProcessName.apply) - - implicit val versionIdMapping: BaseColumnType[VersionId] = - MappedColumnType.base[VersionId, Long](_.value, VersionId(_)) - - implicit val ProcessActionIdTypedType: TypedType[ProcessActionId] = - MappedColumnType.base[ProcessActionId, UUID]( - _.value, - ProcessActionId(_) - ) - - abstract class PeriodicProcessesTable[ENTITY <: PeriodicProcessEntity](tag: Tag) - extends Table[ENTITY](tag, "periodic_processes") { - - def id: Rep[PeriodicProcessId] = column[PeriodicProcessId]("id", O.PrimaryKey, O.AutoInc) - - def processName: Rep[ProcessName] = column[ProcessName]("process_name", NotNull) - - def processVersionId: Rep[VersionId] = column[VersionId]("process_version_id", NotNull) - - def processingType: Rep[String] = column[String]("processing_type", NotNull) - - def jarFileName: Rep[String] = column[String]("jar_file_name", NotNull) - - def scheduleProperty: Rep[String] = column[String]("schedule_property", NotNull) - - def active: Rep[Boolean] = column[Boolean]("active", NotNull) - - def createdAt: Rep[LocalDateTime] = column[LocalDateTime]("created_at", NotNull) - - def processActionId: Rep[Option[ProcessActionId]] = column[Option[ProcessActionId]]("process_action_id") - - } - - class PeriodicProcessesWithJsonTable(tag: Tag) extends PeriodicProcessesTable[PeriodicProcessEntityWithJson](tag) { - - def processJson: Rep[String] = column[String]("process_json", NotNull) - - def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) - - override def * : ProvenShape[PeriodicProcessEntityWithJson] = ( - id, - processName, - processVersionId, - processingType, - processJson, - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) <> ( - (PeriodicProcessEntity.createWithJson _).tupled, - (e: PeriodicProcessEntityWithJson) => - PeriodicProcessEntityWithJson.unapply(e).map { - case ( - id, - processName, - versionId, - processingType, - processJson, - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) => - ( - id, - processName, - versionId, - processingType, - processJson.asJson.noSpaces, - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) - } - ) - - } - - class PeriodicProcessWithoutJson(tag: Tag) extends PeriodicProcessesTable[PeriodicProcessEntityWithoutJson](tag) { - - override def * : ProvenShape[PeriodicProcessEntityWithoutJson] = ( - id, - processName, - processVersionId, - processingType, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) <> ( - (PeriodicProcessEntity.createWithoutJson _).tupled, - (e: PeriodicProcessEntityWithoutJson) => - PeriodicProcessEntityWithoutJson.unapply(e).map { - case ( - id, - processName, - versionId, - processingType, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) => - ( - id, - processName, - versionId, - processingType, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) - } - ) - - } - - object PeriodicProcessesWithJson extends TableQuery(new PeriodicProcessesWithJsonTable(_)) - - object PeriodicProcessesWithoutJson extends TableQuery(new PeriodicProcessWithoutJson(_)) - -} - -object PeriodicProcessEntity { - - def createWithJson( - id: PeriodicProcessId, - processName: ProcessName, - processVersionId: VersionId, - processingType: String, - processJson: String, - inputConfigDuringExecutionJson: String, - jarFileName: String, - scheduleProperty: String, - active: Boolean, - createdAt: LocalDateTime, - processActionId: Option[ProcessActionId] - ): PeriodicProcessEntityWithJson = - PeriodicProcessEntityWithJson( - id, - processName, - processVersionId, - processingType, - ProcessMarshaller.fromJsonUnsafe(processJson), - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) - - def createWithoutJson( - id: PeriodicProcessId, - processName: ProcessName, - processVersionId: VersionId, - processingType: String, - jarFileName: String, - scheduleProperty: String, - active: Boolean, - createdAt: LocalDateTime, - processActionId: Option[ProcessActionId] - ): PeriodicProcessEntityWithoutJson = - PeriodicProcessEntityWithoutJson( - id, - processName, - processVersionId, - processingType, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) - -} - -trait PeriodicProcessEntity { - - def id: PeriodicProcessId - - def processName: ProcessName - - def processVersionId: VersionId - - def processingType: String - - def jarFileName: String - - def scheduleProperty: String - - def active: Boolean - - def createdAt: LocalDateTime - - def processActionId: Option[ProcessActionId] - -} - -case class PeriodicProcessEntityWithJson( - id: PeriodicProcessId, - processName: ProcessName, - processVersionId: VersionId, - processingType: String, - processJson: CanonicalProcess, - inputConfigDuringExecutionJson: String, - jarFileName: String, - scheduleProperty: String, - active: Boolean, - createdAt: LocalDateTime, - processActionId: Option[ProcessActionId] -) extends PeriodicProcessEntity - -case class PeriodicProcessEntityWithoutJson( - id: PeriodicProcessId, - processName: ProcessName, - processVersionId: VersionId, - processingType: String, - jarFileName: String, - scheduleProperty: String, - active: Boolean, - createdAt: LocalDateTime, - processActionId: Option[ProcessActionId] -) extends PeriodicProcessEntity diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkJarManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala similarity index 50% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkJarManager.scala rename to engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala index 47b33e7bc93..e7fcdc26562 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkJarManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala @@ -2,12 +2,12 @@ package pl.touk.nussknacker.engine.management.periodic.flink import com.typesafe.scalalogging.LazyLogging import org.apache.flink.api.common.JobID -import pl.touk.nussknacker.engine.{BaseModelData, newdeployment} import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{DeploymentWithRuntimeParams, RuntimeParams} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.common.periodic.PeriodicDeploymentHandler import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData -import pl.touk.nussknacker.engine.management.periodic.{JarManager, PeriodicBatchConfig} +import pl.touk.nussknacker.engine.management.periodic.flink.FlinkPeriodicDeploymentHandler.jarFileNameRuntimeParam import pl.touk.nussknacker.engine.management.rest.{FlinkClient, HttpFlinkClient} import pl.touk.nussknacker.engine.management.{ FlinkConfig, @@ -16,20 +16,23 @@ import pl.touk.nussknacker.engine.management.{ FlinkStreamingRestManager } import pl.touk.nussknacker.engine.modelconfig.InputConfigDuringExecution +import pl.touk.nussknacker.engine.{BaseModelData, newdeployment} import sttp.client3.SttpBackend import java.nio.file.{Files, Path, Paths} import scala.concurrent.{ExecutionContext, Future} -private[periodic] object FlinkJarManager { +private[periodic] object FlinkPeriodicDeploymentHandler { + + val jarFileNameRuntimeParam = "jarFileName" - def apply(flinkConfig: FlinkConfig, periodicBatchConfig: PeriodicBatchConfig, modelData: BaseModelData)( + def apply(flinkConfig: FlinkConfig, jarsDir: String, modelData: BaseModelData)( implicit backend: SttpBackend[Future, Any], ec: ExecutionContext - ): JarManager = { - new FlinkJarManager( + ): PeriodicDeploymentHandler = { + new FlinkPeriodicDeploymentHandler( flinkClient = HttpFlinkClient.createUnsafe(flinkConfig), - jarsDir = Paths.get(periodicBatchConfig.jarsDir), + jarsDir = Paths.get(jarsDir), inputConfigDuringExecution = modelData.inputConfigDuringExecution, modelJarProvider = new FlinkModelJarProvider(modelData.modelClassLoaderUrls) ) @@ -38,27 +41,25 @@ private[periodic] object FlinkJarManager { } // Used by [[PeriodicProcessService]]. -private[periodic] class FlinkJarManager( +class FlinkPeriodicDeploymentHandler( flinkClient: FlinkClient, jarsDir: Path, inputConfigDuringExecution: InputConfigDuringExecution, modelJarProvider: FlinkModelJarProvider -) extends JarManager +) extends PeriodicDeploymentHandler with LazyLogging { import scala.concurrent.ExecutionContext.Implicits.global - override def prepareDeploymentWithJar( + override def prepareDeploymentWithRuntimeParams( processVersion: ProcessVersion, - canonicalProcess: CanonicalProcess - ): Future[DeploymentWithJarData.WithCanonicalProcess] = { + ): Future[DeploymentWithRuntimeParams] = { logger.info(s"Prepare deployment for scenario: $processVersion") copyJarToLocalDir(processVersion).map { jarFileName => - DeploymentWithJarData.WithCanonicalProcess( + DeploymentWithRuntimeParams( processVersion = processVersion, - process = canonicalProcess, inputConfigDuringExecutionJson = inputConfigDuringExecution.serialized, - jarFileName = jarFileName + runtimeParams = RuntimeParams(Map(jarFileNameRuntimeParam -> jarFileName)) ) } } @@ -73,36 +74,52 @@ private[periodic] class FlinkJarManager( jarFileName } - override def deployWithJar( - deploymentWithJarData: DeploymentWithJarData.WithCanonicalProcess, - deploymentData: DeploymentData + override def deployWithRuntimeParams( + deployment: DeploymentWithRuntimeParams, + deploymentData: DeploymentData, + canonicalProcess: CanonicalProcess, ): Future[Option[ExternalDeploymentId]] = { - val processVersion = deploymentWithJarData.processVersion - logger.info( - s"Deploying scenario ${processVersion.processName}, version id: ${processVersion.versionId} and jar: ${deploymentWithJarData.jarFileName}" - ) - val jarFile = jarsDir.resolve(deploymentWithJarData.jarFileName).toFile - val args = FlinkDeploymentManager.prepareProgramArgs( - deploymentWithJarData.inputConfigDuringExecutionJson, - processVersion, - deploymentData, - deploymentWithJarData.process - ) - flinkClient.runProgram( - jarFile, - FlinkStreamingRestManager.MainClassName, - args, - None, - deploymentData.deploymentId.toNewDeploymentIdOpt.map(toJobId) - ) + val processVersion = deployment.processVersion + deployment.runtimeParams.params.get(jarFileNameRuntimeParam) match { + case Some(jarFileName) => + logger.info( + s"Deploying scenario ${processVersion.processName}, version id: ${processVersion.versionId} and jar: $jarFileName" + ) + val jarFile = jarsDir.resolve(jarFileName).toFile + val args = FlinkDeploymentManager.prepareProgramArgs( + deployment.inputConfigDuringExecutionJson, + processVersion, + deploymentData, + canonicalProcess, + ) + flinkClient.runProgram( + jarFile, + FlinkStreamingRestManager.MainClassName, + args, + None, + deploymentData.deploymentId.toNewDeploymentIdOpt.map(toJobId) + ) + case None => + logger.error( + s"Cannot deploy scenario ${processVersion.processName}, version id: ${processVersion.versionId}: jar file name not present" + ) + Future.successful(None) + } } - override def deleteJar(jarFileName: String): Future[Unit] = { - logger.info(s"Deleting jar: $jarFileName") - for { - _ <- deleteLocalJar(jarFileName) - _ <- flinkClient.deleteJarIfExists(jarFileName) - } yield () + override def cleanAfterDeployment(runtimeParams: RuntimeParams): Future[Unit] = { + runtimeParams.params.get(jarFileNameRuntimeParam) match { + case Some(jarFileName) => + logger.info(s"Deleting jar: $jarFileName") + for { + _ <- deleteLocalJar(jarFileName) + _ <- flinkClient.deleteJarIfExists(jarFileName) + } yield () + case None => + logger.warn(s"Jar file name not present among runtime params: ${runtimeParams}") + Future.unit + } + } private def deleteLocalJar(jarFileName: String): Future[Unit] = Future { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala new file mode 100644 index 00000000000..ce889eb3a23 --- /dev/null +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala @@ -0,0 +1,37 @@ +package pl.touk.nussknacker.engine.management.periodic.flink + +import com.typesafe.config.Config +import pl.touk.nussknacker.engine.common.periodic.{ + PeriodicBatchConfig, + PeriodicDeploymentHandler, + PeriodicDeploymentManagerProvider +} +import pl.touk.nussknacker.engine.management.{FlinkConfig, FlinkStreamingDeploymentManagerProvider} +import pl.touk.nussknacker.engine.util.config.ConfigEnrichments.RichConfig +import pl.touk.nussknacker.engine.{BaseModelData, DeploymentManagerDependencies} + +class FlinkPeriodicDeploymentManagerProvider + extends PeriodicDeploymentManagerProvider( + name = "flinkPeriodic", + delegate = new FlinkStreamingDeploymentManagerProvider(), + ) { + + override protected def createPeriodicDeploymentHandler( + modelData: BaseModelData, + dependencies: DeploymentManagerDependencies, + config: Config + ): PeriodicDeploymentHandler = { + import dependencies._ + import net.ceedubs.ficus.Ficus._ + import net.ceedubs.ficus.readers.ArbitraryTypeReader._ + val periodicBatchConfig = config.as[PeriodicBatchConfig]("deploymentManager") + val flinkConfig = config.rootAs[FlinkConfig] + + FlinkPeriodicDeploymentHandler( + flinkConfig = flinkConfig, + jarsDir = periodicBatchConfig.jarsDir, + modelData = modelData, + ) + } + +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/DeploymentWithJarData.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/DeploymentWithJarData.scala deleted file mode 100644 index 6491b9506b8..00000000000 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/DeploymentWithJarData.scala +++ /dev/null @@ -1,25 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic.model - -import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess - -sealed trait DeploymentWithJarData { - def processVersion: ProcessVersion - def jarFileName: String -} - -object DeploymentWithJarData { - - final case class WithCanonicalProcess( - processVersion: ProcessVersion, - jarFileName: String, - process: CanonicalProcess, - inputConfigDuringExecutionJson: String, - ) extends DeploymentWithJarData - - final case class WithoutCanonicalProcess( - processVersion: ProcessVersion, - jarFileName: String - ) extends DeploymentWithJarData - -} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala deleted file mode 100644 index 88f620eff9e..00000000000 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala +++ /dev/null @@ -1,52 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic.model - -import pl.touk.nussknacker.engine.management.periodic.{MultipleScheduleProperty, SingleScheduleProperty} -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import slick.lifted.MappedTo - -import java.time.{Clock, LocalDateTime} - -// TODO: We should separate schedules concept from deployments - fully switch to ScheduleData and ScheduleDeploymentData -case class PeriodicProcessDeployment[DeploymentData <: DeploymentWithJarData]( - id: PeriodicProcessDeploymentId, - periodicProcess: PeriodicProcess[DeploymentData], - createdAt: LocalDateTime, - runAt: LocalDateTime, - scheduleName: ScheduleName, - retriesLeft: Int, - nextRetryAt: Option[LocalDateTime], - state: PeriodicProcessDeploymentState -) { - - def nextRunAt(clock: Clock): Either[String, Option[LocalDateTime]] = - (periodicProcess.scheduleProperty, scheduleName.value) match { - case (MultipleScheduleProperty(schedules), Some(name)) => - schedules.get(name).toRight(s"Failed to find schedule: $scheduleName").flatMap(_.nextRunAt(clock)) - case (e: SingleScheduleProperty, None) => e.nextRunAt(clock) - case (schedule, name) => Left(s"Schedule name: $name mismatch with schedule: $schedule") - } - - def display: String = - s"${periodicProcess.processVersion} with scheduleName=${scheduleName.display} and deploymentId=$id" - -} - -case class PeriodicProcessDeploymentState( - deployedAt: Option[LocalDateTime], - completedAt: Option[LocalDateTime], - status: PeriodicProcessDeploymentStatus -) - -case class PeriodicProcessDeploymentId(value: Long) extends AnyVal with MappedTo[Long] { - override def toString: String = value.toString -} - -object PeriodicProcessDeploymentStatus extends Enumeration { - type PeriodicProcessDeploymentStatus = Value - - val Scheduled, Deployed, Finished, Failed, RetryingDeploy, FailedOnDeploy = Value -} - -case class ScheduleName(value: Option[String]) { - def display: String = value.getOrElse("[default]") -} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala deleted file mode 100644 index 4ded31dd0d1..00000000000 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala +++ /dev/null @@ -1,38 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic - -import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData - -import scala.concurrent.Future - -class JarManagerStub extends JarManager { - - var deployWithJarFuture: Future[Option[ExternalDeploymentId]] = Future.successful(None) - var lastDeploymentWithJarData: Option[DeploymentWithJarData.WithCanonicalProcess] = None - - override def prepareDeploymentWithJar( - processVersion: ProcessVersion, - canonicalProcess: CanonicalProcess - ): Future[DeploymentWithJarData.WithCanonicalProcess] = { - Future.successful( - model.DeploymentWithJarData.WithCanonicalProcess( - processVersion = processVersion, - process = canonicalProcess, - inputConfigDuringExecutionJson = "", - jarFileName = "" - ) - ) - } - - override def deployWithJar( - deploymentWithJarData: DeploymentWithJarData.WithCanonicalProcess, - deploymentData: DeploymentData, - ): Future[Option[ExternalDeploymentId]] = { - lastDeploymentWithJarData = Some(deploymentWithJarData) - deployWithJarFuture - } - - override def deleteJar(jarFileName: String): Future[Unit] = Future.successful(()) -} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyExtractorTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyExtractorTest.scala similarity index 88% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyExtractorTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyExtractorTest.scala index 5874bd1bb26..d4d248ac99e 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyExtractorTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyExtractorTest.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import org.scalatest.Inside import org.scalatest.funsuite.AnyFunSuite @@ -6,8 +6,9 @@ import org.scalatest.matchers.should.Matchers import pl.touk.nussknacker.engine.api.NodeId import pl.touk.nussknacker.engine.api.parameter.ParameterName import pl.touk.nussknacker.engine.build.ScenarioBuilder +import pl.touk.nussknacker.engine.common.periodic.cron.CronParameterValidator +import pl.touk.nussknacker.engine.common.periodic.{CronScheduleProperty, CronSchedulePropertyExtractor, MultipleScheduleProperty} import pl.touk.nussknacker.engine.graph.expression.Expression -import pl.touk.nussknacker.engine.management.periodic.cron.CronParameterValidator import pl.touk.nussknacker.test.{EitherValuesDetailedMessage, ValidatedValuesDetailedMessage} class CronSchedulePropertyExtractorTest diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyTest.scala similarity index 88% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyTest.scala index 5fff9dcb12d..e1c06033c7e 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyTest.scala @@ -1,8 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink -import java.time.{Clock, LocalDateTime, ZoneId, ZonedDateTime} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import pl.touk.nussknacker.engine.common.periodic.CronScheduleProperty + +import java.time.{Clock, LocalDateTime, ZoneId, ZonedDateTime} class CronSchedulePropertyTest extends AnyFunSuite with Matchers { diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActorTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala similarity index 66% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActorTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala index ac646390e38..fff133f4df9 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActorTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala @@ -1,15 +1,15 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import akka.actor.{ActorRef, ActorSystem} import akka.testkit.{TestKit, TestKitBase, TestProbe} -import org.scalatest.LoneElement._ import org.scalatest.BeforeAndAfterAll +import org.scalatest.LoneElement._ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.DeploymentActor.CheckToBeDeployed +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment +import pl.touk.nussknacker.engine.common.periodic.DeploymentActor import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment +import pl.touk.nussknacker.engine.common.periodic.DeploymentActor.CheckToBeDeployed import scala.concurrent.Future import scala.concurrent.duration._ @@ -34,11 +34,11 @@ class DeploymentActorTest extends AnyFunSuite with TestKitBase with Matchers wit } private def shouldFindToBeDeployedScenarios( - result: Future[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] + result: Future[Seq[PeriodicProcessDeployment]] ): Unit = { val probe = TestProbe() var counter = 0 - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] = { + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { counter += 1 probe.ref ! s"invoked $counter" result @@ -55,14 +55,14 @@ class DeploymentActorTest extends AnyFunSuite with TestKitBase with Matchers wit } test("should deploy found scenario") { - val probe = TestProbe() - val waitingDeployment = PeriodicProcessDeploymentGen() - var toBeDeployed: Seq[PeriodicProcessDeployment[WithCanonicalProcess]] = Seq(waitingDeployment) - var actor: ActorRef = null - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] = { + val probe = TestProbe() + val waitingDeployment = PeriodicProcessDeploymentGen() + var toBeDeployed: Seq[PeriodicProcessDeployment] = Seq(waitingDeployment) + var actor: ActorRef = null + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { Future.successful(toBeDeployed) } - def deploy(deployment: PeriodicProcessDeployment[WithCanonicalProcess]): Future[Unit] = { + def deploy(deployment: PeriodicProcessDeployment): Future[Unit] = { probe.ref ! deployment // Simulate periodic check for waiting scenarios while deploying a scenario. actor ! CheckToBeDeployed diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentManagerStub.scala similarity index 92% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentManagerStub.scala index c2e8a4edb4f..daa83490ca5 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentManagerStub.scala @@ -1,10 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentId import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.process.{ProcessIdWithName, ProcessName, VersionId} import pl.touk.nussknacker.engine.deployment.{DeploymentId, ExternalDeploymentId} -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentId import pl.touk.nussknacker.engine.testing.StubbingCommands import scala.concurrent.Future diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/FlinkClientStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkClientStub.scala similarity index 96% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/FlinkClientStub.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkClientStub.scala index 5c386e88208..5b39c80139a 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/FlinkClientStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkClientStub.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import org.apache.flink.configuration.Configuration import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, SavepointResult, WithDataFreshnessStatus} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala new file mode 100644 index 00000000000..0c4d2fc9cbc --- /dev/null +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala @@ -0,0 +1,38 @@ +package pl.touk.nussknacker.engine.management.periodic.flink + +import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{DeploymentWithRuntimeParams, RuntimeParams} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.common.periodic.PeriodicDeploymentHandler +import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} + +import scala.concurrent.Future + +class PeriodicDeploymentHandlerStub extends PeriodicDeploymentHandler { + + var deployWithJarFuture: Future[Option[ExternalDeploymentId]] = Future.successful(None) + var lastDeploymentWithRuntimeParams: Option[DeploymentWithRuntimeParams] = None + + override def prepareDeploymentWithRuntimeParams( + processVersion: ProcessVersion, + ): Future[DeploymentWithRuntimeParams] = { + Future.successful( + DeploymentWithRuntimeParams( + processVersion = processVersion, + inputConfigDuringExecutionJson = "", + runtimeParams = RuntimeParams(Map("jarFileName" -> "")) + ) + ) + } + + override def deployWithRuntimeParams( + deploymentWithJarData: DeploymentWithRuntimeParams, + deploymentData: DeploymentData, + canonicalProcess: CanonicalProcess, + ): Future[Option[ExternalDeploymentId]] = { + lastDeploymentWithRuntimeParams = Some(deploymentWithJarData) + deployWithJarFuture + } + + override def cleanAfterDeployment(runtimeParams: RuntimeParams): Future[Unit] = Future.successful(()) +} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerTest.scala similarity index 56% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerTest.scala index b63433a8795..4b1c9d34aa2 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerTest.scala @@ -1,28 +1,26 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import com.typesafe.config.ConfigFactory import org.scalatest.concurrent.ScalaFutures import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{DeploymentWithRuntimeParams, RuntimeParams} import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} -import pl.touk.nussknacker.engine.api.{MetaData, ProcessVersion, StreamMetaData} -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.common.periodic.PeriodicDeploymentHandler import pl.touk.nussknacker.engine.management.FlinkModelJarProvider -import pl.touk.nussknacker.engine.management.periodic.flink.FlinkJarManager -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData import pl.touk.nussknacker.engine.modelconfig.InputConfigDuringExecution import pl.touk.nussknacker.test.PatientScalaFutures import java.nio.file.{Files, Path, Paths} import scala.concurrent.Future -class JarManagerTest extends AnyFunSuite with Matchers with ScalaFutures with PatientScalaFutures { +class PeriodicDeploymentHandlerTest extends AnyFunSuite with Matchers with ScalaFutures with PatientScalaFutures { private val processName = "test" private val processVersionId = 5 private val processVersion = ProcessVersion.empty.copy(processName = ProcessName(processName), versionId = VersionId(processVersionId)) - private val process = CanonicalProcess(MetaData("foo", StreamMetaData()), Nil) private val jarsDir = Files.createTempDirectory("jars-dir") private val modelJarFileContent = "abc".getBytes @@ -34,14 +32,14 @@ class JarManagerTest extends AnyFunSuite with Matchers with ScalaFutures with Pa private val currentModelUrls = List(currentModelJarFile.toURI.toURL) - private val jarManager = createJarManager(jarsDir = jarsDir) + private val periodicDeploymentHandler = createPeriodicDeploymentHandler(jarsDir = jarsDir) - private def createJarManager( + private def createPeriodicDeploymentHandler( jarsDir: Path, modelJarProvider: FlinkModelJarProvider = new FlinkModelJarProvider(currentModelUrls) - ): JarManager = { + ): PeriodicDeploymentHandler = { - new FlinkJarManager( + new FlinkPeriodicDeploymentHandler( flinkClient = new FlinkClientStub, jarsDir = jarsDir, inputConfigDuringExecution = InputConfigDuringExecution(ConfigFactory.empty()), @@ -50,9 +48,9 @@ class JarManagerTest extends AnyFunSuite with Matchers with ScalaFutures with Pa } test("prepareDeploymentWithJar - should copy to local dir") { - val result = jarManager.prepareDeploymentWithJar(processVersion, process) + val result = periodicDeploymentHandler.prepareDeploymentWithRuntimeParams(processVersion) - val copiedJarFileName = result.futureValue.jarFileName + val copiedJarFileName = result.futureValue.runtimeParams.params("jarFileName") copiedJarFileName should fullyMatch regex s"^$processName-$processVersionId-\\d+\\.jar$$" val copiedJarFile = jarsDir.resolve(copiedJarFileName) Files.exists(copiedJarFile) shouldBe true @@ -60,33 +58,33 @@ class JarManagerTest extends AnyFunSuite with Matchers with ScalaFutures with Pa } test("prepareDeploymentWithJar - should handle disappearing model JAR") { - val modelJarProvider = new FlinkModelJarProvider(currentModelUrls) - val jarManager = createJarManager(jarsDir, modelJarProvider) + val modelJarProvider = new FlinkModelJarProvider(currentModelUrls) + val periodicDeploymentHandler = createPeriodicDeploymentHandler(jarsDir, modelJarProvider) - def verifyAndDeleteJar(result: Future[DeploymentWithJarData.WithCanonicalProcess]): Unit = { - val copiedJarFile = jarsDir.resolve(result.futureValue.jarFileName) + def verifyAndDeleteJar(result: Future[DeploymentWithRuntimeParams]): Unit = { + val copiedJarFile = jarsDir.resolve(result.futureValue.runtimeParams.params("jarFileName")) Files.exists(copiedJarFile) shouldBe true Files.readAllBytes(copiedJarFile) shouldBe modelJarFileContent Files.delete(copiedJarFile) } - verifyAndDeleteJar(jarManager.prepareDeploymentWithJar(processVersion, process)) + verifyAndDeleteJar(periodicDeploymentHandler.prepareDeploymentWithRuntimeParams(processVersion)) modelJarProvider.getJobJar().delete() shouldBe true - verifyAndDeleteJar(jarManager.prepareDeploymentWithJar(processVersion, process)) + verifyAndDeleteJar(periodicDeploymentHandler.prepareDeploymentWithRuntimeParams(processVersion)) } test("prepareDeploymentWithJar - should create jars dir if not exists") { - val tmpDir = System.getProperty("java.io.tmpdir") - val jarsDir = Paths.get(tmpDir, s"jars-dir-not-exists-${System.currentTimeMillis()}") - val jarManager = createJarManager(jarsDir = jarsDir) + val tmpDir = System.getProperty("java.io.tmpdir") + val jarsDir = Paths.get(tmpDir, s"jars-dir-not-exists-${System.currentTimeMillis()}") + val periodicDeploymentHandler = createPeriodicDeploymentHandler(jarsDir = jarsDir) Files.exists(jarsDir) shouldBe false - val result = jarManager.prepareDeploymentWithJar(processVersion, process) + val result = periodicDeploymentHandler.prepareDeploymentWithRuntimeParams(processVersion) - val copiedJarFileName = result.futureValue.jarFileName + val copiedJarFileName = result.futureValue.runtimeParams.params("jarFileName") Files.exists(jarsDir) shouldBe true Files.exists(jarsDir.resolve(copiedJarFileName)) shouldBe true } @@ -96,13 +94,14 @@ class JarManagerTest extends AnyFunSuite with Matchers with ScalaFutures with Pa val jarPath = jarsDir.resolve(jarFileName) Files.copy(currentModelJarFile.toPath, jarPath) - jarManager.deleteJar(jarFileName).futureValue + periodicDeploymentHandler.cleanAfterDeployment(RuntimeParams(Map("jarFileName" -> jarFileName))).futureValue Files.exists(jarPath) shouldBe false } test("deleteJar - should handle not existing file") { - val result = jarManager.deleteJar("unknown.jar").futureValue + val result = + periodicDeploymentHandler.cleanAfterDeployment(RuntimeParams(Map("jarFileName" -> "unknown.jar"))).futureValue result shouldBe (()) } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentManagerTest.scala similarity index 75% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentManagerTest.scala index d4310d1b5ec..7480a5cc93e 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentManagerTest.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import org.scalatest.concurrent.ScalaFutures import org.scalatest.funsuite.AnyFunSuite @@ -7,20 +7,22 @@ import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.{Inside, OptionValues} import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, ProcessName, VersionId} import pl.touk.nussknacker.engine.api.{MetaData, ProcessVersion, StreamMetaData} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{DeploymentData, User} -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.PeriodicProcessStatus -import pl.touk.nussknacker.engine.management.periodic.PeriodicStateStatus.{ScheduledStatus, WaitingForScheduleStatus} -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.service.{ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.PeriodicProcessStatus +import pl.touk.nussknacker.engine.common.periodic.PeriodicStateStatus.{ScheduledStatus, WaitingForScheduleStatus} +import pl.touk.nussknacker.engine.common.periodic.service.{ DefaultAdditionalDeploymentDataProvider, EmptyListener, ProcessConfigEnricher } +import pl.touk.nussknacker.engine.common.periodic._ +import pl.touk.nussknacker.engine.deployment.{DeploymentData, User} +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesManager import pl.touk.nussknacker.test.PatientScalaFutures import java.time.{Clock, LocalDateTime, ZoneOffset} @@ -59,15 +61,15 @@ class PeriodicDeploymentManagerTest ) class Fixture(executionConfig: PeriodicExecutionConfig = PeriodicExecutionConfig()) { - val repository = new db.InMemPeriodicProcessesRepository(processingType = "testProcessingType") + val manager = new InMemPeriodicProcessesManager(processingType = "testProcessingType") val delegateDeploymentManagerStub = new DeploymentManagerStub - val jarManagerStub = new JarManagerStub + val periodicDeploymentHandlerStub = new PeriodicDeploymentHandlerStub val preparedDeploymentData = DeploymentData.withDeploymentId(UUID.randomUUID().toString) val periodicProcessService = new PeriodicProcessService( delegateDeploymentManager = delegateDeploymentManagerStub, - jarManager = jarManagerStub, - scheduledProcessesRepository = repository, + periodicDeploymentHandler = periodicDeploymentHandlerStub, + periodicProcessesManager = manager, periodicProcessListener = EmptyListener, additionalDeploymentDataProvider = DefaultAdditionalDeploymentDataProvider, deploymentRetryConfig = DeploymentRetryConfig(), @@ -76,7 +78,7 @@ class PeriodicDeploymentManagerTest processConfigEnricher = ProcessConfigEnricher.identity, clock = Clock.systemDefaultZone(), new ProcessingTypeActionServiceStub, - Map.empty + Map.empty, ) val periodicDeploymentManager = new PeriodicDeploymentManager( @@ -84,6 +86,8 @@ class PeriodicDeploymentManagerTest service = periodicProcessService, repository = repository, schedulePropertyExtractor = CronSchedulePropertyExtractor(), + EmptyPeriodicCustomActionsProvider, + "testProcessingType", toClose = () => () ) @@ -119,7 +123,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should return not deployed for scenario with different processing type") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled, processingType = "other") + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled, processingType = "other") val state = f.getMergedStatusDetails.status @@ -128,7 +132,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be scheduled when scenario scheduled and no job on Flink") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) val statusDetails = f.getMergedStatusDetails statusDetails.status shouldBe a[ScheduledStatus] @@ -151,7 +155,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be scheduled when scenario scheduled and job finished on Flink") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails @@ -164,8 +168,8 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be finished when scenario finished and job finished on Flink") { val f = new Fixture - val periodicProcessId = f.repository.addOnlyProcess(processName, CronScheduleProperty("0 0 0 1 1 ? 1970")) - val deploymentId = f.repository.addOnlyDeployment( + val periodicProcessId = f.manager.addOnlyProcess(processName, CronScheduleProperty("0 0 0 1 1 ? 1970")) + val deploymentId = f.manager.addOnlyDeployment( periodicProcessId, PeriodicProcessDeploymentStatus.Finished, LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC) @@ -185,7 +189,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be running when scenario deployed and job running on Flink") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails @@ -197,7 +201,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be waiting for reschedule if job finished on Flink but scenario is still deployed") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails @@ -209,7 +213,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be failed after unsuccessful deployment") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Failed) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Failed) val statusDetails = f.getMergedStatusDetails statusDetails.status shouldBe ProblemStateStatus.Failed @@ -250,13 +254,13 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities.loneElement.active shouldBe true - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Scheduled + f.manager.processEntities.loneElement.active shouldBe true + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Scheduled } test("deploy - should not cancel current schedule after trying to deploy with past date") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) f.periodicDeploymentManager .processCommand( @@ -270,13 +274,13 @@ class PeriodicDeploymentManagerTest .failed .futureValue - f.repository.processEntities.loneElement.active shouldBe true - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Scheduled + f.manager.processEntities.loneElement.active shouldBe true + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Scheduled } test("deploy - should cancel existing scenario if already scheduled") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) f.periodicDeploymentManager .processCommand( @@ -289,13 +293,13 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities should have size 2 - f.repository.processEntities.map(_.active) shouldBe List(false, true) + f.manager.processEntities should have size 2 + f.manager.processEntities.map(_.active) shouldBe List(false, true) } test("should get status of failed job") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails @@ -307,7 +311,7 @@ class PeriodicDeploymentManagerTest test("should redeploy failed scenario") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) val statusDetailsBeforeRedeploy = f.getMergedStatusDetails statusDetailsBeforeRedeploy.status shouldBe ProblemStateStatus.Failed @@ -331,8 +335,8 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities.map(_.active) shouldBe List(false, true) - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.processEntities.map(_.active) shouldBe List(false, true) + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Failed, PeriodicProcessDeploymentStatus.Scheduled ) @@ -352,7 +356,7 @@ class PeriodicDeploymentManagerTest test("should redeploy scheduled scenario") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) f.periodicDeploymentManager .processCommand( @@ -365,8 +369,8 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities.map(_.active) shouldBe List(false, true) - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.processEntities.map(_.active) shouldBe List(false, true) + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Scheduled, PeriodicProcessDeploymentStatus.Scheduled ) @@ -374,7 +378,7 @@ class PeriodicDeploymentManagerTest test("should redeploy running scenario") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails f.getAllowedActions(statusDetails, processVersion.versionId, None, Some(processVersion.versionId)) shouldBe List( @@ -392,8 +396,8 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities.map(_.active) shouldBe List(false, true) - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.processEntities.map(_.active) shouldBe List(false, true) + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Deployed, PeriodicProcessDeploymentStatus.Scheduled ) @@ -401,7 +405,7 @@ class PeriodicDeploymentManagerTest test("should redeploy finished scenario") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails f.getAllowedActions(statusDetails, processVersion.versionId, None, Some(processVersion.versionId)) shouldBe List( @@ -419,8 +423,8 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities.map(_.active) shouldBe List(false, true) - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.processEntities.map(_.active) shouldBe List(false, true) + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Finished, PeriodicProcessDeploymentStatus.Scheduled ) @@ -428,55 +432,55 @@ class PeriodicDeploymentManagerTest test("should cancel failed job after RescheduleActor handles finished") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) // this one is cyclically called by RescheduleActor f.periodicProcessService.handleFinished.futureValue f.getMergedStatusDetails.status shouldEqual ProblemStateStatus.Failed - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed - f.repository.processEntities.loneElement.active shouldBe true + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.processEntities.loneElement.active shouldBe true f.periodicDeploymentManager.processCommand(DMCancelScenarioCommand(processName, User("test", "Tester"))).futureValue - f.repository.processEntities.loneElement.active shouldBe false - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.processEntities.loneElement.active shouldBe false + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed f.getMergedStatusDetails.status shouldEqual SimpleStateStatus.Canceled } test("should reschedule failed job after RescheduleActor handles finished when configured") { val f = new Fixture(executionConfig = PeriodicExecutionConfig(rescheduleOnFailure = true)) - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) // this one is cyclically called by RescheduleActor f.periodicProcessService.handleFinished.futureValue f.getMergedStatusDetails.status shouldBe a[ScheduledStatus] - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Failed, PeriodicProcessDeploymentStatus.Scheduled ) - f.repository.processEntities.loneElement.active shouldBe true + f.manager.processEntities.loneElement.active shouldBe true } test("should cancel failed job before RescheduleActor handles finished") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) f.periodicDeploymentManager.processCommand(DMCancelScenarioCommand(processName, User("test", "Tester"))).futureValue - f.repository.processEntities.loneElement.active shouldBe false - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.processEntities.loneElement.active shouldBe false + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed f.getMergedStatusDetails.status shouldEqual SimpleStateStatus.Canceled } test("should cancel failed scenario after disappeared from Flink console") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) // this one is cyclically called by RescheduleActor @@ -486,22 +490,22 @@ class PeriodicDeploymentManagerTest f.delegateDeploymentManagerStub.jobStatus = None f.getMergedStatusDetails.status shouldEqual ProblemStateStatus.Failed - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed - f.repository.processEntities.loneElement.active shouldBe true + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.processEntities.loneElement.active shouldBe true f.periodicDeploymentManager.processCommand(DMCancelScenarioCommand(processName, User("test", "Tester"))).futureValue - f.repository.processEntities.loneElement.active shouldBe false - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.processEntities.loneElement.active shouldBe false + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed f.getMergedStatusDetails.status shouldBe SimpleStateStatus.Canceled } test("should take into account only latest deployments of active schedules during merged status computation") { val f = new Fixture - val processId = f.repository.addOnlyProcess(processName) + val processId = f.manager.addOnlyProcess(processName) val firstDeploymentRunAt = LocalDateTime.of(2023, 1, 1, 10, 0) - f.repository.addOnlyDeployment(processId, PeriodicProcessDeploymentStatus.Failed, firstDeploymentRunAt) - f.repository.addOnlyDeployment( + f.manager.addOnlyDeployment(processId, PeriodicProcessDeploymentStatus.Failed, firstDeploymentRunAt) + f.manager.addOnlyDeployment( processId, PeriodicProcessDeploymentStatus.Finished, firstDeploymentRunAt.plusHours(1) @@ -514,13 +518,13 @@ class PeriodicDeploymentManagerTest "should take into account only latest inactive schedule request (periodic process) during merged status computation" ) { val f = new Fixture - val firstProcessId = f.repository.addOnlyProcess(processName) - f.repository.addOnlyDeployment(firstProcessId, PeriodicProcessDeploymentStatus.Failed) - f.repository.markInactive(firstProcessId) + val firstProcessId = f.manager.addOnlyProcess(processName) + f.manager.addOnlyDeployment(firstProcessId, PeriodicProcessDeploymentStatus.Failed) + f.manager.markInactive(firstProcessId) - val secProcessId = f.repository.addOnlyProcess(processName) - f.repository.addOnlyDeployment(secProcessId, PeriodicProcessDeploymentStatus.Finished) - f.repository.markInactive(secProcessId) + val secProcessId = f.manager.addOnlyProcess(processName) + f.manager.addOnlyDeployment(secProcessId, PeriodicProcessDeploymentStatus.Finished) + f.manager.markInactive(secProcessId) f.getMergedStatusDetails.status shouldBe SimpleStateStatus.Finished } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessDeploymentGen.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala similarity index 53% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessDeploymentGen.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala index cf813b7b011..ccbc60e06d3 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessDeploymentGen.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala @@ -1,14 +1,6 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.{ - PeriodicProcessDeployment, - PeriodicProcessDeploymentId, - PeriodicProcessDeploymentState, - PeriodicProcessDeploymentStatus, - ScheduleName -} +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import java.time.LocalDateTime @@ -16,7 +8,7 @@ object PeriodicProcessDeploymentGen { val now: LocalDateTime = LocalDateTime.now() - def apply(): PeriodicProcessDeployment[WithCanonicalProcess] = { + def apply(): PeriodicProcessDeployment = { PeriodicProcessDeployment( id = PeriodicProcessDeploymentId(42), periodicProcess = PeriodicProcessGen(), diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessGen.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala similarity index 58% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessGen.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala index 5d1062d4bb6..73c40fbf67d 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessGen.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala @@ -1,24 +1,23 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager.CronScheduleProperty +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.CronSchedulePropertyExtractor.CronPropertyDefaultName -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.{DeploymentWithJarData, PeriodicProcess, PeriodicProcessId} +import pl.touk.nussknacker.engine.common.periodic.CronSchedulePropertyExtractor.CronPropertyDefaultName import java.time.LocalDateTime object PeriodicProcessGen { - def apply(): PeriodicProcess[WithCanonicalProcess] = { + def apply(): PeriodicProcess = { PeriodicProcess( id = PeriodicProcessId(42), - deploymentData = DeploymentWithJarData.WithCanonicalProcess( + deploymentData = DeploymentWithRuntimeParams( processVersion = ProcessVersion.empty, - process = buildCanonicalProcess(), inputConfigDuringExecutionJson = "{}", - jarFileName = "jar-file-name.jar" + runtimeParams = RuntimeParams(Map("jarFileName" -> "jar-file-name.jar")) ), scheduleProperty = CronScheduleProperty("0 0 * * * ?"), active = true, diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala similarity index 72% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala index 9b6569f972f..37aae4481e9 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala @@ -1,26 +1,28 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import com.typesafe.config.{ConfigFactory, ConfigValueFactory} +import org.scalatest.OptionValues import org.scalatest.concurrent.ScalaFutures import org.scalatest.exceptions.TestFailedException -import org.scalatest.prop.TableDrivenPropertyChecks -import org.scalatest.OptionValues import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import org.scalatest.prop.TableDrivenPropertyChecks import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId, ProcessingTypeActionServiceStub} +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{ + PeriodicProcessDeployment, + PeriodicProcessDeploymentStatus +} import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus -import pl.touk.nussknacker.engine.api.process.ProcessName +import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId, ProcessingTypeActionServiceStub} +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.PeriodicProcessStatus -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessDeployment -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model.{PeriodicProcessDeployment, PeriodicProcessDeploymentStatus} -import pl.touk.nussknacker.engine.management.periodic.service.ProcessConfigEnricher.EnrichedProcessConfig -import pl.touk.nussknacker.engine.management.periodic.service.{ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.PeriodicProcessStatus +import pl.touk.nussknacker.engine.common.periodic._ +import pl.touk.nussknacker.engine.common.periodic.service.ProcessConfigEnricher.EnrichedProcessConfig +import pl.touk.nussknacker.engine.common.periodic.service.{ AdditionalDeploymentDataProvider, DeployedEvent, FailedOnDeployEvent, @@ -32,6 +34,8 @@ import pl.touk.nussknacker.engine.management.periodic.service.{ ScheduledEvent } import pl.touk.nussknacker.test.PatientScalaFutures +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesManager +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesManager.createPeriodicProcessDeployment import java.time.temporal.ChronoField import java.time.{Clock, LocalDate, LocalDateTime} @@ -58,15 +62,19 @@ class PeriodicProcessServiceTest private val cronInFuture = CronScheduleProperty(s"0 0 6 6 9 ? ${yearNow + 1}") private val cronInPast = CronScheduleProperty(s"0 0 6 6 9 ? ${yearNow - 1}") - private val canonicalProcess = ScenarioBuilder - .streaming(processName.value) - .source("start", "source") - .emptySink("end", "KafkaSink") + private val processVersion = ProcessVersion( + versionId = VersionId(1), + processName = processName, + processId = ProcessId(1), + labels = List.empty, + user = "testUser", + modelVersion = None, + ) class Fixture { - val repository = new db.InMemPeriodicProcessesRepository(processingType = "testProcessingType") + val manager = new InMemPeriodicProcessesManager(processingType = "testProcessingType") val delegateDeploymentManagerStub = new DeploymentManagerStub - val jarManagerStub = new JarManagerStub + val periodicDeploymentHandlerStub = new PeriodicDeploymentHandlerStub val events = new ArrayBuffer[PeriodicProcessEvent]() val additionalData = Map("testMap" -> "testValue") @@ -74,8 +82,8 @@ class PeriodicProcessServiceTest val periodicProcessService = new PeriodicProcessService( delegateDeploymentManager = delegateDeploymentManagerStub, - jarManager = jarManagerStub, - scheduledProcessesRepository = repository, + periodicDeploymentHandler = periodicDeploymentHandlerStub, + periodicProcessesManager = manager, new PeriodicProcessListener { override def onPeriodicProcessEvent: PartialFunction[PeriodicProcessEvent, Unit] = { case k => @@ -86,7 +94,7 @@ class PeriodicProcessServiceTest additionalDeploymentDataProvider = new AdditionalDeploymentDataProvider { override def prepareAdditionalData( - runDetails: PeriodicProcessDeployment[WithCanonicalProcess] + runDetails: PeriodicProcessDeployment ): Map[String, String] = additionalData + ("runId" -> runDetails.id.value.toString) @@ -103,7 +111,7 @@ class PeriodicProcessServiceTest EnrichedProcessConfig( initialScheduleData.inputConfigDuringExecution.withValue( "processName", - ConfigValueFactory.fromAnyRef(initialScheduleData.canonicalProcess.name.value) + ConfigValueFactory.fromAnyRef(processName.value) ) ) ) @@ -123,19 +131,19 @@ class PeriodicProcessServiceTest }, Clock.systemDefaultZone(), actionService, - Map.empty + Map.empty, ) } test("findToBeDeployed - should return scheduled and to be retried scenarios") { val fWithNoRetries = new Fixture - fWithNoRetries.repository.addActiveProcess( + fWithNoRetries.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.FailedOnDeploy, deployMaxRetries = 0 ) - val scheduledId1 = fWithNoRetries.repository.addActiveProcess( + val scheduledId1 = fWithNoRetries.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Scheduled, deployMaxRetries = 0 @@ -143,12 +151,12 @@ class PeriodicProcessServiceTest fWithNoRetries.periodicProcessService.findToBeDeployed.futureValue.map(_.id) shouldBe List(scheduledId1) val fWithRetries = new Fixture - val failedId2 = fWithRetries.repository.addActiveProcess( + val failedId2 = fWithRetries.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.FailedOnDeploy, deployMaxRetries = 1 ) - val scheduledId2 = fWithRetries.repository.addActiveProcess( + val scheduledId2 = fWithRetries.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Scheduled, deployMaxRetries = 1 @@ -158,7 +166,7 @@ class PeriodicProcessServiceTest test("findToBeDeployed - should not return scenarios with different processing type") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled, processingType = "other") + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled, processingType = "other") f.periodicProcessService.findToBeDeployed.futureValue shouldBe Symbol("empty") } @@ -166,7 +174,7 @@ class PeriodicProcessServiceTest // Flink job could disappear from Flink console. test("handleFinished - should reschedule scenario if Flink job is missing") { val f = new Fixture - f.repository.addActiveProcess( + f.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Deployed, runAt = LocalDateTime.now().minusMinutes(11), @@ -175,23 +183,23 @@ class PeriodicProcessServiceTest f.periodicProcessService.handleFinished.futureValue - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true - f.repository.deploymentEntities should have size 2 - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.deploymentEntities should have size 2 + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Finished, PeriodicProcessDeploymentStatus.Scheduled ) val finished :: scheduled :: Nil = - f.repository.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList + f.manager.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList f.events.toList shouldBe List(FinishedEvent(finished, None), ScheduledEvent(scheduled, firstSchedule = false)) } // Flink job could not be available in Flink console if checked too quickly after submit. test("handleFinished - shouldn't reschedule scenario if Flink job is missing but not deployed for long enough") { val f = new Fixture - f.repository.addActiveProcess( + f.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Deployed, runAt = LocalDateTime.now().minusSeconds(11), @@ -200,15 +208,15 @@ class PeriodicProcessServiceTest f.periodicProcessService.handleFinished.futureValue - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true - f.repository.deploymentEntities should have size 1 - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.deploymentEntities should have size 1 + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Deployed ) val deployed :: Nil = - f.repository.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList + f.manager.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList f.events.toList shouldBe List.empty } @@ -216,7 +224,7 @@ class PeriodicProcessServiceTest val f = new Fixture val processActionId = randomProcessActionId val deploymentId = - f.repository.addActiveProcess( + f.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Deployed, processActionId = Some(processActionId) @@ -227,16 +235,16 @@ class PeriodicProcessServiceTest f.actionService.sentActionIds shouldBe Nil - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true - f.repository.deploymentEntities should have size 2 - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.deploymentEntities should have size 2 + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Finished, PeriodicProcessDeploymentStatus.Scheduled ) val finished :: scheduled :: Nil = - f.repository.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList + f.manager.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList f.events.toList shouldBe List( FinishedEvent(finished, f.delegateDeploymentManagerStub.jobStatus), ScheduledEvent(scheduled, firstSchedule = false) @@ -247,7 +255,7 @@ class PeriodicProcessServiceTest val f = new Fixture val processActionId = randomProcessActionId val deploymentId = - f.repository.addActiveProcess( + f.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Deployed, processActionId = Some(processActionId) @@ -263,7 +271,7 @@ class PeriodicProcessServiceTest test("handleFinished - should deactivate process if there are no future schedules") { val f = new Fixture val processActionId = randomProcessActionId - val deploymentId = f.repository.addActiveProcess( + val deploymentId = f.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Deployed, scheduleProperty = cronInPast, @@ -275,26 +283,29 @@ class PeriodicProcessServiceTest f.actionService.sentActionIds shouldBe List(processActionId) - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe false - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Finished + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Finished // TODO: active should be false val event = - createPeriodicProcessDeployment(processEntity.copy(active = true), f.repository.deploymentEntities.loneElement) + createPeriodicProcessDeployment( + processEntity.copy(active = true), + f.manager.deploymentEntities.loneElement, + ) f.events.loneElement shouldBe FinishedEvent(event, f.delegateDeploymentManagerStub.jobStatus) } test("handleFinished - should not deactivate process if there is future schedule") { val f = new Fixture val scheduleProperty = MultipleScheduleProperty(Map("schedule1" -> cronInPast, "schedule2" -> cronInFuture)) - val periodicProcessId = f.repository.addOnlyProcess(processName, scheduleProperty) - f.repository.addOnlyDeployment( + val periodicProcessId = f.manager.addOnlyProcess(processName, scheduleProperty) + f.manager.addOnlyDeployment( periodicProcessId, status = PeriodicProcessDeploymentStatus.Deployed, scheduleName = Some("schedule1"), deployedAt = Some(LocalDateTime.now().minusMinutes(10)) ) - f.repository.addOnlyDeployment( + f.manager.addOnlyDeployment( periodicProcessId, status = PeriodicProcessDeploymentStatus.Scheduled, scheduleName = Some("schedule2") @@ -304,22 +315,22 @@ class PeriodicProcessServiceTest f.actionService.sentActionIds shouldBe Nil - f.repository.processEntities.loneElement.active shouldBe true - f.repository.deploymentEntities.map( + f.manager.processEntities.loneElement.active shouldBe true + f.manager.deploymentEntities.map( _.status ) should contain only (PeriodicProcessDeploymentStatus.Finished, PeriodicProcessDeploymentStatus.Scheduled) } test("handleFinished - should not reschedule scenario with different processing type") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed, processingType = "other") + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed, processingType = "other") f.periodicProcessService.handleFinished.futureValue - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true - f.repository.deploymentEntities should have size 1 - f.repository.deploymentEntities.map(_.status) shouldBe List(PeriodicProcessDeploymentStatus.Deployed) + f.manager.deploymentEntities should have size 1 + f.manager.deploymentEntities.map(_.status) shouldBe List(PeriodicProcessDeploymentStatus.Deployed) f.events.toList shouldBe Symbol("empty") } @@ -327,75 +338,80 @@ class PeriodicProcessServiceTest val f = new Fixture f.periodicProcessService - .schedule(CronScheduleProperty("0 0 * * * ?"), ProcessVersion.empty, canonicalProcess, randomProcessActionId) + .schedule(CronScheduleProperty("0 0 * * * ?"), processVersion, randomProcessActionId) .futureValue - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true ConfigFactory .parseString(processEntity.inputConfigDuringExecutionJson) .getString("processName") shouldBe processName.value - val deploymentEntity = f.repository.deploymentEntities.loneElement + val deploymentEntity = f.manager.deploymentEntities.loneElement deploymentEntity.status shouldBe PeriodicProcessDeploymentStatus.Scheduled f.events.toList shouldBe List( - ScheduledEvent(createPeriodicProcessDeployment(processEntity, deploymentEntity), firstSchedule = true) + ScheduledEvent( + createPeriodicProcessDeployment(processEntity, deploymentEntity), + firstSchedule = true + ) ) } test("handleFinished - should mark as failed for failed Flink job") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) f.periodicProcessService.handleFinished.futureValue // Process is still active and has to be manually canceled by user. - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed - val expectedDetails = createPeriodicProcessDeployment(processEntity, f.repository.deploymentEntities.head) + val expectedDetails = + createPeriodicProcessDeployment(processEntity, f.manager.deploymentEntities.head) f.events.toList shouldBe List(FailedOnRunEvent(expectedDetails, f.delegateDeploymentManagerStub.jobStatus)) } test("deploy - should deploy and mark as so") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) val toSchedule = createPeriodicProcessDeployment( - f.repository.processEntities.loneElement, - f.repository.deploymentEntities.loneElement + f.manager.processEntities.loneElement, + f.manager.deploymentEntities.loneElement, ) f.periodicProcessService.deploy(toSchedule).futureValue - val deploymentEntity = f.repository.deploymentEntities.loneElement + val deploymentEntity = f.manager.deploymentEntities.loneElement deploymentEntity.status shouldBe PeriodicProcessDeploymentStatus.Deployed ConfigFactory - .parseString(f.jarManagerStub.lastDeploymentWithJarData.value.inputConfigDuringExecutionJson) + .parseString(f.periodicDeploymentHandlerStub.lastDeploymentWithRuntimeParams.value.inputConfigDuringExecutionJson) .getString("runAt") shouldBe deploymentEntity.runAt.toString - val expectedDetails = createPeriodicProcessDeployment(f.repository.processEntities.loneElement, deploymentEntity) + val expectedDetails = + createPeriodicProcessDeployment(f.manager.processEntities.loneElement, deploymentEntity) f.events.toList shouldBe List(DeployedEvent(expectedDetails, None)) } test("deploy - should handle failed deployment") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) - f.jarManagerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.periodicDeploymentHandlerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) val toSchedule = createPeriodicProcessDeployment( - f.repository.processEntities.loneElement, - f.repository.deploymentEntities.loneElement + f.manager.processEntities.loneElement, + f.manager.deploymentEntities.loneElement, ) f.periodicProcessService.deploy(toSchedule).futureValue - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.FailedOnDeploy + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.FailedOnDeploy val expectedDetails = createPeriodicProcessDeployment( - f.repository.processEntities.loneElement, - f.repository.deploymentEntities.loneElement + f.manager.processEntities.loneElement, + f.manager.deploymentEntities.loneElement, ) f.events.toList shouldBe List(FailedOnDeployEvent(expectedDetails, None)) } @@ -405,7 +421,7 @@ class PeriodicProcessServiceTest def tryToSchedule(schedule: ScheduleProperty): Unit = f.periodicProcessService - .schedule(schedule, ProcessVersion.empty, canonicalProcess, randomProcessActionId) + .schedule(schedule, processVersion, randomProcessActionId) .futureValue tryToSchedule(cronInFuture) shouldBe (()) @@ -472,9 +488,9 @@ class PeriodicProcessServiceTest val f = new Fixture val now = LocalDateTime.now() val scheduleProperty = MultipleScheduleProperty(Map("schedule1" -> cronInPast, "schedule2" -> cronInFuture)) - val periodicProcessId = f.repository.addOnlyProcess(processName, scheduleProperty) + val periodicProcessId = f.manager.addOnlyProcess(processName, scheduleProperty) statuses.zip(schedules).zipWithIndex.foreach { case ((status, schedule), index) => - f.repository.addOnlyDeployment( + f.manager.addOnlyDeployment( periodicProcessId, status = status, runAt = now.plusDays(index), diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManagerTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessStateDefinitionManagerTest.scala similarity index 84% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManagerTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessStateDefinitionManagerTest.scala index 77e551054a2..6cebe251781 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManagerTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessStateDefinitionManagerTest.scala @@ -1,15 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.api.deployment.ProcessStateDefinitionManager.ProcessStatus -import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName -import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus -import pl.touk.nussknacker.engine.api.process.VersionId -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.{DeploymentStatus, PeriodicProcessStatus} -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessStateDefinitionManager.statusTooltip -import pl.touk.nussknacker.engine.management.periodic.PeriodicStateStatus.ScheduledStatus -import pl.touk.nussknacker.engine.management.periodic.model._ +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.{DeploymentStatus, PeriodicProcessStatus} +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessStateDefinitionManager.statusTooltip import java.time.LocalDateTime import java.util.concurrent.atomic.AtomicLong diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActorTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/RescheduleFinishedActorTest.scala similarity index 91% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActorTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/RescheduleFinishedActorTest.scala index 720364a78d4..f92a0959d8d 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActorTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/RescheduleFinishedActorTest.scala @@ -1,10 +1,11 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import akka.actor.ActorSystem import akka.testkit.{TestKit, TestKitBase, TestProbe} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatest.BeforeAndAfterAll +import pl.touk.nussknacker.engine.common.periodic.RescheduleFinishedActor import scala.concurrent.Future import scala.concurrent.duration._ diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/UtilsSpec.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/UtilsSpec.scala similarity index 94% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/UtilsSpec.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/UtilsSpec.scala index a6d53da9c16..c539f6c8085 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/UtilsSpec.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/UtilsSpec.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import akka.actor.{Actor, ActorSystem, Props} import akka.testkit.TestKit @@ -6,7 +6,8 @@ import com.typesafe.scalalogging.LazyLogging import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike -import pl.touk.nussknacker.engine.management.periodic.Utils.createActorWithRetry +import pl.touk.nussknacker.engine.common.periodic.Utils +import pl.touk.nussknacker.engine.common.periodic.Utils.createActorWithRetry import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala similarity index 57% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala index 8c4b39c08e0..5dd16e1f31d 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala @@ -1,20 +1,17 @@ -package pl.touk.nussknacker.engine.management.periodic.db +package pl.touk.nussknacker.engine.management.periodic.flink.db -import cats.{Id, Monad} import io.circe.syntax.EncoderOps +import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic._ -import pl.touk.nussknacker.engine.management.periodic.db.InMemPeriodicProcessesRepository.{ - DeploymentIdSequence, - ProcessIdSequence -} -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessDeployment -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model._ +import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} +import pl.touk.nussknacker.engine.common.periodic.{CronScheduleProperty, ScheduleProperty, SingleScheduleProperty} +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesManager._ import java.time.chrono.ChronoLocalDateTime import java.time.{LocalDateTime, ZoneId} @@ -24,23 +21,19 @@ import scala.collection.mutable.ListBuffer import scala.concurrent.Future import scala.util.Random -object InMemPeriodicProcessesRepository { - private val ProcessIdSequence = new AtomicLong(0) - private val DeploymentIdSequence = new AtomicLong(0) -} - -class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicProcessesRepository { +class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProcessesManager { var processEntities: mutable.ListBuffer[PeriodicProcessEntityWithJson] = ListBuffer.empty var deploymentEntities: mutable.ListBuffer[PeriodicProcessDeploymentEntity] = ListBuffer.empty - private implicit val localDateOrdering: Ordering[LocalDateTime] = Ordering.by(identity[ChronoLocalDateTime[_]]) - - override type Action[T] = Id[T] - - override implicit def monad: Monad[Id] = cats.catsInstancesForId + private def canonicalProcess(processName: ProcessName) = { + ScenarioBuilder + .streaming(processName.value) + .source("start", "source") + .emptySink("end", "KafkaSink") + } - override def run[T](action: Id[T]): Future[T] = Future.successful(action) + private implicit val localDateOrdering: Ordering[LocalDateTime] = Ordering.by(identity[ChronoLocalDateTime[_]]) def addActiveProcess( processName: ProcessName, @@ -74,12 +67,8 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP processName = processName, processVersionId = VersionId.initialVersionId, processingType = processingType, - processJson = ScenarioBuilder - .streaming(processName.value) - .source("start", "source") - .emptySink("end", "KafkaSink"), inputConfigDuringExecutionJson = "{}", - jarFileName = "", + runtimeParams = RuntimeParams(Map("jarFileName" -> "")), scheduleProperty = scheduleProperty.asJson.noSpaces, active = true, createdAt = LocalDateTime.now(), @@ -117,46 +106,46 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP override def getSchedulesState( scenarioName: ProcessName, after: Option[LocalDateTime], - ): Action[SchedulesState] = { + ): Future[SchedulesState] = Future.successful { val filteredProcesses = processEntities.filter { pe => pe.processName == scenarioName && deploymentEntities.exists(d => d.periodicProcessId == pe.id) }.toSeq getLatestDeploymentsForPeriodicProcesses(filteredProcesses, deploymentsPerScheduleMaxCount = Int.MaxValue) } - override def markInactive(processId: PeriodicProcessId): Unit = + override def markInactive(processId: PeriodicProcessId): Future[Unit] = Future.successful { processEntities.zipWithIndex .find { case (process, _) => process.id == processId } .foreach { case (process, index) => processEntities.update(index, process.copy(active = false)) } + } override def create( - deploymentWithJarData: DeploymentWithJarData.WithCanonicalProcess, - scheduleProperty: ScheduleProperty, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, - ): PeriodicProcess[WithCanonicalProcess] = { + ): Future[PeriodicProcess] = Future.successful { val id = PeriodicProcessId(Random.nextLong()) val periodicProcess = PeriodicProcessEntityWithJson( id = id, - processName = deploymentWithJarData.processVersion.processName, - processVersionId = deploymentWithJarData.processVersion.versionId, + processName = deploymentWithRuntimeParams.processVersion.processName, + processVersionId = deploymentWithRuntimeParams.processVersion.versionId, processingType = processingType, - processJson = deploymentWithJarData.process, - inputConfigDuringExecutionJson = deploymentWithJarData.inputConfigDuringExecutionJson, - jarFileName = deploymentWithJarData.jarFileName, - scheduleProperty = scheduleProperty.asJson.noSpaces, + inputConfigDuringExecutionJson = deploymentWithRuntimeParams.inputConfigDuringExecutionJson, + runtimeParams = deploymentWithRuntimeParams.runtimeParams, + scheduleProperty = fromApi(scheduleProperty).asJson.noSpaces, active = true, createdAt = LocalDateTime.now(), processActionId = Some(processActionId) ) processEntities += periodicProcess - PeriodicProcessesRepository.createPeriodicProcessWithJson(periodicProcess) + createPeriodicProcess(periodicProcess) } override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( - expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus] - ): Action[SchedulesState] = { + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + ): Future[SchedulesState] = Future.successful { val filteredProcesses = processEntities.filter { pe => pe.processingType == processingType && deploymentEntities.exists(d => d.periodicProcessId == pe.id && expectedDeploymentStatuses.contains(d.status)) @@ -166,18 +155,19 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP override def getLatestDeploymentsForActiveSchedules( processName: ProcessName, - deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] = + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = Future.successful { getLatestDeploymentsForPeriodicProcesses( processEntities(processName).filter(_.active), deploymentsPerScheduleMaxCount ) + } override def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, - deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] = { + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = Future.successful { val filteredProcesses = processEntities(processName).filterNot(_.active).sortBy(_.createdAt).takeRight(inactiveProcessesMaxCount) getLatestDeploymentsForPeriodicProcesses(filteredProcesses, deploymentsPerScheduleMaxCount) @@ -197,43 +187,41 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP val ds = deployments .sortBy(d => -d.runAt.atZone(ZoneId.systemDefault()).toInstant.toEpochMilli) .take(deploymentsPerScheduleMaxCount) - .map(ScheduleDeploymentData(_)) + .map(scheduleDeploymentData(_)) .toList - scheduleId -> ScheduleData(PeriodicProcessesRepository.createPeriodicProcessWithoutJson(process), ds) + scheduleId -> ScheduleData(createPeriodicProcess(process), ds) } } yield deploymentGroupedByScheduleName).toMap) - override def findToBeDeployed: Seq[PeriodicProcessDeployment[WithCanonicalProcess]] = { + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { val scheduled = findActive(PeriodicProcessDeploymentStatus.Scheduled) readyToRun(scheduled) } - override def findToBeRetried: Action[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] = { + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = { val toBeRetried = findActive(PeriodicProcessDeploymentStatus.FailedOnDeploy).filter(_.retriesLeft > 0) readyToRun(toBeRetried) } - override def findProcessData(id: PeriodicProcessDeploymentId): PeriodicProcessDeployment[WithCanonicalProcess] = + override def findProcessData( + id: PeriodicProcessDeploymentId, + ): Future[PeriodicProcessDeployment] = Future.successful { (for { d <- deploymentEntities if d.id == id p <- processEntities if p.id == d.periodicProcessId } yield createPeriodicProcessDeployment(p, d)).head - - override def findProcessData(processName: ProcessName): Seq[PeriodicProcess[WithCanonicalProcess]] = - processEntities(processName) - .filter(_.active) - .map(PeriodicProcessesRepository.createPeriodicProcessWithJson) + } private def processEntities(processName: ProcessName): Seq[PeriodicProcessEntityWithJson] = processEntities .filter(process => process.processName == processName && process.processingType == processingType) .toSeq - override def markDeployed(id: PeriodicProcessDeploymentId): Unit = { + override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = Future.successful { update(id)(_.copy(status = PeriodicProcessDeploymentStatus.Deployed, deployedAt = Some(LocalDateTime.now()))) } - override def markFinished(id: PeriodicProcessDeploymentId): Unit = { + override def markFinished(id: PeriodicProcessDeploymentId): Future[Unit] = Future.successful { update(id)(_.copy(status = PeriodicProcessDeploymentStatus.Finished, completedAt = Some(LocalDateTime.now()))) } @@ -242,7 +230,7 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP status: PeriodicProcessDeploymentStatus, deployRetries: Int, retryAt: Option[LocalDateTime] - ): Action[Unit] = { + ): Future[Unit] = Future.successful { update(id)( _.copy( status = status, @@ -253,7 +241,7 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP ) } - override def markFailed(id: PeriodicProcessDeploymentId): Unit = { + override def markFailed(id: PeriodicProcessDeploymentId): Future[Unit] = Future.successful { update(id)( _.copy( status = PeriodicProcessDeploymentStatus.Failed, @@ -266,8 +254,8 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP id: PeriodicProcessId, scheduleName: ScheduleName, runAt: LocalDateTime, - deployMaxRetries: Int - ): PeriodicProcessDeployment[WithCanonicalProcess] = { + deployMaxRetries: Int, + ): Future[PeriodicProcessDeployment] = Future.successful { val deploymentEntity = PeriodicProcessDeploymentEntity( id = PeriodicProcessDeploymentId(Random.nextLong()), periodicProcessId = id, @@ -281,7 +269,8 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP status = PeriodicProcessDeploymentStatus.Scheduled ) deploymentEntities += deploymentEntity - createPeriodicProcessDeployment(processEntities.find(_.id == id).head, deploymentEntity) + val processEntity = processEntities.find(_.id == id).head + createPeriodicProcessDeployment(processEntity, deploymentEntity) } private def update( @@ -294,26 +283,129 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP } } - private def findActive( - status: PeriodicProcessDeploymentStatus - ): Seq[PeriodicProcessDeployment[WithCanonicalProcess]] = + private def findActive(status: PeriodicProcessDeploymentStatus): Seq[PeriodicProcessDeployment] = findActive( Seq(status) ) private def findActive( statusList: Seq[PeriodicProcessDeploymentStatus] - ): Seq[PeriodicProcessDeployment[WithCanonicalProcess]] = + ): Seq[PeriodicProcessDeployment] = (for { p <- processEntities if p.active && p.processingType == processingType d <- deploymentEntities if d.periodicProcessId == p.id && statusList.contains(d.status) } yield createPeriodicProcessDeployment(p, d)).toSeq private def readyToRun( - deployments: Seq[PeriodicProcessDeployment[WithCanonicalProcess]] - ): Seq[PeriodicProcessDeployment[WithCanonicalProcess]] = { + deployments: Seq[PeriodicProcessDeployment] + ): Future[Seq[PeriodicProcessDeployment]] = { val now = LocalDateTime.now() - deployments.filter(d => d.runAt.isBefore(now) || d.runAt.isEqual(now)) + Future.successful(deployments.filter(d => d.runAt.isBefore(now) || d.runAt.isEqual(now))) + } + + override def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId + ): Future[Option[CanonicalProcess]] = Future.successful(Some(canonicalProcess(processName))) + +} + +object InMemPeriodicProcessesManager { + + private val ProcessIdSequence = new AtomicLong(0) + private val DeploymentIdSequence = new AtomicLong(0) + + final case class PeriodicProcessEntity( + id: PeriodicProcessId, + processName: ProcessName, + processVersionId: VersionId, + processingType: String, + inputConfigDuringExecutionJson: String, + runtimeParams: RuntimeParams, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[ProcessActionId] + ) + + case class PeriodicProcessDeploymentEntity( + id: PeriodicProcessDeploymentId, + periodicProcessId: PeriodicProcessId, + createdAt: LocalDateTime, + runAt: LocalDateTime, + scheduleName: Option[String], + deployedAt: Option[LocalDateTime], + completedAt: Option[LocalDateTime], + retriesLeft: Int, + nextRetryAt: Option[LocalDateTime], + status: PeriodicProcessDeploymentStatus + ) + + def createPeriodicProcessDeployment( + processEntity: PeriodicProcessEntity, + processDeploymentEntity: PeriodicProcessDeploymentEntity, + ): PeriodicProcessDeployment = { + val process = createPeriodicProcess(processEntity) + PeriodicProcessDeployment( + processDeploymentEntity.id, + process, + processDeploymentEntity.createdAt, + processDeploymentEntity.runAt, + ScheduleName(processDeploymentEntity.scheduleName), + processDeploymentEntity.retriesLeft, + processDeploymentEntity.nextRetryAt, + createPeriodicDeploymentState(processDeploymentEntity) + ) + } + + def createPeriodicDeploymentState( + processDeploymentEntity: PeriodicProcessDeploymentEntity + ): PeriodicProcessDeploymentState = { + PeriodicProcessDeploymentState( + processDeploymentEntity.deployedAt, + processDeploymentEntity.completedAt, + processDeploymentEntity.status + ) + } + + def createPeriodicProcess(processEntity: PeriodicProcessEntity): PeriodicProcess = { + val processVersion = createProcessVersion(processEntity) + val scheduleProperty = prepareScheduleProperty(processEntity) + PeriodicProcess( + processEntity.id, + DeploymentWithRuntimeParams( + processVersion = processVersion, + inputConfigDuringExecutionJson = processEntity.inputConfigDuringExecutionJson, + runtimeParams = processEntity.runtimeParams, + ), + scheduleProperty, + processEntity.active, + processEntity.createdAt, + processEntity.processActionId + ) + } + + private def prepareScheduleProperty(processEntity: PeriodicProcessEntity) = { + val scheduleProperty = io.circe.parser + .decode[ScheduleProperty](processEntity.scheduleProperty) + .fold(e => throw new IllegalArgumentException(e), identity) + toApi(scheduleProperty) + } + + private def createProcessVersion(processEntity: PeriodicProcessEntity): ProcessVersion = { + ProcessVersion.empty.copy(versionId = processEntity.processVersionId, processName = processEntity.processName) + } + + private def scheduleDeploymentData(deployment: PeriodicProcessDeploymentEntity): ScheduleDeploymentData = { + ScheduleDeploymentData( + deployment.id, + deployment.createdAt, + deployment.runAt, + deployment.deployedAt, + deployment.retriesLeft, + deployment.nextRetryAt, + createPeriodicDeploymentState(deployment) + ) } } diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala index 931b35d5794..4a5f1202cf9 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala @@ -3,23 +3,17 @@ package pl.touk.nussknacker.engine.management.streaming import akka.actor.ActorSystem import org.asynchttpclient.DefaultAsyncHttpClientConfig import pl.touk.nussknacker.engine.api.component.DesignerWideComponentId +import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.api.deployment.{ DeploymentManager, NoOpScenarioActivityManager, ProcessingTypeActionServiceStub, - ProcessingTypeDeployedScenariosProviderStub, - ScenarioActivityManager + ProcessingTypeDeployedScenariosProviderStub } import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode import pl.touk.nussknacker.engine.management.FlinkStreamingDeploymentManagerProvider -import pl.touk.nussknacker.engine.{ - ConfigWithUnresolvedVersion, - DeploymentManagerDependencies, - ModelData, - ModelDependencies, - ProcessingTypeConfig -} -import sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend +import pl.touk.nussknacker.engine._ +import _root_.sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend object FlinkStreamingDeploymentManagerProviderHelper { @@ -43,6 +37,7 @@ object FlinkStreamingDeploymentManagerProviderHelper { new ProcessingTypeDeployedScenariosProviderStub(List.empty), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, actorSystem.dispatcher, actorSystem, backend diff --git a/engine/flink/management/src/test/scala/pl/touk/nussknacker/engine/management/FlinkRestManagerSpec.scala b/engine/flink/management/src/test/scala/pl/touk/nussknacker/engine/management/FlinkRestManagerSpec.scala index f379f73891b..722764a7496 100644 --- a/engine/flink/management/src/test/scala/pl/touk/nussknacker/engine/management/FlinkRestManagerSpec.scala +++ b/engine/flink/management/src/test/scala/pl/touk/nussknacker/engine/management/FlinkRestManagerSpec.scala @@ -14,18 +14,13 @@ import pl.touk.nussknacker.engine.api.component.NodesDeploymentData import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.deployment.inconsistency.InconsistentStateDetector +import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.api.{MetaData, ProcessVersion, StreamMetaData} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{ - AdditionalModelConfigs, - DeploymentData, - DeploymentId, - ExternalDeploymentId, - User -} +import pl.touk.nussknacker.engine.deployment._ import pl.touk.nussknacker.engine.management.rest.HttpFlinkClient import pl.touk.nussknacker.engine.management.rest.flinkRestModel._ import pl.touk.nussknacker.engine.testing.LocalModelData @@ -553,6 +548,7 @@ class FlinkRestManagerSpec extends AnyFunSuite with Matchers with PatientScalaFu new ProcessingTypeDeployedScenariosProviderStub(List.empty), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, ExecutionContext.global, ActorSystem(getClass.getSimpleName), sttpBackend diff --git a/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala b/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala index 0d28fae5775..fc846926fd5 100644 --- a/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala +++ b/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala @@ -8,19 +8,9 @@ import org.scalatest.matchers.should.Matchers import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy import pl.touk.nussknacker.engine.api.deployment.cache.ScenarioStateCachingConfig +import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus -import pl.touk.nussknacker.engine.api.deployment.{ - DMCancelScenarioCommand, - DMRunDeploymentCommand, - DataFreshnessPolicy, - DeployedScenarioData, - DeploymentManager, - DeploymentUpdateStrategy, - NoOpScenarioActivityManager, - ProcessingTypeActionServiceStub, - ProcessingTypeDeployedScenariosProviderStub, - ScenarioActivityManager -} +import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess @@ -56,6 +46,7 @@ class RequestResponseEmbeddedDeploymentManagerTest new ProcessingTypeDeployedScenariosProviderStub(initiallyDeployedScenarios), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, as.dispatcher, as, SttpBackendStub.asynchronousFuture diff --git a/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/BaseK8sDeploymentManagerTest.scala b/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/BaseK8sDeploymentManagerTest.scala index 05b006824bc..4de62eb847d 100644 --- a/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/BaseK8sDeploymentManagerTest.scala +++ b/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/BaseK8sDeploymentManagerTest.scala @@ -9,8 +9,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy -import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider +import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.DeploymentData import pl.touk.nussknacker.engine.{DeploymentManagerDependencies, ModelData} @@ -61,6 +62,7 @@ class BaseK8sDeploymentManagerTest new ProcessingTypeDeployedScenariosProviderStub(List.empty), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, system.dispatcher, system, backend diff --git a/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManagerOnMocksTest.scala b/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManagerOnMocksTest.scala index 16b645f8582..f1837cc3a05 100644 --- a/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManagerOnMocksTest.scala +++ b/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManagerOnMocksTest.scala @@ -9,13 +9,12 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatest.{BeforeAndAfterAll, Inside, OptionValues} import pl.touk.nussknacker.engine.DeploymentManagerDependencies +import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.api.deployment.{ DataFreshnessPolicy, NoOpScenarioActivityManager, ProcessingTypeActionServiceStub, - ProcessingTypeDeployedScenariosProvider, - ProcessingTypeDeployedScenariosProviderStub, - ScenarioActivityManager + ProcessingTypeDeployedScenariosProviderStub } import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.testing.LocalModelData @@ -75,6 +74,7 @@ class K8sDeploymentManagerOnMocksTest new ProcessingTypeDeployedScenariosProviderStub(List.empty), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, system.dispatcher, system, SttpBackendStub.asynchronousFuture From 1eaccffb83e1bd6f295c6ec5b8a1c69ed6439e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Thu, 19 Dec 2024 09:17:23 +0100 Subject: [PATCH 02/11] merge fixes --- .../img/toolbarButtons/run-off-schedule.svg | 7 + .../periodic/PeriodicProcessesManager.scala | 34 ++-- .../model/DeploymentWithRuntimeParams.scala | 24 ++- .../periodic/model/PeriodicProcess.scala | 4 +- .../model/PeriodicProcessDeployment.scala | 4 +- .../periodic/model/SchedulesState.scala | 11 +- .../ui/db/entity/PeriodicProcessesTable.scala | 164 ++++++++++++++++-- ...ositoryBasedPeriodicProcessesManager.scala | 17 +- .../PeriodicProcessesRepository.scala | 115 +++++++----- .../repository/ScenarioActionRepository.scala | 1 - ...eriodicProcessServiceIntegrationTest.scala | 8 +- .../common/periodic/DeploymentActor.scala | 16 +- .../periodic/PeriodicDeploymentHandler.scala | 4 +- .../periodic/PeriodicDeploymentManager.scala | 18 +- .../periodic/PeriodicProcessService.scala | 30 ++-- .../AdditionalDeploymentDataProvider.scala | 6 +- .../service/PeriodicProcessListener.scala | 14 +- .../service/ProcessConfigEnricher.scala | 5 +- .../utils}/DeterministicUUIDFromLong.scala | 2 +- .../FlinkPeriodicDeploymentHandler.scala | 6 +- .../periodic/flink/DeploymentActorTest.scala | 18 +- .../flink/PeriodicDeploymentHandlerStub.scala | 10 +- .../flink/PeriodicDeploymentManagerTest.scala | 5 +- .../flink/PeriodicProcessDeploymentGen.scala | 3 +- .../periodic/flink/PeriodicProcessGen.scala | 5 +- .../flink/PeriodicProcessServiceTest.scala | 3 +- ...dicProcessStateDefinitionManagerTest.scala | 6 + .../db/InMemPeriodicProcessesManager.scala | 67 ++++--- 28 files changed, 414 insertions(+), 193 deletions(-) create mode 100644 designer/client/src/assets/img/toolbarButtons/run-off-schedule.svg rename engine/{flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/util => common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/utils}/DeterministicUUIDFromLong.scala (93%) diff --git a/designer/client/src/assets/img/toolbarButtons/run-off-schedule.svg b/designer/client/src/assets/img/toolbarButtons/run-off-schedule.svg new file mode 100644 index 00000000000..c9b3eb0110d --- /dev/null +++ b/designer/client/src/assets/img/toolbarButtons/run-off-schedule.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala index 457072ae5a0..b12aba58b25 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala @@ -13,10 +13,10 @@ import scala.concurrent.Future trait PeriodicProcessesManager { def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, scheduleProperty: ScheduleProperty, processActionId: ProcessActionId, - ): Future[PeriodicProcess] + ): Future[PeriodicProcess[DeploymentWithRuntimeParams.WithConfig]] def markInactive(processId: PeriodicProcessId): Future[Unit] @@ -25,13 +25,15 @@ trait PeriodicProcessesManager { scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int, - ): Future[PeriodicProcessDeployment] + ): Future[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]] - def findProcessData(id: PeriodicProcessDeploymentId): Future[PeriodicProcessDeployment] + def findProcessData( + id: PeriodicProcessDeploymentId + ): Future[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]] - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]] - def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] + def findToBeRetried: Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]] def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] @@ -47,7 +49,8 @@ trait PeriodicProcessesManager { ): Future[Unit] def getSchedulesState( - scenarioName: ProcessName + scenarioName: ProcessName, + after: Option[LocalDateTime], ): Future[SchedulesState] def getLatestDeploymentsForActiveSchedules( @@ -87,10 +90,10 @@ object PeriodicProcessesManager { object NoOpPeriodicProcessesManager extends PeriodicProcessesManager { override def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, scheduleProperty: ScheduleProperty, processActionId: ProcessActionId, - ): Future[PeriodicProcess] = notImplemented + ): Future[PeriodicProcess[DeploymentWithRuntimeParams.WithConfig]] = notImplemented override def markInactive(processId: PeriodicProcessId): Future[Unit] = notImplemented @@ -99,15 +102,17 @@ object NoOpPeriodicProcessesManager extends PeriodicProcessesManager { scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Future[PeriodicProcessDeployment] = notImplemented + ): Future[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]] = notImplemented override def findProcessData( id: PeriodicProcessDeploymentId, - ): Future[PeriodicProcessDeployment] = notImplemented + ): Future[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]] = notImplemented - override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = notImplemented + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]] = + notImplemented - override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = notImplemented + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]] = + notImplemented override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = notImplemented @@ -122,7 +127,8 @@ object NoOpPeriodicProcessesManager extends PeriodicProcessesManager { retryAt: Option[LocalDateTime] ): Future[Unit] = notImplemented - override def getSchedulesState(scenarioName: ProcessName): Future[SchedulesState] = notImplemented + override def getSchedulesState(scenarioName: ProcessName, after: Option[LocalDateTime]): Future[SchedulesState] = + notImplemented override def getLatestDeploymentsForActiveSchedules( processName: ProcessName, diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala index 84811ae0c9a..b8a3abf218c 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala @@ -2,10 +2,24 @@ package pl.touk.nussknacker.engine.api.deployment.periodic.model import pl.touk.nussknacker.engine.api.ProcessVersion -case class DeploymentWithRuntimeParams( - processVersion: ProcessVersion, - inputConfigDuringExecutionJson: String, - runtimeParams: RuntimeParams, -) +sealed trait DeploymentWithRuntimeParams { + def processVersion: ProcessVersion + def runtimeParams: RuntimeParams +} + +object DeploymentWithRuntimeParams { + + final case class WithConfig( + processVersion: ProcessVersion, + runtimeParams: RuntimeParams, + inputConfigDuringExecutionJson: String, + ) extends DeploymentWithRuntimeParams + + final case class WithoutConfig( + processVersion: ProcessVersion, + runtimeParams: RuntimeParams, + ) extends DeploymentWithRuntimeParams + +} final case class RuntimeParams(params: Map[String, String]) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala index ab1c666a008..6201b140a3e 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala @@ -8,9 +8,9 @@ import java.time.LocalDateTime case class PeriodicProcessId(value: Long) -case class PeriodicProcess( +case class PeriodicProcess[DeploymentData <: DeploymentWithRuntimeParams]( id: PeriodicProcessId, - deploymentData: DeploymentWithRuntimeParams, + deploymentData: DeploymentData, scheduleProperty: ScheduleProperty, active: Boolean, createdAt: LocalDateTime, diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala index e67b46142dc..10206e6e3b3 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala @@ -5,9 +5,9 @@ import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessD import java.time.LocalDateTime // TODO: We should separate schedules concept from deployments - fully switch to ScheduleData and ScheduleDeploymentData -case class PeriodicProcessDeployment( +case class PeriodicProcessDeployment[DeploymentData <: DeploymentWithRuntimeParams]( id: PeriodicProcessDeploymentId, - periodicProcess: PeriodicProcess, + periodicProcess: PeriodicProcess[DeploymentData], createdAt: LocalDateTime, runAt: LocalDateTime, scheduleName: ScheduleName, diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala index ecfddb07ab0..964e4406893 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala @@ -1,5 +1,6 @@ package pl.touk.nussknacker.engine.api.deployment.periodic.model +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithoutConfig import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap @@ -34,7 +35,7 @@ case class SchedulesState(schedules: Map[ScheduleId, ScheduleData]) { // For most operations it will contain only one latest deployment but for purpose of statuses of historical deployments // it has list instead of one element. // This structure should contain SingleScheduleProperty as well. See note above -case class ScheduleData(process: PeriodicProcess, latestDeployments: List[ScheduleDeploymentData]) +case class ScheduleData(process: PeriodicProcess[WithoutConfig], latestDeployments: List[ScheduleDeploymentData]) // To identify schedule we need scheduleName - None for SingleScheduleProperty and Some(key) for MultipleScheduleProperty keys // Also we need PeriodicProcessId to distinguish between active schedules and some inactive from the past for the same PeriodicProcessId @@ -52,9 +53,9 @@ case class ScheduleDeploymentData( ) { def toFullDeploymentData( - process: PeriodicProcess, + process: PeriodicProcess[DeploymentWithRuntimeParams.WithoutConfig], scheduleName: ScheduleName - ): PeriodicProcessDeployment = + ): PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithoutConfig] = PeriodicProcessDeployment(id, process, createdAt, runAt, scheduleName, retriesLeft, nextRetryAt, state) def display = s"deploymentId=$id" @@ -63,6 +64,6 @@ case class ScheduleDeploymentData( // These below are temporary structures, see notice next to SchedulesState case class PeriodicProcessScheduleData( - process: PeriodicProcess, - deployments: List[PeriodicProcessDeployment] + process: PeriodicProcess[DeploymentWithRuntimeParams.WithoutConfig], + deployments: List[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithoutConfig]] ) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala index 0d3b85155c4..23046e4db7f 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala @@ -5,6 +5,8 @@ import io.circe.syntax.EncoderOps import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.model.{PeriodicProcessId, RuntimeParams} import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.marshall.ProcessMarshaller import slick.jdbc.JdbcProfile import slick.lifted.ProvenShape import slick.sql.SqlProfile.ColumnOption.NotNull @@ -12,7 +14,7 @@ import slick.sql.SqlProfile.ColumnOption.NotNull import java.time.LocalDateTime import java.util.UUID -trait PeriodicProcessesTableFactory extends BaseEntityFactory{ +trait PeriodicProcessesTableFactory extends BaseEntityFactory { protected val profile: JdbcProfile @@ -21,7 +23,7 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory{ implicit val periodicProcessIdMapping: BaseColumnType[PeriodicProcessId] = MappedColumnType.base[PeriodicProcessId, Long](_.value, PeriodicProcessId.apply) - private implicit val ProcessActionIdTypedType: BaseColumnType[ProcessActionId] = + private implicit val processActionIdTypedType: BaseColumnType[ProcessActionId] = MappedColumnType.base[ProcessActionId, UUID]( _.value, ProcessActionId(_) @@ -37,7 +39,8 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory{ } ) - class PeriodicProcessesTable(tag: Tag) extends Table[PeriodicProcessEntity](tag, "periodic_processes") { + abstract class PeriodicProcessesTable[ENTITY <: PeriodicProcessEntity](tag: Tag) + extends Table[ENTITY](tag, "periodic_processes") { def id: Rep[PeriodicProcessId] = column[PeriodicProcessId]("id", O.PrimaryKey, O.AutoInc) @@ -47,10 +50,6 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory{ def processingType: Rep[String] = column[String]("processing_type", NotNull) - def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) - - // This is a legacy column left after migrating periodic processes to core - // The periodic deployment manager is now decoupled from Flink, and its runtime params are stored in runtime_params column def jarFileName: Rep[Option[String]] = column[Option[String]]("jar_file_name") def runtimeParams: Rep[RuntimeParams] = column[RuntimeParams]("runtime_params") @@ -63,7 +62,13 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory{ def processActionId: Rep[Option[ProcessActionId]] = column[Option[ProcessActionId]]("process_action_id") - override def * : ProvenShape[PeriodicProcessEntity] = ( + } + + class PeriodicProcessesWithJsonTable(tag: Tag) extends PeriodicProcessesTable[PeriodicProcessEntityWithJson](tag) { + + def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) + + override def * : ProvenShape[PeriodicProcessEntityWithJson] = ( id, processName, processVersionId, @@ -77,7 +82,7 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory{ processActionId ) <> ( tuple => - PeriodicProcessEntity( + PeriodicProcessEntityWithJson( id = tuple._1, processName = tuple._2, processVersionId = tuple._3, @@ -90,8 +95,8 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory{ createdAt = tuple._10, processActionId = tuple._11, ), - (e: PeriodicProcessEntity) => - PeriodicProcessEntity.unapply(e).map { + (e: PeriodicProcessEntityWithJson) => + PeriodicProcessEntityWithJson.unapply(e).map { case ( id, processName, @@ -122,11 +127,130 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory{ } - object PeriodicProcesses extends TableQuery(new PeriodicProcessesTable(_)) + class PeriodicProcessWithoutJson(tag: Tag) extends PeriodicProcessesTable[PeriodicProcessEntityWithoutJson](tag) { + + override def * : ProvenShape[PeriodicProcessEntityWithoutJson] = ( + id, + processName, + processVersionId, + processingType, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) <> ( + (PeriodicProcessEntity.createWithoutJson _).tupled, + (e: PeriodicProcessEntityWithoutJson) => + PeriodicProcessEntityWithoutJson.unapply(e).map { + case ( + id, + processName, + versionId, + processingType, + jarFileName, + scheduleProperty, + active, + createdAt, + processActionId + ) => + ( + id, + processName, + versionId, + processingType, + jarFileName, + scheduleProperty, + active, + createdAt, + processActionId + ) + } + ) + + } + + object PeriodicProcessesWithJson extends TableQuery(new PeriodicProcessesWithJsonTable(_)) + + object PeriodicProcessesWithoutJson extends TableQuery(new PeriodicProcessWithoutJson(_)) + +} + +object PeriodicProcessEntity { + + def createWithJson( + id: PeriodicProcessId, + processName: ProcessName, + processVersionId: VersionId, + processingType: String, + inputConfigDuringExecutionJson: String, + runtimeParams: RuntimeParams, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[ProcessActionId] + ): PeriodicProcessEntityWithJson = + PeriodicProcessEntityWithJson( + id, + processName, + processVersionId, + processingType, + inputConfigDuringExecutionJson, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) + + def createWithoutJson( + id: PeriodicProcessId, + processName: ProcessName, + processVersionId: VersionId, + processingType: String, + runtimeParams: RuntimeParams, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[ProcessActionId] + ): PeriodicProcessEntityWithoutJson = + PeriodicProcessEntityWithoutJson( + id, + processName, + processVersionId, + processingType, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) + +} + +trait PeriodicProcessEntity { + + def id: PeriodicProcessId + + def processName: ProcessName + + def processVersionId: VersionId + + def processingType: String + + def runtimeParams: RuntimeParams + + def scheduleProperty: String + + def active: Boolean + + def createdAt: LocalDateTime + + def processActionId: Option[ProcessActionId] } -final case class PeriodicProcessEntity( +case class PeriodicProcessEntityWithJson( id: PeriodicProcessId, processName: ProcessName, processVersionId: VersionId, @@ -137,4 +261,16 @@ final case class PeriodicProcessEntity( active: Boolean, createdAt: LocalDateTime, processActionId: Option[ProcessActionId] -) +) extends PeriodicProcessEntity + +case class PeriodicProcessEntityWithoutJson( + id: PeriodicProcessId, + processName: ProcessName, + processVersionId: VersionId, + processingType: String, + runtimeParams: RuntimeParams, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[ProcessActionId] +) extends PeriodicProcessEntity diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala index 957b11dc6ba..429cdd45ada 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala @@ -2,6 +2,7 @@ package pl.touk.nussknacker.ui.process.periodic import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} @@ -20,10 +21,10 @@ class RepositoryBasedPeriodicProcessesManager( import periodicProcessesRepository._ override def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, - ): Future[PeriodicProcess] = + ): Future[PeriodicProcess[DeploymentWithRuntimeParams.WithConfig]] = periodicProcessesRepository .create(deploymentWithRuntimeParams, scheduleProperty, processActionId, processingType) .run @@ -36,18 +37,18 @@ class RepositoryBasedPeriodicProcessesManager( scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Future[PeriodicProcessDeployment] = + ): Future[PeriodicProcessDeployment[WithConfig]] = periodicProcessesRepository.schedule(id, scheduleName, runAt, deployMaxRetries).run override def findProcessData( id: PeriodicProcessDeploymentId, - ): Future[PeriodicProcessDeployment] = + ): Future[PeriodicProcessDeployment[WithConfig]] = periodicProcessesRepository.findProcessData(id).run - override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = periodicProcessesRepository.findToBeDeployed(processingType).run - override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = periodicProcessesRepository.findToBeRetried(processingType).run override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = @@ -66,8 +67,8 @@ class RepositoryBasedPeriodicProcessesManager( retryAt: Option[LocalDateTime] ): Future[Unit] = periodicProcessesRepository.markFailedOnDeployWithStatus(id, status, deployRetries, retryAt).run - override def getSchedulesState(scenarioName: ProcessName): Future[SchedulesState] = - periodicProcessesRepository.getSchedulesState(scenarioName).run + override def getSchedulesState(scenarioName: ProcessName, after: Option[LocalDateTime]): Future[SchedulesState] = + periodicProcessesRepository.getSchedulesState(scenarioName, after).run override def getLatestDeploymentsForActiveSchedules( processName: ProcessName, diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala index 6cb2531a9c6..66f147919d4 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala @@ -9,6 +9,7 @@ import io.circe.syntax.EncoderOps import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.{WithConfig, WithoutConfig} import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} @@ -27,10 +28,10 @@ import scala.language.higherKinds object PeriodicProcessesRepository { def createPeriodicProcessDeployment( - processEntity: PeriodicProcessEntity, - processDeploymentEntity: PeriodicProcessDeploymentEntity, - ): PeriodicProcessDeployment = { - val process = createPeriodicProcess(processEntity) + processEntity: PeriodicProcessEntityWithJson, + processDeploymentEntity: PeriodicProcessDeploymentEntity + ): PeriodicProcessDeployment[WithConfig] = { + val process = createPeriodicProcessWithJson(processEntity) PeriodicProcessDeployment( processDeploymentEntity.id, process, @@ -53,17 +54,37 @@ object PeriodicProcessesRepository { ) } - def createPeriodicProcess(processEntity: PeriodicProcessEntity): PeriodicProcess = { + def createPeriodicProcessWithJson( + processEntity: PeriodicProcessEntityWithJson + ): PeriodicProcess[WithConfig] = { val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, - DeploymentWithRuntimeParams( + DeploymentWithRuntimeParams.WithConfig( processVersion = processVersion, inputConfigDuringExecutionJson = processEntity.inputConfigDuringExecutionJson, runtimeParams = processEntity.runtimeParams, ), - scheduleProperty, + toApi(scheduleProperty), + processEntity.active, + processEntity.createdAt, + processEntity.processActionId + ) + } + + def createPeriodicProcessWithoutJson( + processEntity: PeriodicProcessEntity + ): PeriodicProcess[WithoutConfig] = { + val processVersion = createProcessVersion(processEntity) + val scheduleProperty = prepareScheduleProperty(processEntity) + PeriodicProcess( + processEntity.id, + DeploymentWithRuntimeParams.WithoutConfig( + processVersion = processVersion, + runtimeParams = processEntity.runtimeParams, + ), + toApi(scheduleProperty), processEntity.active, processEntity.createdAt, processEntity.processActionId @@ -73,7 +94,7 @@ object PeriodicProcessesRepository { private def prepareScheduleProperty(processEntity: PeriodicProcessEntity) = { val scheduleProperty = decode[ScheduleProperty](processEntity.scheduleProperty) .fold(e => throw new IllegalArgumentException(e), identity) - toApi(scheduleProperty) + scheduleProperty } private def createProcessVersion(processEntity: PeriodicProcessEntity): ProcessVersion = { @@ -102,11 +123,11 @@ trait PeriodicProcessesRepository { ): Action[SchedulesState] def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, processingType: String, - ): Action[PeriodicProcess] + ): Action[PeriodicProcess[DeploymentWithRuntimeParams.WithConfig]] def getLatestDeploymentsForActiveSchedules( processName: ProcessName, @@ -121,16 +142,16 @@ trait PeriodicProcessesRepository { processingType: String, ): Action[SchedulesState] - def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment]] + def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment[WithConfig]]] - def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment]] + def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment[WithConfig]]] def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], processingType: String, ): Action[SchedulesState] - def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment] + def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment[WithConfig]] def markDeployed(id: PeriodicProcessDeploymentId): Action[Unit] @@ -150,12 +171,12 @@ trait PeriodicProcessesRepository { scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Action[PeriodicProcessDeployment] + ): Action[PeriodicProcessDeployment[WithConfig]] def fetchCanonicalProcess( processName: ProcessName, versionId: VersionId, - ): Action[Option[WithCanonicalProcess]] + ): Action[Option[CanonicalProcess]] } @@ -183,7 +204,7 @@ class SlickPeriodicProcessesRepository( scenarioName: ProcessName, afterOpt: Option[LocalDateTime], ): Action[SchedulesState] = { - PeriodicProcesses + PeriodicProcessesWithoutJson .filter(_.processName === scenarioName) .join(PeriodicProcessDeployments) .on(_.id === _.periodicProcessId) @@ -193,12 +214,12 @@ class SlickPeriodicProcessesRepository( } override def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, processingType: String, - ): Action[PeriodicProcess] = { - val processEntity = PeriodicProcessEntity( + ): Action[PeriodicProcess[DeploymentWithRuntimeParams.WithConfig]] = { + val processEntity = PeriodicProcessEntityWithJson( id = PeriodicProcessId(-1), processName = deploymentWithRuntimeParams.processVersion.processName, processVersionId = deploymentWithRuntimeParams.processVersion.versionId, @@ -210,32 +231,34 @@ class SlickPeriodicProcessesRepository( createdAt = now(), Some(processActionId) ) - ((PeriodicProcesses returning PeriodicProcesses into ((_, id) => id)) += processEntity) - .map(PeriodicProcessesRepository.createPeriodicProcess) + ((PeriodicProcessesWithJson returning PeriodicProcessesWithJson into ((_, id) => id)) += processEntity) + .map(PeriodicProcessesRepository.createPeriodicProcessWithJson) } private def now(): LocalDateTime = LocalDateTime.now(clock) - override def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment]] = findProcesses( - activePeriodicProcessWithDeploymentQuery(processingType) - .filter { case (_, d) => - d.runAt <= now() && - d.status === (PeriodicProcessDeploymentStatus.Scheduled: PeriodicProcessDeploymentStatus) - } - ) + override def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment[WithConfig]]] = + findProcesses( + activePeriodicProcessWithDeploymentQuery(processingType) + .filter { case (_, d) => + d.runAt <= now() && + d.status === (PeriodicProcessDeploymentStatus.Scheduled: PeriodicProcessDeploymentStatus) + } + ) - override def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment]] = findProcesses( - activePeriodicProcessWithDeploymentQuery(processingType) - .filter { case (_, d) => - d.nextRetryAt <= now() && - d.status === (PeriodicProcessDeploymentStatus.RetryingDeploy: PeriodicProcessDeploymentStatus) - } - ) + override def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment[WithConfig]]] = + findProcesses( + activePeriodicProcessWithDeploymentQuery(processingType) + .filter { case (_, d) => + d.nextRetryAt <= now() && + d.status === (PeriodicProcessDeploymentStatus.RetryingDeploy: PeriodicProcessDeploymentStatus) + } + ) private def findProcesses( query: Query[ - (PeriodicProcessesTable, PeriodicProcessDeploymentsTable), - (PeriodicProcessEntity, PeriodicProcessDeploymentEntity), + (PeriodicProcessesWithJsonTable, PeriodicProcessDeploymentsTable), + (PeriodicProcessEntityWithJson, PeriodicProcessDeploymentEntity), Seq ] ) = { @@ -248,9 +271,9 @@ class SlickPeriodicProcessesRepository( }) } - override def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment] = + override def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment[WithConfig]] = findProcesses( - (PeriodicProcesses join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) + (PeriodicProcessesWithJson join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) .filter { case (_, deployment) => deployment.id === id } ).map(_.head) @@ -298,7 +321,7 @@ class SlickPeriodicProcessesRepository( expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], processingType: String, ): Action[SchedulesState] = { - val processesHavingDeploymentsWithMatchingStatus = PeriodicProcesses.filter(p => + val processesHavingDeploymentsWithMatchingStatus = PeriodicProcessesWithoutJson.filter(p => p.active && PeriodicProcessDeployments .filter(d => d.periodicProcessId === p.id && d.status.inSet(expectedDeploymentStatuses)) @@ -316,7 +339,7 @@ class SlickPeriodicProcessesRepository( deploymentsPerScheduleMaxCount: Int, processingType: String, ): Action[SchedulesState] = { - val activeProcessesQuery = PeriodicProcesses.filter(p => p.processName === processName && p.active) + val activeProcessesQuery = PeriodicProcessesWithoutJson.filter(p => p.processName === processName && p.active) getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount, processingType) } @@ -326,7 +349,7 @@ class SlickPeriodicProcessesRepository( deploymentsPerScheduleMaxCount: Int, processingType: String, ): Action[SchedulesState] = { - val filteredProcessesQuery = PeriodicProcesses + val filteredProcessesQuery = PeriodicProcessesWithoutJson .filter(p => p.processName === processName && !p.active) .sortBy(_.createdAt.desc) .take(inactiveProcessesMaxCount) @@ -425,7 +448,7 @@ class SlickPeriodicProcessesRepository( scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Action[PeriodicProcessDeployment] = { + ): Action[PeriodicProcessDeployment[WithConfig]] = { val deploymentEntity = PeriodicProcessDeploymentEntity( id = PeriodicProcessDeploymentId(-1), periodicProcessId = id, @@ -445,7 +468,7 @@ class SlickPeriodicProcessesRepository( override def markInactive(processId: PeriodicProcessId): Action[Unit] = { val q = for { - p <- PeriodicProcesses if p.id === processId + p <- PeriodicProcessesWithoutJson if p.id === processId } yield p.active val update = q.update(false) update.map(_ => ()) @@ -463,7 +486,7 @@ class SlickPeriodicProcessesRepository( } private def activePeriodicProcessWithDeploymentQuery(processingType: String) = { - (PeriodicProcesses.filter(p => p.active === true && p.processingType === processingType) + (PeriodicProcessesWithJson.filter(p => p.active === true && p.processingType === processingType) join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) } @@ -473,7 +496,7 @@ class SlickPeriodicProcessesRepository( .map { case (process, deployment) => val scheduleId = ScheduleId(process.id, ScheduleName(deployment.scheduleName)) val scheduleDataWithoutDeployment = - (scheduleId, PeriodicProcessesRepository.createPeriodicProcess(process)) + (scheduleId, PeriodicProcessesRepository.createPeriodicProcessWithoutJson(process)) val scheduleDeployment = scheduleDeploymentData(deployment) (scheduleDataWithoutDeployment, scheduleDeployment) } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala index 5c9e6f83ef5..68aecc26a42 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala @@ -7,7 +7,6 @@ import pl.touk.nussknacker.engine.api.Comment import pl.touk.nussknacker.engine.api.deployment.ProcessActionState.ProcessActionState import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, ProcessingType, VersionId} -import pl.touk.nussknacker.engine.common.periodic.InstantBatchCustomAction import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap import pl.touk.nussknacker.ui.app.BuildInfo import pl.touk.nussknacker.ui.db.entity.{ diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala index 9232b146cfa..92aae13cffd 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -66,6 +66,8 @@ class PeriodicProcessServiceIntegrationTest private val processName = ProcessName("test") + private val processIdWithName = ProcessIdWithName(ProcessId(1), processName) + private val startTime = Instant.parse("2021-04-06T13:18:00Z") // we truncate to millis, as HSQL stores with that precision... @@ -83,8 +85,8 @@ class PeriodicProcessServiceIntegrationTest )(testCode: Fixture => Any): Unit = { def runTestCodeWithDbConfig = { testCode( - new Fixture(testDbRef, deploymentRetryConfig, executionConfig, maxFetchedPeriodicScenarioActivities) - ) + new Fixture(testDbRef, deploymentRetryConfig, executionConfig, maxFetchedPeriodicScenarioActivities) + ) } logger.debug("Running test with database") runTestCodeWithDbConfig @@ -612,7 +614,7 @@ class PeriodicProcessServiceIntegrationTest inactiveStates.latestDeploymentForSchedule(schedule1).state.status shouldBe PeriodicProcessDeploymentStatus.Finished inactiveStates.latestDeploymentForSchedule(schedule2).state.status shouldBe PeriodicProcessDeploymentStatus.Finished - val activities = service.getScenarioActivitiesSpecificToPeriodicProcess(processIdWithName).futureValue + val activities = service.getScenarioActivitiesSpecificToPeriodicProcess(processIdWithName, None).futureValue val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] val secondActivity = activities(1).asInstanceOf[ScenarioActivity.PerformedScheduledExecution] activities shouldBe List( diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala index ba383b46eab..d68fdc7dea3 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala @@ -2,7 +2,7 @@ package pl.touk.nussknacker.engine.common.periodic import akka.actor.{Actor, Props, Timers} import com.typesafe.scalalogging.LazyLogging -import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{DeploymentWithRuntimeParams, PeriodicProcessDeployment} import pl.touk.nussknacker.engine.common.periodic.DeploymentActor.{ CheckToBeDeployed, DeploymentCompleted, @@ -20,8 +20,8 @@ object DeploymentActor { } private[engine] def props( - findToBeDeployed: => Future[Seq[PeriodicProcessDeployment]], - deploy: PeriodicProcessDeployment => Future[Unit], + findToBeDeployed: => Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]], + deploy: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig] => Future[Unit], interval: FiniteDuration ) = { Props(new DeploymentActor(findToBeDeployed, deploy, interval)) @@ -29,14 +29,14 @@ object DeploymentActor { private[engine] case object CheckToBeDeployed - private case class WaitingForDeployment(ids: List[PeriodicProcessDeployment]) + private case class WaitingForDeployment(ids: List[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]) private case object DeploymentCompleted } class DeploymentActor( - findToBeDeployed: => Future[Seq[PeriodicProcessDeployment]], - deploy: PeriodicProcessDeployment => Future[Unit], + findToBeDeployed: => Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]], + deploy: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig] => Future[Unit], interval: FiniteDuration ) extends Actor with Timers @@ -72,7 +72,9 @@ class DeploymentActor( } } - private def receiveOngoingDeployment(runDetails: PeriodicProcessDeployment): Receive = { + private def receiveOngoingDeployment( + runDetails: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig] + ): Receive = { case CheckToBeDeployed => logger.debug(s"Still waiting for ${runDetails.display} to be deployed") case DeploymentCompleted => diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala index 81a0b8bd3f2..5d9456be4cf 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala @@ -11,10 +11,10 @@ trait PeriodicDeploymentHandler { def prepareDeploymentWithRuntimeParams( processVersion: ProcessVersion, - ): Future[DeploymentWithRuntimeParams] + ): Future[DeploymentWithRuntimeParams.WithConfig] def deployWithRuntimeParams( - deploymentWithJarData: DeploymentWithRuntimeParams, + deploymentWithJarData: DeploymentWithRuntimeParams.WithConfig, deploymentData: DeploymentData, canonicalProcess: CanonicalProcess, ): Future[Option[ExternalDeploymentId]] diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala index 162b98de32e..df429c86341 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala @@ -15,7 +15,7 @@ import pl.touk.nussknacker.engine.common.periodic.service.{ PeriodicProcessListenerFactory, ProcessConfigEnricherFactory } -import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, ExternalDeploymentId} +import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, ExternalDeploymentId, RunOffScheduleResult} import java.time.{Clock, Instant} import scala.concurrent.{ExecutionContext, Future} @@ -67,12 +67,6 @@ object PeriodicDeploymentManager { dependencies.actorSystem ) - val customActionsProvider = customActionsProviderFactory.create( - periodicProcessesManager, - service, - periodicBatchConfig.processingType - ) - val toClose = () => { runSafely(listener.close()) // deploymentActor and rescheduleFinishedActor just call methods from PeriodicProcessService on interval, @@ -83,9 +77,8 @@ object PeriodicDeploymentManager { new PeriodicDeploymentManager( delegate, service, - scheduledProcessesRepository, + periodicProcessesManager, schedulePropertyExtractorFactory(originalConfig), - customActionsProvider, periodicBatchConfig.processingType, toClose ) @@ -96,9 +89,8 @@ object PeriodicDeploymentManager { class PeriodicDeploymentManager private[engine] ( val delegate: DeploymentManager, service: PeriodicProcessService, - repository: PeriodicProcessesRepository, + periodicProcessesManager: PeriodicProcessesManager, schedulePropertyExtractor: SchedulePropertyExtractor, - customActionsProvider: PeriodicCustomActionsProvider, processingType: String, toClose: () => Unit )(implicit val ec: ExecutionContext) @@ -106,8 +98,6 @@ class PeriodicDeploymentManager private[engine] ( with ManagerSpecificScenarioActivitiesStoredByManager with LazyLogging { - import repository._ - override def processCommand[Result](command: DMScenarioCommand[Result]): Future[Result] = command match { case command: DMValidateScenarioCommand => validate(command) @@ -270,7 +260,7 @@ class PeriodicDeploymentManager private[engine] ( .map(_.groupedByPeriodicProcess.headOption.flatMap(_.deployments.headOption)) ) processDeploymentWithProcessJson <- OptionT.liftF( - repository.findProcessData(processDeployment.id).run + periodicProcessesManager.findProcessData(processDeployment.id) ) _ <- OptionT.liftF(service.deploy(processDeploymentWithProcessJson)) } yield () diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala index bfbd375cb38..3cfbef3ca9f 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala @@ -11,6 +11,7 @@ import pl.touk.nussknacker.engine.api.component.{ import pl.touk.nussknacker.engine.api.deployment.StateStatus.StatusName import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.{WithConfig, WithoutConfig} import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus @@ -26,6 +27,7 @@ import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.{ import pl.touk.nussknacker.engine.common.periodic.PeriodicStateStatus.{ScheduledStatus, WaitingForScheduleStatus} import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} import pl.touk.nussknacker.engine.common.periodic.service._ +import pl.touk.nussknacker.engine.common.periodic.utils.DeterministicUUIDFromLong import pl.touk.nussknacker.engine.deployment.{AdditionalModelConfigs, DeploymentData, DeploymentId} import pl.touk.nussknacker.engine.util.AdditionalComponentConfigsForRuntimeExtractor @@ -69,7 +71,10 @@ class PeriodicProcessService( processIdWithName: ProcessIdWithName, after: Option[Instant], ): Future[List[ScenarioActivity]] = for { - schedulesState <- periodicProcessesManager.getSchedulesState(processIdWithName.name) + schedulesState <- periodicProcessesManager.getSchedulesState( + processIdWithName.name, + after.map(localDateTimeAtSystemDefaultZone) + ) groupedByProcess = schedulesState.groupedByPeriodicProcess deployments = groupedByProcess.flatMap(_.deployments) deploymentsWithStatuses = deployments.flatMap(d => scheduledExecutionStatusAndDateFinished(d).map((d, _))) @@ -163,7 +168,7 @@ class PeriodicProcessService( private def initialSchedule( scheduleMap: ScheduleProperty, scheduleDates: List[(ScheduleName, Option[LocalDateTime])], - deploymentWithJarData: DeploymentWithRuntimeParams, + deploymentWithJarData: DeploymentWithRuntimeParams.WithConfig, processActionId: ProcessActionId, ): Future[Unit] = { periodicProcessesManager @@ -184,7 +189,7 @@ class PeriodicProcessService( .map(_ => ()) } - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]] = { for { toBeDeployed <- periodicProcessesManager.findToBeDeployed.flatMap { toDeployList => Future.sequence(toDeployList.map(checkIfNotRunning)).map(_.flatten) @@ -198,8 +203,8 @@ class PeriodicProcessService( // Currently we don't allow simultaneous runs of one scenario - only sequential, so if other schedule kicks in, it'll have to wait // TODO: we show allow to deploy scenarios with different scheduleName to be deployed simultaneous private def checkIfNotRunning( - toDeploy: PeriodicProcessDeployment - ): Future[Option[PeriodicProcessDeployment]] = { + toDeploy: PeriodicProcessDeployment[WithConfig] + ): Future[Option[PeriodicProcessDeployment[WithConfig]]] = { delegateDeploymentManager .getProcessStates(toDeploy.periodicProcess.processVersion.processName)(DataFreshnessPolicy.Fresh) .map( @@ -344,7 +349,10 @@ class PeriodicProcessService( scheduleActions.flatten.sequence.as(emptyCallback) } - private def nextRunAt(deployment: PeriodicProcessDeployment, clock: Clock): Either[String, Option[LocalDateTime]] = + private def nextRunAt( + deployment: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithoutConfig], + clock: Clock + ): Either[String, Option[LocalDateTime]] = (fromApi(deployment.periodicProcess.scheduleProperty), deployment.scheduleName.value) match { case (MultipleScheduleProperty(schedules), Some(name)) => schedules.get(name).toRight(s"Failed to find schedule: $deployment.scheduleName").flatMap(_.nextRunAt(clock)) @@ -361,7 +369,7 @@ class PeriodicProcessService( } private def handleFailedDeployment( - deployment: PeriodicProcessDeployment, + deployment: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig], state: Option[StatusDetails] ): Future[Unit] = { def calculateNextRetryAt = now().plus(deploymentRetryConfig.deployRetryPenalize.toMillis, ChronoUnit.MILLIS) @@ -405,7 +413,9 @@ class PeriodicProcessService( _ <- activeSchedules.groupedByPeriodicProcess.map(p => deactivateAction(p.process)).sequence.runWithCallbacks } yield runningDeploymentsForSchedules.map(deployment => DeploymentId(deployment.toString)) - private def deactivateAction(process: PeriodicProcess): Future[Callback] = { + private def deactivateAction( + process: PeriodicProcess[DeploymentWithRuntimeParams.WithoutConfig] + ): Future[Callback] = { logger.info(s"Deactivate periodic process id: ${process.id.value}") for { _ <- periodicProcessesManager.markInactive(process.id) @@ -424,7 +434,7 @@ class PeriodicProcessService( .map(_ => ()) } - def deploy(deployment: PeriodicProcessDeployment): Future[Unit] = { + def deploy(deployment: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]): Future[Unit] = { // TODO: set status before deployment? val id = deployment.id val deploymentData = DeploymentData( @@ -568,7 +578,7 @@ class PeriodicProcessService( } private def scheduledExecutionStatusAndDateFinished( - entity: PeriodicProcessDeployment, + entity: PeriodicProcessDeployment[WithoutConfig], ): Option[FinishedScheduledExecutionMetadata] = { for { status <- entity.state.status match { diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala index 2fcdfcaab12..b48985f7ae9 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala @@ -1,19 +1,19 @@ package pl.touk.nussknacker.engine.common.periodic.service +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import java.time.format.DateTimeFormatter trait AdditionalDeploymentDataProvider { - def prepareAdditionalData(runDetails: PeriodicProcessDeployment): Map[String, String] + def prepareAdditionalData(runDetails: PeriodicProcessDeployment[WithConfig]): Map[String, String] } object DefaultAdditionalDeploymentDataProvider extends AdditionalDeploymentDataProvider { - override def prepareAdditionalData(runDetails: PeriodicProcessDeployment): Map[String, String] = { + override def prepareAdditionalData(runDetails: PeriodicProcessDeployment[WithConfig]): Map[String, String] = { Map( "deploymentId" -> runDetails.id.value.toString, "runAt" -> runDetails.runAt.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala index 80ef76f6564..e1d89442069 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala @@ -2,6 +2,7 @@ package pl.touk.nussknacker.engine.common.periodic.service import com.typesafe.config.Config import pl.touk.nussknacker.engine.api.deployment.StatusDetails +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.ExternalDeploymentId @@ -21,28 +22,29 @@ trait PeriodicProcessListenerFactory { } sealed trait PeriodicProcessEvent { - val deployment: PeriodicProcessDeployment + val deployment: PeriodicProcessDeployment[WithConfig] } case class DeployedEvent( - deployment: PeriodicProcessDeployment, + deployment: PeriodicProcessDeployment[WithConfig], externalDeploymentId: Option[ExternalDeploymentId] ) extends PeriodicProcessEvent -case class FinishedEvent(deployment: PeriodicProcessDeployment, processState: Option[StatusDetails]) +case class FinishedEvent(deployment: PeriodicProcessDeployment[WithConfig], processState: Option[StatusDetails]) extends PeriodicProcessEvent case class FailedOnDeployEvent( - deployment: PeriodicProcessDeployment, + deployment: PeriodicProcessDeployment[WithConfig], processState: Option[StatusDetails] ) extends PeriodicProcessEvent case class FailedOnRunEvent( - deployment: PeriodicProcessDeployment, + deployment: PeriodicProcessDeployment[WithConfig], processState: Option[StatusDetails] ) extends PeriodicProcessEvent -case class ScheduledEvent(deployment: PeriodicProcessDeployment, firstSchedule: Boolean) extends PeriodicProcessEvent +case class ScheduledEvent(deployment: PeriodicProcessDeployment[WithConfig], firstSchedule: Boolean) + extends PeriodicProcessEvent object EmptyListener extends EmptyListener diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala index 5f439a0601b..579bd9c2500 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala @@ -1,9 +1,8 @@ package pl.touk.nussknacker.engine.common.periodic.service import com.typesafe.config.{Config, ConfigFactory} +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess import pl.touk.nussknacker.engine.common.periodic.service.ProcessConfigEnricher.{ DeployData, EnrichedProcessConfig, @@ -44,7 +43,7 @@ object ProcessConfigEnricher { case class DeployData( inputConfigDuringExecutionJson: String, - deployment: PeriodicProcessDeployment + deployment: PeriodicProcessDeployment[WithConfig] ) extends ProcessConfigEnricherInputData case class EnrichedProcessConfig(inputConfigDuringExecutionJson: String) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/util/DeterministicUUIDFromLong.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/utils/DeterministicUUIDFromLong.scala similarity index 93% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/util/DeterministicUUIDFromLong.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/utils/DeterministicUUIDFromLong.scala index 3f1a80b09b4..218964b2820 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/util/DeterministicUUIDFromLong.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/utils/DeterministicUUIDFromLong.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic.util +package pl.touk.nussknacker.engine.common.periodic.utils import java.util.UUID diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala index e7fcdc26562..29e5524dd91 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala @@ -53,10 +53,10 @@ class FlinkPeriodicDeploymentHandler( override def prepareDeploymentWithRuntimeParams( processVersion: ProcessVersion, - ): Future[DeploymentWithRuntimeParams] = { + ): Future[DeploymentWithRuntimeParams.WithConfig] = { logger.info(s"Prepare deployment for scenario: $processVersion") copyJarToLocalDir(processVersion).map { jarFileName => - DeploymentWithRuntimeParams( + DeploymentWithRuntimeParams.WithConfig( processVersion = processVersion, inputConfigDuringExecutionJson = inputConfigDuringExecution.serialized, runtimeParams = RuntimeParams(Map(jarFileNameRuntimeParam -> jarFileName)) @@ -75,7 +75,7 @@ class FlinkPeriodicDeploymentHandler( } override def deployWithRuntimeParams( - deployment: DeploymentWithRuntimeParams, + deployment: DeploymentWithRuntimeParams.WithConfig, deploymentData: DeploymentData, canonicalProcess: CanonicalProcess, ): Future[Option[ExternalDeploymentId]] = { diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala index fff133f4df9..4a6384aaafc 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala @@ -6,9 +6,9 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.LoneElement._ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment import pl.touk.nussknacker.engine.common.periodic.DeploymentActor -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess import pl.touk.nussknacker.engine.common.periodic.DeploymentActor.CheckToBeDeployed import scala.concurrent.Future @@ -34,11 +34,11 @@ class DeploymentActorTest extends AnyFunSuite with TestKitBase with Matchers wit } private def shouldFindToBeDeployedScenarios( - result: Future[Seq[PeriodicProcessDeployment]] + result: Future[Seq[PeriodicProcessDeployment[WithConfig]]] ): Unit = { val probe = TestProbe() var counter = 0 - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = { counter += 1 probe.ref ! s"invoked $counter" result @@ -55,14 +55,14 @@ class DeploymentActorTest extends AnyFunSuite with TestKitBase with Matchers wit } test("should deploy found scenario") { - val probe = TestProbe() - val waitingDeployment = PeriodicProcessDeploymentGen() - var toBeDeployed: Seq[PeriodicProcessDeployment] = Seq(waitingDeployment) - var actor: ActorRef = null - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { + val probe = TestProbe() + val waitingDeployment = PeriodicProcessDeploymentGen() + var toBeDeployed: Seq[PeriodicProcessDeployment[WithConfig]] = Seq(waitingDeployment) + var actor: ActorRef = null + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = { Future.successful(toBeDeployed) } - def deploy(deployment: PeriodicProcessDeployment): Future[Unit] = { + def deploy(deployment: PeriodicProcessDeployment[WithConfig]): Future[Unit] = { probe.ref ! deployment // Simulate periodic check for waiting scenarios while deploying a scenario. actor ! CheckToBeDeployed diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala index 0c4d2fc9cbc..bf537de09e3 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala @@ -10,14 +10,14 @@ import scala.concurrent.Future class PeriodicDeploymentHandlerStub extends PeriodicDeploymentHandler { - var deployWithJarFuture: Future[Option[ExternalDeploymentId]] = Future.successful(None) - var lastDeploymentWithRuntimeParams: Option[DeploymentWithRuntimeParams] = None + var deployWithJarFuture: Future[Option[ExternalDeploymentId]] = Future.successful(None) + var lastDeploymentWithRuntimeParams: Option[DeploymentWithRuntimeParams.WithConfig] = None override def prepareDeploymentWithRuntimeParams( processVersion: ProcessVersion, - ): Future[DeploymentWithRuntimeParams] = { + ): Future[DeploymentWithRuntimeParams.WithConfig] = { Future.successful( - DeploymentWithRuntimeParams( + DeploymentWithRuntimeParams.WithConfig( processVersion = processVersion, inputConfigDuringExecutionJson = "", runtimeParams = RuntimeParams(Map("jarFileName" -> "")) @@ -26,7 +26,7 @@ class PeriodicDeploymentHandlerStub extends PeriodicDeploymentHandler { } override def deployWithRuntimeParams( - deploymentWithJarData: DeploymentWithRuntimeParams, + deploymentWithJarData: DeploymentWithRuntimeParams.WithConfig, deploymentData: DeploymentData, canonicalProcess: CanonicalProcess, ): Future[Option[ExternalDeploymentId]] = { diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentManagerTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentManagerTest.scala index 7480a5cc93e..08a5f72442d 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentManagerTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentManagerTest.scala @@ -15,12 +15,12 @@ import pl.touk.nussknacker.engine.api.{MetaData, ProcessVersion, StreamMetaData} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.PeriodicProcessStatus import pl.touk.nussknacker.engine.common.periodic.PeriodicStateStatus.{ScheduledStatus, WaitingForScheduleStatus} +import pl.touk.nussknacker.engine.common.periodic._ import pl.touk.nussknacker.engine.common.periodic.service.{ DefaultAdditionalDeploymentDataProvider, EmptyListener, ProcessConfigEnricher } -import pl.touk.nussknacker.engine.common.periodic._ import pl.touk.nussknacker.engine.deployment.{DeploymentData, User} import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesManager import pl.touk.nussknacker.test.PatientScalaFutures @@ -84,9 +84,8 @@ class PeriodicDeploymentManagerTest val periodicDeploymentManager = new PeriodicDeploymentManager( delegate = delegateDeploymentManagerStub, service = periodicProcessService, - repository = repository, + periodicProcessesManager = manager, schedulePropertyExtractor = CronSchedulePropertyExtractor(), - EmptyPeriodicCustomActionsProvider, "testProcessingType", toClose = () => () ) diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala index ccbc60e06d3..e4130b97d3e 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala @@ -1,5 +1,6 @@ package pl.touk.nussknacker.engine.management.periodic.flink +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import java.time.LocalDateTime @@ -8,7 +9,7 @@ object PeriodicProcessDeploymentGen { val now: LocalDateTime = LocalDateTime.now() - def apply(): PeriodicProcessDeployment = { + def apply(): PeriodicProcessDeployment[WithConfig] = { PeriodicProcessDeployment( id = PeriodicProcessDeploymentId(42), periodicProcess = PeriodicProcessGen(), diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala index 73c40fbf67d..b96dc3072e6 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala @@ -2,6 +2,7 @@ package pl.touk.nussknacker.engine.management.periodic.flink import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager.CronScheduleProperty +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess @@ -11,10 +12,10 @@ import java.time.LocalDateTime object PeriodicProcessGen { - def apply(): PeriodicProcess = { + def apply(): PeriodicProcess[WithConfig] = { PeriodicProcess( id = PeriodicProcessId(42), - deploymentData = DeploymentWithRuntimeParams( + deploymentData = DeploymentWithRuntimeParams.WithConfig( processVersion = ProcessVersion.empty, inputConfigDuringExecutionJson = "{}", runtimeParams = RuntimeParams(Map("jarFileName" -> "jar-file-name.jar")) diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala index 37aae4481e9..8b4b5d2feb5 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala @@ -8,6 +8,7 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model.{ PeriodicProcessDeployment, @@ -94,7 +95,7 @@ class PeriodicProcessServiceTest additionalDeploymentDataProvider = new AdditionalDeploymentDataProvider { override def prepareAdditionalData( - runDetails: PeriodicProcessDeployment + runDetails: PeriodicProcessDeployment[WithConfig] ): Map[String, String] = additionalData + ("runId" -> runDetails.id.value.toString) diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessStateDefinitionManagerTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessStateDefinitionManagerTest.scala index 6cebe251781..ff32fd0d4ff 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessStateDefinitionManagerTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessStateDefinitionManagerTest.scala @@ -2,9 +2,15 @@ package pl.touk.nussknacker.engine.management.periodic.flink import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import pl.touk.nussknacker.engine.api.deployment.ProcessStateDefinitionManager.ProcessStatus +import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus +import pl.touk.nussknacker.engine.api.process.VersionId import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.{DeploymentStatus, PeriodicProcessStatus} import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessStateDefinitionManager.statusTooltip +import pl.touk.nussknacker.engine.common.periodic.PeriodicStateStatus +import pl.touk.nussknacker.engine.common.periodic.PeriodicStateStatus.ScheduledStatus import java.time.LocalDateTime import java.util.concurrent.atomic.AtomicLong diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala index 5dd16e1f31d..e9dc4cd6006 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala @@ -4,6 +4,7 @@ import io.circe.syntax.EncoderOps import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.{WithConfig, WithoutConfig} import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} @@ -23,7 +24,7 @@ import scala.util.Random class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProcessesManager { - var processEntities: mutable.ListBuffer[PeriodicProcessEntityWithJson] = ListBuffer.empty + var processEntities: mutable.ListBuffer[PeriodicProcessEntity] = ListBuffer.empty var deploymentEntities: mutable.ListBuffer[PeriodicProcessDeploymentEntity] = ListBuffer.empty private def canonicalProcess(processName: ProcessName) = { @@ -62,7 +63,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc processActionId: Option[ProcessActionId] = None ): PeriodicProcessId = { val id = PeriodicProcessId(ProcessIdSequence.incrementAndGet()) - val entity = PeriodicProcessEntityWithJson( + val entity = PeriodicProcessEntity( id = id, processName = processName, processVersionId = VersionId.initialVersionId, @@ -122,12 +123,12 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc } override def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, - ): Future[PeriodicProcess] = Future.successful { + ): Future[PeriodicProcess[WithConfig]] = Future.successful { val id = PeriodicProcessId(Random.nextLong()) - val periodicProcess = PeriodicProcessEntityWithJson( + val periodicProcess = PeriodicProcessEntity( id = id, processName = deploymentWithRuntimeParams.processVersion.processName, processVersionId = deploymentWithRuntimeParams.processVersion.versionId, @@ -140,7 +141,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc processActionId = Some(processActionId) ) processEntities += periodicProcess - createPeriodicProcess(periodicProcess) + createPeriodicProcessWithJson(periodicProcess) } override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( @@ -189,30 +190,30 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc .take(deploymentsPerScheduleMaxCount) .map(scheduleDeploymentData(_)) .toList - scheduleId -> ScheduleData(createPeriodicProcess(process), ds) + scheduleId -> ScheduleData(createPeriodicProcessWithoutJson(process), ds) } } yield deploymentGroupedByScheduleName).toMap) - override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = { val scheduled = findActive(PeriodicProcessDeploymentStatus.Scheduled) readyToRun(scheduled) } - override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = { + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = { val toBeRetried = findActive(PeriodicProcessDeploymentStatus.FailedOnDeploy).filter(_.retriesLeft > 0) readyToRun(toBeRetried) } override def findProcessData( id: PeriodicProcessDeploymentId, - ): Future[PeriodicProcessDeployment] = Future.successful { + ): Future[PeriodicProcessDeployment[WithConfig]] = Future.successful { (for { d <- deploymentEntities if d.id == id p <- processEntities if p.id == d.periodicProcessId } yield createPeriodicProcessDeployment(p, d)).head } - private def processEntities(processName: ProcessName): Seq[PeriodicProcessEntityWithJson] = + private def processEntities(processName: ProcessName): Seq[PeriodicProcessEntity] = processEntities .filter(process => process.processName == processName && process.processingType == processingType) .toSeq @@ -255,7 +256,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int, - ): Future[PeriodicProcessDeployment] = Future.successful { + ): Future[PeriodicProcessDeployment[WithConfig]] = Future.successful { val deploymentEntity = PeriodicProcessDeploymentEntity( id = PeriodicProcessDeploymentId(Random.nextLong()), periodicProcessId = id, @@ -283,22 +284,22 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc } } - private def findActive(status: PeriodicProcessDeploymentStatus): Seq[PeriodicProcessDeployment] = + private def findActive(status: PeriodicProcessDeploymentStatus): Seq[PeriodicProcessDeployment[WithConfig]] = findActive( Seq(status) ) private def findActive( statusList: Seq[PeriodicProcessDeploymentStatus] - ): Seq[PeriodicProcessDeployment] = + ): Seq[PeriodicProcessDeployment[WithConfig]] = (for { p <- processEntities if p.active && p.processingType == processingType d <- deploymentEntities if d.periodicProcessId == p.id && statusList.contains(d.status) } yield createPeriodicProcessDeployment(p, d)).toSeq private def readyToRun( - deployments: Seq[PeriodicProcessDeployment] - ): Future[Seq[PeriodicProcessDeployment]] = { + deployments: Seq[PeriodicProcessDeployment[WithConfig]] + ): Future[Seq[PeriodicProcessDeployment[WithConfig]]] = { val now = LocalDateTime.now() Future.successful(deployments.filter(d => d.runAt.isBefore(now) || d.runAt.isEqual(now))) } @@ -343,9 +344,9 @@ object InMemPeriodicProcessesManager { def createPeriodicProcessDeployment( processEntity: PeriodicProcessEntity, - processDeploymentEntity: PeriodicProcessDeploymentEntity, - ): PeriodicProcessDeployment = { - val process = createPeriodicProcess(processEntity) + processDeploymentEntity: PeriodicProcessDeploymentEntity + ): PeriodicProcessDeployment[WithConfig] = { + val process = createPeriodicProcessWithJson(processEntity) PeriodicProcessDeployment( processDeploymentEntity.id, process, @@ -368,17 +369,37 @@ object InMemPeriodicProcessesManager { ) } - def createPeriodicProcess(processEntity: PeriodicProcessEntity): PeriodicProcess = { + def createPeriodicProcessWithJson( + processEntity: PeriodicProcessEntity + ): PeriodicProcess[WithConfig] = { val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, - DeploymentWithRuntimeParams( + DeploymentWithRuntimeParams.WithConfig( processVersion = processVersion, inputConfigDuringExecutionJson = processEntity.inputConfigDuringExecutionJson, runtimeParams = processEntity.runtimeParams, ), - scheduleProperty, + toApi(scheduleProperty), + processEntity.active, + processEntity.createdAt, + processEntity.processActionId + ) + } + + def createPeriodicProcessWithoutJson( + processEntity: PeriodicProcessEntity + ): PeriodicProcess[WithoutConfig] = { + val processVersion = createProcessVersion(processEntity) + val scheduleProperty = prepareScheduleProperty(processEntity) + PeriodicProcess( + processEntity.id, + DeploymentWithRuntimeParams.WithoutConfig( + processVersion = processVersion, + runtimeParams = processEntity.runtimeParams, + ), + toApi(scheduleProperty), processEntity.active, processEntity.createdAt, processEntity.processActionId @@ -389,7 +410,7 @@ object InMemPeriodicProcessesManager { val scheduleProperty = io.circe.parser .decode[ScheduleProperty](processEntity.scheduleProperty) .fold(e => throw new IllegalArgumentException(e), identity) - toApi(scheduleProperty) + scheduleProperty } private def createProcessVersion(processEntity: PeriodicProcessEntity): ProcessVersion = { From 1fba2a7831ccc21ca137f1470281b55dc69696e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Fri, 20 Dec 2024 00:36:30 +0100 Subject: [PATCH 03/11] merge fixes --- .../periodic/PeriodicProcessesManager.scala | 18 ++++++++ ...ositoryBasedPeriodicProcessesManager.scala | 21 +++++++++- .../PeriodicProcessesRepository.scala | 18 ++++---- .../PeriodicDeploymentManagerProvider.scala | 2 +- .../periodic/PeriodicProcessService.scala | 23 +++++------ .../PeriodicProcessesFetchingTest.scala | 33 +++++++++------ .../db/InMemPeriodicProcessesManager.scala | 41 ++++++++----------- 7 files changed, 99 insertions(+), 57 deletions(-) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala index b12aba58b25..176e5e23de4 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala @@ -58,12 +58,21 @@ trait PeriodicProcessesManager { deploymentsPerScheduleMaxCount: Int, ): Future[SchedulesState] + def getLatestDeploymentsForActiveSchedules( + deploymentsPerScheduleMaxCount: Int, + ): Future[Map[ProcessName, SchedulesState]] + def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int, ): Future[SchedulesState] + def getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + ): Future[Map[ProcessName, SchedulesState]] + def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], ): Future[SchedulesState] @@ -135,12 +144,21 @@ object NoOpPeriodicProcessesManager extends PeriodicProcessesManager { deploymentsPerScheduleMaxCount: Int, ): Future[SchedulesState] = notImplemented + override def getLatestDeploymentsForActiveSchedules( + deploymentsPerScheduleMaxCount: Int + ): Future[Map[ProcessName, SchedulesState]] = notImplemented + override def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int, ): Future[SchedulesState] = notImplemented + override def getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int + ): Future[Map[ProcessName, SchedulesState]] = notImplemented + override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], ): Future[SchedulesState] = notImplemented diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala index 429cdd45ada..23ae15bf537 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala @@ -5,7 +5,7 @@ import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManag import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ -import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.ui.process.repository.PeriodicProcessesRepository @@ -78,6 +78,13 @@ class RepositoryBasedPeriodicProcessesManager( .getLatestDeploymentsForActiveSchedules(processName, deploymentsPerScheduleMaxCount, processingType) .run + override def getLatestDeploymentsForActiveSchedules( + deploymentsPerScheduleMaxCount: Int, + ): Future[Map[ProcessName, SchedulesState]] = + periodicProcessesRepository + .getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount, processingType) + .run + override def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, @@ -92,6 +99,18 @@ class RepositoryBasedPeriodicProcessesManager( ) .run + override def getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + ): Future[Map[ProcessName, SchedulesState]] = + periodicProcessesRepository + .getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount, + deploymentsPerScheduleMaxCount, + processingType, + ) + .run + override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], ): Future[SchedulesState] = periodicProcessesRepository diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala index e245ee6b3b5..3f99a62eecb 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala @@ -10,7 +10,6 @@ import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.{WithConfig, WithoutConfig} -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessWithoutJson import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} @@ -18,6 +17,7 @@ import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} import pl.touk.nussknacker.engine.common.periodic._ import pl.touk.nussknacker.ui.db.entity._ +import pl.touk.nussknacker.ui.process.repository.PeriodicProcessesRepository.createPeriodicProcessWithoutJson import slick.dbio.{DBIOAction, Effect, NoStream} import slick.jdbc.PostgresProfile.api._ import slick.jdbc.{JdbcBackend, JdbcProfile} @@ -137,7 +137,8 @@ trait PeriodicProcessesRepository { ): Action[SchedulesState] def getLatestDeploymentsForActiveSchedules( - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[Map[ProcessName, SchedulesState]] def getLatestDeploymentsForLatestInactiveSchedules( @@ -149,7 +150,8 @@ trait PeriodicProcessesRepository { def getLatestDeploymentsForLatestInactiveSchedules( inactiveProcessesMaxCount: Int, - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[Map[ProcessName, SchedulesState]] def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment[WithConfig]]] @@ -355,10 +357,11 @@ class SlickPeriodicProcessesRepository( } override def getLatestDeploymentsForActiveSchedules( - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[Map[ProcessName, SchedulesState]] = { val activeProcessesQuery = PeriodicProcessesWithoutJson.filter(_.active) - getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount) + getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount, processingType) } override def getLatestDeploymentsForLatestInactiveSchedules( @@ -377,13 +380,14 @@ class SlickPeriodicProcessesRepository( override def getLatestDeploymentsForLatestInactiveSchedules( inactiveProcessesMaxCount: Int, - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[Map[ProcessName, SchedulesState]] = { val filteredProcessesQuery = PeriodicProcessesWithoutJson .filter(!_.active) .sortBy(_.createdAt.desc) .take(inactiveProcessesMaxCount) - getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount) + getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount, processingType) } private def getLatestDeploymentsForEachSchedule( diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala index f5ce739688b..0d740f67f3e 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala @@ -45,7 +45,7 @@ abstract class PeriodicDeploymentManagerProvider( scenarioStateCacheTTL: Option[FiniteDuration], ): ValidatedNel[String, DeploymentManager] = { logger.info("Creating periodic scenario manager") - delegate.createDeploymentManagerWithCapabilities(modelData, dependencies, config, scenarioStateCacheTTL).map { + delegate.createDeploymentManager(modelData, dependencies, config, scenarioStateCacheTTL).map { delegateDeploymentManager => import net.ceedubs.ficus.Ficus._ import net.ceedubs.ficus.readers.ArbitraryTypeReader._ diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala index 1fed20fda86..fbdd56ac728 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala @@ -621,30 +621,27 @@ class PeriodicProcessService( def getLatestDeploymentsForActiveSchedules( deploymentsPerScheduleMaxCount: Int ): Future[Map[ProcessName, SchedulesState]] = - scheduledProcessesRepository.getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount).run + periodicProcessesManager.getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount) def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int ): Future[SchedulesState] = - periodicProcessesManager - .getLatestDeploymentsForLatestInactiveSchedules( - processName, - inactiveProcessesMaxCount, - deploymentsPerScheduleMaxCount, - ) + periodicProcessesManager.getLatestDeploymentsForLatestInactiveSchedules( + processName, + inactiveProcessesMaxCount, + deploymentsPerScheduleMaxCount, + ) def getLatestDeploymentsForLatestInactiveSchedules( inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int ): Future[Map[ProcessName, SchedulesState]] = - scheduledProcessesRepository - .getLatestDeploymentsForLatestInactiveSchedules( - inactiveProcessesMaxCount, - deploymentsPerScheduleMaxCount - ) - .run + periodicProcessesManager.getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount, + deploymentsPerScheduleMaxCount + ) implicit class RuntimeStatusesExt(runtimeStatuses: List[StatusDetails]) { diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala index a5278f94586..325e17c116a 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala @@ -6,16 +6,19 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.{Inside, OptionValues} import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.process.ProcessName -import pl.touk.nussknacker.engine.deployment.DeploymentData -import pl.touk.nussknacker.engine.management.periodic.db.InMemPeriodicProcessesRepository.getLatestDeploymentQueryCount -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.service.{ +import pl.touk.nussknacker.engine.common.periodic._ +import pl.touk.nussknacker.engine.common.periodic.service.{ DefaultAdditionalDeploymentDataProvider, EmptyListener, ProcessConfigEnricher } +import pl.touk.nussknacker.engine.deployment.DeploymentData +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesManager +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesRepository.getLatestDeploymentQueryCount +import pl.touk.nussknacker.engine.management.periodic.flink.{DeploymentManagerStub, PeriodicDeploymentHandlerStub} import pl.touk.nussknacker.test.PatientScalaFutures import java.time.Clock @@ -37,15 +40,16 @@ class PeriodicProcessesFetchingTest private def processName(n: Int) = ProcessName(s"test$n") class Fixture(executionConfig: PeriodicExecutionConfig = PeriodicExecutionConfig()) { - val repository = new db.InMemPeriodicProcessesRepository(processingType = "testProcessingType") + val processingType = "testProcessingType" + val periodicProcessesManager = new InMemPeriodicProcessesManager(processingType) val delegateDeploymentManagerStub = new DeploymentManagerStub - val jarManagerStub = new JarManagerStub + val periodicDeploymentHandlerStub = new PeriodicDeploymentHandlerStub val preparedDeploymentData = DeploymentData.withDeploymentId(UUID.randomUUID().toString) val periodicProcessService = new PeriodicProcessService( delegateDeploymentManager = delegateDeploymentManagerStub, - jarManager = jarManagerStub, - scheduledProcessesRepository = repository, + periodicDeploymentHandler = periodicDeploymentHandlerStub, + periodicProcessesManager = periodicProcessesManager, periodicProcessListener = EmptyListener, additionalDeploymentDataProvider = DefaultAdditionalDeploymentDataProvider, deploymentRetryConfig = DeploymentRetryConfig(), @@ -60,8 +64,9 @@ class PeriodicProcessesFetchingTest val periodicDeploymentManager = new PeriodicDeploymentManager( delegate = delegateDeploymentManagerStub, service = periodicProcessService, - repository = repository, + periodicProcessesManager = periodicProcessesManager, schedulePropertyExtractor = CronSchedulePropertyExtractor(), + processingType = processingType, toClose = () => () ) @@ -76,7 +81,8 @@ class PeriodicProcessesFetchingTest f.delegateDeploymentManagerStub.setEmptyStateStatus() for (i <- 1 to n) { - val deploymentId = f.repository.addActiveProcess(processName(i), PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = + f.periodicProcessesManager.addActiveProcess(processName(i), PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.addStateStatus(processName(i), SimpleStateStatus.Running, Some(deploymentId)) } @@ -91,14 +97,17 @@ class PeriodicProcessesFetchingTest getLatestDeploymentQueryCount.get() shouldEqual 2 * n } - test("getStatusDetails - should perform 2 db queries for N periodic processes when fetching all at once") { + test( + "getAllProcessesStates - should perform 2 db queries for N periodic processes when fetching all at once" + ) { val f = new Fixture val n = 10 f.delegateDeploymentManagerStub.setEmptyStateStatus() for (i <- 1 to n) { - val deploymentId = f.repository.addActiveProcess(processName(i), PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = + f.periodicProcessesManager.addActiveProcess(processName(i), PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.addStateStatus(processName(i), SimpleStateStatus.Running, Some(deploymentId)) } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala index c09bcc9c010..9ccb30c9c36 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala @@ -9,16 +9,11 @@ import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessD import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.build.ScenarioBuilder -import pl.touk.nussknacker.engine.management.periodic._ -import pl.touk.nussknacker.engine.management.periodic.db.InMemPeriodicProcessesRepository.{ - DeploymentIdSequence, - ProcessIdSequence, - getLatestDeploymentQueryCount -} -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessDeployment -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model._ +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} +import pl.touk.nussknacker.engine.common.periodic.{CronScheduleProperty, ScheduleProperty, SingleScheduleProperty} +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesManager._ +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesRepository.getLatestDeploymentQueryCount import java.time.chrono.ChronoLocalDateTime import java.time.{LocalDateTime, ZoneId} @@ -29,12 +24,11 @@ import scala.concurrent.Future import scala.util.Random object InMemPeriodicProcessesRepository { - private val ProcessIdSequence = new AtomicLong(0) - private val DeploymentIdSequence = new AtomicLong(0) - val getLatestDeploymentQueryCount = new AtomicLong(0) } +class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProcessesManager { + var processEntities: mutable.ListBuffer[PeriodicProcessEntity] = ListBuffer.empty var deploymentEntities: mutable.ListBuffer[PeriodicProcessDeploymentEntity] = ListBuffer.empty @@ -167,8 +161,8 @@ object InMemPeriodicProcessesRepository { override def getLatestDeploymentsForActiveSchedules( processName: ProcessName, - deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] = { + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = Future.successful { getLatestDeploymentQueryCount.incrementAndGet() getLatestDeploymentsForPeriodicProcesses( processEntities(processName).filter(_.active), @@ -178,7 +172,7 @@ object InMemPeriodicProcessesRepository { override def getLatestDeploymentsForActiveSchedules( deploymentsPerScheduleMaxCount: Int - ): Action[Map[ProcessName, SchedulesState]] = { + ): Future[Map[ProcessName, SchedulesState]] = Future.successful { getLatestDeploymentQueryCount.incrementAndGet() allProcessEntities.map { case (processName, list) => processName -> getLatestDeploymentsForPeriodicProcesses( @@ -202,7 +196,7 @@ object InMemPeriodicProcessesRepository { override def getLatestDeploymentsForLatestInactiveSchedules( inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - ): Action[Map[ProcessName, SchedulesState]] = { + ): Future[Map[ProcessName, SchedulesState]] = Future.successful { getLatestDeploymentQueryCount.incrementAndGet() allProcessEntities.map { case (processName, list) => processName -> getLatestDeploymentsForPeriodicProcesses( @@ -215,7 +209,7 @@ object InMemPeriodicProcessesRepository { private def getLatestDeploymentsForPeriodicProcesses( processes: Seq[PeriodicProcessEntity], deploymentsPerScheduleMaxCount: Int - ): SchedulesState = + ): SchedulesState = { SchedulesState((for { process <- processes deploymentGroupedByScheduleName <- deploymentEntities @@ -231,6 +225,7 @@ object InMemPeriodicProcessesRepository { scheduleId -> ScheduleData(createPeriodicProcessWithoutJson(process), ds) } } yield deploymentGroupedByScheduleName).toMap) + } override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = { val scheduled = findActive(PeriodicProcessDeploymentStatus.Scheduled) @@ -251,16 +246,16 @@ object InMemPeriodicProcessesRepository { } yield createPeriodicProcessDeployment(p, d)).head } - private def allProcessEntities: Map[ProcessName, Seq[PeriodicProcessEntity]] = + private def processEntities(processName: ProcessName): Seq[PeriodicProcessEntity] = processEntities - .filter(process => process.processingType == processingType) + .filter(process => process.processName == processName && process.processingType == processingType) .toSeq - .groupBy(_.processName) - private def processEntities(processName: ProcessName): Seq[PeriodicProcessEntity] = + private def allProcessEntities: Map[ProcessName, Seq[PeriodicProcessEntity]] = processEntities - .filter(process => process.processName == processName && process.processingType == processingType) + .filter(process => process.processingType == processingType) .toSeq + .groupBy(_.processName) override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = Future.successful { update(id)(_.copy(status = PeriodicProcessDeploymentStatus.Deployed, deployedAt = Some(LocalDateTime.now()))) From e734cc121bdd4f3af3d306a052e20a245b94c419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Fri, 20 Dec 2024 11:03:33 +0100 Subject: [PATCH 04/11] isolate config json --- .../periodic/PeriodicProcessesManager.scala | 36 ++++++---- .../model/DeploymentWithRuntimeParams.scala | 23 ++---- .../periodic/model/PeriodicProcess.scala | 4 +- .../model/PeriodicProcessDeployment.scala | 4 +- .../periodic/model/SchedulesState.scala | 11 ++- .../ui/db/entity/PeriodicProcessesTable.scala | 5 +- ...ositoryBasedPeriodicProcessesManager.scala | 28 +++++--- .../PeriodicProcessesRepository.scala | 70 +++++++++++-------- ...eriodicProcessServiceIntegrationTest.scala | 32 --------- .../common/periodic/DeploymentActor.scala | 12 ++-- .../periodic/PeriodicDeploymentHandler.scala | 8 ++- .../periodic/PeriodicProcessService.scala | 58 +++++++-------- .../AdditionalDeploymentDataProvider.scala | 5 +- .../service/PeriodicProcessListener.scala | 15 ++-- .../service/ProcessConfigEnricher.scala | 3 +- .../FlinkPeriodicDeploymentHandler.scala | 13 ++-- .../periodic/flink/DeploymentActorTest.scala | 17 +++-- .../flink/PeriodicDeploymentHandlerStub.scala | 20 ++++-- .../flink/PeriodicProcessDeploymentGen.scala | 3 +- .../periodic/flink/PeriodicProcessGen.scala | 6 +- .../flink/PeriodicProcessServiceTest.scala | 5 +- .../db/InMemPeriodicProcessesManager.scala | 41 ++++++----- 22 files changed, 210 insertions(+), 209 deletions(-) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala index 176e5e23de4..8264f8a0085 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala @@ -13,10 +13,11 @@ import scala.concurrent.Future trait PeriodicProcessesManager { def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, scheduleProperty: ScheduleProperty, processActionId: ProcessActionId, - ): Future[PeriodicProcess[DeploymentWithRuntimeParams.WithConfig]] + ): Future[PeriodicProcess] def markInactive(processId: PeriodicProcessId): Future[Unit] @@ -25,15 +26,15 @@ trait PeriodicProcessesManager { scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int, - ): Future[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]] + ): Future[PeriodicProcessDeployment] def findProcessData( id: PeriodicProcessDeploymentId - ): Future[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]] + ): Future[PeriodicProcessDeployment] - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]] + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] - def findToBeRetried: Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]] + def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] @@ -82,6 +83,11 @@ trait PeriodicProcessesManager { versionId: VersionId, ): Future[Option[CanonicalProcess]] + def fetchInputConfigDuringExecutionJson( + processName: ProcessName, + versionId: VersionId, + ): Future[Option[String]] + } object PeriodicProcessesManager { @@ -99,10 +105,11 @@ object PeriodicProcessesManager { object NoOpPeriodicProcessesManager extends PeriodicProcessesManager { override def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, scheduleProperty: ScheduleProperty, processActionId: ProcessActionId, - ): Future[PeriodicProcess[DeploymentWithRuntimeParams.WithConfig]] = notImplemented + ): Future[PeriodicProcess] = notImplemented override def markInactive(processId: PeriodicProcessId): Future[Unit] = notImplemented @@ -111,16 +118,16 @@ object NoOpPeriodicProcessesManager extends PeriodicProcessesManager { scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Future[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]] = notImplemented + ): Future[PeriodicProcessDeployment] = notImplemented override def findProcessData( id: PeriodicProcessDeploymentId, - ): Future[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]] = notImplemented + ): Future[PeriodicProcessDeployment] = notImplemented - override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]] = + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = notImplemented - override def findToBeRetried: Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]] = + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = notImplemented override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = notImplemented @@ -168,6 +175,11 @@ object NoOpPeriodicProcessesManager extends PeriodicProcessesManager { versionId: VersionId ): Future[Option[CanonicalProcess]] = notImplemented + override def fetchInputConfigDuringExecutionJson( + processName: ProcessName, + versionId: VersionId + ): Future[Option[String]] = notImplemented + private def notImplemented: Future[Nothing] = Future.failed(new NotImplementedError()) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala index b8a3abf218c..bbc4364fd88 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala @@ -2,24 +2,9 @@ package pl.touk.nussknacker.engine.api.deployment.periodic.model import pl.touk.nussknacker.engine.api.ProcessVersion -sealed trait DeploymentWithRuntimeParams { - def processVersion: ProcessVersion - def runtimeParams: RuntimeParams -} - -object DeploymentWithRuntimeParams { - - final case class WithConfig( - processVersion: ProcessVersion, - runtimeParams: RuntimeParams, - inputConfigDuringExecutionJson: String, - ) extends DeploymentWithRuntimeParams - - final case class WithoutConfig( - processVersion: ProcessVersion, - runtimeParams: RuntimeParams, - ) extends DeploymentWithRuntimeParams - -} +final case class DeploymentWithRuntimeParams( + processVersion: ProcessVersion, + runtimeParams: RuntimeParams, +) final case class RuntimeParams(params: Map[String, String]) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala index 6201b140a3e..ab1c666a008 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala @@ -8,9 +8,9 @@ import java.time.LocalDateTime case class PeriodicProcessId(value: Long) -case class PeriodicProcess[DeploymentData <: DeploymentWithRuntimeParams]( +case class PeriodicProcess( id: PeriodicProcessId, - deploymentData: DeploymentData, + deploymentData: DeploymentWithRuntimeParams, scheduleProperty: ScheduleProperty, active: Boolean, createdAt: LocalDateTime, diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala index 10206e6e3b3..e67b46142dc 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala @@ -5,9 +5,9 @@ import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessD import java.time.LocalDateTime // TODO: We should separate schedules concept from deployments - fully switch to ScheduleData and ScheduleDeploymentData -case class PeriodicProcessDeployment[DeploymentData <: DeploymentWithRuntimeParams]( +case class PeriodicProcessDeployment( id: PeriodicProcessDeploymentId, - periodicProcess: PeriodicProcess[DeploymentData], + periodicProcess: PeriodicProcess, createdAt: LocalDateTime, runAt: LocalDateTime, scheduleName: ScheduleName, diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala index 964e4406893..ecfddb07ab0 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala @@ -1,6 +1,5 @@ package pl.touk.nussknacker.engine.api.deployment.periodic.model -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithoutConfig import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap @@ -35,7 +34,7 @@ case class SchedulesState(schedules: Map[ScheduleId, ScheduleData]) { // For most operations it will contain only one latest deployment but for purpose of statuses of historical deployments // it has list instead of one element. // This structure should contain SingleScheduleProperty as well. See note above -case class ScheduleData(process: PeriodicProcess[WithoutConfig], latestDeployments: List[ScheduleDeploymentData]) +case class ScheduleData(process: PeriodicProcess, latestDeployments: List[ScheduleDeploymentData]) // To identify schedule we need scheduleName - None for SingleScheduleProperty and Some(key) for MultipleScheduleProperty keys // Also we need PeriodicProcessId to distinguish between active schedules and some inactive from the past for the same PeriodicProcessId @@ -53,9 +52,9 @@ case class ScheduleDeploymentData( ) { def toFullDeploymentData( - process: PeriodicProcess[DeploymentWithRuntimeParams.WithoutConfig], + process: PeriodicProcess, scheduleName: ScheduleName - ): PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithoutConfig] = + ): PeriodicProcessDeployment = PeriodicProcessDeployment(id, process, createdAt, runAt, scheduleName, retriesLeft, nextRetryAt, state) def display = s"deploymentId=$id" @@ -64,6 +63,6 @@ case class ScheduleDeploymentData( // These below are temporary structures, see notice next to SchedulesState case class PeriodicProcessScheduleData( - process: PeriodicProcess[DeploymentWithRuntimeParams.WithoutConfig], - deployments: List[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithoutConfig]] + process: PeriodicProcess, + deployments: List[PeriodicProcessDeployment] ) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala index 23046e4db7f..135a2606cf0 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala @@ -127,7 +127,8 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory { } - class PeriodicProcessWithoutJson(tag: Tag) extends PeriodicProcessesTable[PeriodicProcessEntityWithoutJson](tag) { + class PeriodicProcessesWithoutJsonTable(tag: Tag) + extends PeriodicProcessesTable[PeriodicProcessEntityWithoutJson](tag) { override def * : ProvenShape[PeriodicProcessEntityWithoutJson] = ( id, @@ -172,7 +173,7 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory { object PeriodicProcessesWithJson extends TableQuery(new PeriodicProcessesWithJsonTable(_)) - object PeriodicProcessesWithoutJson extends TableQuery(new PeriodicProcessWithoutJson(_)) + object PeriodicProcessesWithoutJson extends TableQuery(new PeriodicProcessesWithoutJsonTable(_)) } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala index 23ae15bf537..fab081133fb 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala @@ -2,7 +2,6 @@ package pl.touk.nussknacker.ui.process.periodic import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} @@ -21,12 +20,19 @@ class RepositoryBasedPeriodicProcessesManager( import periodicProcessesRepository._ override def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, - ): Future[PeriodicProcess[DeploymentWithRuntimeParams.WithConfig]] = + ): Future[PeriodicProcess] = periodicProcessesRepository - .create(deploymentWithRuntimeParams, scheduleProperty, processActionId, processingType) + .create( + deploymentWithRuntimeParams, + inputConfigDuringExecutionJson, + scheduleProperty, + processActionId, + processingType + ) .run override def markInactive(processId: PeriodicProcessId): Future[Unit] = @@ -37,18 +43,18 @@ class RepositoryBasedPeriodicProcessesManager( scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Future[PeriodicProcessDeployment[WithConfig]] = + ): Future[PeriodicProcessDeployment] = periodicProcessesRepository.schedule(id, scheduleName, runAt, deployMaxRetries).run override def findProcessData( id: PeriodicProcessDeploymentId, - ): Future[PeriodicProcessDeployment[WithConfig]] = + ): Future[PeriodicProcessDeployment] = periodicProcessesRepository.findProcessData(id).run - override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = periodicProcessesRepository.findToBeDeployed(processingType).run - override def findToBeRetried: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = periodicProcessesRepository.findToBeRetried(processingType).run override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = @@ -123,4 +129,10 @@ class RepositoryBasedPeriodicProcessesManager( ): Future[Option[CanonicalProcess]] = periodicProcessesRepository.fetchCanonicalProcess(processName, versionId).run + override def fetchInputConfigDuringExecutionJson( + processName: ProcessName, + versionId: VersionId + ): Future[Option[String]] = + periodicProcessesRepository.fetchInputConfigDuringExecutionJson(processName, versionId).run + } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala index 3f99a62eecb..df98d2c6e46 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala @@ -9,7 +9,6 @@ import io.circe.syntax.EncoderOps import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.{WithConfig, WithoutConfig} import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} @@ -29,9 +28,9 @@ import scala.language.higherKinds object PeriodicProcessesRepository { def createPeriodicProcessDeployment( - processEntity: PeriodicProcessEntityWithJson, + processEntity: PeriodicProcessEntityWithoutJson, processDeploymentEntity: PeriodicProcessDeploymentEntity - ): PeriodicProcessDeployment[WithConfig] = { + ): PeriodicProcessDeployment = { val process = createPeriodicProcessWithJson(processEntity) PeriodicProcessDeployment( processDeploymentEntity.id, @@ -56,15 +55,14 @@ object PeriodicProcessesRepository { } def createPeriodicProcessWithJson( - processEntity: PeriodicProcessEntityWithJson - ): PeriodicProcess[WithConfig] = { + processEntity: PeriodicProcessEntityWithoutJson + ): PeriodicProcess = { val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, - DeploymentWithRuntimeParams.WithConfig( + DeploymentWithRuntimeParams( processVersion = processVersion, - inputConfigDuringExecutionJson = processEntity.inputConfigDuringExecutionJson, runtimeParams = processEntity.runtimeParams, ), toApi(scheduleProperty), @@ -76,12 +74,12 @@ object PeriodicProcessesRepository { def createPeriodicProcessWithoutJson( processEntity: PeriodicProcessEntity - ): PeriodicProcess[WithoutConfig] = { + ): PeriodicProcess = { val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, - DeploymentWithRuntimeParams.WithoutConfig( + DeploymentWithRuntimeParams( processVersion = processVersion, runtimeParams = processEntity.runtimeParams, ), @@ -124,11 +122,12 @@ trait PeriodicProcessesRepository { ): Action[SchedulesState] def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, processingType: String, - ): Action[PeriodicProcess[DeploymentWithRuntimeParams.WithConfig]] + ): Action[PeriodicProcess] def getLatestDeploymentsForActiveSchedules( processName: ProcessName, @@ -154,16 +153,16 @@ trait PeriodicProcessesRepository { processingType: String, ): Action[Map[ProcessName, SchedulesState]] - def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment[WithConfig]]] + def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment]] - def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment[WithConfig]]] + def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment]] def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], processingType: String, ): Action[SchedulesState] - def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment[WithConfig]] + def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment] def markDeployed(id: PeriodicProcessDeploymentId): Action[Unit] @@ -183,13 +182,18 @@ trait PeriodicProcessesRepository { scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Action[PeriodicProcessDeployment[WithConfig]] + ): Action[PeriodicProcessDeployment] def fetchCanonicalProcess( processName: ProcessName, versionId: VersionId, ): Action[Option[CanonicalProcess]] + def fetchInputConfigDuringExecutionJson( + processName: ProcessName, + versionId: VersionId + ): Action[Option[String]] + } class SlickPeriodicProcessesRepository( @@ -226,17 +230,18 @@ class SlickPeriodicProcessesRepository( } override def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, processingType: String, - ): Action[PeriodicProcess[DeploymentWithRuntimeParams.WithConfig]] = { + ): Action[PeriodicProcess] = { val processEntity = PeriodicProcessEntityWithJson( id = PeriodicProcessId(-1), processName = deploymentWithRuntimeParams.processVersion.processName, processVersionId = deploymentWithRuntimeParams.processVersion.versionId, processingType = processingType, - inputConfigDuringExecutionJson = deploymentWithRuntimeParams.inputConfigDuringExecutionJson, + inputConfigDuringExecutionJson = inputConfigDuringExecutionJson, runtimeParams = deploymentWithRuntimeParams.runtimeParams, scheduleProperty = fromApi(scheduleProperty).asJson.noSpaces, active = true, @@ -249,7 +254,7 @@ class SlickPeriodicProcessesRepository( private def now(): LocalDateTime = LocalDateTime.now(clock) - override def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment[WithConfig]]] = + override def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment]] = findProcesses( activePeriodicProcessWithDeploymentQuery(processingType) .filter { case (_, d) => @@ -258,7 +263,7 @@ class SlickPeriodicProcessesRepository( } ) - override def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment[WithConfig]]] = + override def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment]] = findProcesses( activePeriodicProcessWithDeploymentQuery(processingType) .filter { case (_, d) => @@ -269,8 +274,8 @@ class SlickPeriodicProcessesRepository( private def findProcesses( query: Query[ - (PeriodicProcessesWithJsonTable, PeriodicProcessDeploymentsTable), - (PeriodicProcessEntityWithJson, PeriodicProcessDeploymentEntity), + (PeriodicProcessesWithoutJsonTable, PeriodicProcessDeploymentsTable), + (PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity), Seq ] ) = { @@ -283,9 +288,9 @@ class SlickPeriodicProcessesRepository( }) } - override def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment[WithConfig]] = + override def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment] = findProcesses( - (PeriodicProcessesWithJson join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) + (PeriodicProcessesWithoutJson join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) .filter { case (_, deployment) => deployment.id === id } ).map(_.head) @@ -391,7 +396,7 @@ class SlickPeriodicProcessesRepository( } private def getLatestDeploymentsForEachSchedule( - periodicProcessesQuery: Query[PeriodicProcessWithoutJson, PeriodicProcessEntityWithoutJson, Seq], + periodicProcessesQuery: Query[PeriodicProcessesWithoutJsonTable, PeriodicProcessEntityWithoutJson, Seq], deploymentsPerScheduleMaxCount: Int, processingType: String, ): Action[Map[ProcessName, SchedulesState]] = { @@ -406,7 +411,7 @@ class SlickPeriodicProcessesRepository( } private def getLatestDeploymentsForEachSchedulePostgres( - periodicProcessesQuery: Query[PeriodicProcessWithoutJson, PeriodicProcessEntityWithoutJson, Seq], + periodicProcessesQuery: Query[PeriodicProcessesWithoutJsonTable, PeriodicProcessEntityWithoutJson, Seq], deploymentsPerScheduleMaxCount: Int ): Action[Seq[(PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity)]] = { // To effectively limit deployments to given count for each schedule in one query, we use window functions in slick @@ -440,7 +445,7 @@ class SlickPeriodicProcessesRepository( // If we decided to support more databases, we should consider some optimization like extracting periodic_schedule table // with foreign key to periodic_process and with schedule_name column - it would reduce number of queries private def getLatestDeploymentsForEachScheduleJdbcGeneric( - periodicProcessesQuery: Query[PeriodicProcessWithoutJson, PeriodicProcessEntityWithoutJson, Seq], + periodicProcessesQuery: Query[PeriodicProcessesWithoutJsonTable, PeriodicProcessEntityWithoutJson, Seq], deploymentsPerScheduleMaxCount: Int ): Action[Seq[(PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity)]] = { // It is debug instead of warn to not bloast logs when e.g. for some reasons is used hsql under the hood @@ -482,7 +487,7 @@ class SlickPeriodicProcessesRepository( scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Action[PeriodicProcessDeployment[WithConfig]] = { + ): Action[PeriodicProcessDeployment] = { val deploymentEntity = PeriodicProcessDeploymentEntity( id = PeriodicProcessDeploymentId(-1), periodicProcessId = id, @@ -519,8 +524,15 @@ class SlickPeriodicProcessesRepository( .map(_.flatMap(_._2.json)) } + def fetchInputConfigDuringExecutionJson(processName: ProcessName, versionId: VersionId): Action[Option[String]] = + PeriodicProcessesWithJson + .filter(p => p.processName === processName && p.processVersionId === versionId) + .map(_.inputConfigDuringExecutionJson) + .result + .headOption + private def activePeriodicProcessWithDeploymentQuery(processingType: String) = { - (PeriodicProcessesWithJson.filter(p => p.active === true && p.processingType === processingType) + (PeriodicProcessesWithoutJson.filter(p => p.active === true && p.processingType === processingType) join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala index a01595e2b8f..2cffdc87628 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -617,38 +617,6 @@ class PeriodicProcessServiceIntegrationTest .futureValue inactiveStates.latestDeploymentForSchedule(schedule1).state.status shouldBe PeriodicProcessDeploymentStatus.Finished inactiveStates.latestDeploymentForSchedule(schedule2).state.status shouldBe PeriodicProcessDeploymentStatus.Finished - - val activities = service.getScenarioActivitiesSpecificToPeriodicProcess(processIdWithName, None).futureValue - val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] - val secondActivity = activities(1).asInstanceOf[ScenarioActivity.PerformedScheduledExecution] - activities shouldBe List( - ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(processIdWithName.id.value), - scenarioActivityId = firstActivity.scenarioActivityId, - user = ScenarioUser(None, UserName("Nussknacker"), None, None), - date = firstActivity.date, - scenarioVersionId = Some(ScenarioVersionId(1)), - dateFinished = firstActivity.dateFinished, - scheduleName = "schedule1", - scheduledExecutionStatus = ScheduledExecutionStatus.Finished, - createdAt = firstActivity.createdAt, - retriesLeft = None, - nextRetryAt = None - ), - ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(processIdWithName.id.value), - scenarioActivityId = secondActivity.scenarioActivityId, - user = ScenarioUser(None, UserName("Nussknacker"), None, None), - date = secondActivity.date, - scenarioVersionId = Some(ScenarioVersionId(1)), - dateFinished = secondActivity.dateFinished, - scheduleName = "schedule2", - scheduledExecutionStatus = ScheduledExecutionStatus.Finished, - createdAt = secondActivity.createdAt, - retriesLeft = None, - nextRetryAt = None - ), - ) } it should "handle failed event handler" in withFixture() { f => diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala index d68fdc7dea3..25898ce535a 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala @@ -20,8 +20,8 @@ object DeploymentActor { } private[engine] def props( - findToBeDeployed: => Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]], - deploy: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig] => Future[Unit], + findToBeDeployed: => Future[Seq[PeriodicProcessDeployment]], + deploy: PeriodicProcessDeployment => Future[Unit], interval: FiniteDuration ) = { Props(new DeploymentActor(findToBeDeployed, deploy, interval)) @@ -29,14 +29,14 @@ object DeploymentActor { private[engine] case object CheckToBeDeployed - private case class WaitingForDeployment(ids: List[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]) + private case class WaitingForDeployment(ids: List[PeriodicProcessDeployment]) private case object DeploymentCompleted } class DeploymentActor( - findToBeDeployed: => Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]], - deploy: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig] => Future[Unit], + findToBeDeployed: => Future[Seq[PeriodicProcessDeployment]], + deploy: PeriodicProcessDeployment => Future[Unit], interval: FiniteDuration ) extends Actor with Timers @@ -73,7 +73,7 @@ class DeploymentActor( } private def receiveOngoingDeployment( - runDetails: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig] + runDetails: PeriodicProcessDeployment ): Receive = { case CheckToBeDeployed => logger.debug(s"Still waiting for ${runDetails.display} to be deployed") diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala index 5d9456be4cf..0e0780e4c41 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala @@ -4,6 +4,7 @@ import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.periodic.model.{DeploymentWithRuntimeParams, RuntimeParams} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} +import pl.touk.nussknacker.engine.modelconfig.InputConfigDuringExecution import scala.concurrent.Future @@ -11,10 +12,13 @@ trait PeriodicDeploymentHandler { def prepareDeploymentWithRuntimeParams( processVersion: ProcessVersion, - ): Future[DeploymentWithRuntimeParams.WithConfig] + ): Future[DeploymentWithRuntimeParams] + + def provideInputConfigDuringExecutionJson(): Future[InputConfigDuringExecution] def deployWithRuntimeParams( - deploymentWithJarData: DeploymentWithRuntimeParams.WithConfig, + deploymentWithJarData: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, deploymentData: DeploymentData, canonicalProcess: CanonicalProcess, ): Future[Option[ExternalDeploymentId]] diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala index fbdd56ac728..ec14e806a0d 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala @@ -11,7 +11,6 @@ import pl.touk.nussknacker.engine.api.component.{ import pl.touk.nussknacker.engine.api.deployment.StateStatus.StatusName import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.{WithConfig, WithoutConfig} import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus @@ -144,22 +143,20 @@ class PeriodicProcessService( processActionId: ProcessActionId, ): Future[Unit] = { logger.info("Scheduling periodic scenario: {} on {}", processVersion, scheduleDates) + for { + inputConfigDuringExecution <- periodicDeploymentHandler.provideInputConfigDuringExecutionJson() deploymentWithJarData <- periodicDeploymentHandler.prepareDeploymentWithRuntimeParams( processVersion, ) enrichedProcessConfig <- processConfigEnricher.onInitialSchedule( - ProcessConfigEnricher.InitialScheduleData( - deploymentWithJarData.inputConfigDuringExecutionJson - ) - ) - enrichedDeploymentWithJarData = deploymentWithJarData.copy(inputConfigDuringExecutionJson = - enrichedProcessConfig.inputConfigDuringExecutionJson + ProcessConfigEnricher.InitialScheduleData(inputConfigDuringExecution.serialized) ) _ <- initialSchedule( scheduleProperty, scheduleDates, - enrichedDeploymentWithJarData, + deploymentWithJarData, + enrichedProcessConfig.inputConfigDuringExecutionJson, processActionId, ) } yield () @@ -168,11 +165,12 @@ class PeriodicProcessService( private def initialSchedule( scheduleMap: ScheduleProperty, scheduleDates: List[(ScheduleName, Option[LocalDateTime])], - deploymentWithJarData: DeploymentWithRuntimeParams.WithConfig, + deploymentWithJarData: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, processActionId: ProcessActionId, ): Future[Unit] = { periodicProcessesManager - .create(deploymentWithJarData, toApi(scheduleMap), processActionId) + .create(deploymentWithJarData, inputConfigDuringExecutionJson, toApi(scheduleMap), processActionId) .flatMap { process => scheduleDates.collect { case (name, Some(date)) => @@ -189,7 +187,7 @@ class PeriodicProcessService( .map(_ => ()) } - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]]] = { + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { for { toBeDeployed <- periodicProcessesManager.findToBeDeployed.flatMap { toDeployList => Future.sequence(toDeployList.map(checkIfNotRunning)).map(_.flatten) @@ -203,8 +201,8 @@ class PeriodicProcessService( // Currently we don't allow simultaneous runs of one scenario - only sequential, so if other schedule kicks in, it'll have to wait // TODO: we show allow to deploy scenarios with different scheduleName to be deployed simultaneous private def checkIfNotRunning( - toDeploy: PeriodicProcessDeployment[WithConfig] - ): Future[Option[PeriodicProcessDeployment[WithConfig]]] = { + toDeploy: PeriodicProcessDeployment + ): Future[Option[PeriodicProcessDeployment]] = { delegateDeploymentManager .getProcessStates(toDeploy.periodicProcess.processVersion.processName)(DataFreshnessPolicy.Fresh) .map( @@ -350,7 +348,7 @@ class PeriodicProcessService( } private def nextRunAt( - deployment: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithoutConfig], + deployment: PeriodicProcessDeployment, clock: Clock ): Either[String, Option[LocalDateTime]] = (fromApi(deployment.periodicProcess.scheduleProperty), deployment.scheduleName.value) match { @@ -369,7 +367,7 @@ class PeriodicProcessService( } private def handleFailedDeployment( - deployment: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig], + deployment: PeriodicProcessDeployment, state: Option[StatusDetails] ): Future[Unit] = { def calculateNextRetryAt = now().plus(deploymentRetryConfig.deployRetryPenalize.toMillis, ChronoUnit.MILLIS) @@ -414,7 +412,7 @@ class PeriodicProcessService( } yield runningDeploymentsForSchedules.map(deployment => DeploymentId(deployment.toString)) private def deactivateAction( - process: PeriodicProcess[DeploymentWithRuntimeParams.WithoutConfig] + process: PeriodicProcess ): Future[Callback] = { logger.info(s"Deactivate periodic process id: ${process.id.value}") for { @@ -434,7 +432,7 @@ class PeriodicProcessService( .map(_ => ()) } - def deploy(deployment: PeriodicProcessDeployment[DeploymentWithRuntimeParams.WithConfig]): Future[Unit] = { + def deploy(deployment: PeriodicProcessDeployment): Future[Unit] = { // TODO: set status before deployment? val id = deployment.id val deploymentData = DeploymentData( @@ -454,23 +452,27 @@ class PeriodicProcessService( ) processName = deploymentWithJarData.processVersion.processName versionId = deploymentWithJarData.processVersion.versionId - canonicalProcessOrError <- periodicProcessesManager.fetchCanonicalProcess(processName, versionId) - canonicalProcess = canonicalProcessOrError.getOrElse { + canonicalProcessOpt <- periodicProcessesManager.fetchCanonicalProcess(processName, versionId) + canonicalProcess = canonicalProcessOpt.getOrElse { throw new PeriodicProcessException( s"Could not fetch CanonicalProcess for processName=$processName, versionId=$versionId" ) } - enrichedProcessConfig <- processConfigEnricher.onDeploy( - ProcessConfigEnricher.DeployData( - deploymentWithJarData.inputConfigDuringExecutionJson, - deployment, - ) + inputConfigDuringExecutionJsonOpt <- periodicProcessesManager.fetchInputConfigDuringExecutionJson( + processName, + versionId, ) - enrichedDeploymentWithJarData = deploymentWithJarData.copy(inputConfigDuringExecutionJson = - enrichedProcessConfig.inputConfigDuringExecutionJson + inputConfigDuringExecutionJson = inputConfigDuringExecutionJsonOpt.getOrElse { + throw new PeriodicProcessException( + s"Could not fetch inputConfigDuringExecutionJson for processName=${processName}, versionId=${versionId}" + ) + } + enrichedProcessConfig <- processConfigEnricher.onDeploy( + ProcessConfigEnricher.DeployData(inputConfigDuringExecutionJson, deployment) ) externalDeploymentId <- periodicDeploymentHandler.deployWithRuntimeParams( - enrichedDeploymentWithJarData, + deploymentWithJarData, + enrichedProcessConfig.inputConfigDuringExecutionJson, deploymentData, canonicalProcess, ) @@ -652,7 +654,7 @@ class PeriodicProcessService( } private def scheduledExecutionStatusAndDateFinished( - entity: PeriodicProcessDeployment[WithoutConfig], + entity: PeriodicProcessDeployment, ): Option[FinishedScheduledExecutionMetadata] = { for { status <- entity.state.status match { diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala index b48985f7ae9..9d438ce84a1 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala @@ -1,19 +1,18 @@ package pl.touk.nussknacker.engine.common.periodic.service -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment import java.time.format.DateTimeFormatter trait AdditionalDeploymentDataProvider { - def prepareAdditionalData(runDetails: PeriodicProcessDeployment[WithConfig]): Map[String, String] + def prepareAdditionalData(runDetails: PeriodicProcessDeployment): Map[String, String] } object DefaultAdditionalDeploymentDataProvider extends AdditionalDeploymentDataProvider { - override def prepareAdditionalData(runDetails: PeriodicProcessDeployment[WithConfig]): Map[String, String] = { + override def prepareAdditionalData(runDetails: PeriodicProcessDeployment): Map[String, String] = { Map( "deploymentId" -> runDetails.id.value.toString, "runAt" -> runDetails.runAt.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala index e1d89442069..bdd999f36e6 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala @@ -2,9 +2,7 @@ package pl.touk.nussknacker.engine.common.periodic.service import com.typesafe.config.Config import pl.touk.nussknacker.engine.api.deployment.StatusDetails -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.ExternalDeploymentId /* @@ -22,29 +20,28 @@ trait PeriodicProcessListenerFactory { } sealed trait PeriodicProcessEvent { - val deployment: PeriodicProcessDeployment[WithConfig] + val deployment: PeriodicProcessDeployment } case class DeployedEvent( - deployment: PeriodicProcessDeployment[WithConfig], + deployment: PeriodicProcessDeployment, externalDeploymentId: Option[ExternalDeploymentId] ) extends PeriodicProcessEvent -case class FinishedEvent(deployment: PeriodicProcessDeployment[WithConfig], processState: Option[StatusDetails]) +case class FinishedEvent(deployment: PeriodicProcessDeployment, processState: Option[StatusDetails]) extends PeriodicProcessEvent case class FailedOnDeployEvent( - deployment: PeriodicProcessDeployment[WithConfig], + deployment: PeriodicProcessDeployment, processState: Option[StatusDetails] ) extends PeriodicProcessEvent case class FailedOnRunEvent( - deployment: PeriodicProcessDeployment[WithConfig], + deployment: PeriodicProcessDeployment, processState: Option[StatusDetails] ) extends PeriodicProcessEvent -case class ScheduledEvent(deployment: PeriodicProcessDeployment[WithConfig], firstSchedule: Boolean) - extends PeriodicProcessEvent +case class ScheduledEvent(deployment: PeriodicProcessDeployment, firstSchedule: Boolean) extends PeriodicProcessEvent object EmptyListener extends EmptyListener diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala index 579bd9c2500..bdaee9be8bc 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala @@ -1,7 +1,6 @@ package pl.touk.nussknacker.engine.common.periodic.service import com.typesafe.config.{Config, ConfigFactory} -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment import pl.touk.nussknacker.engine.common.periodic.service.ProcessConfigEnricher.{ DeployData, @@ -43,7 +42,7 @@ object ProcessConfigEnricher { case class DeployData( inputConfigDuringExecutionJson: String, - deployment: PeriodicProcessDeployment[WithConfig] + deployment: PeriodicProcessDeployment ) extends ProcessConfigEnricherInputData case class EnrichedProcessConfig(inputConfigDuringExecutionJson: String) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala index 29e5524dd91..ff13d15a536 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala @@ -53,17 +53,19 @@ class FlinkPeriodicDeploymentHandler( override def prepareDeploymentWithRuntimeParams( processVersion: ProcessVersion, - ): Future[DeploymentWithRuntimeParams.WithConfig] = { + ): Future[DeploymentWithRuntimeParams] = { logger.info(s"Prepare deployment for scenario: $processVersion") copyJarToLocalDir(processVersion).map { jarFileName => - DeploymentWithRuntimeParams.WithConfig( + DeploymentWithRuntimeParams( processVersion = processVersion, - inputConfigDuringExecutionJson = inputConfigDuringExecution.serialized, runtimeParams = RuntimeParams(Map(jarFileNameRuntimeParam -> jarFileName)) ) } } + override def provideInputConfigDuringExecutionJson(): Future[InputConfigDuringExecution] = + Future.successful(inputConfigDuringExecution) + private def copyJarToLocalDir(processVersion: ProcessVersion): Future[String] = Future { jarsDir.toFile.mkdirs() val jarFileName = @@ -75,7 +77,8 @@ class FlinkPeriodicDeploymentHandler( } override def deployWithRuntimeParams( - deployment: DeploymentWithRuntimeParams.WithConfig, + deployment: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, deploymentData: DeploymentData, canonicalProcess: CanonicalProcess, ): Future[Option[ExternalDeploymentId]] = { @@ -87,7 +90,7 @@ class FlinkPeriodicDeploymentHandler( ) val jarFile = jarsDir.resolve(jarFileName).toFile val args = FlinkDeploymentManager.prepareProgramArgs( - deployment.inputConfigDuringExecutionJson, + inputConfigDuringExecutionJson, processVersion, deploymentData, canonicalProcess, diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala index 4a6384aaafc..bda0b268a96 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala @@ -6,7 +6,6 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.LoneElement._ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment import pl.touk.nussknacker.engine.common.periodic.DeploymentActor import pl.touk.nussknacker.engine.common.periodic.DeploymentActor.CheckToBeDeployed @@ -34,11 +33,11 @@ class DeploymentActorTest extends AnyFunSuite with TestKitBase with Matchers wit } private def shouldFindToBeDeployedScenarios( - result: Future[Seq[PeriodicProcessDeployment[WithConfig]]] + result: Future[Seq[PeriodicProcessDeployment]] ): Unit = { val probe = TestProbe() var counter = 0 - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = { + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { counter += 1 probe.ref ! s"invoked $counter" result @@ -55,14 +54,14 @@ class DeploymentActorTest extends AnyFunSuite with TestKitBase with Matchers wit } test("should deploy found scenario") { - val probe = TestProbe() - val waitingDeployment = PeriodicProcessDeploymentGen() - var toBeDeployed: Seq[PeriodicProcessDeployment[WithConfig]] = Seq(waitingDeployment) - var actor: ActorRef = null - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = { + val probe = TestProbe() + val waitingDeployment = PeriodicProcessDeploymentGen() + var toBeDeployed: Seq[PeriodicProcessDeployment] = Seq(waitingDeployment) + var actor: ActorRef = null + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { Future.successful(toBeDeployed) } - def deploy(deployment: PeriodicProcessDeployment[WithConfig]): Future[Unit] = { + def deploy(deployment: PeriodicProcessDeployment): Future[Unit] = { probe.ref ! deployment // Simulate periodic check for waiting scenarios while deploying a scenario. actor ! CheckToBeDeployed diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala index bf537de09e3..6df57e9f9c0 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala @@ -1,38 +1,46 @@ package pl.touk.nussknacker.engine.management.periodic.flink +import com.typesafe.config.ConfigFactory import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.periodic.model.{DeploymentWithRuntimeParams, RuntimeParams} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.common.periodic.PeriodicDeploymentHandler import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} +import pl.touk.nussknacker.engine.modelconfig.InputConfigDuringExecution import scala.concurrent.Future class PeriodicDeploymentHandlerStub extends PeriodicDeploymentHandler { - var deployWithJarFuture: Future[Option[ExternalDeploymentId]] = Future.successful(None) - var lastDeploymentWithRuntimeParams: Option[DeploymentWithRuntimeParams.WithConfig] = None + var deployWithJarFuture: Future[Option[ExternalDeploymentId]] = Future.successful(None) + var lastDeploymentWithRuntimeParams: Option[DeploymentWithRuntimeParams] = None + var lastInputConfigDuringExecutionJson: Option[String] = None override def prepareDeploymentWithRuntimeParams( processVersion: ProcessVersion, - ): Future[DeploymentWithRuntimeParams.WithConfig] = { + ): Future[DeploymentWithRuntimeParams] = { Future.successful( - DeploymentWithRuntimeParams.WithConfig( + DeploymentWithRuntimeParams( processVersion = processVersion, - inputConfigDuringExecutionJson = "", runtimeParams = RuntimeParams(Map("jarFileName" -> "")) ) ) } + override def provideInputConfigDuringExecutionJson(): Future[InputConfigDuringExecution] = + Future.successful(InputConfigDuringExecution(ConfigFactory.parseString(""))) + override def deployWithRuntimeParams( - deploymentWithJarData: DeploymentWithRuntimeParams.WithConfig, + deploymentWithJarData: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, deploymentData: DeploymentData, canonicalProcess: CanonicalProcess, ): Future[Option[ExternalDeploymentId]] = { lastDeploymentWithRuntimeParams = Some(deploymentWithJarData) + lastInputConfigDuringExecutionJson = Some(inputConfigDuringExecutionJson) deployWithJarFuture } override def cleanAfterDeployment(runtimeParams: RuntimeParams): Future[Unit] = Future.successful(()) + } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala index e4130b97d3e..ccbc60e06d3 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala @@ -1,6 +1,5 @@ package pl.touk.nussknacker.engine.management.periodic.flink -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import java.time.LocalDateTime @@ -9,7 +8,7 @@ object PeriodicProcessDeploymentGen { val now: LocalDateTime = LocalDateTime.now() - def apply(): PeriodicProcessDeployment[WithConfig] = { + def apply(): PeriodicProcessDeployment = { PeriodicProcessDeployment( id = PeriodicProcessDeploymentId(42), periodicProcess = PeriodicProcessGen(), diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala index b96dc3072e6..5e63507337b 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala @@ -2,7 +2,6 @@ package pl.touk.nussknacker.engine.management.periodic.flink import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager.CronScheduleProperty -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess @@ -12,12 +11,11 @@ import java.time.LocalDateTime object PeriodicProcessGen { - def apply(): PeriodicProcess[WithConfig] = { + def apply(): PeriodicProcess = { PeriodicProcess( id = PeriodicProcessId(42), - deploymentData = DeploymentWithRuntimeParams.WithConfig( + deploymentData = DeploymentWithRuntimeParams( processVersion = ProcessVersion.empty, - inputConfigDuringExecutionJson = "{}", runtimeParams = RuntimeParams(Map("jarFileName" -> "jar-file-name.jar")) ), scheduleProperty = CronScheduleProperty("0 0 * * * ?"), diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala index 25f58cf1e95..a530171f7be 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala @@ -8,7 +8,6 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.WithConfig import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model.{ PeriodicProcessDeployment, @@ -95,7 +94,7 @@ class PeriodicProcessServiceTest additionalDeploymentDataProvider = new AdditionalDeploymentDataProvider { override def prepareAdditionalData( - runDetails: PeriodicProcessDeployment[WithConfig] + runDetails: PeriodicProcessDeployment ): Map[String, String] = additionalData + ("runId" -> runDetails.id.value.toString) @@ -396,7 +395,7 @@ class PeriodicProcessServiceTest val deploymentEntity = f.manager.deploymentEntities.loneElement deploymentEntity.status shouldBe PeriodicProcessDeploymentStatus.Deployed ConfigFactory - .parseString(f.periodicDeploymentHandlerStub.lastDeploymentWithRuntimeParams.value.inputConfigDuringExecutionJson) + .parseString(f.periodicDeploymentHandlerStub.lastInputConfigDuringExecutionJson.value) .getString("runAt") shouldBe deploymentEntity.runAt.toString val expectedDetails = diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala index 9ccb30c9c36..0f95f511475 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala @@ -4,7 +4,6 @@ import io.circe.syntax.EncoderOps import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager -import pl.touk.nussknacker.engine.api.deployment.periodic.model.DeploymentWithRuntimeParams.{WithConfig, WithoutConfig} import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} @@ -128,17 +127,18 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc } override def create( - deploymentWithRuntimeParams: DeploymentWithRuntimeParams.WithConfig, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, - ): Future[PeriodicProcess[WithConfig]] = Future.successful { + ): Future[PeriodicProcess] = Future.successful { val id = PeriodicProcessId(Random.nextLong()) val periodicProcess = PeriodicProcessEntity( id = id, processName = deploymentWithRuntimeParams.processVersion.processName, processVersionId = deploymentWithRuntimeParams.processVersion.versionId, processingType = processingType, - inputConfigDuringExecutionJson = deploymentWithRuntimeParams.inputConfigDuringExecutionJson, + inputConfigDuringExecutionJson = inputConfigDuringExecutionJson, runtimeParams = deploymentWithRuntimeParams.runtimeParams, scheduleProperty = fromApi(scheduleProperty).asJson.noSpaces, active = true, @@ -227,19 +227,19 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc } yield deploymentGroupedByScheduleName).toMap) } - override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = { + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { val scheduled = findActive(PeriodicProcessDeploymentStatus.Scheduled) readyToRun(scheduled) } - override def findToBeRetried: Future[Seq[PeriodicProcessDeployment[WithConfig]]] = { + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = { val toBeRetried = findActive(PeriodicProcessDeploymentStatus.FailedOnDeploy).filter(_.retriesLeft > 0) readyToRun(toBeRetried) } override def findProcessData( id: PeriodicProcessDeploymentId, - ): Future[PeriodicProcessDeployment[WithConfig]] = Future.successful { + ): Future[PeriodicProcessDeployment] = Future.successful { (for { d <- deploymentEntities if d.id == id p <- processEntities if p.id == d.periodicProcessId @@ -295,7 +295,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int, - ): Future[PeriodicProcessDeployment[WithConfig]] = Future.successful { + ): Future[PeriodicProcessDeployment] = Future.successful { val deploymentEntity = PeriodicProcessDeploymentEntity( id = PeriodicProcessDeploymentId(Random.nextLong()), periodicProcessId = id, @@ -323,22 +323,22 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc } } - private def findActive(status: PeriodicProcessDeploymentStatus): Seq[PeriodicProcessDeployment[WithConfig]] = + private def findActive(status: PeriodicProcessDeploymentStatus): Seq[PeriodicProcessDeployment] = findActive( Seq(status) ) private def findActive( statusList: Seq[PeriodicProcessDeploymentStatus] - ): Seq[PeriodicProcessDeployment[WithConfig]] = + ): Seq[PeriodicProcessDeployment] = (for { p <- processEntities if p.active && p.processingType == processingType d <- deploymentEntities if d.periodicProcessId == p.id && statusList.contains(d.status) } yield createPeriodicProcessDeployment(p, d)).toSeq private def readyToRun( - deployments: Seq[PeriodicProcessDeployment[WithConfig]] - ): Future[Seq[PeriodicProcessDeployment[WithConfig]]] = { + deployments: Seq[PeriodicProcessDeployment] + ): Future[Seq[PeriodicProcessDeployment]] = { val now = LocalDateTime.now() Future.successful(deployments.filter(d => d.runAt.isBefore(now) || d.runAt.isEqual(now))) } @@ -348,6 +348,12 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc versionId: VersionId ): Future[Option[CanonicalProcess]] = Future.successful(Some(canonicalProcess(processName))) + override def fetchInputConfigDuringExecutionJson( + processName: ProcessName, + versionId: VersionId + ): Future[Option[String]] = + Future.successful(Some("{}")) + } object InMemPeriodicProcessesManager { @@ -384,7 +390,7 @@ object InMemPeriodicProcessesManager { def createPeriodicProcessDeployment( processEntity: PeriodicProcessEntity, processDeploymentEntity: PeriodicProcessDeploymentEntity - ): PeriodicProcessDeployment[WithConfig] = { + ): PeriodicProcessDeployment = { val process = createPeriodicProcessWithJson(processEntity) PeriodicProcessDeployment( processDeploymentEntity.id, @@ -410,14 +416,13 @@ object InMemPeriodicProcessesManager { def createPeriodicProcessWithJson( processEntity: PeriodicProcessEntity - ): PeriodicProcess[WithConfig] = { + ): PeriodicProcess = { val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, - DeploymentWithRuntimeParams.WithConfig( + DeploymentWithRuntimeParams( processVersion = processVersion, - inputConfigDuringExecutionJson = processEntity.inputConfigDuringExecutionJson, runtimeParams = processEntity.runtimeParams, ), toApi(scheduleProperty), @@ -429,12 +434,12 @@ object InMemPeriodicProcessesManager { def createPeriodicProcessWithoutJson( processEntity: PeriodicProcessEntity - ): PeriodicProcess[WithoutConfig] = { + ): PeriodicProcess = { val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, - DeploymentWithRuntimeParams.WithoutConfig( + DeploymentWithRuntimeParams( processVersion = processVersion, runtimeParams = processEntity.runtimeParams, ), From 5926de530a1f769b8a6d8b624655f353afb80888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Mon, 23 Dec 2024 19:38:50 +0100 Subject: [PATCH 05/11] isolate config json --- .../PeriodicProcessesManagerProvider.scala | 2 - ...dicDeploymentManagerTablesDefinition.scala | 69 ++----- .../ui/db/entity/PeriodicProcessesTable.scala | 168 +++--------------- ...ositoryBasedPeriodicProcessesManager.scala | 1 - ...asedPeriodicProcessesManagerProvider.scala | 2 - .../PeriodicProcessesRepository.scala | 92 +++++----- ...eriodicProcessServiceIntegrationTest.scala | 3 +- .../PeriodicDeploymentManagerProvider.scala | 2 +- 8 files changed, 85 insertions(+), 254 deletions(-) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManagerProvider.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManagerProvider.scala index c756a754d7f..3b465cfeff9 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManagerProvider.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManagerProvider.scala @@ -3,7 +3,6 @@ package pl.touk.nussknacker.engine.api.deployment.periodic trait PeriodicProcessesManagerProvider { def provide( - deploymentManagerName: String, processingType: String, ): PeriodicProcessesManager @@ -12,7 +11,6 @@ trait PeriodicProcessesManagerProvider { object NoOpPeriodicProcessesManagerProvider extends PeriodicProcessesManagerProvider { override def provide( - deploymentManagerName: String, processingType: String ): PeriodicProcessesManager = NoOpPeriodicProcessesManager diff --git a/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala b/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala index fd1b42362f4..79f297f350d 100644 --- a/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala +++ b/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala @@ -90,7 +90,9 @@ object V1_060__PeriodicDeploymentManagerTablesDefinition { class PeriodicProcessesTable(tag: Tag) extends Table[PeriodicProcessEntity](tag, "periodic_processes") { - def id: Rep[Long] = column[Long]("id", O.PrimaryKey, O.AutoInc) + def periodicProcessId: Rep[Long] = column[Long]("id", O.Unique, O.AutoInc) + + def processId: Rep[Option[Long]] = column[Option[Long]]("process_id") def processName: Rep[String] = column[String]("process_name", NotNull) @@ -98,12 +100,6 @@ object V1_060__PeriodicDeploymentManagerTablesDefinition { def processingType: Rep[String] = column[String]("processing_type", NotNull) - def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) - - // This is a legacy column left after migrating periodic processes to core - // The periodic deployment manager is now decoupled from Flink, and its runtime params are stored in runtime_params column - def jarFileName: Rep[Option[String]] = column[Option[String]]("jar_file_name") - def runtimeParams: Rep[String] = column[String]("runtime_params") def scheduleProperty: Rep[String] = column[String]("schedule_property", NotNull) @@ -114,75 +110,36 @@ object V1_060__PeriodicDeploymentManagerTablesDefinition { def processActionId: Rep[Option[UUID]] = column[Option[UUID]]("process_action_id") + def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) + override def * : ProvenShape[PeriodicProcessEntity] = ( - id, + periodicProcessId, + processId, processName, processVersionId, processingType, - inputConfigDuringExecutionJson, - jarFileName, runtimeParams, scheduleProperty, active, createdAt, - processActionId - ) <> ( - tuple => - PeriodicProcessEntity( - id = tuple._1, - processName = tuple._2, - processVersionId = tuple._3, - processingType = tuple._4, - inputConfigDuringExecutionJson = tuple._5, - runtimeParams = tuple._7, - scheduleProperty = tuple._8, - active = tuple._9, - createdAt = tuple._10, - processActionId = tuple._11, - ), - (e: PeriodicProcessEntity) => - PeriodicProcessEntity.unapply(e).map { - case ( - id, - processName, - versionId, - processingType, - inputConfigDuringExecutionJson, - runtimeParams, - scheduleProperty, - active, - createdAt, - processActionId - ) => - ( - id, - processName, - versionId, - processingType, - inputConfigDuringExecutionJson, - None, - runtimeParams, - scheduleProperty, - active, - createdAt, - processActionId - ) - } - ) + processActionId, + inputConfigDuringExecutionJson, + ) <> (PeriodicProcessEntity.apply _ tupled, PeriodicProcessEntity.unapply) } case class PeriodicProcessEntity( id: Long, + processId: Option[Long], processName: String, processVersionId: Long, processingType: String, - inputConfigDuringExecutionJson: String, runtimeParams: String, scheduleProperty: String, active: Boolean, createdAt: LocalDateTime, - processActionId: Option[UUID] + processActionId: Option[UUID], + inputConfigDuringExecutionJson: String, ) } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala index 135a2606cf0..60c2d07eb8b 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala @@ -4,9 +4,7 @@ import io.circe.Decoder import io.circe.syntax.EncoderOps import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.model.{PeriodicProcessId, RuntimeParams} -import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.marshall.ProcessMarshaller +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import slick.jdbc.JdbcProfile import slick.lifted.ProvenShape import slick.sql.SqlProfile.ColumnOption.NotNull @@ -44,14 +42,14 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory { def id: Rep[PeriodicProcessId] = column[PeriodicProcessId]("id", O.PrimaryKey, O.AutoInc) + def processId: Rep[Option[ProcessId]] = column[Option[ProcessId]]("process_id") + def processName: Rep[ProcessName] = column[ProcessName]("process_name", NotNull) def processVersionId: Rep[VersionId] = column[VersionId]("process_version_id", NotNull) def processingType: Rep[String] = column[String]("processing_type", NotNull) - def jarFileName: Rep[Option[String]] = column[Option[String]]("jar_file_name") - def runtimeParams: Rep[RuntimeParams] = column[RuntimeParams]("runtime_params") def scheduleProperty: Rep[String] = column[String]("schedule_property", NotNull) @@ -64,74 +62,33 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory { } - class PeriodicProcessesWithJsonTable(tag: Tag) extends PeriodicProcessesTable[PeriodicProcessEntityWithJson](tag) { + class PeriodicProcessesWithInputConfigJsonTable(tag: Tag) + extends PeriodicProcessesTable[PeriodicProcessEntityWithInputConfigJson](tag) { def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) - override def * : ProvenShape[PeriodicProcessEntityWithJson] = ( + override def * : ProvenShape[PeriodicProcessEntityWithInputConfigJson] = ( id, + processId, processName, processVersionId, processingType, - inputConfigDuringExecutionJson, - jarFileName, runtimeParams, scheduleProperty, active, createdAt, - processActionId - ) <> ( - tuple => - PeriodicProcessEntityWithJson( - id = tuple._1, - processName = tuple._2, - processVersionId = tuple._3, - processingType = tuple._4, - inputConfigDuringExecutionJson = tuple._5, - runtimeParams = - RuntimeParams(tuple._6.map(f => Map("jarFileName" -> f)).getOrElse(Map.empty) ++ tuple._7.params), - scheduleProperty = tuple._8, - active = tuple._9, - createdAt = tuple._10, - processActionId = tuple._11, - ), - (e: PeriodicProcessEntityWithJson) => - PeriodicProcessEntityWithJson.unapply(e).map { - case ( - id, - processName, - versionId, - processingType, - inputConfigDuringExecutionJson, - runtimeParams, - scheduleProperty, - active, - createdAt, - processActionId - ) => - ( - id, - processName, - versionId, - processingType, - inputConfigDuringExecutionJson, - None, - runtimeParams, - scheduleProperty, - active, - createdAt, - processActionId - ) - } - ) + processActionId, + inputConfigDuringExecutionJson, + ) <> (PeriodicProcessEntityWithInputConfigJson.apply _ tupled, PeriodicProcessEntityWithInputConfigJson.unapply) } - class PeriodicProcessesWithoutJsonTable(tag: Tag) - extends PeriodicProcessesTable[PeriodicProcessEntityWithoutJson](tag) { + class PeriodicProcessesWithoutInputConfigJsonTable(tag: Tag) + extends PeriodicProcessesTable[PeriodicProcessEntityWithoutInputConfigJson](tag) { - override def * : ProvenShape[PeriodicProcessEntityWithoutJson] = ( + override def * : ProvenShape[PeriodicProcessEntityWithoutInputConfigJson] = ( id, + processId, processName, processVersionId, processingType, @@ -140,92 +97,13 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory { active, createdAt, processActionId - ) <> ( - (PeriodicProcessEntity.createWithoutJson _).tupled, - (e: PeriodicProcessEntityWithoutJson) => - PeriodicProcessEntityWithoutJson.unapply(e).map { - case ( - id, - processName, - versionId, - processingType, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) => - ( - id, - processName, - versionId, - processingType, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) - } - ) + ) <> (PeriodicProcessEntityWithoutInputConfigJson.apply _ tupled, PeriodicProcessEntityWithoutInputConfigJson.unapply) } - object PeriodicProcessesWithJson extends TableQuery(new PeriodicProcessesWithJsonTable(_)) - - object PeriodicProcessesWithoutJson extends TableQuery(new PeriodicProcessesWithoutJsonTable(_)) - -} - -object PeriodicProcessEntity { - - def createWithJson( - id: PeriodicProcessId, - processName: ProcessName, - processVersionId: VersionId, - processingType: String, - inputConfigDuringExecutionJson: String, - runtimeParams: RuntimeParams, - scheduleProperty: String, - active: Boolean, - createdAt: LocalDateTime, - processActionId: Option[ProcessActionId] - ): PeriodicProcessEntityWithJson = - PeriodicProcessEntityWithJson( - id, - processName, - processVersionId, - processingType, - inputConfigDuringExecutionJson, - runtimeParams, - scheduleProperty, - active, - createdAt, - processActionId - ) + object PeriodicProcessesWithoutInputConfig extends TableQuery(new PeriodicProcessesWithoutInputConfigJsonTable(_)) - def createWithoutJson( - id: PeriodicProcessId, - processName: ProcessName, - processVersionId: VersionId, - processingType: String, - runtimeParams: RuntimeParams, - scheduleProperty: String, - active: Boolean, - createdAt: LocalDateTime, - processActionId: Option[ProcessActionId] - ): PeriodicProcessEntityWithoutJson = - PeriodicProcessEntityWithoutJson( - id, - processName, - processVersionId, - processingType, - runtimeParams, - scheduleProperty, - active, - createdAt, - processActionId - ) + object PeriodicProcessesWithInputConfig extends TableQuery(new PeriodicProcessesWithInputConfigJsonTable(_)) } @@ -233,6 +111,8 @@ trait PeriodicProcessEntity { def id: PeriodicProcessId + def processId: Option[ProcessId] + def processName: ProcessName def processVersionId: VersionId @@ -251,21 +131,23 @@ trait PeriodicProcessEntity { } -case class PeriodicProcessEntityWithJson( +case class PeriodicProcessEntityWithInputConfigJson( id: PeriodicProcessId, + processId: Option[ProcessId], processName: ProcessName, processVersionId: VersionId, processingType: String, - inputConfigDuringExecutionJson: String, runtimeParams: RuntimeParams, scheduleProperty: String, active: Boolean, createdAt: LocalDateTime, - processActionId: Option[ProcessActionId] + processActionId: Option[ProcessActionId], + inputConfigDuringExecutionJson: String, ) extends PeriodicProcessEntity -case class PeriodicProcessEntityWithoutJson( +case class PeriodicProcessEntityWithoutInputConfigJson( id: PeriodicProcessId, + processId: Option[ProcessId], processName: ProcessName, processVersionId: VersionId, processingType: String, diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala index fab081133fb..c3cc9572d24 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala @@ -12,7 +12,6 @@ import java.time.LocalDateTime import scala.concurrent.Future class RepositoryBasedPeriodicProcessesManager( - deploymentManagerName: String, processingType: String, periodicProcessesRepository: PeriodicProcessesRepository, ) extends PeriodicProcessesManager { diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala index f2b576e35a0..b4de6525cb7 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala @@ -8,11 +8,9 @@ class RepositoryBasedPeriodicProcessesManagerProvider( ) extends PeriodicProcessesManagerProvider { override def provide( - deploymentManagerName: String, processingType: String ): PeriodicProcessesManager = { new RepositoryBasedPeriodicProcessesManager( - deploymentManagerName, processingType, periodicProcessesRepository ) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala index df98d2c6e46..112ee184444 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala @@ -16,7 +16,7 @@ import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} import pl.touk.nussknacker.engine.common.periodic._ import pl.touk.nussknacker.ui.db.entity._ -import pl.touk.nussknacker.ui.process.repository.PeriodicProcessesRepository.createPeriodicProcessWithoutJson +import pl.touk.nussknacker.ui.process.repository.PeriodicProcessesRepository.createPeriodicProcess import slick.dbio.{DBIOAction, Effect, NoStream} import slick.jdbc.PostgresProfile.api._ import slick.jdbc.{JdbcBackend, JdbcProfile} @@ -28,10 +28,10 @@ import scala.language.higherKinds object PeriodicProcessesRepository { def createPeriodicProcessDeployment( - processEntity: PeriodicProcessEntityWithoutJson, + processEntity: PeriodicProcessEntity, processDeploymentEntity: PeriodicProcessDeploymentEntity ): PeriodicProcessDeployment = { - val process = createPeriodicProcessWithJson(processEntity) + val process = createPeriodicProcess(processEntity) PeriodicProcessDeployment( processDeploymentEntity.id, process, @@ -54,25 +54,7 @@ object PeriodicProcessesRepository { ) } - def createPeriodicProcessWithJson( - processEntity: PeriodicProcessEntityWithoutJson - ): PeriodicProcess = { - val processVersion = createProcessVersion(processEntity) - val scheduleProperty = prepareScheduleProperty(processEntity) - PeriodicProcess( - processEntity.id, - DeploymentWithRuntimeParams( - processVersion = processVersion, - runtimeParams = processEntity.runtimeParams, - ), - toApi(scheduleProperty), - processEntity.active, - processEntity.createdAt, - processEntity.processActionId - ) - } - - def createPeriodicProcessWithoutJson( + def createPeriodicProcess( processEntity: PeriodicProcessEntity ): PeriodicProcess = { val processVersion = createProcessVersion(processEntity) @@ -220,7 +202,7 @@ class SlickPeriodicProcessesRepository( scenarioName: ProcessName, afterOpt: Option[LocalDateTime], ): Action[SchedulesState] = { - PeriodicProcessesWithoutJson + PeriodicProcessesWithoutInputConfig .filter(_.processName === scenarioName) .join(PeriodicProcessDeployments) .on(_.id === _.periodicProcessId) @@ -236,20 +218,23 @@ class SlickPeriodicProcessesRepository( processActionId: ProcessActionId, processingType: String, ): Action[PeriodicProcess] = { - val processEntity = PeriodicProcessEntityWithJson( + val processEntity = PeriodicProcessEntityWithInputConfigJson( id = PeriodicProcessId(-1), + processId = Some(deploymentWithRuntimeParams.processVersion.processId), processName = deploymentWithRuntimeParams.processVersion.processName, processVersionId = deploymentWithRuntimeParams.processVersion.versionId, processingType = processingType, - inputConfigDuringExecutionJson = inputConfigDuringExecutionJson, runtimeParams = deploymentWithRuntimeParams.runtimeParams, scheduleProperty = fromApi(scheduleProperty).asJson.noSpaces, active = true, createdAt = now(), - Some(processActionId) + Some(processActionId), + inputConfigDuringExecutionJson = inputConfigDuringExecutionJson, ) - ((PeriodicProcessesWithJson returning PeriodicProcessesWithJson into ((_, id) => id)) += processEntity) - .map(PeriodicProcessesRepository.createPeriodicProcessWithJson) + ((PeriodicProcessesWithInputConfig returning PeriodicProcessesWithInputConfig into ((_, id) => + id + )) += processEntity) + .map(PeriodicProcessesRepository.createPeriodicProcess) } private def now(): LocalDateTime = LocalDateTime.now(clock) @@ -274,8 +259,8 @@ class SlickPeriodicProcessesRepository( private def findProcesses( query: Query[ - (PeriodicProcessesWithoutJsonTable, PeriodicProcessDeploymentsTable), - (PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity), + (PeriodicProcessesWithoutInputConfigJsonTable, PeriodicProcessDeploymentsTable), + (PeriodicProcessEntityWithoutInputConfigJson, PeriodicProcessDeploymentEntity), Seq ] ) = { @@ -290,7 +275,7 @@ class SlickPeriodicProcessesRepository( override def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment] = findProcesses( - (PeriodicProcessesWithoutJson join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) + (PeriodicProcessesWithoutInputConfig join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) .filter { case (_, deployment) => deployment.id === id } ).map(_.head) @@ -338,7 +323,7 @@ class SlickPeriodicProcessesRepository( expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], processingType: String, ): Action[SchedulesState] = { - val processesHavingDeploymentsWithMatchingStatus = PeriodicProcessesWithoutJson.filter(p => + val processesHavingDeploymentsWithMatchingStatus = PeriodicProcessesWithoutInputConfig.filter(p => p.active && PeriodicProcessDeployments .filter(d => d.periodicProcessId === p.id && d.status.inSet(expectedDeploymentStatuses)) @@ -356,7 +341,8 @@ class SlickPeriodicProcessesRepository( deploymentsPerScheduleMaxCount: Int, processingType: String, ): Action[SchedulesState] = { - val activeProcessesQuery = PeriodicProcessesWithoutJson.filter(p => p.processName === processName && p.active) + val activeProcessesQuery = + PeriodicProcessesWithoutInputConfig.filter(p => p.processName === processName && p.active) getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount, processingType) .map(_.getOrElse(processName, SchedulesState(Map.empty))) } @@ -365,7 +351,7 @@ class SlickPeriodicProcessesRepository( deploymentsPerScheduleMaxCount: Int, processingType: String, ): Action[Map[ProcessName, SchedulesState]] = { - val activeProcessesQuery = PeriodicProcessesWithoutJson.filter(_.active) + val activeProcessesQuery = PeriodicProcessesWithoutInputConfig.filter(_.active) getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount, processingType) } @@ -375,7 +361,7 @@ class SlickPeriodicProcessesRepository( deploymentsPerScheduleMaxCount: Int, processingType: String, ): Action[SchedulesState] = { - val filteredProcessesQuery = PeriodicProcessesWithoutJson + val filteredProcessesQuery = PeriodicProcessesWithoutInputConfig .filter(p => p.processName === processName && !p.active) .sortBy(_.createdAt.desc) .take(inactiveProcessesMaxCount) @@ -388,7 +374,7 @@ class SlickPeriodicProcessesRepository( deploymentsPerScheduleMaxCount: Int, processingType: String, ): Action[Map[ProcessName, SchedulesState]] = { - val filteredProcessesQuery = PeriodicProcessesWithoutJson + val filteredProcessesQuery = PeriodicProcessesWithoutInputConfig .filter(!_.active) .sortBy(_.createdAt.desc) .take(inactiveProcessesMaxCount) @@ -396,7 +382,11 @@ class SlickPeriodicProcessesRepository( } private def getLatestDeploymentsForEachSchedule( - periodicProcessesQuery: Query[PeriodicProcessesWithoutJsonTable, PeriodicProcessEntityWithoutJson, Seq], + periodicProcessesQuery: Query[ + PeriodicProcessesWithoutInputConfigJsonTable, + PeriodicProcessEntityWithoutInputConfigJson, + Seq + ], deploymentsPerScheduleMaxCount: Int, processingType: String, ): Action[Map[ProcessName, SchedulesState]] = { @@ -411,9 +401,13 @@ class SlickPeriodicProcessesRepository( } private def getLatestDeploymentsForEachSchedulePostgres( - periodicProcessesQuery: Query[PeriodicProcessesWithoutJsonTable, PeriodicProcessEntityWithoutJson, Seq], + periodicProcessesQuery: Query[ + PeriodicProcessesWithoutInputConfigJsonTable, + PeriodicProcessEntityWithoutInputConfigJson, + Seq + ], deploymentsPerScheduleMaxCount: Int - ): Action[Seq[(PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity)]] = { + ): Action[Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)]] = { // To effectively limit deployments to given count for each schedule in one query, we use window functions in slick import ExPostgresProfile.api._ import com.github.tminglei.slickpg.window.PgWindowFuncSupport.WindowFunctions._ @@ -445,9 +439,13 @@ class SlickPeriodicProcessesRepository( // If we decided to support more databases, we should consider some optimization like extracting periodic_schedule table // with foreign key to periodic_process and with schedule_name column - it would reduce number of queries private def getLatestDeploymentsForEachScheduleJdbcGeneric( - periodicProcessesQuery: Query[PeriodicProcessesWithoutJsonTable, PeriodicProcessEntityWithoutJson, Seq], + periodicProcessesQuery: Query[ + PeriodicProcessesWithoutInputConfigJsonTable, + PeriodicProcessEntityWithoutInputConfigJson, + Seq + ], deploymentsPerScheduleMaxCount: Int - ): Action[Seq[(PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity)]] = { + ): Action[Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)]] = { // It is debug instead of warn to not bloast logs when e.g. for some reasons is used hsql under the hood logger.debug( "WARN: Using not optimized version of getLatestDeploymentsForEachSchedule that not uses window functions" @@ -507,7 +505,7 @@ class SlickPeriodicProcessesRepository( override def markInactive(processId: PeriodicProcessId): Action[Unit] = { val q = for { - p <- PeriodicProcessesWithoutJson if p.id === processId + p <- PeriodicProcessesWithoutInputConfig if p.id === processId } yield p.active val update = q.update(false) update.map(_ => ()) @@ -525,19 +523,19 @@ class SlickPeriodicProcessesRepository( } def fetchInputConfigDuringExecutionJson(processName: ProcessName, versionId: VersionId): Action[Option[String]] = - PeriodicProcessesWithJson + PeriodicProcessesWithInputConfig .filter(p => p.processName === processName && p.processVersionId === versionId) .map(_.inputConfigDuringExecutionJson) .result .headOption private def activePeriodicProcessWithDeploymentQuery(processingType: String) = { - (PeriodicProcessesWithoutJson.filter(p => p.active === true && p.processingType === processingType) + (PeriodicProcessesWithoutInputConfig.filter(p => p.active === true && p.processingType === processingType) join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) } private def toSchedulesState( - list: Seq[(PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity)] + list: Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)] ): Map[ProcessName, SchedulesState] = { list .groupBy(_._1.processName) @@ -545,7 +543,7 @@ class SlickPeriodicProcessesRepository( } private def toSchedulesStateForSinglePeriodicProcess( - list: Seq[(PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity)] + list: Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)] ): SchedulesState = { SchedulesState( list @@ -559,7 +557,7 @@ class SlickPeriodicProcessesRepository( .toGroupedMap .toList .map { case ((scheduleId, process), deployments) => - scheduleId -> ScheduleData(createPeriodicProcessWithoutJson(process), deployments) + scheduleId -> ScheduleData(createPeriodicProcess(process), deployments) } .toMap ) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala index 2cffdc87628..0fa8b000857 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -105,7 +105,6 @@ class PeriodicProcessServiceIntegrationTest def periodicProcessService( currentTime: Instant, - deploymentManagerName: String = "testPeriodicDeploymentManager", processingType: String = processingType ) = new PeriodicProcessService( @@ -113,7 +112,7 @@ class PeriodicProcessServiceIntegrationTest periodicDeploymentHandler = periodicDeploymentHandlerStub, periodicProcessesManager = new RepositoryBasedPeriodicProcessesManagerProvider( new SlickPeriodicProcessesRepository(dbRef.db, dbRef.profile, fixedClock(currentTime)) - ).provide(deploymentManagerName, processingType), + ).provide(processingType), periodicProcessListener = new PeriodicProcessListener { override def onPeriodicProcessEvent: PartialFunction[PeriodicProcessEvent, Unit] = { diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala index 0d740f67f3e..d133f237ad8 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala @@ -61,7 +61,7 @@ abstract class PeriodicDeploymentManagerProvider( EmptyPeriodicProcessListenerFactory, DefaultAdditionalDeploymentDataProvider, dependencies, - dependencies.periodicProcessesManagerProvider.provide(name, periodicBatchConfig.processingType) + dependencies.periodicProcessesManagerProvider.provide(periodicBatchConfig.processingType) ) } From a088f26c76a583a9ea50c6bb5bbe7026ceb6f2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Mon, 30 Dec 2024 14:22:59 +0100 Subject: [PATCH 06/11] legacy repo --- build.sbt | 3 +- .../periodic/PeriodicProcessesManager.scala | 2 + ...ositoryBasedPeriodicProcessesManager.scala | 1 + ...eriodicProcessServiceIntegrationTest.scala | 99 ++- .../common/periodic/PeriodicBatchConfig.scala | 2 +- .../periodic/PeriodicDeploymentManager.scala | 1 + .../PeriodicDeploymentManagerProvider.scala | 11 +- .../periodic/PeriodicProcessService.scala | 17 +- engine/flink/management/periodic/README.md | 2 +- ...inkPeriodicDeploymentManagerProvider.scala | 26 + .../flink/db/LegacyDbInitializer.scala | 56 ++ ...riodicProcessDeploymentsTableFactory.scala | 80 +++ .../LegacyPeriodicProcessesRepository.scala | 598 ++++++++++++++++++ .../LegacyPeriodicProcessesTableFactory.scala | 259 ++++++++ ...ositoryBasedPeriodicProcessesManager.scala | 138 ++++ .../flink/PeriodicProcessServiceTest.scala | 9 +- .../db/InMemPeriodicProcessesManager.scala | 1 + 17 files changed, 1280 insertions(+), 25 deletions(-) create mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyDbInitializer.scala create mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessDeploymentsTableFactory.scala create mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesRepository.scala create mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesTableFactory.scala create mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyRepositoryBasedPeriodicProcessesManager.scala diff --git a/build.sbt b/build.sbt index 11fae00bcea..8acd5bd9fdc 100644 --- a/build.sbt +++ b/build.sbt @@ -644,6 +644,7 @@ lazy val flinkPeriodicDeploymentManager = (project in flink("management/periodic name := "nussknacker-flink-periodic-manager", libraryDependencies ++= { Seq( + "org.flywaydb" % "flyway-core" % flywayV, "com.typesafe.slick" %% "slick-hikaricp" % slickV % "provided, test", "org.hsqldb" % "hsqldb" % hsqldbV % Test, "com.typesafe.akka" %% "akka-testkit" % akkaV % Test, @@ -1791,7 +1792,7 @@ lazy val commonPeriodicDeploymentManager = (project in engine("common/periodic-d libraryDependencies ++= { Seq( "com.typesafe.akka" %% "akka-actor" % akkaV, - "org.typelevel" %% "cats-core" % catsV % Provided, + "org.typelevel" %% "cats-core" % catsV, "com.cronutils" % "cron-utils" % cronParserV, "com.softwaremill.retry" %% "retry" % retryV, "com.github.tminglei" %% "slick-pg" % slickPgV, diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala index 8264f8a0085..1480edee63a 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala @@ -15,6 +15,7 @@ trait PeriodicProcessesManager { def create( deploymentWithRuntimeParams: DeploymentWithRuntimeParams, inputConfigDuringExecutionJson: String, + canonicalProcess: CanonicalProcess, scheduleProperty: ScheduleProperty, processActionId: ProcessActionId, ): Future[PeriodicProcess] @@ -107,6 +108,7 @@ object NoOpPeriodicProcessesManager extends PeriodicProcessesManager { override def create( deploymentWithRuntimeParams: DeploymentWithRuntimeParams, inputConfigDuringExecutionJson: String, + canonicalProcess: CanonicalProcess, scheduleProperty: ScheduleProperty, processActionId: ProcessActionId, ): Future[PeriodicProcess] = notImplemented diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala index c3cc9572d24..4af3bc1751c 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala @@ -21,6 +21,7 @@ class RepositoryBasedPeriodicProcessesManager( override def create( deploymentWithRuntimeParams: DeploymentWithRuntimeParams, inputConfigDuringExecutionJson: String, + canonicalProcess: CanonicalProcess, scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, ): Future[PeriodicProcess] = diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala index 0fa8b000857..d27e4cd27c0 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -5,6 +5,7 @@ import com.cronutils.builder.CronBuilder import com.cronutils.model.CronType import com.cronutils.model.definition.CronDefinitionBuilder import com.cronutils.model.field.expression.FieldExpressionFactory.{on, questionMark} +import com.typesafe.config.{Config, ConfigFactory} import com.typesafe.scalalogging.LazyLogging import org.scalatest.LoneElement._ import org.scalatest.OptionValues @@ -12,9 +13,9 @@ import org.scalatest.concurrent.ScalaFutures import org.scalatest.exceptions.TestFailedException import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import org.scalatest.time.{Millis, Seconds, Span} import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.api.deployment.periodic.{PeriodicProcessesManager, PeriodicProcessesManagerProvider} import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, ProcessName, VersionId} @@ -23,13 +24,17 @@ import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.PeriodicProcessStatus import pl.touk.nussknacker.engine.common.periodic._ import pl.touk.nussknacker.engine.common.periodic.service._ +import pl.touk.nussknacker.engine.management.periodic.flink.db.{ + LegacyDbInitializer, + LegacyRepositoryBasedPeriodicProcessesManager, + SlickLegacyPeriodicProcessesRepository +} import pl.touk.nussknacker.engine.management.periodic.flink.{DeploymentManagerStub, PeriodicDeploymentHandlerStub} import pl.touk.nussknacker.test.PatientScalaFutures import pl.touk.nussknacker.test.base.db.WithPostgresDbTesting import pl.touk.nussknacker.test.base.it.WithClock import pl.touk.nussknacker.test.utils.domain.TestFactory.newWriteProcessRepository import pl.touk.nussknacker.test.utils.scalas.DBIOActionValues -import pl.touk.nussknacker.ui.db.DbRef import pl.touk.nussknacker.ui.process.repository.ProcessRepository.CreateProcessAction import pl.touk.nussknacker.ui.process.repository.{ DBIOActionRunner, @@ -37,12 +42,15 @@ import pl.touk.nussknacker.ui.process.repository.{ SlickPeriodicProcessesRepository } import pl.touk.nussknacker.ui.security.api.AdminUser +import slick.jdbc +import slick.jdbc.JdbcProfile import java.time._ import java.time.temporal.ChronoUnit import java.util.UUID import scala.collection.mutable.ArrayBuffer import scala.concurrent.Future +import scala.jdk.CollectionConverters._ //Integration test with both in-memory hsql and postgres from test containers class PeriodicProcessServiceIntegrationTest @@ -68,6 +76,8 @@ class PeriodicProcessServiceIntegrationTest private val processIdWithName = ProcessIdWithName(ProcessId(1), processName) + private val sampleProcess = CanonicalProcess(MetaData(processName.value, StreamMetaData()), Nil) + private val startTime = Instant.parse("2021-04-06T13:18:00Z") // we truncate to millis, as HSQL stores with that precision... @@ -83,17 +93,64 @@ class PeriodicProcessServiceIntegrationTest executionConfig: PeriodicExecutionConfig = PeriodicExecutionConfig(), maxFetchedPeriodicScenarioActivities: Option[Int] = None, )(testCode: Fixture => Any): Unit = { - def runTestCodeWithDbConfig = { - testCode( - new Fixture(testDbRef, deploymentRetryConfig, executionConfig, maxFetchedPeriodicScenarioActivities) - ) + val postgresConfig = ConfigFactory.parseMap( + Map( + "user" -> container.username, + "password" -> container.password, + "url" -> container.jdbcUrl, + "driver" -> "org.postgresql.Driver", + "schema" -> UUID.randomUUID().toString + ).asJava + ) + + val hsqlConfig = ConfigFactory.parseMap( + Map( + "url" -> s"jdbc:hsqldb:mem:periodic-${UUID.randomUUID().toString};sql.syntax_ora=true", + "user" -> "SA", + "password" -> "" + ).asJava + ) + + def runWithLegacyRepository(dbConfig: Config): Unit = { + val (db: jdbc.JdbcBackend.DatabaseDef, dbProfile: JdbcProfile) = LegacyDbInitializer.init(dbConfig) + val provider = (currentTime: Instant) => + new PeriodicProcessesManagerProvider { + override def provide(processingType: String): PeriodicProcessesManager = { + val repository = new SlickLegacyPeriodicProcessesRepository(db, dbProfile, fixedClock(currentTime)) + new LegacyRepositoryBasedPeriodicProcessesManager(processingType, repository) + } + } + try { + testCode( + new Fixture(provider, deploymentRetryConfig, executionConfig, maxFetchedPeriodicScenarioActivities) + ) + } finally { + db.close() + } + } + + def runTestCodeWithNuDb() = { + val provider = (currentTime: Instant) => + new RepositoryBasedPeriodicProcessesManagerProvider( + new SlickPeriodicProcessesRepository(testDbRef.db, testDbRef.profile, fixedClock(currentTime)) + ) + testCode(new Fixture(provider, deploymentRetryConfig, executionConfig, maxFetchedPeriodicScenarioActivities)) } - logger.debug("Running test with database") - runTestCodeWithDbConfig + + def testHeader(str: String) = "\n\n" + "*" * 100 + s"\n***** $str\n" + "*" * 100 + "\n" + logger.info(testHeader("Running test with legacy hsql-based repository")) + runWithLegacyRepository(hsqlConfig) + cleanDB() + logger.info(testHeader("Running test with legacy postgres-based repository")) + runWithLegacyRepository(postgresConfig) + cleanDB() + logger.info(testHeader("Running test with Nu database")) + runTestCodeWithNuDb() + cleanDB() } class Fixture( - dbRef: DbRef, + periodicProcessesManagerProvider: Instant => PeriodicProcessesManagerProvider, deploymentRetryConfig: DeploymentRetryConfig, executionConfig: PeriodicExecutionConfig, maxFetchedPeriodicScenarioActivities: Option[Int], @@ -110,9 +167,7 @@ class PeriodicProcessServiceIntegrationTest new PeriodicProcessService( delegateDeploymentManager = delegateDeploymentManagerStub, periodicDeploymentHandler = periodicDeploymentHandlerStub, - periodicProcessesManager = new RepositoryBasedPeriodicProcessesManagerProvider( - new SlickPeriodicProcessesRepository(dbRef.db, dbRef.profile, fixedClock(currentTime)) - ).provide(processingType), + periodicProcessesManager = periodicProcessesManagerProvider(currentTime).provide(processingType), periodicProcessListener = new PeriodicProcessListener { override def onPeriodicProcessEvent: PartialFunction[PeriodicProcessEvent, Unit] = { @@ -131,7 +186,7 @@ class PeriodicProcessServiceIntegrationTest Map.empty, ) - def writeProcessRepository: DBProcessRepository = newWriteProcessRepository(dbRef, clock) + def writeProcessRepository: DBProcessRepository = newWriteProcessRepository(testDbRef, clock) def prepareProcess(processName: ProcessName): DB[ProcessIdWithName] = { val canonicalProcess = CanonicalProcess(MetaData(processName.value, StreamMetaData()), Nil) @@ -164,12 +219,14 @@ class PeriodicProcessServiceIntegrationTest def otherProcessingTypeService = f.periodicProcessService(currentTime, processingType = "other") val otherProcessName = ProcessName("other") - val processIdWithName = f.prepareProcess(processName).dbioActionValues + val processIdWithName = f.prepareProcess(processName).dbioActionValues + val otherProcessIdWithName = f.prepareProcess(otherProcessName).dbioActionValues service .schedule( cronEveryHour, ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + sampleProcess, randomProcessActionId, ) .futureValue @@ -177,6 +234,7 @@ class PeriodicProcessServiceIntegrationTest .schedule( cronEvery30Minutes, ProcessVersion.empty.copy(processName = every30MinutesProcessName), + sampleProcess, randomProcessActionId, ) .futureValue @@ -184,13 +242,15 @@ class PeriodicProcessServiceIntegrationTest .schedule( cronEvery4Hours, ProcessVersion.empty.copy(processName = every4HoursProcessName), + sampleProcess, randomProcessActionId, ) .futureValue otherProcessingTypeService .schedule( cronEveryHour, - ProcessVersion.empty.copy(processName = otherProcessName), + ProcessVersion.empty.copy(processName = otherProcessIdWithName.name), + sampleProcess, randomProcessActionId, ) .futureValue @@ -206,7 +266,7 @@ class PeriodicProcessServiceIntegrationTest PeriodicProcessDeploymentStatus.Scheduled ) afterSchedule.latestDeployments.head.runAt shouldBe localTime(expectedScheduleTime) - service.getLatestDeploymentsForActiveSchedules(otherProcessName).futureValue shouldBe empty + service.getLatestDeploymentsForActiveSchedules(otherProcessIdWithName.name).futureValue shouldBe empty currentTime = timeToTriggerCheck @@ -288,6 +348,7 @@ class PeriodicProcessServiceIntegrationTest .schedule( cronEveryHour, ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + sampleProcess, randomProcessActionId, ) .futureValue @@ -346,6 +407,7 @@ class PeriodicProcessServiceIntegrationTest ) ), ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + sampleProcess, randomProcessActionId, ) .futureValue @@ -360,6 +422,7 @@ class PeriodicProcessServiceIntegrationTest ) ), ProcessVersion.empty.copy(processName = ProcessName("other")), + sampleProcess, randomProcessActionId, ) .futureValue @@ -412,6 +475,7 @@ class PeriodicProcessServiceIntegrationTest ) ), ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + sampleProcess, randomProcessActionId, ) .futureValue @@ -545,6 +609,7 @@ class PeriodicProcessServiceIntegrationTest ) ), ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + sampleProcess, randomProcessActionId, ) .futureValue @@ -638,6 +703,7 @@ class PeriodicProcessServiceIntegrationTest service.schedule( cronEveryHour, ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + sampleProcess, randomProcessActionId, ) } @@ -669,6 +735,7 @@ class PeriodicProcessServiceIntegrationTest .schedule( cronEveryHour, ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + sampleProcess, randomProcessActionId, ) .futureValue diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicBatchConfig.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicBatchConfig.scala index 138a166ace1..d98aa77d529 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicBatchConfig.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicBatchConfig.scala @@ -16,7 +16,7 @@ import scala.concurrent.duration._ * @param maxFetchedPeriodicScenarioActivities Optional limit of number of latest periodic-related Scenario Activities that are returned by Periodic DM. */ case class PeriodicBatchConfig( - db: Config, + db: Option[Config], processingType: String, rescheduleCheckInterval: FiniteDuration = 13 seconds, deployInterval: FiniteDuration = 17 seconds, diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala index 8f1b2994f1b..a93e97f6452 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala @@ -131,6 +131,7 @@ class PeriodicDeploymentManager private[engine] ( .schedule( scheduleProperty, processVersion, + canonicalProcess, deploymentData.deploymentId.toActionIdOpt.getOrElse( throw new IllegalArgumentException(s"deploymentData.deploymentId should be valid ProcessActionId") ), diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala index d133f237ad8..da637191c3f 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala @@ -6,6 +6,7 @@ import com.typesafe.scalalogging.LazyLogging import pl.touk.nussknacker.engine.api.component.ScenarioPropertyConfig import pl.touk.nussknacker.engine.api.definition.{MandatoryParameterValidator, StringParameterEditor} import pl.touk.nussknacker.engine.api.deployment.DeploymentManager +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.common.periodic.cron.CronParameterValidator import pl.touk.nussknacker.engine.common.periodic.service._ import pl.touk.nussknacker.engine.deployment.EngineSetupName @@ -38,6 +39,11 @@ abstract class PeriodicDeploymentManagerProvider( config: Config, ): PeriodicDeploymentHandler + protected def createPeriodicProcessesManagerProvider( + dependencies: DeploymentManagerDependencies, + periodicBatchConfig: PeriodicBatchConfig, + ): PeriodicProcessesManagerProvider = dependencies.periodicProcessesManagerProvider + override def createDeploymentManager( modelData: BaseModelData, dependencies: DeploymentManagerDependencies, @@ -49,7 +55,8 @@ abstract class PeriodicDeploymentManagerProvider( delegateDeploymentManager => import net.ceedubs.ficus.Ficus._ import net.ceedubs.ficus.readers.ArbitraryTypeReader._ - val periodicBatchConfig = config.as[PeriodicBatchConfig]("deploymentManager") + val periodicBatchConfig = config.as[PeriodicBatchConfig]("deploymentManager") + val periodicProcessesManagerProvider = createPeriodicProcessesManagerProvider(dependencies, periodicBatchConfig) PeriodicDeploymentManager( delegate = delegateDeploymentManager, @@ -61,7 +68,7 @@ abstract class PeriodicDeploymentManagerProvider( EmptyPeriodicProcessListenerFactory, DefaultAdditionalDeploymentDataProvider, dependencies, - dependencies.periodicProcessesManagerProvider.provide(periodicBatchConfig.processingType) + periodicProcessesManagerProvider.provide(periodicBatchConfig.processingType) ) } diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala index ec14e806a0d..406acd5ef26 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala @@ -16,6 +16,7 @@ import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus import pl.touk.nussknacker.engine.api.process.{ProcessIdWithName, ProcessName} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.{ DeploymentStatus, EngineStatusesToReschedule, @@ -105,12 +106,15 @@ class PeriodicProcessService( def schedule( schedule: ScheduleProperty, processVersion: ProcessVersion, + canonicalProcess: CanonicalProcess, processActionId: ProcessActionId, beforeSchedule: => Future[Unit] = Future.unit ): Future[Unit] = { prepareInitialScheduleDates(schedule) match { case Right(scheduleDates) => - beforeSchedule.flatMap(_ => scheduleWithInitialDates(schedule, processVersion, scheduleDates, processActionId)) + beforeSchedule.flatMap(_ => + scheduleWithInitialDates(schedule, processVersion, canonicalProcess, scheduleDates, processActionId) + ) case Left(error) => Future.failed(error) } @@ -139,6 +143,7 @@ class PeriodicProcessService( private def scheduleWithInitialDates( scheduleProperty: ScheduleProperty, processVersion: ProcessVersion, + canonicalProcess: CanonicalProcess, scheduleDates: List[(ScheduleName, Option[LocalDateTime])], processActionId: ProcessActionId, ): Future[Unit] = { @@ -156,6 +161,7 @@ class PeriodicProcessService( scheduleProperty, scheduleDates, deploymentWithJarData, + canonicalProcess, enrichedProcessConfig.inputConfigDuringExecutionJson, processActionId, ) @@ -166,11 +172,18 @@ class PeriodicProcessService( scheduleMap: ScheduleProperty, scheduleDates: List[(ScheduleName, Option[LocalDateTime])], deploymentWithJarData: DeploymentWithRuntimeParams, + canonicalProcess: CanonicalProcess, inputConfigDuringExecutionJson: String, processActionId: ProcessActionId, ): Future[Unit] = { periodicProcessesManager - .create(deploymentWithJarData, inputConfigDuringExecutionJson, toApi(scheduleMap), processActionId) + .create( + deploymentWithJarData, + inputConfigDuringExecutionJson, + canonicalProcess, + toApi(scheduleMap), + processActionId + ) .flatMap { process => scheduleDates.collect { case (name, Some(date)) => diff --git a/engine/flink/management/periodic/README.md b/engine/flink/management/periodic/README.md index caa2148a016..ba42e85fdae 100644 --- a/engine/flink/management/periodic/README.md +++ b/engine/flink/management/periodic/README.md @@ -23,7 +23,7 @@ scenario is scheduled to be run again according to the schedule. Use `deploymentManager` with the following properties: -- `db` - Nussknacker db configuration. +- `db` - optional configuration of the legacy, custom Periodic DM database; deprecated and will be removed in the future; when config not present, then the default Nu database will be used - `processingType` - processing type of scenarios to be managed by this instance of the periodic engine. - `rescheduleCheckInterval` - frequency of checking finished scenarios to be rescheduled. Optional. - `deployInterval` - frequency of checking scenarios to be deployed on Flink cluster. Optional. diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala index ce889eb3a23..80d685628c8 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala @@ -1,14 +1,24 @@ package pl.touk.nussknacker.engine.management.periodic.flink import com.typesafe.config.Config +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.common.periodic.{ PeriodicBatchConfig, PeriodicDeploymentHandler, PeriodicDeploymentManagerProvider } +import pl.touk.nussknacker.engine.management.periodic.flink.db.{ + LegacyDbInitializer, + LegacyRepositoryBasedPeriodicProcessesManager, + SlickLegacyPeriodicProcessesRepository +} import pl.touk.nussknacker.engine.management.{FlinkConfig, FlinkStreamingDeploymentManagerProvider} import pl.touk.nussknacker.engine.util.config.ConfigEnrichments.RichConfig import pl.touk.nussknacker.engine.{BaseModelData, DeploymentManagerDependencies} +import slick.jdbc +import slick.jdbc.JdbcProfile + +import java.time.Clock class FlinkPeriodicDeploymentManagerProvider extends PeriodicDeploymentManagerProvider( @@ -34,4 +44,20 @@ class FlinkPeriodicDeploymentManagerProvider ) } + override protected def createPeriodicProcessesManagerProvider( + dependencies: DeploymentManagerDependencies, + periodicBatchConfig: PeriodicBatchConfig, + ): PeriodicProcessesManagerProvider = { + import dependencies._ + periodicBatchConfig.db match { + case None => + dependencies.periodicProcessesManagerProvider + case Some(customDbConfig) => + val clock = Clock.systemDefaultZone() + val (db: jdbc.JdbcBackend.DatabaseDef, dbProfile: JdbcProfile) = LegacyDbInitializer.init(customDbConfig) + val repository = new SlickLegacyPeriodicProcessesRepository(db, dbProfile, clock) + (processingType: String) => new LegacyRepositoryBasedPeriodicProcessesManager(processingType, repository) + } + } + } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyDbInitializer.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyDbInitializer.scala new file mode 100644 index 00000000000..222618f5fdc --- /dev/null +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyDbInitializer.scala @@ -0,0 +1,56 @@ +package pl.touk.nussknacker.engine.management.periodic.flink.db + +import com.github.tminglei.slickpg.ExPostgresProfile +import com.typesafe.config.Config +import com.typesafe.scalalogging.LazyLogging +import net.ceedubs.ficus.readers.ValueReader +import org.flywaydb.core.Flyway +import org.flywaydb.core.api.configuration.FluentConfiguration +import org.flywaydb.core.internal.database.postgresql.PostgreSQLDatabaseType +import slick.jdbc.{HsqldbProfile, JdbcBackend, JdbcProfile, PostgresProfile} + +object LegacyDbInitializer extends LazyLogging { + + def init(configDb: Config): (JdbcBackend.DatabaseDef, JdbcProfile) = { + import net.ceedubs.ficus.Ficus._ + val url = configDb.as[String]("url") + val profile = chooseDbProfile(url) + logger.info("Applying db migrations") + + // we want to set property on FluentConfiguration only if there is property in config + implicit class OptionalConfig(config: FluentConfiguration) { + def setOptional[A](name: String, setAction: (FluentConfiguration, A) => FluentConfiguration)( + implicit reader: ValueReader[Option[A]] + ): FluentConfiguration = { + configDb.getAs[A](name).fold(config)(setAction(config, _)) + } + } + + Flyway + .configure() + .locations( + (profile match { + case _: HsqldbProfile => Array("db/batch_periodic/migration/hsql", "db/batch_periodic/migration/common") + case _: PostgresProfile => Array("db/batch_periodic/migration/postgres", "db/batch_periodic/migration/common") + case _ => + throw new IllegalArgumentException(s"Unsupported database url: $url. Use either PostgreSQL or HSQLDB.") + }): _* + ) + .dataSource(url, configDb.as[String]("user"), configDb.as[String]("password")) + .setOptional[String]("schema", _.schemas(_)) + .setOptional[String]("table", _.table(_)) + .baselineOnMigrate(true) + .load() + .migrate() + + (JdbcBackend.Database.forConfig(path = "", configDb), profile) + } + + private def chooseDbProfile(dbUrl: String): JdbcProfile = { + dbUrl match { + case url if (new PostgreSQLDatabaseType).handlesJDBCUrl(url) => ExPostgresProfile + case _ => HsqldbProfile + } + } + +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessDeploymentsTableFactory.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessDeploymentsTableFactory.scala new file mode 100644 index 00000000000..38d83ad4353 --- /dev/null +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessDeploymentsTableFactory.scala @@ -0,0 +1,80 @@ +package pl.touk.nussknacker.engine.management.periodic.flink.db + +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{ + PeriodicProcessDeploymentId, + PeriodicProcessDeploymentStatus, + PeriodicProcessId +} +import slick.jdbc.{JdbcProfile, JdbcType} +import slick.lifted.ProvenShape +import slick.sql.SqlProfile.ColumnOption.NotNull + +import java.time.LocalDateTime + +trait LegacyPeriodicProcessDeploymentsTableFactory extends LegacyPeriodicProcessesTableFactory { + + protected val profile: JdbcProfile + + import profile.api._ + + implicit val periodicProcessDeploymentIdMapping: BaseColumnType[PeriodicProcessDeploymentId] = + MappedColumnType.base[PeriodicProcessDeploymentId, Long](_.value, PeriodicProcessDeploymentId.apply) + + implicit val periodicProcessDeploymentStatusColumnTyped: JdbcType[PeriodicProcessDeploymentStatus] = + MappedColumnType.base[PeriodicProcessDeploymentStatus, String](_.toString, PeriodicProcessDeploymentStatus.withName) + + class PeriodicProcessDeploymentsTable(tag: Tag) + extends Table[PeriodicProcessDeploymentEntity](tag, "periodic_process_deployments") { + + def id: Rep[PeriodicProcessDeploymentId] = column[PeriodicProcessDeploymentId]("id", O.PrimaryKey, O.AutoInc) + + def periodicProcessId: Rep[PeriodicProcessId] = column[PeriodicProcessId]("periodic_process_id", NotNull) + + def createdAt: Rep[LocalDateTime] = column[LocalDateTime]("created_at", NotNull) + + def runAt: Rep[LocalDateTime] = column[LocalDateTime]("run_at", NotNull) + + def scheduleName: Rep[Option[String]] = column[Option[String]]("schedule_name") + + def deployedAt: Rep[Option[LocalDateTime]] = column[Option[LocalDateTime]]("deployed_at") + + def completedAt: Rep[Option[LocalDateTime]] = column[Option[LocalDateTime]]("completed_at") + + def retriesLeft: Rep[Int] = column[Int]("retries_left") + + def nextRetryAt: Rep[Option[LocalDateTime]] = column[Option[LocalDateTime]]("next_retry_at") + + def status: Rep[PeriodicProcessDeploymentStatus] = column[PeriodicProcessDeploymentStatus]("status", NotNull) + + override def * : ProvenShape[PeriodicProcessDeploymentEntity] = ( + id, + periodicProcessId, + createdAt, + runAt, + scheduleName, + deployedAt, + completedAt, + retriesLeft, + nextRetryAt, + status + ) <> + ((PeriodicProcessDeploymentEntity.apply _).tupled, PeriodicProcessDeploymentEntity.unapply) + + } + + object PeriodicProcessDeployments extends TableQuery(new PeriodicProcessDeploymentsTable(_)) +} + +case class PeriodicProcessDeploymentEntity( + id: PeriodicProcessDeploymentId, + periodicProcessId: PeriodicProcessId, + createdAt: LocalDateTime, + runAt: LocalDateTime, + scheduleName: Option[String], + deployedAt: Option[LocalDateTime], + completedAt: Option[LocalDateTime], + retriesLeft: Int, + nextRetryAt: Option[LocalDateTime], + status: PeriodicProcessDeploymentStatus +) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesRepository.scala new file mode 100644 index 00000000000..efadc01d2c0 --- /dev/null +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesRepository.scala @@ -0,0 +1,598 @@ +package pl.touk.nussknacker.engine.management.periodic.flink.db + +import cats.Monad +import com.github.tminglei.slickpg.ExPostgresProfile +import com.typesafe.scalalogging.LazyLogging +import io.circe.parser +import io.circe.parser.decode +import io.circe.syntax.EncoderOps +import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} +import pl.touk.nussknacker.engine.common.periodic._ +import pl.touk.nussknacker.engine.management.periodic.flink.FlinkPeriodicDeploymentHandler.jarFileNameRuntimeParam +import pl.touk.nussknacker.engine.management.periodic.flink.db.LegacyPeriodicProcessesRepository.createPeriodicProcess +import pl.touk.nussknacker.engine.marshall.ProcessMarshaller +import slick.dbio.{DBIOAction, Effect, NoStream} +import slick.jdbc.PostgresProfile.api._ +import slick.jdbc.{JdbcBackend, JdbcProfile} + +import java.time.{Clock, LocalDateTime} +import scala.concurrent.{ExecutionContext, Future} +import scala.language.higherKinds + +object LegacyPeriodicProcessesRepository { + + def createPeriodicProcessDeployment( + processEntity: PeriodicProcessEntity, + processDeploymentEntity: PeriodicProcessDeploymentEntity + ): PeriodicProcessDeployment = { + val process = createPeriodicProcess(processEntity) + PeriodicProcessDeployment( + processDeploymentEntity.id, + process, + processDeploymentEntity.createdAt, + processDeploymentEntity.runAt, + ScheduleName(processDeploymentEntity.scheduleName), + processDeploymentEntity.retriesLeft, + processDeploymentEntity.nextRetryAt, + createPeriodicDeploymentState(processDeploymentEntity) + ) + } + + def createPeriodicDeploymentState( + processDeploymentEntity: PeriodicProcessDeploymentEntity + ): PeriodicProcessDeploymentState = { + PeriodicProcessDeploymentState( + processDeploymentEntity.deployedAt, + processDeploymentEntity.completedAt, + processDeploymentEntity.status + ) + } + + def createPeriodicProcess( + processEntity: PeriodicProcessEntity + ): PeriodicProcess = { + val processVersion = createProcessVersion(processEntity) + val scheduleProperty = prepareScheduleProperty(processEntity) + PeriodicProcess( + processEntity.id, + DeploymentWithRuntimeParams( + processVersion = processVersion, + runtimeParams = RuntimeParams(Map(jarFileNameRuntimeParam + "a" -> processEntity.jarFileName)), + ), + toApi(scheduleProperty), + processEntity.active, + processEntity.createdAt, + processEntity.processActionId + ) + } + + private def prepareScheduleProperty(processEntity: PeriodicProcessEntity) = { + val scheduleProperty = decode[ScheduleProperty](processEntity.scheduleProperty) + .fold(e => throw new IllegalArgumentException(e), identity) + scheduleProperty + } + + private def createProcessVersion(processEntity: PeriodicProcessEntity): ProcessVersion = { + ProcessVersion.empty.copy(versionId = processEntity.processVersionId, processName = processEntity.processName) + } + +} + +trait LegacyPeriodicProcessesRepository { + + type Action[_] + + implicit def monad: Monad[Action] + + implicit class RunOps[T](action: Action[T]) { + def run: Future[T] = LegacyPeriodicProcessesRepository.this.run(action) + } + + def run[T](action: Action[T]): Future[T] + + def markInactive(processId: PeriodicProcessId): Action[Unit] + + def getSchedulesState( + scenarioName: ProcessName, + after: Option[LocalDateTime], + ): Action[SchedulesState] + + def create( + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, + canonicalProcess: CanonicalProcess, + scheduleProperty: PeriodicProcessesManager.ScheduleProperty, + processActionId: ProcessActionId, + processingType: String, + ): Action[PeriodicProcess] + + def getLatestDeploymentsForActiveSchedules( + processName: ProcessName, + deploymentsPerScheduleMaxCount: Int, + processingType: String, + ): Action[SchedulesState] + + def getLatestDeploymentsForActiveSchedules( + deploymentsPerScheduleMaxCount: Int, + processingType: String, + ): Action[Map[ProcessName, SchedulesState]] + + def getLatestDeploymentsForLatestInactiveSchedules( + processName: ProcessName, + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + processingType: String, + ): Action[SchedulesState] + + def getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + processingType: String, + ): Action[Map[ProcessName, SchedulesState]] + + def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment]] + + def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment]] + + def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + processingType: String, + ): Action[SchedulesState] + + def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment] + + def markDeployed(id: PeriodicProcessDeploymentId): Action[Unit] + + def markFinished(id: PeriodicProcessDeploymentId): Action[Unit] + + def markFailedOnDeployWithStatus( + id: PeriodicProcessDeploymentId, + status: PeriodicProcessDeploymentStatus, + deployRetries: Int, + retryAt: Option[LocalDateTime] + ): Action[Unit] + + def markFailed(id: PeriodicProcessDeploymentId): Action[Unit] + + def schedule( + id: PeriodicProcessId, + scheduleName: ScheduleName, + runAt: LocalDateTime, + deployMaxRetries: Int + ): Action[PeriodicProcessDeployment] + + def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId, + ): Action[Option[CanonicalProcess]] + + def fetchInputConfigDuringExecutionJson( + processName: ProcessName, + versionId: VersionId + ): Action[Option[String]] + +} + +class SlickLegacyPeriodicProcessesRepository( + db: JdbcBackend.DatabaseDef, + override val profile: JdbcProfile, + clock: Clock, +)(implicit ec: ExecutionContext) + extends LegacyPeriodicProcessesRepository + with LegacyPeriodicProcessesTableFactory + with LegacyPeriodicProcessDeploymentsTableFactory + with LazyLogging { + + import pl.touk.nussknacker.engine.util.Implicits._ + + type Action[T] = DBIOActionInstances.DB[T] + + override implicit def monad: Monad[Action] = DBIOActionInstances.dbMonad + + override def run[T](action: DBIOAction[T, NoStream, Effect.All]): Future[T] = db.run(action.transactionally) + + override def getSchedulesState( + scenarioName: ProcessName, + afterOpt: Option[LocalDateTime], + ): Action[SchedulesState] = { + PeriodicProcessesWithoutJson + .filter(_.processName === scenarioName) + .join(PeriodicProcessDeployments) + .on(_.id === _.periodicProcessId) + .filterOpt(afterOpt)((entities, after) => entities._2.completedAt > after) + .result + .map(toSchedulesStateForSinglePeriodicProcess) + } + + override def create( + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, + canonicalProcess: CanonicalProcess, + scheduleProperty: PeriodicProcessesManager.ScheduleProperty, + processActionId: ProcessActionId, + processingType: String, + ): Action[PeriodicProcess] = { + val processEntity = PeriodicProcessEntityWithJson( + id = PeriodicProcessId(-1), + processName = deploymentWithRuntimeParams.processVersion.processName, + processVersionId = deploymentWithRuntimeParams.processVersion.versionId, + processingType = processingType, + jarFileName = deploymentWithRuntimeParams.runtimeParams.params(jarFileNameRuntimeParam), + scheduleProperty = fromApi(scheduleProperty).asJson.noSpaces, + active = true, + createdAt = now(), + processActionId = Some(processActionId), + inputConfigDuringExecutionJson = inputConfigDuringExecutionJson, + processJson = canonicalProcess, + ) + ((PeriodicProcessesWithJson returning PeriodicProcessesWithJson into ((_, id) => id)) += processEntity) + .map(LegacyPeriodicProcessesRepository.createPeriodicProcess) + } + + private def now(): LocalDateTime = LocalDateTime.now(clock) + + override def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment]] = + findProcesses( + activePeriodicProcessWithDeploymentQuery(processingType) + .filter { case (_, d) => + d.runAt <= now() && + d.status === (PeriodicProcessDeploymentStatus.Scheduled: PeriodicProcessDeploymentStatus) + } + ) + + override def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment]] = + findProcesses( + activePeriodicProcessWithDeploymentQuery(processingType) + .filter { case (_, d) => + d.nextRetryAt <= now() && + d.status === (PeriodicProcessDeploymentStatus.RetryingDeploy: PeriodicProcessDeploymentStatus) + } + ) + + private def findProcesses( + query: Query[ + (PeriodicProcessWithoutJson, PeriodicProcessDeploymentsTable), + (PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity), + Seq + ] + ) = { + query.result + .map(_.map { case (periodicProcess, periodicDeployment) => + LegacyPeriodicProcessesRepository.createPeriodicProcessDeployment( + periodicProcess, + periodicDeployment, + ) + }) + } + + override def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment] = + findProcesses( + (PeriodicProcessesWithoutJson join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) + .filter { case (_, deployment) => deployment.id === id } + ).map(_.head) + + override def markDeployed(id: PeriodicProcessDeploymentId): Action[Unit] = { + val q = for { + d <- PeriodicProcessDeployments if d.id === id + } yield (d.status, d.deployedAt) + val update = q.update((PeriodicProcessDeploymentStatus.Deployed, Some(now()))) + update.map(_ => ()) + } + + override def markFailed(id: PeriodicProcessDeploymentId): Action[Unit] = { + updateCompleted(id, PeriodicProcessDeploymentStatus.Failed) + } + + override def markFinished(id: PeriodicProcessDeploymentId): Action[Unit] = { + updateCompleted(id, PeriodicProcessDeploymentStatus.Finished) + } + + override def markFailedOnDeployWithStatus( + id: PeriodicProcessDeploymentId, + status: PeriodicProcessDeploymentStatus, + retriesLeft: Int, + retryAt: Option[LocalDateTime] + ): Action[Unit] = { + val q = for { + d <- PeriodicProcessDeployments if d.id === id + } yield (d.status, d.completedAt, d.retriesLeft, d.nextRetryAt) + val update = q.update((status, Some(now()), retriesLeft, retryAt)) + update.map(_ => ()) + } + + private def updateCompleted( + id: PeriodicProcessDeploymentId, + status: PeriodicProcessDeploymentStatus + ): Action[Unit] = { + val q = for { + d <- PeriodicProcessDeployments if d.id === id + } yield (d.status, d.completedAt) + val update = q.update((status, Some(now()))) + update.map(_ => ()) + } + + override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + processingType: String, + ): Action[SchedulesState] = { + val processesHavingDeploymentsWithMatchingStatus = PeriodicProcessesWithoutJson.filter(p => + p.active && + PeriodicProcessDeployments + .filter(d => d.periodicProcessId === p.id && d.status.inSet(expectedDeploymentStatuses)) + .exists + ) + getLatestDeploymentsForEachSchedule( + processesHavingDeploymentsWithMatchingStatus, + deploymentsPerScheduleMaxCount = 1, + processingType = processingType, + ).map(_.values.headOption.getOrElse(SchedulesState(Map.empty))) + } + + override def getLatestDeploymentsForActiveSchedules( + processName: ProcessName, + deploymentsPerScheduleMaxCount: Int, + processingType: String, + ): Action[SchedulesState] = { + val activeProcessesQuery = + PeriodicProcessesWithoutJson.filter(p => p.processName === processName && p.active) + getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount, processingType) + .map(_.getOrElse(processName, SchedulesState(Map.empty))) + } + + override def getLatestDeploymentsForActiveSchedules( + deploymentsPerScheduleMaxCount: Int, + processingType: String, + ): Action[Map[ProcessName, SchedulesState]] = { + val activeProcessesQuery = PeriodicProcessesWithoutJson.filter(_.active) + getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount, processingType) + } + + override def getLatestDeploymentsForLatestInactiveSchedules( + processName: ProcessName, + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + processingType: String, + ): Action[SchedulesState] = { + val filteredProcessesQuery = PeriodicProcessesWithoutJson + .filter(p => p.processName === processName && !p.active) + .sortBy(_.createdAt.desc) + .take(inactiveProcessesMaxCount) + getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount, processingType) + .map(_.getOrElse(processName, SchedulesState(Map.empty))) + } + + override def getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + processingType: String, + ): Action[Map[ProcessName, SchedulesState]] = { + val filteredProcessesQuery = PeriodicProcessesWithoutJson + .filter(!_.active) + .sortBy(_.createdAt.desc) + .take(inactiveProcessesMaxCount) + getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount, processingType) + } + + private def getLatestDeploymentsForEachSchedule( + periodicProcessesQuery: Query[ + PeriodicProcessWithoutJson, + PeriodicProcessEntityWithoutJson, + Seq + ], + deploymentsPerScheduleMaxCount: Int, + processingType: String, + ): Action[Map[ProcessName, SchedulesState]] = { + val filteredPeriodicProcessQuery = periodicProcessesQuery.filter(p => p.processingType === processingType) + val latestDeploymentsForSchedules = profile match { + case _: ExPostgresProfile => + getLatestDeploymentsForEachSchedulePostgres(filteredPeriodicProcessQuery, deploymentsPerScheduleMaxCount) + case _ => + getLatestDeploymentsForEachScheduleJdbcGeneric(filteredPeriodicProcessQuery, deploymentsPerScheduleMaxCount) + } + latestDeploymentsForSchedules.map(toSchedulesState) + } + + private def getLatestDeploymentsForEachSchedulePostgres( + periodicProcessesQuery: Query[ + PeriodicProcessWithoutJson, + PeriodicProcessEntityWithoutJson, + Seq + ], + deploymentsPerScheduleMaxCount: Int + ): Action[Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)]] = { + // To effectively limit deployments to given count for each schedule in one query, we use window functions in slick + import ExPostgresProfile.api._ + import com.github.tminglei.slickpg.window.PgWindowFuncSupport.WindowFunctions._ + + val deploymentsForProcesses = + periodicProcessesQuery join PeriodicProcessDeployments on (_.id === _.periodicProcessId) + deploymentsForProcesses + .map { case (process, deployment) => + ( + rowNumber() :: Over + .partitionBy((deployment.periodicProcessId, deployment.scheduleName)) + .sortBy( + deployment.runAt.desc, + deployment.createdAt.desc + ), // Remember to change DeploymentStatus.ordering accordingly + process, + deployment + ) + } + .subquery + .filter(_._1 <= deploymentsPerScheduleMaxCount.longValue()) + .map { case (_, process, deployment) => + (process, deployment) + } + .result + } + + // This variant of method is much less optimal than postgres one. It is highly recommended to use postgres with periodics + // If we decided to support more databases, we should consider some optimization like extracting periodic_schedule table + // with foreign key to periodic_process and with schedule_name column - it would reduce number of queries + private def getLatestDeploymentsForEachScheduleJdbcGeneric( + periodicProcessesQuery: Query[ + PeriodicProcessWithoutJson, + PeriodicProcessEntityWithoutJson, + Seq + ], + deploymentsPerScheduleMaxCount: Int + ): Action[Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)]] = { + // It is debug instead of warn to not bloast logs when e.g. for some reasons is used hsql under the hood + logger.debug( + "WARN: Using not optimized version of getLatestDeploymentsForEachSchedule that not uses window functions" + ) + for { + processes <- periodicProcessesQuery.result + schedulesForProcesses <- + DBIO + .sequence(processes.map { process => + PeriodicProcessDeployments + .filter(_.periodicProcessId === process.id) + .map(_.scheduleName) + .distinct + .result + .map(_.map((process, _))) + }) + .map(_.flatten) + deploymentsForSchedules <- + DBIO + .sequence(schedulesForProcesses.map { case (process, scheduleName) => + PeriodicProcessDeployments + // In SQL when you compare nulls, you will get always false + .filter(deployment => + deployment.periodicProcessId === process.id && (deployment.scheduleName === scheduleName || deployment.scheduleName.isEmpty && scheduleName.isEmpty) + ) + .sortBy(a => (a.runAt.desc, a.createdAt.desc)) // Remember to change DeploymentStatus.ordering accordingly + .take(deploymentsPerScheduleMaxCount) + .result + .map(_.map((process, _))) + }) + .map(_.flatten) + } yield deploymentsForSchedules + } + + override def schedule( + id: PeriodicProcessId, + scheduleName: ScheduleName, + runAt: LocalDateTime, + deployMaxRetries: Int + ): Action[PeriodicProcessDeployment] = { + val deploymentEntity = PeriodicProcessDeploymentEntity( + id = PeriodicProcessDeploymentId(-1), + periodicProcessId = id, + createdAt = now(), + runAt = runAt, + scheduleName = scheduleName.value, + deployedAt = None, + completedAt = None, + retriesLeft = deployMaxRetries, + nextRetryAt = None, + status = PeriodicProcessDeploymentStatus.Scheduled + ) + ((PeriodicProcessDeployments returning PeriodicProcessDeployments.map(_.id) into ((_, id) => + id + )) += deploymentEntity).flatMap(findProcessData) + } + + override def markInactive(processId: PeriodicProcessId): Action[Unit] = { + val q = for { + p <- PeriodicProcessesWithoutJson if p.id === processId + } yield p.active + val update = q.update(false) + update.map(_ => ()) + } + + def fetchCanonicalProcess(processName: ProcessName, versionId: VersionId): Action[Option[CanonicalProcess]] = { + PeriodicProcessesWithJson + .filter(p => p.processName === processName && p.processVersionId === versionId) + .map(_.processJson) + .result + .headOption + .map(_.flatMap(parser.parse(_).toOption)) + .map(_.flatMap(ProcessMarshaller.fromJson(_).toOption)) + } + + def fetchInputConfigDuringExecutionJson(processName: ProcessName, versionId: VersionId): Action[Option[String]] = + PeriodicProcessesWithJson + .filter(p => p.processName === processName && p.processVersionId === versionId) + .map(_.inputConfigDuringExecutionJson) + .result + .headOption + + private def activePeriodicProcessWithDeploymentQuery(processingType: String) = { + (PeriodicProcessesWithoutJson.filter(p => p.active === true && p.processingType === processingType) + join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) + } + + private def toSchedulesState( + list: Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)] + ): Map[ProcessName, SchedulesState] = { + list + .groupBy(_._1.processName) + .map { case (processName, list) => processName -> toSchedulesStateForSinglePeriodicProcess(list) } + } + + private def toSchedulesStateForSinglePeriodicProcess( + list: Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)] + ): SchedulesState = { + SchedulesState( + list + .map { case (process, deployment) => + val scheduleId = ScheduleId(process.id, ScheduleName(deployment.scheduleName)) + val scheduleData = (scheduleId, process) + val scheduleDeployment = scheduleDeploymentData(deployment) + (scheduleData, scheduleDeployment) + } + .toList + .toGroupedMap + .toList + .map { case ((scheduleId, process), deployments) => + scheduleId -> ScheduleData(createPeriodicProcess(process), deployments) + } + .toMap + ) + } + + private def scheduleDeploymentData(deployment: PeriodicProcessDeploymentEntity): ScheduleDeploymentData = { + ScheduleDeploymentData( + deployment.id, + deployment.createdAt, + deployment.runAt, + deployment.deployedAt, + deployment.retriesLeft, + deployment.nextRetryAt, + LegacyPeriodicProcessesRepository.createPeriodicDeploymentState(deployment) + ) + } + +} + +//Copied from designer/server. +object DBIOActionInstances { + + type DB[A] = DBIOAction[A, NoStream, Effect.All] + + implicit def dbMonad(implicit ec: ExecutionContext): Monad[DB] = new Monad[DB] { + + override def pure[A](x: A) = DBIO.successful(x) + + override def flatMap[A, B](fa: DB[A])(f: (A) => DB[B]) = fa.flatMap(f) + + // this is *not* tail recursive + override def tailRecM[A, B](a: A)(f: (A) => DB[Either[A, B]]): DB[B] = + f(a).flatMap { + case Right(r) => pure(r) + case Left(l) => tailRecM(l)(f) + } + + } + +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesTableFactory.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesTableFactory.scala new file mode 100644 index 00000000000..87f8c144646 --- /dev/null +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesTableFactory.scala @@ -0,0 +1,259 @@ +package pl.touk.nussknacker.engine.management.periodic.flink.db + +import io.circe.syntax.EncoderOps +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessId +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.marshall.ProcessMarshaller +import slick.jdbc.JdbcProfile +import slick.lifted.ProvenShape +import slick.sql.SqlProfile.ColumnOption.NotNull + +import java.time.LocalDateTime +import java.util.UUID + +trait LegacyPeriodicProcessesTableFactory { + + protected val profile: JdbcProfile + + import profile.api._ + + implicit val periodicProcessIdMapping: BaseColumnType[PeriodicProcessId] = + MappedColumnType.base[PeriodicProcessId, Long](_.value, PeriodicProcessId.apply) + + implicit val processNameMapping: BaseColumnType[ProcessName] = + MappedColumnType.base[ProcessName, String](_.value, ProcessName.apply) + + implicit val versionIdMapping: BaseColumnType[VersionId] = + MappedColumnType.base[VersionId, Long](_.value, VersionId.apply) + + implicit val processActionIdMapping: BaseColumnType[ProcessActionId] = + MappedColumnType.base[ProcessActionId, UUID](_.value, ProcessActionId.apply) + + abstract class PeriodicProcessesTable[ENTITY <: PeriodicProcessEntity](tag: Tag) + extends Table[ENTITY](tag, "periodic_processes") { + + def id: Rep[PeriodicProcessId] = column[PeriodicProcessId]("id", O.PrimaryKey, O.AutoInc) + + def processName: Rep[ProcessName] = column[ProcessName]("process_name", NotNull) + + def processVersionId: Rep[VersionId] = column[VersionId]("process_version_id", NotNull) + + def processingType: Rep[String] = column[String]("processing_type", NotNull) + + def jarFileName: Rep[String] = column[String]("jar_file_name", NotNull) + + def scheduleProperty: Rep[String] = column[String]("schedule_property", NotNull) + + def active: Rep[Boolean] = column[Boolean]("active", NotNull) + + def createdAt: Rep[LocalDateTime] = column[LocalDateTime]("created_at", NotNull) + + def processActionId: Rep[Option[ProcessActionId]] = column[Option[ProcessActionId]]("process_action_id") + + } + + class PeriodicProcessesWithJsonTable(tag: Tag) extends PeriodicProcessesTable[PeriodicProcessEntityWithJson](tag) { + + def processJson: Rep[String] = column[String]("process_json", NotNull) + + def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) + + override def * : ProvenShape[PeriodicProcessEntityWithJson] = ( + id, + processName, + processVersionId, + processingType, + processJson, + inputConfigDuringExecutionJson, + jarFileName, + scheduleProperty, + active, + createdAt, + processActionId + ) <> ( + (PeriodicProcessEntity.createWithJson _).tupled, + (e: PeriodicProcessEntityWithJson) => + PeriodicProcessEntityWithJson.unapply(e).map { + case ( + id, + processName, + versionId, + processingType, + processJson, + inputConfigDuringExecutionJson, + jarFileName, + scheduleProperty, + active, + createdAt, + processActionId + ) => + ( + id, + processName, + versionId, + processingType, + processJson.asJson.noSpaces, + inputConfigDuringExecutionJson, + jarFileName, + scheduleProperty, + active, + createdAt, + processActionId + ) + } + ) + + } + + class PeriodicProcessWithoutJson(tag: Tag) extends PeriodicProcessesTable[PeriodicProcessEntityWithoutJson](tag) { + + override def * : ProvenShape[PeriodicProcessEntityWithoutJson] = ( + id, + processName, + processVersionId, + processingType, + jarFileName, + scheduleProperty, + active, + createdAt, + processActionId + ) <> ( + (PeriodicProcessEntity.createWithoutJson _).tupled, + (e: PeriodicProcessEntityWithoutJson) => + PeriodicProcessEntityWithoutJson.unapply(e).map { + case ( + id, + processName, + versionId, + processingType, + jarFileName, + scheduleProperty, + active, + createdAt, + processActionId + ) => + ( + id, + processName, + versionId, + processingType, + jarFileName, + scheduleProperty, + active, + createdAt, + processActionId + ) + } + ) + + } + + object PeriodicProcessesWithJson extends TableQuery(new PeriodicProcessesWithJsonTable(_)) + + object PeriodicProcessesWithoutJson extends TableQuery(new PeriodicProcessWithoutJson(_)) + +} + +object PeriodicProcessEntity { + + def createWithJson( + id: PeriodicProcessId, + processName: ProcessName, + processVersionId: VersionId, + processingType: String, + processJson: String, + inputConfigDuringExecutionJson: String, + jarFileName: String, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[ProcessActionId] + ): PeriodicProcessEntityWithJson = + PeriodicProcessEntityWithJson( + id, + processName, + processVersionId, + processingType, + ProcessMarshaller.fromJsonUnsafe(processJson), + inputConfigDuringExecutionJson, + jarFileName, + scheduleProperty, + active, + createdAt, + processActionId + ) + + def createWithoutJson( + id: PeriodicProcessId, + processName: ProcessName, + processVersionId: VersionId, + processingType: String, + jarFileName: String, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[ProcessActionId] + ): PeriodicProcessEntityWithoutJson = + PeriodicProcessEntityWithoutJson( + id, + processName, + processVersionId, + processingType, + jarFileName, + scheduleProperty, + active, + createdAt, + processActionId + ) + +} + +trait PeriodicProcessEntity { + + def id: PeriodicProcessId + + def processName: ProcessName + + def processVersionId: VersionId + + def processingType: String + + def jarFileName: String + + def scheduleProperty: String + + def active: Boolean + + def createdAt: LocalDateTime + + def processActionId: Option[ProcessActionId] + +} + +case class PeriodicProcessEntityWithJson( + id: PeriodicProcessId, + processName: ProcessName, + processVersionId: VersionId, + processingType: String, + processJson: CanonicalProcess, + inputConfigDuringExecutionJson: String, + jarFileName: String, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[ProcessActionId] +) extends PeriodicProcessEntity + +case class PeriodicProcessEntityWithoutJson( + id: PeriodicProcessId, + processName: ProcessName, + processVersionId: VersionId, + processingType: String, + jarFileName: String, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[ProcessActionId] +) extends PeriodicProcessEntity diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyRepositoryBasedPeriodicProcessesManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyRepositoryBasedPeriodicProcessesManager.scala new file mode 100644 index 00000000000..9fb278b85e0 --- /dev/null +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyRepositoryBasedPeriodicProcessesManager.scala @@ -0,0 +1,138 @@ +package pl.touk.nussknacker.engine.management.periodic.flink.db + +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess + +import java.time.LocalDateTime +import scala.concurrent.Future + +class LegacyRepositoryBasedPeriodicProcessesManager( + processingType: String, + periodicProcessesRepository: LegacyPeriodicProcessesRepository, +) extends PeriodicProcessesManager { + + import periodicProcessesRepository._ + + override def create( + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + inputConfigDuringExecutionJson: String, + canonicalProcess: CanonicalProcess, + scheduleProperty: PeriodicProcessesManager.ScheduleProperty, + processActionId: ProcessActionId, + ): Future[PeriodicProcess] = + periodicProcessesRepository + .create( + deploymentWithRuntimeParams, + inputConfigDuringExecutionJson, + canonicalProcess, + scheduleProperty, + processActionId, + processingType + ) + .run + + override def markInactive(processId: PeriodicProcessId): Future[Unit] = + periodicProcessesRepository.markInactive(processId).run + + override def schedule( + id: PeriodicProcessId, + scheduleName: ScheduleName, + runAt: LocalDateTime, + deployMaxRetries: Int + ): Future[PeriodicProcessDeployment] = + periodicProcessesRepository.schedule(id, scheduleName, runAt, deployMaxRetries).run + + override def findProcessData( + id: PeriodicProcessDeploymentId, + ): Future[PeriodicProcessDeployment] = + periodicProcessesRepository.findProcessData(id).run + + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = + periodicProcessesRepository.findToBeDeployed(processingType).run + + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = + periodicProcessesRepository.findToBeRetried(processingType).run + + override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = + periodicProcessesRepository.markDeployed(id).run + + override def markFinished(id: PeriodicProcessDeploymentId): Future[Unit] = + periodicProcessesRepository.markFinished(id).run + + override def markFailed(id: PeriodicProcessDeploymentId): Future[Unit] = + periodicProcessesRepository.markFailed(id).run + + override def markFailedOnDeployWithStatus( + id: PeriodicProcessDeploymentId, + status: PeriodicProcessDeploymentStatus, + deployRetries: Int, + retryAt: Option[LocalDateTime] + ): Future[Unit] = periodicProcessesRepository.markFailedOnDeployWithStatus(id, status, deployRetries, retryAt).run + + override def getSchedulesState(scenarioName: ProcessName, after: Option[LocalDateTime]): Future[SchedulesState] = + periodicProcessesRepository.getSchedulesState(scenarioName, after).run + + override def getLatestDeploymentsForActiveSchedules( + processName: ProcessName, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = + periodicProcessesRepository + .getLatestDeploymentsForActiveSchedules(processName, deploymentsPerScheduleMaxCount, processingType) + .run + + override def getLatestDeploymentsForActiveSchedules( + deploymentsPerScheduleMaxCount: Int, + ): Future[Map[ProcessName, SchedulesState]] = + periodicProcessesRepository + .getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount, processingType) + .run + + override def getLatestDeploymentsForLatestInactiveSchedules( + processName: ProcessName, + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = + periodicProcessesRepository + .getLatestDeploymentsForLatestInactiveSchedules( + processName, + inactiveProcessesMaxCount, + deploymentsPerScheduleMaxCount, + processingType, + ) + .run + + override def getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + ): Future[Map[ProcessName, SchedulesState]] = + periodicProcessesRepository + .getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount, + deploymentsPerScheduleMaxCount, + processingType, + ) + .run + + override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + ): Future[SchedulesState] = periodicProcessesRepository + .findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus(expectedDeploymentStatuses, processingType) + .run + + override def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId + ): Future[Option[CanonicalProcess]] = + periodicProcessesRepository.fetchCanonicalProcess(processName, versionId).run + + override def fetchInputConfigDuringExecutionJson( + processName: ProcessName, + versionId: VersionId + ): Future[Option[String]] = + periodicProcessesRepository.fetchInputConfigDuringExecutionJson(processName, versionId).run + +} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala index a530171f7be..3c43bee612e 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala @@ -62,6 +62,11 @@ class PeriodicProcessServiceTest private val cronInFuture = CronScheduleProperty(s"0 0 6 6 9 ? ${yearNow + 1}") private val cronInPast = CronScheduleProperty(s"0 0 6 6 9 ? ${yearNow - 1}") + private val canonicalProcess = ScenarioBuilder + .streaming(processName.value) + .source("start", "source") + .emptySink("end", "KafkaSink") + private val processVersion = ProcessVersion( versionId = VersionId(1), processName = processName, @@ -341,7 +346,7 @@ class PeriodicProcessServiceTest val f = new Fixture f.periodicProcessService - .schedule(CronScheduleProperty("0 0 * * * ?"), processVersion, randomProcessActionId) + .schedule(CronScheduleProperty("0 0 * * * ?"), processVersion, canonicalProcess, randomProcessActionId) .futureValue val processEntity = f.manager.processEntities.loneElement @@ -429,7 +434,7 @@ class PeriodicProcessServiceTest def tryToSchedule(schedule: ScheduleProperty): Unit = f.periodicProcessService - .schedule(schedule, processVersion, randomProcessActionId) + .schedule(schedule, processVersion, canonicalProcess, randomProcessActionId) .futureValue tryToSchedule(cronInFuture) shouldBe (()) diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala index 0f95f511475..a958cf29ed9 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala @@ -129,6 +129,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc override def create( deploymentWithRuntimeParams: DeploymentWithRuntimeParams, inputConfigDuringExecutionJson: String, + canonicalProcess: CanonicalProcess, scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, ): Future[PeriodicProcess] = Future.successful { From 0b59f09327fed028057adfb561bab0009976e305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Mon, 30 Dec 2024 18:31:00 +0100 Subject: [PATCH 07/11] real ProcessVersion --- .../DeploymentManagerDependencies.scala | 8 +- .../periodic/PeriodicProcessesManager.scala | 11 +- .../model/DeploymentWithRuntimeParams.scala | 6 +- .../periodic/model/PeriodicProcess.scala | 5 +- .../model/PeriodicProcessDeployment.scala | 2 +- .../periodic/model/SchedulesState.scala | 2 +- ...ositoryBasedPeriodicProcessesManager.scala | 13 +- ...asedPeriodicProcessesManagerProvider.scala | 8 +- .../DBFetchingProcessRepository.scala | 25 +++- .../FetchingProcessRepository.scala | 11 +- .../PeriodicProcessesRepository.scala | 34 +---- .../repository/ScenarioActionRepository.scala | 130 ++++++++++++++++-- .../server/AkkaHttpBasedRouteProvider.scala | 33 +++-- .../mock/MockFetchingProcessRepository.scala | 19 ++- ...eriodicProcessServiceIntegrationTest.scala | 40 ++++-- .../periodic/PeriodicDeploymentHandler.scala | 1 + .../periodic/PeriodicProcessService.scala | 24 ++-- .../FlinkPeriodicDeploymentHandler.scala | 10 +- ...inkPeriodicDeploymentManagerProvider.scala | 7 +- .../LegacyPeriodicProcessesRepository.scala | 22 +-- ...ositoryBasedPeriodicProcessesManager.scala | 14 +- .../flink/PeriodicDeploymentHandlerStub.scala | 5 +- .../periodic/flink/PeriodicProcessGen.scala | 6 +- .../db/InMemPeriodicProcessesManager.scala | 29 ++-- ...amingDeploymentManagerProviderHelper.scala | 4 +- ...esponseEmbeddedDeploymentManagerTest.scala | 2 +- 26 files changed, 323 insertions(+), 148 deletions(-) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala index 4f7242b16be..7e410cb6e11 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala @@ -2,12 +2,8 @@ package pl.touk.nussknacker.engine import akka.actor.ActorSystem import pl.touk.nussknacker.engine.api.component.{ComponentAdditionalConfig, DesignerWideComponentId} -import pl.touk.nussknacker.engine.api.deployment.periodic.{PeriodicProcessesManager, PeriodicProcessesManagerProvider} -import pl.touk.nussknacker.engine.api.deployment.{ - ProcessingTypeActionService, - ProcessingTypeDeployedScenariosProvider, - ScenarioActivityManager -} +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManagerProvider +import pl.touk.nussknacker.engine.api.deployment._ import sttp.client3.SttpBackend import scala.concurrent.{ExecutionContext, Future} diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala index 1480edee63a..53ccecaa0c7 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala @@ -1,5 +1,6 @@ package pl.touk.nussknacker.engine.api.deployment.periodic +import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager.ScheduleProperty import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus @@ -79,10 +80,10 @@ trait PeriodicProcessesManager { expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], ): Future[SchedulesState] - def fetchCanonicalProcess( + def fetchCanonicalProcessWithVersion( processName: ProcessName, - versionId: VersionId, - ): Future[Option[CanonicalProcess]] + versionId: VersionId + ): Future[Option[(CanonicalProcess, ProcessVersion)]] def fetchInputConfigDuringExecutionJson( processName: ProcessName, @@ -172,10 +173,10 @@ object NoOpPeriodicProcessesManager extends PeriodicProcessesManager { expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], ): Future[SchedulesState] = notImplemented - override def fetchCanonicalProcess( + override def fetchCanonicalProcessWithVersion( processName: ProcessName, versionId: VersionId - ): Future[Option[CanonicalProcess]] = notImplemented + ): Future[Option[(CanonicalProcess, ProcessVersion)]] = notImplemented override def fetchInputConfigDuringExecutionJson( processName: ProcessName, diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala index bbc4364fd88..059259c15dc 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala @@ -1,9 +1,11 @@ package pl.touk.nussknacker.engine.api.deployment.periodic.model -import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} final case class DeploymentWithRuntimeParams( - processVersion: ProcessVersion, + processId: Option[ProcessId], + processName: ProcessName, + versionId: VersionId, runtimeParams: RuntimeParams, ) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala index ab1c666a008..2a9eaf540b5 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala @@ -1,6 +1,5 @@ package pl.touk.nussknacker.engine.api.deployment.periodic.model -import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager.ScheduleProperty @@ -15,6 +14,4 @@ case class PeriodicProcess( active: Boolean, createdAt: LocalDateTime, processActionId: Option[ProcessActionId] -) { - val processVersion: ProcessVersion = deploymentData.processVersion -} +) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala index e67b46142dc..ae432c0dc89 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala @@ -17,7 +17,7 @@ case class PeriodicProcessDeployment( ) { def display: String = - s"${periodicProcess.processVersion} with scheduleName=${scheduleName.display} and deploymentId=$id" + s"Process with id=${periodicProcess.deploymentData.processId}, name=${periodicProcess.deploymentData.processName}, versionId=${periodicProcess.deploymentData.versionId}, scheduleName=${scheduleName.display} and deploymentId=$id" } diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala index ecfddb07ab0..4f013482e55 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala @@ -18,7 +18,7 @@ case class SchedulesState(schedules: Map[ScheduleId, ScheduleData]) { def isEmpty: Boolean = schedules.isEmpty def groupByProcessName: Map[ProcessName, SchedulesState] = - schedules.groupBy(_._2.process.processVersion.processName).mapValuesNow(SchedulesState) + schedules.groupBy(_._2.process.deploymentData.processName).mapValuesNow(SchedulesState) lazy val groupedByPeriodicProcess: List[PeriodicProcessScheduleData] = schedules.toList.groupBy(_._2.process).toList.map { case (periodicProcess, groupedSchedules) => diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala index 4af3bc1751c..7e51d71e142 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala @@ -1,12 +1,14 @@ package pl.touk.nussknacker.ui.process.periodic +import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.ui.process.repository.PeriodicProcessesRepository +import pl.touk.nussknacker.ui.process.repository.{FetchingProcessRepository, PeriodicProcessesRepository} +import pl.touk.nussknacker.ui.security.api.{AdminUser, NussknackerInternalUser} import java.time.LocalDateTime import scala.concurrent.Future @@ -14,6 +16,7 @@ import scala.concurrent.Future class RepositoryBasedPeriodicProcessesManager( processingType: String, periodicProcessesRepository: PeriodicProcessesRepository, + fetchingProcessRepository: FetchingProcessRepository[Future] ) extends PeriodicProcessesManager { import periodicProcessesRepository._ @@ -123,11 +126,13 @@ class RepositoryBasedPeriodicProcessesManager( .findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus(expectedDeploymentStatuses, processingType) .run - override def fetchCanonicalProcess( + override def fetchCanonicalProcessWithVersion( processName: ProcessName, versionId: VersionId - ): Future[Option[CanonicalProcess]] = - periodicProcessesRepository.fetchCanonicalProcess(processName, versionId).run + ): Future[Option[(CanonicalProcess, ProcessVersion)]] = { + implicit val user: AdminUser = NussknackerInternalUser.instance + fetchingProcessRepository.getCanonicalProcessWithVersion(processName, versionId) + } override def fetchInputConfigDuringExecutionJson( processName: ProcessName, diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala index b4de6525cb7..d8b61062523 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala @@ -1,10 +1,13 @@ package pl.touk.nussknacker.ui.process.periodic import pl.touk.nussknacker.engine.api.deployment.periodic.{PeriodicProcessesManager, PeriodicProcessesManagerProvider} -import pl.touk.nussknacker.ui.process.repository.PeriodicProcessesRepository +import pl.touk.nussknacker.ui.process.repository.{FetchingProcessRepository, PeriodicProcessesRepository} + +import scala.concurrent.Future class RepositoryBasedPeriodicProcessesManagerProvider( periodicProcessesRepository: PeriodicProcessesRepository, + fetchingProcessRepository: FetchingProcessRepository[Future], ) extends PeriodicProcessesManagerProvider { override def provide( @@ -12,7 +15,8 @@ class RepositoryBasedPeriodicProcessesManagerProvider( ): PeriodicProcessesManager = { new RepositoryBasedPeriodicProcessesManager( processingType, - periodicProcessesRepository + periodicProcessesRepository, + fetchingProcessRepository, ) } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/DBFetchingProcessRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/DBFetchingProcessRepository.scala index c0ff82ef570..55b49e81bd0 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/DBFetchingProcessRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/DBFetchingProcessRepository.scala @@ -5,8 +5,10 @@ import cats.data.OptionT import cats.instances.future._ import com.typesafe.scalalogging.LazyLogging import db.util.DBIOActionInstances._ +import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.{ProcessAction, ProcessActionState, ScenarioActionName} import pl.touk.nussknacker.engine.api.process._ +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.ui.db.DbRef import pl.touk.nussknacker.ui.db.entity._ import pl.touk.nussknacker.ui.process.label.ScenarioLabel @@ -29,12 +31,13 @@ object DBFetchingProcessRepository { def createFutureRepository( dbRef: DbRef, - actionRepository: ScenarioActionRepository, + actionReadOnlyRepository: ScenarioActionReadOnlyRepository, scenarioLabelsRepository: ScenarioLabelsRepository )( implicit ec: ExecutionContext ) = - new DBFetchingProcessRepository[Future](dbRef, actionRepository, scenarioLabelsRepository) with BasicRepository + new DBFetchingProcessRepository[Future](dbRef, actionReadOnlyRepository, scenarioLabelsRepository) + with BasicRepository } @@ -43,7 +46,7 @@ object DBFetchingProcessRepository { // to the resource on the services side abstract class DBFetchingProcessRepository[F[_]: Monad]( protected val dbRef: DbRef, - actionRepository: ScenarioActionRepository, + actionRepository: ScenarioActionReadOnlyRepository, scenarioLabelsRepository: ScenarioLabelsRepository, )(protected implicit val ec: ExecutionContext) extends FetchingProcessRepository[F] @@ -51,6 +54,22 @@ abstract class DBFetchingProcessRepository[F[_]: Monad]( import api._ + override def getCanonicalProcessWithVersion( + processName: ProcessName, + versionId: VersionId + )( + implicit user: LoggedUser, + ): F[Option[(CanonicalProcess, ProcessVersion)]] = { + val result = for { + processId <- OptionT(fetchProcessId(processName)) + details <- OptionT(fetchProcessDetailsForId[CanonicalProcess](processId, versionId)) + } yield ( + details.json, + details.toEngineProcessVersion, + ) + result.value + } + override def fetchLatestProcessesDetails[PS: ScenarioShapeFetchStrategy]( query: ScenarioQuery )(implicit loggedUser: LoggedUser, ec: ExecutionContext): F[List[ScenarioWithDetailsEntity[PS]]] = { diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/FetchingProcessRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/FetchingProcessRepository.scala index db88f7dd0a3..c404f705218 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/FetchingProcessRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/FetchingProcessRepository.scala @@ -1,7 +1,9 @@ package pl.touk.nussknacker.ui.process.repository import cats.Monad -import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, ProcessName, ProcessingType, VersionId} +import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.process._ +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.ui.process.ScenarioQuery import pl.touk.nussknacker.ui.security.api.LoggedUser @@ -23,6 +25,13 @@ abstract class FetchingProcessRepository[F[_]: Monad] extends ProcessDBQueryRepo query: ScenarioQuery )(implicit loggedUser: LoggedUser, ec: ExecutionContext): F[List[ScenarioWithDetailsEntity[PS]]] + def getCanonicalProcessWithVersion( + processName: ProcessName, + versionId: VersionId + )( + implicit user: LoggedUser, + ): F[Option[(CanonicalProcess, ProcessVersion)]] + def fetchProcessId(processName: ProcessName)(implicit ec: ExecutionContext): F[Option[ProcessId]] def fetchProcessName(processId: ProcessId)(implicit ec: ExecutionContext): F[Option[ProcessName]] diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala index 112ee184444..360a11d4974 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala @@ -6,7 +6,6 @@ import com.github.tminglei.slickpg.ExPostgresProfile import com.typesafe.scalalogging.LazyLogging import io.circe.parser.decode import io.circe.syntax.EncoderOps -import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus @@ -57,12 +56,13 @@ object PeriodicProcessesRepository { def createPeriodicProcess( processEntity: PeriodicProcessEntity ): PeriodicProcess = { - val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, DeploymentWithRuntimeParams( - processVersion = processVersion, + processId = processEntity.processId, + processName = processEntity.processName, + versionId = processEntity.processVersionId, runtimeParams = processEntity.runtimeParams, ), toApi(scheduleProperty), @@ -78,10 +78,6 @@ object PeriodicProcessesRepository { scheduleProperty } - private def createProcessVersion(processEntity: PeriodicProcessEntity): ProcessVersion = { - ProcessVersion.empty.copy(versionId = processEntity.processVersionId, processName = processEntity.processName) - } - } trait PeriodicProcessesRepository { @@ -166,11 +162,6 @@ trait PeriodicProcessesRepository { deployMaxRetries: Int ): Action[PeriodicProcessDeployment] - def fetchCanonicalProcess( - processName: ProcessName, - versionId: VersionId, - ): Action[Option[CanonicalProcess]] - def fetchInputConfigDuringExecutionJson( processName: ProcessName, versionId: VersionId @@ -186,8 +177,6 @@ class SlickPeriodicProcessesRepository( extends PeriodicProcessesRepository with PeriodicProcessesTableFactory with PeriodicProcessDeploymentsTableFactory - with ProcessVersionEntityFactory - with ProcessEntityFactory with LazyLogging { import pl.touk.nussknacker.engine.util.Implicits._ @@ -220,9 +209,9 @@ class SlickPeriodicProcessesRepository( ): Action[PeriodicProcess] = { val processEntity = PeriodicProcessEntityWithInputConfigJson( id = PeriodicProcessId(-1), - processId = Some(deploymentWithRuntimeParams.processVersion.processId), - processName = deploymentWithRuntimeParams.processVersion.processName, - processVersionId = deploymentWithRuntimeParams.processVersion.versionId, + processId = deploymentWithRuntimeParams.processId, + processName = deploymentWithRuntimeParams.processName, + processVersionId = deploymentWithRuntimeParams.versionId, processingType = processingType, runtimeParams = deploymentWithRuntimeParams.runtimeParams, scheduleProperty = fromApi(scheduleProperty).asJson.noSpaces, @@ -511,17 +500,6 @@ class SlickPeriodicProcessesRepository( update.map(_ => ()) } - def fetchCanonicalProcess(processName: ProcessName, versionId: VersionId): Action[Option[CanonicalProcess]] = { - processesTable - .filter(_.name === processName) - .join(processVersionsTable) - .on((process, version) => process.id === version.processId) - .filter { case (_, version) => version.id === versionId } - .result - .headOption - .map(_.flatMap(_._2.json)) - } - def fetchInputConfigDuringExecutionJson(processName: ProcessName, versionId: VersionId): Action[Option[String]] = PeriodicProcessesWithInputConfig .filter(p => p.processName === processName && p.processVersionId === versionId) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala index 68aecc26a42..958066515ed 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala @@ -32,7 +32,7 @@ import scala.concurrent.ExecutionContext // 2. At the moment, the old ScenarioActionRepository // - handles those activities, which underlying operations may be long and may be in progress // 3. Eventually, the new ScenarioActivityRepository should be aware of the state of the underlying operation, and should replace this repository -trait ScenarioActionRepository extends LockableTable { +trait ScenarioActionRepository extends ScenarioActionReadOnlyRepository with LockableTable { def addInstantAction( processId: ProcessId, @@ -80,6 +80,10 @@ trait ScenarioActionRepository extends LockableTable { def deleteInProgressActions(): DB[Unit] +} + +trait ScenarioActionReadOnlyRepository extends LockableTable { + def getInProgressActionNames(processId: ProcessId): DB[Set[ScenarioActionName]] def getInProgressActionNames( @@ -110,10 +114,11 @@ trait ScenarioActionRepository extends LockableTable { } class DbScenarioActionRepository private ( - protected val dbRef: DbRef, + override protected val dbRef: DbRef, buildInfos: ProcessingTypeDataProvider[Map[String, String], _] )(override implicit val executionContext: ExecutionContext) - extends DbioRepository + extends DbScenarioActionReadOnlyRepository(dbRef) + with DbioRepository with NuTables with DbLockableTable with ScenarioActionRepository @@ -340,6 +345,111 @@ class DbScenarioActionRepository private ( } yield updateCount == 1 } + override def deleteInProgressActions(): DB[Unit] = { + run(scenarioActivityTable.filter(_.state === ProcessActionState.InProgress).delete.map(_ => ())) + } + + private def toFinishedProcessAction( + activityEntity: ScenarioActivityEntityData + ): Option[ProcessAction] = actionName(activityEntity.activityType).flatMap { actionName => + (for { + processVersionId <- activityEntity.scenarioVersion + .map(_.value) + .map(VersionId.apply) + .toRight(s"Process version not available for finished action: $activityEntity") + performedAt = activityEntity.finishedAt.getOrElse(activityEntity.createdAt).toInstant + state <- activityEntity.state + .toRight(s"State not available for finished action: $activityEntity") + } yield ProcessAction( + id = ProcessActionId(activityEntity.activityId.value), + processId = ProcessId(activityEntity.scenarioId.value), + processVersionId = processVersionId, + createdAt = activityEntity.createdAt.toInstant, + performedAt = performedAt, + user = activityEntity.userName.value, + actionName = actionName, + state = state, + failureMessage = activityEntity.errorMessage, + commentId = activityEntity.comment.map(_ => activityEntity.id), + comment = activityEntity.comment.map(_.value), + buildInfo = activityEntity.buildInfo.flatMap(BuildInfo.parseJson).getOrElse(BuildInfo.empty) + )).left.map { error => + logger.error(s"Could not interpret ScenarioActivity entity as ProcessAction: [$error]") + error + }.toOption + } + + private def activityId(actionId: ProcessActionId) = + ScenarioActivityId(actionId.value) + + private def actionName(activityType: ScenarioActivityType): Option[ScenarioActionName] = { + activityType match { + case ScenarioActivityType.ScenarioCreated => + None + case ScenarioActivityType.ScenarioArchived => + Some(ScenarioActionName.Archive) + case ScenarioActivityType.ScenarioUnarchived => + Some(ScenarioActionName.UnArchive) + case ScenarioActivityType.ScenarioDeployed => + Some(ScenarioActionName.Deploy) + case ScenarioActivityType.ScenarioPaused => + Some(ScenarioActionName.Pause) + case ScenarioActivityType.ScenarioCanceled => + Some(ScenarioActionName.Cancel) + case ScenarioActivityType.ScenarioModified => + None + case ScenarioActivityType.ScenarioNameChanged => + Some(ScenarioActionName.Rename) + case ScenarioActivityType.CommentAdded => + None + case ScenarioActivityType.AttachmentAdded => + None + case ScenarioActivityType.ChangedProcessingMode => + None + case ScenarioActivityType.IncomingMigration => + None + case ScenarioActivityType.OutgoingMigration => + None + case ScenarioActivityType.PerformedSingleExecution => + Some(ScenarioActionName.RunOffSchedule) + case ScenarioActivityType.PerformedScheduledExecution => + None + case ScenarioActivityType.AutomaticUpdate => + None + case ScenarioActivityType.CustomAction(name) => + Some(ScenarioActionName(name)) + } + } + +} + +object DbScenarioActionRepository { + + def create(dbRef: DbRef, buildInfos: ProcessingTypeDataProvider[Map[String, String], _])( + implicit executionContext: ExecutionContext, + ): ScenarioActionRepository = { + new ScenarioActionRepositoryAuditLogDecorator( + new DbScenarioActionRepository(dbRef, buildInfos) + ) + } + +} + +class DbScenarioActionReadOnlyRepository( + protected val dbRef: DbRef, +)(override implicit val executionContext: ExecutionContext) + extends DbioRepository + with NuTables + with DbLockableTable + with ScenarioActionReadOnlyRepository + with LazyLogging { + + import profile.api._ + + override type ENTITY = ScenarioActivityEntityFactory#ScenarioActivityEntity + + override protected def table: TableQuery[ScenarioActivityEntityFactory#ScenarioActivityEntity] = scenarioActivityTable + override def getInProgressActionNames(processId: ProcessId): DB[Set[ScenarioActionName]] = { val query = scenarioActivityTable .filter(action => action.scenarioId === processId && action.state === ProcessActionState.InProgress) @@ -391,10 +501,6 @@ class DbScenarioActionRepository private ( ) } - override def deleteInProgressActions(): DB[Unit] = { - run(scenarioActivityTable.filter(_.state === ProcessActionState.InProgress).delete.map(_ => ())) - } - override def getLastActionPerProcess( actionState: Set[ProcessActionState], actionNamesOpt: Option[Set[ScenarioActionName]] @@ -553,14 +659,12 @@ class DbScenarioActionRepository private ( } -object DbScenarioActionRepository { +object DbScenarioActionReadOnlyRepository { - def create(dbRef: DbRef, buildInfos: ProcessingTypeDataProvider[Map[String, String], _])( + def create(dbRef: DbRef)( implicit executionContext: ExecutionContext, - ): ScenarioActionRepository = { - new ScenarioActionRepositoryAuditLogDecorator( - new DbScenarioActionRepository(dbRef, buildInfos) - ) + ): ScenarioActionReadOnlyRepository = { + new DbScenarioActionReadOnlyRepository(dbRef) } } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala index 55148cc2091..092237d424a 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala @@ -25,10 +25,10 @@ import pl.touk.nussknacker.ui.config.scenariotoolbar.CategoriesScenarioToolbarsC import pl.touk.nussknacker.ui.config.{ AttachmentsConfig, ComponentLinksConfigExtractor, + DesignerConfig, FeatureTogglesConfig, UsageStatisticsReportsConfig } -import pl.touk.nussknacker.ui.config.DesignerConfig import pl.touk.nussknacker.ui.db.DbRef import pl.touk.nussknacker.ui.db.timeseries.FEStatisticsRepository import pl.touk.nussknacker.ui.definition.component.{ComponentServiceProcessingTypeData, DefaultComponentService} @@ -65,10 +65,7 @@ import pl.touk.nussknacker.ui.process.newdeployment.synchronize.{ DeploymentsStatusesSynchronizer } import pl.touk.nussknacker.ui.process.newdeployment.{DeploymentRepository, DeploymentService} -import pl.touk.nussknacker.ui.process.periodic.{ - RepositoryBasedPeriodicProcessesManager, - RepositoryBasedPeriodicProcessesManagerProvider -} +import pl.touk.nussknacker.ui.process.periodic.RepositoryBasedPeriodicProcessesManagerProvider import pl.touk.nussknacker.ui.process.processingtype.ProcessingTypeData import pl.touk.nussknacker.ui.process.processingtype.loader.ProcessingTypeDataLoader import pl.touk.nussknacker.ui.process.processingtype.provider.ReloadableProcessingTypeDataProvider @@ -134,12 +131,20 @@ class AkkaHttpBasedRouteProvider( deploymentRepository = new DeploymentRepository(dbRef, Clock.systemDefaultZone()) scenarioActivityRepository = DbScenarioActivityRepository.create(dbRef, designerClock) periodicProcessesRepository = new SlickPeriodicProcessesRepository(dbRef.db, dbRef.profile, designerClock) - dbioRunner = DBIOActionRunner(dbRef) + actionReadOnlyRepository = DbScenarioActionReadOnlyRepository.create(dbRef) + scenarioLabelsRepository = new ScenarioLabelsRepository(dbRef) + futureProcessRepository = DBFetchingProcessRepository.createFutureRepository( + dbRef, + actionReadOnlyRepository, + scenarioLabelsRepository + ) + dbioRunner = DBIOActionRunner(dbRef) processingTypeDataProvider <- prepareProcessingTypeDataReload( additionalUIConfigProvider, actionServiceSupplier, scenarioActivityRepository, periodicProcessesRepository, + futureProcessRepository, dbioRunner, sttpBackend, featureTogglesConfig, @@ -178,8 +183,6 @@ class AkkaHttpBasedRouteProvider( val scenarioLabelsRepository = new ScenarioLabelsRepository(dbRef) val processRepository = DBFetchingProcessRepository.create(dbRef, actionRepository, scenarioLabelsRepository) // TODO: get rid of Future based repositories - it is easier to use everywhere one implementation - DBIOAction based which allows transactions handling - val futureProcessRepository = - DBFetchingProcessRepository.createFutureRepository(dbRef, actionRepository, scenarioLabelsRepository) val writeProcessRepository = ProcessRepository.create(dbRef, designerClock, scenarioActivityRepository, scenarioLabelsRepository, migrations) @@ -711,6 +714,7 @@ class AkkaHttpBasedRouteProvider( actionServiceProvider: Supplier[ActionService], scenarioActivityRepository: ScenarioActivityRepository, periodicProcessesRepository: PeriodicProcessesRepository, + fetchingProcessRepository: FetchingProcessRepository[Future], dbioActionRunner: DBIOActionRunner, sttpBackend: SttpBackend[Future, Any], featureTogglesConfig: FeatureTogglesConfig @@ -729,11 +733,11 @@ class AkkaHttpBasedRouteProvider( actionServiceProvider, scenarioActivityRepository, periodicProcessesRepository, - dbioActionRunner, - sttpBackend, - _ - ), - + fetchingProcessRepository, + dbioActionRunner, + sttpBackend, + _ + ), ) new ReloadableProcessingTypeDataProvider(laodProcessingTypeDataIO) } @@ -747,6 +751,7 @@ class AkkaHttpBasedRouteProvider( actionServiceProvider: Supplier[ActionService], scenarioActivityRepository: ScenarioActivityRepository, periodicProcessesRepository: PeriodicProcessesRepository, + fetchingProcessRepository: FetchingProcessRepository[Future], dbioActionRunner: DBIOActionRunner, sttpBackend: SttpBackend[Future, Any], processingType: ProcessingType @@ -762,7 +767,7 @@ class AkkaHttpBasedRouteProvider( scenarioActivityRepository, dbioActionRunner, ), - new RepositoryBasedPeriodicProcessesManagerProvider(periodicProcessesRepository), + new RepositoryBasedPeriodicProcessesManagerProvider(periodicProcessesRepository, fetchingProcessRepository), system.dispatcher, system, sttpBackend, diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockFetchingProcessRepository.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockFetchingProcessRepository.scala index b027c7317cf..cd11115ea55 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockFetchingProcessRepository.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockFetchingProcessRepository.scala @@ -1,6 +1,8 @@ package pl.touk.nussknacker.test.mock +import cats.data.OptionT import cats.instances.future._ +import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName import pl.touk.nussknacker.engine.api.graph.ScenarioGraph import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, ProcessName, VersionId} @@ -43,6 +45,19 @@ class MockFetchingProcessRepository private ( extends FetchingProcessRepository[Future] with BasicRepository { + override def getCanonicalProcessWithVersion(processName: ProcessName, versionId: VersionId)( + implicit user: LoggedUser + ): Future[Option[(CanonicalProcess, ProcessVersion)]] = { + val result = for { + processId <- OptionT(fetchProcessId(processName)) + details <- OptionT(fetchProcessDetailsForId[CanonicalProcess](processId, versionId)) + } yield ( + details.json, + details.toEngineProcessVersion, + ) + result.value + } + override def fetchLatestProcessesDetails[PS: ScenarioShapeFetchStrategy]( q: ScenarioQuery )(implicit loggedUser: LoggedUser, ec: ExecutionContext): Future[List[ScenarioWithDetailsEntity[PS]]] = @@ -91,8 +106,8 @@ class MockFetchingProcessRepository private ( val shapeStrategy: ScenarioShapeFetchStrategy[PS] = implicitly[ScenarioShapeFetchStrategy[PS]] shapeStrategy match { - case NotFetch => process.copy(json = ().asInstanceOf[PS]) - case FetchCanonical => process.asInstanceOf[ScenarioWithDetailsEntity[PS]] + case NotFetch => process.copy(json = ()) + case FetchCanonical => process case FetchScenarioGraph => process .mapScenario(canonical => CanonicalProcessConverter.toScenarioGraph(canonical)) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala index d27e4cd27c0..7b60c17bd00 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -33,6 +33,7 @@ import pl.touk.nussknacker.engine.management.periodic.flink.{DeploymentManagerSt import pl.touk.nussknacker.test.PatientScalaFutures import pl.touk.nussknacker.test.base.db.WithPostgresDbTesting import pl.touk.nussknacker.test.base.it.WithClock +import pl.touk.nussknacker.test.utils.domain.TestFactory import pl.touk.nussknacker.test.utils.domain.TestFactory.newWriteProcessRepository import pl.touk.nussknacker.test.utils.scalas.DBIOActionValues import pl.touk.nussknacker.ui.process.repository.ProcessRepository.CreateProcessAction @@ -93,6 +94,16 @@ class PeriodicProcessServiceIntegrationTest executionConfig: PeriodicExecutionConfig = PeriodicExecutionConfig(), maxFetchedPeriodicScenarioActivities: Option[Int] = None, )(testCode: Fixture => Any): Unit = { + val repositoryBasedProvider = (currentTime: Instant) => + new RepositoryBasedPeriodicProcessesManagerProvider( + new SlickPeriodicProcessesRepository( + testDbRef.db, + testDbRef.profile, + fixedClock(currentTime), + ), + TestFactory.newFutureFetchingScenarioRepository(testDbRef) + ) + val postgresConfig = ConfigFactory.parseMap( Map( "user" -> container.username, @@ -117,7 +128,11 @@ class PeriodicProcessServiceIntegrationTest new PeriodicProcessesManagerProvider { override def provide(processingType: String): PeriodicProcessesManager = { val repository = new SlickLegacyPeriodicProcessesRepository(db, dbProfile, fixedClock(currentTime)) - new LegacyRepositoryBasedPeriodicProcessesManager(processingType, repository) + new LegacyRepositoryBasedPeriodicProcessesManager( + processingType, + repository, + repositoryBasedProvider(currentTime).provide(processingType) + ) } } try { @@ -129,12 +144,15 @@ class PeriodicProcessServiceIntegrationTest } } - def runTestCodeWithNuDb() = { - val provider = (currentTime: Instant) => - new RepositoryBasedPeriodicProcessesManagerProvider( - new SlickPeriodicProcessesRepository(testDbRef.db, testDbRef.profile, fixedClock(currentTime)) + def runTestCodeWithNuDb(): Unit = { + testCode( + new Fixture( + repositoryBasedProvider, + deploymentRetryConfig, + executionConfig, + maxFetchedPeriodicScenarioActivities ) - testCode(new Fixture(provider, deploymentRetryConfig, executionConfig, maxFetchedPeriodicScenarioActivities)) + ) } def testHeader(str: String) = "\n\n" + "*" * 100 + s"\n***** $str\n" + "*" * 100 + "\n" @@ -259,7 +277,7 @@ class PeriodicProcessServiceIntegrationTest stateAfterSchedule should have size 1 val afterSchedule = stateAfterSchedule.firstScheduleData - afterSchedule.process.processVersion.processName shouldBe processName + afterSchedule.process.deploymentData.processName shouldBe processName afterSchedule.latestDeployments.head.state shouldBe PeriodicProcessDeploymentState( None, None, @@ -272,9 +290,9 @@ class PeriodicProcessServiceIntegrationTest val allToDeploy = service.findToBeDeployed.futureValue allToDeploy.map( - _.periodicProcess.processVersion.processName + _.periodicProcess.deploymentData.processName ) should contain only (processName, every30MinutesProcessName) - val toDeploy = allToDeploy.find(_.periodicProcess.processVersion.processName == processName).value + val toDeploy = allToDeploy.find(_.periodicProcess.deploymentData.processName == processName).value service.deploy(toDeploy).futureValue otherProcessingTypeService.deploy(otherProcessingTypeService.findToBeDeployed.futureValue.loneElement).futureValue @@ -299,7 +317,7 @@ class PeriodicProcessServiceIntegrationTest service.handleFinished.futureValue val toDeployAfterFinish = service.findToBeDeployed.futureValue - toDeployAfterFinish.map(_.periodicProcess.processVersion.processName) should contain only every30MinutesProcessName + toDeployAfterFinish.map(_.periodicProcess.deploymentData.processName) should contain only every30MinutesProcessName service.deactivate(processName).futureValue service.getLatestDeploymentsForActiveSchedules(processName).futureValue shouldBe empty val inactiveStates = service @@ -440,7 +458,7 @@ class PeriodicProcessServiceIntegrationTest val allToDeploy = service.findToBeDeployed.futureValue allToDeploy should have length 4 - val toDeploy = allToDeploy.filter(_.periodicProcess.processVersion.processName == processName) + val toDeploy = allToDeploy.filter(_.periodicProcess.deploymentData.processName == processName) toDeploy should have length 2 toDeploy.head.runAt shouldBe localTime(expectedScheduleTime.plus(5, ChronoUnit.MINUTES)) toDeploy.head.scheduleName.value shouldBe Some(scheduleMinute5) diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala index 0e0780e4c41..7b72a9ffd07 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala @@ -21,6 +21,7 @@ trait PeriodicDeploymentHandler { inputConfigDuringExecutionJson: String, deploymentData: DeploymentData, canonicalProcess: CanonicalProcess, + processVersion: ProcessVersion, ): Future[Option[ExternalDeploymentId]] def cleanAfterDeployment( diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala index 406acd5ef26..7d28d8fec25 100644 --- a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala @@ -88,7 +88,7 @@ class PeriodicProcessService( scenarioActivityId = ScenarioActivityId(DeterministicUUIDFromLong.longUUID(deployment.id.value)), user = ScenarioUser.internalNuUser, date = metadata.dateDeployed.getOrElse(metadata.dateFinished), - scenarioVersionId = Some(ScenarioVersionId.from(deployment.periodicProcess.processVersion.versionId)), + scenarioVersionId = Some(ScenarioVersionId.from(deployment.periodicProcess.deploymentData.versionId)), scheduledExecutionStatus = metadata.status, dateFinished = metadata.dateFinished, scheduleName = deployment.scheduleName.display, @@ -217,7 +217,7 @@ class PeriodicProcessService( toDeploy: PeriodicProcessDeployment ): Future[Option[PeriodicProcessDeployment]] = { delegateDeploymentManager - .getProcessStates(toDeploy.periodicProcess.processVersion.processName)(DataFreshnessPolicy.Fresh) + .getProcessStates(toDeploy.periodicProcess.deploymentData.processName)(DataFreshnessPolicy.Fresh) .map( _.value .map(_.status) @@ -461,14 +461,17 @@ class PeriodicProcessService( val deploymentWithJarData = deployment.periodicProcess.deploymentData val deploymentAction = for { _ <- Future.successful( - logger.info("Deploying scenario {} for deployment id {}", deploymentWithJarData.processVersion, id) + logger.info("Deploying scenario {} for deployment id {}", deploymentWithJarData, id) ) - processName = deploymentWithJarData.processVersion.processName - versionId = deploymentWithJarData.processVersion.versionId - canonicalProcessOpt <- periodicProcessesManager.fetchCanonicalProcess(processName, versionId) - canonicalProcess = canonicalProcessOpt.getOrElse { + processName = deploymentWithJarData.processName + versionId = deploymentWithJarData.versionId + canonicalProcessWithVersionOpt <- periodicProcessesManager.fetchCanonicalProcessWithVersion( + processName, + versionId + ) + canonicalProcessWithVersion = canonicalProcessWithVersionOpt.getOrElse { throw new PeriodicProcessException( - s"Could not fetch CanonicalProcess for processName=$processName, versionId=$versionId" + s"Could not fetch CanonicalProcess with ProcessVersion for processName=$processName, versionId=$versionId" ) } inputConfigDuringExecutionJsonOpt <- periodicProcessesManager.fetchInputConfigDuringExecutionJson( @@ -487,12 +490,13 @@ class PeriodicProcessService( deploymentWithJarData, enrichedProcessConfig.inputConfigDuringExecutionJson, deploymentData, - canonicalProcess, + canonicalProcessWithVersion._1, + canonicalProcessWithVersion._2, ) } yield externalDeploymentId deploymentAction .flatMap { externalDeploymentId => - logger.info("Scenario has been deployed {} for deployment id {}", deploymentWithJarData.processVersion, id) + logger.info("Scenario has been deployed {} for deployment id {}", deploymentWithJarData, id) // TODO: add externalDeploymentId?? periodicProcessesManager .markDeployed(id) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala index ff13d15a536..1adfb936dc7 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala @@ -57,7 +57,9 @@ class FlinkPeriodicDeploymentHandler( logger.info(s"Prepare deployment for scenario: $processVersion") copyJarToLocalDir(processVersion).map { jarFileName => DeploymentWithRuntimeParams( - processVersion = processVersion, + processId = Some(processVersion.processId), + processName = processVersion.processName, + versionId = processVersion.versionId, runtimeParams = RuntimeParams(Map(jarFileNameRuntimeParam -> jarFileName)) ) } @@ -81,12 +83,12 @@ class FlinkPeriodicDeploymentHandler( inputConfigDuringExecutionJson: String, deploymentData: DeploymentData, canonicalProcess: CanonicalProcess, + processVersion: ProcessVersion, ): Future[Option[ExternalDeploymentId]] = { - val processVersion = deployment.processVersion deployment.runtimeParams.params.get(jarFileNameRuntimeParam) match { case Some(jarFileName) => logger.info( - s"Deploying scenario ${processVersion.processName}, version id: ${processVersion.versionId} and jar: $jarFileName" + s"Deploying scenario ${deployment.processName}, version id: ${deployment.versionId} and jar: $jarFileName" ) val jarFile = jarsDir.resolve(jarFileName).toFile val args = FlinkDeploymentManager.prepareProgramArgs( @@ -104,7 +106,7 @@ class FlinkPeriodicDeploymentHandler( ) case None => logger.error( - s"Cannot deploy scenario ${processVersion.processName}, version id: ${processVersion.versionId}: jar file name not present" + s"Cannot deploy scenario ${deployment.processName}, version id: ${deployment.versionId}: jar file name not present" ) Future.successful(None) } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala index 80d685628c8..5c6efe7538a 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala @@ -56,7 +56,12 @@ class FlinkPeriodicDeploymentManagerProvider val clock = Clock.systemDefaultZone() val (db: jdbc.JdbcBackend.DatabaseDef, dbProfile: JdbcProfile) = LegacyDbInitializer.init(customDbConfig) val repository = new SlickLegacyPeriodicProcessesRepository(db, dbProfile, clock) - (processingType: String) => new LegacyRepositoryBasedPeriodicProcessesManager(processingType, repository) + (processingType: String) => + new LegacyRepositoryBasedPeriodicProcessesManager( + processingType, + repository, + dependencies.periodicProcessesManagerProvider.provide(processingType) + ) } } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesRepository.scala index efadc01d2c0..eb555899c8d 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyPeriodicProcessesRepository.scala @@ -6,7 +6,6 @@ import com.typesafe.scalalogging.LazyLogging import io.circe.parser import io.circe.parser.decode import io.circe.syntax.EncoderOps -import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus @@ -58,13 +57,14 @@ object LegacyPeriodicProcessesRepository { def createPeriodicProcess( processEntity: PeriodicProcessEntity ): PeriodicProcess = { - val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, DeploymentWithRuntimeParams( - processVersion = processVersion, - runtimeParams = RuntimeParams(Map(jarFileNameRuntimeParam + "a" -> processEntity.jarFileName)), + processId = None, + processName = processEntity.processName, + versionId = processEntity.processVersionId, + runtimeParams = RuntimeParams(Map(jarFileNameRuntimeParam -> processEntity.jarFileName)), ), toApi(scheduleProperty), processEntity.active, @@ -79,10 +79,6 @@ object LegacyPeriodicProcessesRepository { scheduleProperty } - private def createProcessVersion(processEntity: PeriodicProcessEntity): ProcessVersion = { - ProcessVersion.empty.copy(versionId = processEntity.processVersionId, processName = processEntity.processName) - } - } trait LegacyPeriodicProcessesRepository { @@ -219,12 +215,16 @@ class SlickLegacyPeriodicProcessesRepository( processActionId: ProcessActionId, processingType: String, ): Action[PeriodicProcess] = { + val jarFileName = deploymentWithRuntimeParams.runtimeParams.params.getOrElse( + jarFileNameRuntimeParam, + throw new RuntimeException(s"jarFileName runtime param not present") + ) val processEntity = PeriodicProcessEntityWithJson( id = PeriodicProcessId(-1), - processName = deploymentWithRuntimeParams.processVersion.processName, - processVersionId = deploymentWithRuntimeParams.processVersion.versionId, + processName = deploymentWithRuntimeParams.processName, + processVersionId = deploymentWithRuntimeParams.versionId, processingType = processingType, - jarFileName = deploymentWithRuntimeParams.runtimeParams.params(jarFileNameRuntimeParam), + jarFileName = jarFileName, scheduleProperty = fromApi(scheduleProperty).asJson.noSpaces, active = true, createdAt = now(), diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyRepositoryBasedPeriodicProcessesManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyRepositoryBasedPeriodicProcessesManager.scala index 9fb278b85e0..8b73fe6359a 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyRepositoryBasedPeriodicProcessesManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/LegacyRepositoryBasedPeriodicProcessesManager.scala @@ -1,5 +1,6 @@ package pl.touk.nussknacker.engine.management.periodic.flink.db +import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus @@ -13,6 +14,7 @@ import scala.concurrent.Future class LegacyRepositoryBasedPeriodicProcessesManager( processingType: String, periodicProcessesRepository: LegacyPeriodicProcessesRepository, + underlyingPeriodicProcessesManager: PeriodicProcessesManager, ) extends PeriodicProcessesManager { import periodicProcessesRepository._ @@ -123,16 +125,16 @@ class LegacyRepositoryBasedPeriodicProcessesManager( .findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus(expectedDeploymentStatuses, processingType) .run - override def fetchCanonicalProcess( - processName: ProcessName, - versionId: VersionId - ): Future[Option[CanonicalProcess]] = - periodicProcessesRepository.fetchCanonicalProcess(processName, versionId).run - override def fetchInputConfigDuringExecutionJson( processName: ProcessName, versionId: VersionId ): Future[Option[String]] = periodicProcessesRepository.fetchInputConfigDuringExecutionJson(processName, versionId).run + override def fetchCanonicalProcessWithVersion( + processName: ProcessName, + versionId: VersionId + ): Future[Option[(CanonicalProcess, ProcessVersion)]] = + underlyingPeriodicProcessesManager.fetchCanonicalProcessWithVersion(processName, versionId) + } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala index 6df57e9f9c0..2defbe25e7c 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala @@ -21,7 +21,9 @@ class PeriodicDeploymentHandlerStub extends PeriodicDeploymentHandler { ): Future[DeploymentWithRuntimeParams] = { Future.successful( DeploymentWithRuntimeParams( - processVersion = processVersion, + processId = Some(processVersion.processId), + processName = processVersion.processName, + versionId = processVersion.versionId, runtimeParams = RuntimeParams(Map("jarFileName" -> "")) ) ) @@ -35,6 +37,7 @@ class PeriodicDeploymentHandlerStub extends PeriodicDeploymentHandler { inputConfigDuringExecutionJson: String, deploymentData: DeploymentData, canonicalProcess: CanonicalProcess, + processVersion: ProcessVersion, ): Future[Option[ExternalDeploymentId]] = { lastDeploymentWithRuntimeParams = Some(deploymentWithJarData) lastInputConfigDuringExecutionJson = Some(inputConfigDuringExecutionJson) diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala index 5e63507337b..f57b54baa76 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala @@ -1,8 +1,8 @@ package pl.touk.nussknacker.engine.management.periodic.flink -import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager.CronScheduleProperty import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.common.periodic.CronSchedulePropertyExtractor.CronPropertyDefaultName @@ -15,7 +15,9 @@ object PeriodicProcessGen { PeriodicProcess( id = PeriodicProcessId(42), deploymentData = DeploymentWithRuntimeParams( - processVersion = ProcessVersion.empty, + processId = Some(ProcessId(1)), + processName = ProcessName(""), + versionId = VersionId.initialVersionId, runtimeParams = RuntimeParams(Map("jarFileName" -> "jar-file-name.jar")) ), scheduleProperty = CronScheduleProperty("0 0 * * * ?"), diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala index a958cf29ed9..851a2c37a25 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala @@ -6,7 +6,7 @@ import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.periodic.model._ -import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} @@ -69,6 +69,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc val id = PeriodicProcessId(ProcessIdSequence.incrementAndGet()) val entity = PeriodicProcessEntity( id = id, + processId = None, processName = processName, processVersionId = VersionId.initialVersionId, processingType = processingType, @@ -136,8 +137,9 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc val id = PeriodicProcessId(Random.nextLong()) val periodicProcess = PeriodicProcessEntity( id = id, - processName = deploymentWithRuntimeParams.processVersion.processName, - processVersionId = deploymentWithRuntimeParams.processVersion.versionId, + processId = deploymentWithRuntimeParams.processId, + processName = deploymentWithRuntimeParams.processName, + processVersionId = deploymentWithRuntimeParams.versionId, processingType = processingType, inputConfigDuringExecutionJson = inputConfigDuringExecutionJson, runtimeParams = deploymentWithRuntimeParams.runtimeParams, @@ -344,10 +346,12 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc Future.successful(deployments.filter(d => d.runAt.isBefore(now) || d.runAt.isEqual(now))) } - override def fetchCanonicalProcess( + override def fetchCanonicalProcessWithVersion( processName: ProcessName, versionId: VersionId - ): Future[Option[CanonicalProcess]] = Future.successful(Some(canonicalProcess(processName))) + ): Future[Option[(CanonicalProcess, ProcessVersion)]] = Future.successful { + Some(canonicalProcess(processName), ProcessVersion.empty) + } override def fetchInputConfigDuringExecutionJson( processName: ProcessName, @@ -364,6 +368,7 @@ object InMemPeriodicProcessesManager { final case class PeriodicProcessEntity( id: PeriodicProcessId, + processId: Option[ProcessId], processName: ProcessName, processVersionId: VersionId, processingType: String, @@ -418,12 +423,13 @@ object InMemPeriodicProcessesManager { def createPeriodicProcessWithJson( processEntity: PeriodicProcessEntity ): PeriodicProcess = { - val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, DeploymentWithRuntimeParams( - processVersion = processVersion, + processId = processEntity.processId, + processName = processEntity.processName, + versionId = processEntity.processVersionId, runtimeParams = processEntity.runtimeParams, ), toApi(scheduleProperty), @@ -436,12 +442,13 @@ object InMemPeriodicProcessesManager { def createPeriodicProcessWithoutJson( processEntity: PeriodicProcessEntity ): PeriodicProcess = { - val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, DeploymentWithRuntimeParams( - processVersion = processVersion, + processId = processEntity.processId, + processName = processEntity.processName, + versionId = processEntity.processVersionId, runtimeParams = processEntity.runtimeParams, ), toApi(scheduleProperty), @@ -458,10 +465,6 @@ object InMemPeriodicProcessesManager { scheduleProperty } - private def createProcessVersion(processEntity: PeriodicProcessEntity): ProcessVersion = { - ProcessVersion.empty.copy(versionId = processEntity.processVersionId, processName = processEntity.processName) - } - private def scheduleDeploymentData(deployment: PeriodicProcessDeploymentEntity): ScheduleDeploymentData = { ScheduleDeploymentData( deployment.id, diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala index 4a5f1202cf9..b15fbe1a6f5 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala @@ -1,7 +1,9 @@ package pl.touk.nussknacker.engine.management.streaming +import _root_.sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend import akka.actor.ActorSystem import org.asynchttpclient.DefaultAsyncHttpClientConfig +import pl.touk.nussknacker.engine._ import pl.touk.nussknacker.engine.api.component.DesignerWideComponentId import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.api.deployment.{ @@ -12,8 +14,6 @@ import pl.touk.nussknacker.engine.api.deployment.{ } import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode import pl.touk.nussknacker.engine.management.FlinkStreamingDeploymentManagerProvider -import pl.touk.nussknacker.engine._ -import _root_.sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend object FlinkStreamingDeploymentManagerProviderHelper { diff --git a/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala b/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala index fc846926fd5..14eaeebc06f 100644 --- a/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala +++ b/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala @@ -7,10 +7,10 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy +import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.deployment.cache.ScenarioStateCachingConfig import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus -import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess From e1651d0de04395a03b4d7b0e20cd4941cef013a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Mon, 30 Dec 2024 20:14:49 +0100 Subject: [PATCH 08/11] scala 2.12 and test fix --- ...eriodicProcessServiceIntegrationTest.scala | 2 +- .../flink/PeriodicProcessServiceTest.scala | 1 - .../db/InMemPeriodicProcessesManager.scala | 38 +++++++++---------- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala index 7b60c17bd00..39644b2174f 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -504,7 +504,7 @@ class PeriodicProcessServiceIntegrationTest toDeploy should have length 2 val deployment = toDeploy.find(_.scheduleName.value.contains(firstSchedule)).value - service.deploy(deployment) + service.deploy(deployment).futureValue f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Running, Some(deployment.id)) val toDeployAfterDeploy = service.findToBeDeployed.futureValue diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala index 3c43bee612e..b855048732c 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala @@ -18,7 +18,6 @@ import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.Proble import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId, ProcessingTypeActionServiceStub} import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.build.ScenarioBuilder -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.PeriodicProcessStatus import pl.touk.nussknacker.engine.common.periodic._ import pl.touk.nussknacker.engine.common.periodic.service.ProcessConfigEnricher.EnrichedProcessConfig diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala index 851a2c37a25..5a3194a0c1a 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala @@ -28,8 +28,8 @@ object InMemPeriodicProcessesRepository { class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProcessesManager { - var processEntities: mutable.ListBuffer[PeriodicProcessEntity] = ListBuffer.empty - var deploymentEntities: mutable.ListBuffer[PeriodicProcessDeploymentEntity] = ListBuffer.empty + var processEntities: mutable.ListBuffer[TestPeriodicProcessEntity] = ListBuffer.empty + var deploymentEntities: mutable.ListBuffer[TestPeriodicProcessDeploymentEntity] = ListBuffer.empty private def canonicalProcess(processName: ProcessName) = { ScenarioBuilder @@ -67,7 +67,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc processActionId: Option[ProcessActionId] = None ): PeriodicProcessId = { val id = PeriodicProcessId(ProcessIdSequence.incrementAndGet()) - val entity = PeriodicProcessEntity( + val entity = TestPeriodicProcessEntity( id = id, processId = None, processName = processName, @@ -93,7 +93,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc deployedAt: Option[LocalDateTime] = None, ): PeriodicProcessDeploymentId = { val id = PeriodicProcessDeploymentId(DeploymentIdSequence.incrementAndGet()) - val entity = PeriodicProcessDeploymentEntity( + val entity = TestPeriodicProcessDeploymentEntity( id = id, periodicProcessId = periodicProcessId, createdAt = LocalDateTime.now(), @@ -135,7 +135,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc processActionId: ProcessActionId, ): Future[PeriodicProcess] = Future.successful { val id = PeriodicProcessId(Random.nextLong()) - val periodicProcess = PeriodicProcessEntity( + val periodicProcess = TestPeriodicProcessEntity( id = id, processId = deploymentWithRuntimeParams.processId, processName = deploymentWithRuntimeParams.processName, @@ -210,7 +210,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc } private def getLatestDeploymentsForPeriodicProcesses( - processes: Seq[PeriodicProcessEntity], + processes: Seq[TestPeriodicProcessEntity], deploymentsPerScheduleMaxCount: Int ): SchedulesState = { SchedulesState((for { @@ -249,12 +249,12 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc } yield createPeriodicProcessDeployment(p, d)).head } - private def processEntities(processName: ProcessName): Seq[PeriodicProcessEntity] = + private def processEntities(processName: ProcessName): Seq[TestPeriodicProcessEntity] = processEntities .filter(process => process.processName == processName && process.processingType == processingType) .toSeq - private def allProcessEntities: Map[ProcessName, Seq[PeriodicProcessEntity]] = + private def allProcessEntities: Map[ProcessName, Seq[TestPeriodicProcessEntity]] = processEntities .filter(process => process.processingType == processingType) .toSeq @@ -299,7 +299,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc runAt: LocalDateTime, deployMaxRetries: Int, ): Future[PeriodicProcessDeployment] = Future.successful { - val deploymentEntity = PeriodicProcessDeploymentEntity( + val deploymentEntity = TestPeriodicProcessDeploymentEntity( id = PeriodicProcessDeploymentId(Random.nextLong()), periodicProcessId = id, createdAt = LocalDateTime.now(), @@ -318,7 +318,7 @@ class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProc private def update( id: PeriodicProcessDeploymentId - )(action: PeriodicProcessDeploymentEntity => PeriodicProcessDeploymentEntity): Unit = { + )(action: TestPeriodicProcessDeploymentEntity => TestPeriodicProcessDeploymentEntity): Unit = { deploymentEntities.zipWithIndex .find { case (deployment, _) => deployment.id == id } .foreach { case (deployment, index) => @@ -366,7 +366,7 @@ object InMemPeriodicProcessesManager { private val ProcessIdSequence = new AtomicLong(0) private val DeploymentIdSequence = new AtomicLong(0) - final case class PeriodicProcessEntity( + final case class TestPeriodicProcessEntity( id: PeriodicProcessId, processId: Option[ProcessId], processName: ProcessName, @@ -380,7 +380,7 @@ object InMemPeriodicProcessesManager { processActionId: Option[ProcessActionId] ) - case class PeriodicProcessDeploymentEntity( + case class TestPeriodicProcessDeploymentEntity( id: PeriodicProcessDeploymentId, periodicProcessId: PeriodicProcessId, createdAt: LocalDateTime, @@ -394,8 +394,8 @@ object InMemPeriodicProcessesManager { ) def createPeriodicProcessDeployment( - processEntity: PeriodicProcessEntity, - processDeploymentEntity: PeriodicProcessDeploymentEntity + processEntity: TestPeriodicProcessEntity, + processDeploymentEntity: TestPeriodicProcessDeploymentEntity ): PeriodicProcessDeployment = { val process = createPeriodicProcessWithJson(processEntity) PeriodicProcessDeployment( @@ -411,7 +411,7 @@ object InMemPeriodicProcessesManager { } def createPeriodicDeploymentState( - processDeploymentEntity: PeriodicProcessDeploymentEntity + processDeploymentEntity: TestPeriodicProcessDeploymentEntity ): PeriodicProcessDeploymentState = { PeriodicProcessDeploymentState( processDeploymentEntity.deployedAt, @@ -421,7 +421,7 @@ object InMemPeriodicProcessesManager { } def createPeriodicProcessWithJson( - processEntity: PeriodicProcessEntity + processEntity: TestPeriodicProcessEntity ): PeriodicProcess = { val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( @@ -440,7 +440,7 @@ object InMemPeriodicProcessesManager { } def createPeriodicProcessWithoutJson( - processEntity: PeriodicProcessEntity + processEntity: TestPeriodicProcessEntity ): PeriodicProcess = { val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( @@ -458,14 +458,14 @@ object InMemPeriodicProcessesManager { ) } - private def prepareScheduleProperty(processEntity: PeriodicProcessEntity) = { + private def prepareScheduleProperty(processEntity: TestPeriodicProcessEntity) = { val scheduleProperty = io.circe.parser .decode[ScheduleProperty](processEntity.scheduleProperty) .fold(e => throw new IllegalArgumentException(e), identity) scheduleProperty } - private def scheduleDeploymentData(deployment: PeriodicProcessDeploymentEntity): ScheduleDeploymentData = { + private def scheduleDeploymentData(deployment: TestPeriodicProcessDeploymentEntity): ScheduleDeploymentData = { ScheduleDeploymentData( deployment.id, deployment.createdAt, From 05674307861b0120029c0de12ff9836fd158f546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Tue, 31 Dec 2024 00:46:59 +0100 Subject: [PATCH 09/11] unique table names across schemas --- ...1_060__PeriodicDeploymentManagerTablesDefinition.scala | 4 ++-- .../ui/db/entity/PeriodicProcessDeploymentsTable.scala | 8 ++++++-- .../nussknacker/ui/db/entity/PeriodicProcessesTable.scala | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala b/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala index 79f297f350d..07dac5b0a4b 100644 --- a/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala +++ b/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala @@ -35,7 +35,7 @@ object V1_060__PeriodicDeploymentManagerTablesDefinition { val periodicProcessDeploymentsTable = TableQuery[PeriodicProcessDeploymentsTable] class PeriodicProcessDeploymentsTable(tag: Tag) - extends Table[PeriodicProcessDeploymentEntity](tag, "periodic_process_deployments") { + extends Table[PeriodicProcessDeploymentEntity](tag, "periodic_scenario_deployments") { def id: Rep[Long] = column[Long]("id", O.PrimaryKey, O.AutoInc) @@ -88,7 +88,7 @@ object V1_060__PeriodicDeploymentManagerTablesDefinition { val periodicProcessesTable = TableQuery[PeriodicProcessesTable] - class PeriodicProcessesTable(tag: Tag) extends Table[PeriodicProcessEntity](tag, "periodic_processes") { + class PeriodicProcessesTable(tag: Tag) extends Table[PeriodicProcessEntity](tag, "periodic_scenarios") { def periodicProcessId: Rep[Long] = column[Long]("id", O.Unique, O.AutoInc) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessDeploymentsTable.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessDeploymentsTable.scala index f75cbff1395..3c73a0e8d34 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessDeploymentsTable.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessDeploymentsTable.scala @@ -1,7 +1,11 @@ package pl.touk.nussknacker.ui.db.entity import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.api.deployment.periodic.model.{PeriodicProcessDeploymentId, PeriodicProcessDeploymentStatus, PeriodicProcessId} +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{ + PeriodicProcessDeploymentId, + PeriodicProcessDeploymentStatus, + PeriodicProcessId +} import slick.jdbc.{JdbcProfile, JdbcType} import slick.lifted.ProvenShape import slick.sql.SqlProfile.ColumnOption.NotNull @@ -19,7 +23,7 @@ trait PeriodicProcessDeploymentsTableFactory extends PeriodicProcessesTableFacto MappedColumnType.base[PeriodicProcessDeploymentStatus, String](_.toString, PeriodicProcessDeploymentStatus.withName) class PeriodicProcessDeploymentsTable(tag: Tag) - extends Table[PeriodicProcessDeploymentEntity](tag, "periodic_process_deployments") { + extends Table[PeriodicProcessDeploymentEntity](tag, "periodic_scenario_deployments") { def id: Rep[PeriodicProcessDeploymentId] = column[PeriodicProcessDeploymentId]("id", O.PrimaryKey, O.AutoInc) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala index 60c2d07eb8b..70212f0215d 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala @@ -38,7 +38,7 @@ trait PeriodicProcessesTableFactory extends BaseEntityFactory { ) abstract class PeriodicProcessesTable[ENTITY <: PeriodicProcessEntity](tag: Tag) - extends Table[ENTITY](tag, "periodic_processes") { + extends Table[ENTITY](tag, "periodic_scenarios") { def id: Rep[PeriodicProcessId] = column[PeriodicProcessId]("id", O.PrimaryKey, O.AutoInc) From 2c916bc3a52f74fb7158c8a0d6dc03798cc15e5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Tue, 31 Dec 2024 10:21:19 +0100 Subject: [PATCH 10/11] table names in tests --- .../scala/pl/touk/nussknacker/test/base/db/DbTesting.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala index adf9fd3b3cf..ce16e649949 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala @@ -93,8 +93,8 @@ trait DbTesting extends BeforeAndAfterEach with BeforeAndAfterAll { session.prepareStatement("""delete from "environments"""").execute() session.prepareStatement("""delete from "processes"""").execute() session.prepareStatement("""delete from "fingerprints"""").execute() - session.prepareStatement("""delete from "periodic_processes"""").execute() - session.prepareStatement("""delete from "periodic_process_deployments"""").execute() + session.prepareStatement("""delete from "periodic_scenarios"""").execute() + session.prepareStatement("""delete from "periodic_scenario_deployments"""").execute() } } From bbb6345123c84918bd070a60c09865e19d314d6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Tue, 31 Dec 2024 15:34:51 +0100 Subject: [PATCH 11/11] refactors --- .../ui/process/repository/PeriodicProcessesRepository.scala | 2 +- .../pl/touk/nussknacker/test/mock/MockDeploymentManager.scala | 2 +- .../periodic/PeriodicProcessServiceIntegrationTest.scala | 2 +- designer/server/work/.gitkeep | 0 .../FlinkStreamingDeploymentManagerProviderHelper.scala | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 designer/server/work/.gitkeep diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala index 360a11d4974..4a97c46b980 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala @@ -1,6 +1,6 @@ package pl.touk.nussknacker.ui.process.repository -import _root_.db.util.DBIOActionInstances +import db.util.DBIOActionInstances import cats.Monad import com.github.tminglei.slickpg.ExPostgresProfile import com.typesafe.scalalogging.LazyLogging diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala index 418f95e94aa..35593d8d09a 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala @@ -1,11 +1,11 @@ package pl.touk.nussknacker.test.mock -import _root_.sttp.client3.testing.SttpBackendStub import akka.actor.ActorSystem import cats.data.Validated.valid import cats.data.ValidatedNel import com.google.common.collect.LinkedHashMultimap import com.typesafe.config.Config +import sttp.client3.testing.SttpBackendStub import pl.touk.nussknacker.engine._ import pl.touk.nussknacker.engine.api.definition.{ NotBlankParameterValidator, diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala index 39644b2174f..d0f09780942 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -1,12 +1,12 @@ package pl.touk.nussknacker.ui.process.periodic -import _root_.db.util.DBIOActionInstances.DB import com.cronutils.builder.CronBuilder import com.cronutils.model.CronType import com.cronutils.model.definition.CronDefinitionBuilder import com.cronutils.model.field.expression.FieldExpressionFactory.{on, questionMark} import com.typesafe.config.{Config, ConfigFactory} import com.typesafe.scalalogging.LazyLogging +import db.util.DBIOActionInstances.DB import org.scalatest.LoneElement._ import org.scalatest.OptionValues import org.scalatest.concurrent.ScalaFutures diff --git a/designer/server/work/.gitkeep b/designer/server/work/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala index b15fbe1a6f5..311a9d8cfde 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala @@ -1,8 +1,8 @@ package pl.touk.nussknacker.engine.management.streaming -import _root_.sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend import akka.actor.ActorSystem import org.asynchttpclient.DefaultAsyncHttpClientConfig +import sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend import pl.touk.nussknacker.engine._ import pl.touk.nussknacker.engine.api.component.DesignerWideComponentId import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider