From 53c85fa7ec09ae8694e269b78d5860b398a432ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Fri, 6 Dec 2024 15:51:48 +0100 Subject: [PATCH 01/16] DB optimization draft --- .../management/periodic/DeploymentActor.scala | 25 +-- .../management/periodic/JarManager.scala | 1 + .../periodic/PeriodicProcessService.scala | 82 ++++++--- .../db/PeriodicProcessesRepository.scala | 169 +++++++++++++----- .../periodic/flink/FlinkJarManager.scala | 10 +- .../model/DeploymentWithJarData.scala | 5 +- .../periodic/model/PeriodicProcess.scala | 4 +- .../model/PeriodicProcessDeployment.scala | 17 +- .../periodic/model/SchedulesState.scala | 55 ++++-- .../AdditionalDeploymentDataProvider.scala | 4 +- .../service/PeriodicProcessListener.scala | 13 +- .../service/ProcessConfigEnricher.scala | 2 +- .../periodic/DeploymentActorTest.scala | 21 ++- .../management/periodic/JarManagerStub.scala | 3 +- .../PeriodicProcessDeploymentGen.scala | 44 ++--- .../periodic/PeriodicProcessGen.scala | 3 +- ...eriodicProcessServiceIntegrationTest.scala | 42 ++--- .../periodic/PeriodicProcessServiceTest.scala | 18 +- .../db/InMemPeriodicProcessesRepository.scala | 38 ++-- 19 files changed, 360 insertions(+), 196 deletions(-) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActor.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActor.scala index 091a1cc5052..a204313e4e8 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActor.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActor.scala @@ -8,7 +8,10 @@ import pl.touk.nussknacker.engine.management.periodic.DeploymentActor.{ DeploymentCompleted, WaitingForDeployment } -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment +import pl.touk.nussknacker.engine.management.periodic.model.{ + PeriodicProcessDeployment, + PeriodicProcessDeploymentWithFullProcess +} import scala.concurrent.Future import scala.concurrent.duration._ @@ -21,8 +24,8 @@ object DeploymentActor { } private[periodic] def props( - findToBeDeployed: => Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]], - deploy: PeriodicProcessDeployment[CanonicalProcess] => Future[Unit], + findToBeDeployed: => Future[Seq[PeriodicProcessDeploymentWithFullProcess]], + deploy: PeriodicProcessDeploymentWithFullProcess => Future[Unit], interval: FiniteDuration ) = { Props(new DeploymentActor(findToBeDeployed, deploy, interval)) @@ -30,14 +33,14 @@ object DeploymentActor { private[periodic] case object CheckToBeDeployed - private case class WaitingForDeployment(ids: List[PeriodicProcessDeployment[CanonicalProcess]]) + private case class WaitingForDeployment(ids: List[PeriodicProcessDeploymentWithFullProcess]) private case object DeploymentCompleted } class DeploymentActor( - findToBeDeployed: => Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]], - deploy: PeriodicProcessDeployment[CanonicalProcess] => Future[Unit], + findToBeDeployed: => Future[Seq[PeriodicProcessDeploymentWithFullProcess]], + deploy: PeriodicProcessDeploymentWithFullProcess => Future[Unit], interval: FiniteDuration ) extends Actor with Timers @@ -55,25 +58,25 @@ class DeploymentActor( logger.trace("Checking scenarios to be deployed") findToBeDeployed.onComplete { case Success(runDetailsSeq) => - logger.debug(s"Found ${runDetailsSeq.size} to be deployed: ${runDetailsSeq.map(_.display)}") + logger.debug(s"Found ${runDetailsSeq.size} to be deployed: ${runDetailsSeq.map(_.deployment.display)}") self ! WaitingForDeployment(runDetailsSeq.toList) case Failure(exception) => logger.error("Finding scenarios to be deployed failed unexpectedly", exception) } case WaitingForDeployment(Nil) => case WaitingForDeployment(runDetails :: _) => - logger.info(s"Found a scenario to be deployed: ${runDetails.display}") - context.become(receiveOngoingDeployment(runDetails)) + logger.info(s"Found a scenario to be deployed: ${runDetails.deployment.display}") + context.become(receiveOngoingDeployment(runDetails.deployment)) deploy(runDetails) onComplete { case Success(_) => self ! DeploymentCompleted case Failure(exception) => - logger.error(s"Deployment of ${runDetails.display} failed unexpectedly", exception) + logger.error(s"Deployment of ${runDetails.deployment.display} failed unexpectedly", exception) self ! DeploymentCompleted } } - private def receiveOngoingDeployment(runDetails: PeriodicProcessDeployment[CanonicalProcess]): Receive = { + private def receiveOngoingDeployment(runDetails: PeriodicProcessDeployment): Receive = { case CheckToBeDeployed => logger.debug(s"Still waiting for ${runDetails.display} to be deployed") case DeploymentCompleted => diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala index 255ab8108e9..ad22cb3df90 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala @@ -1,6 +1,7 @@ package pl.touk.nussknacker.engine.management.periodic import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala index e346c92ec08..e0d098cd034 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala @@ -78,7 +78,7 @@ class PeriodicProcessService( scenarioActivityId = ScenarioActivityId.random, user = ScenarioUser.internalNuUser, date = metadata.dateDeployed.getOrElse(metadata.dateFinished), - scenarioVersionId = Some(ScenarioVersionId.from(deployment.periodicProcess.processVersion.versionId)), + scenarioVersionId = Some(ScenarioVersionId.from(deployment.periodicProcessMetadata.versionId)), scheduledExecutionStatus = metadata.status, dateFinished = metadata.dateFinished, scheduleName = deployment.scheduleName.display, @@ -163,7 +163,7 @@ class PeriodicProcessService( scheduledProcessesRepository .schedule(process.id, name, date, deploymentRetryConfig.deployMaxRetries) .flatMap { data => - handleEvent(ScheduledEvent(data, firstSchedule = true)) + handleEvent(ScheduledEvent(data.deployment, firstSchedule = true)) } case (name, None) => logger.warn(s"Schedule $name does not have date to schedule") @@ -174,7 +174,7 @@ class PeriodicProcessService( .map(_ => ()) } - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = { + def findToBeDeployed: Future[Seq[PeriodicProcessDeploymentWithFullProcess]] = { for { toBeDeployed <- scheduledProcessesRepository.findToBeDeployed.run.flatMap { toDeployList => Future.sequence(toDeployList.map(checkIfNotRunning)).map(_.flatten) @@ -182,22 +182,24 @@ class PeriodicProcessService( // We retry scenarios that failed on deployment. Failure recovery of running scenarios should be handled by Flink's restart strategy toBeRetried <- scheduledProcessesRepository.findToBeRetried.run // We don't block scheduled deployments by retries - } yield toBeDeployed.sortBy(d => (d.runAt, d.createdAt)) ++ toBeRetried.sortBy(d => (d.nextRetryAt, d.createdAt)) + } yield toBeDeployed.sortBy(d => (d.deployment.runAt, d.deployment.createdAt)) ++ toBeRetried.sortBy(d => + (d.deployment.nextRetryAt, d.deployment.createdAt) + ) } // Currently we don't allow simultaneous runs of one scenario - only sequential, so if other schedule kicks in, it'll have to wait // TODO: we show allow to deploy scenarios with different scheduleName to be deployed simultaneous private def checkIfNotRunning( - toDeploy: PeriodicProcessDeployment[CanonicalProcess] - ): Future[Option[PeriodicProcessDeployment[CanonicalProcess]]] = { + toDeploy: PeriodicProcessDeploymentWithFullProcess + ): Future[Option[PeriodicProcessDeploymentWithFullProcess]] = { delegateDeploymentManager - .getProcessStates(toDeploy.periodicProcess.processVersion.processName)(DataFreshnessPolicy.Fresh) + .getProcessStates(toDeploy.deployment.periodicProcessMetadata.processName)(DataFreshnessPolicy.Fresh) .map( _.value .map(_.status) .find(SimpleStateStatus.DefaultFollowingDeployStatuses.contains) .map { _ => - logger.debug(s"Deferring run of ${toDeploy.display} as scenario is currently running") + logger.debug(s"Deferring run of ${toDeploy.deployment.display} as scenario is currently running") None } .getOrElse(Some(toDeploy)) @@ -298,9 +300,14 @@ class PeriodicProcessService( case Right(Some(futureDate)) => logger.info(s"Rescheduling ${deployment.display} to $futureDate") val action = scheduledProcessesRepository - .schedule(process.id, deployment.scheduleName, futureDate, deploymentRetryConfig.deployMaxRetries) + .schedule( + periodicProcessMetadata.id, + deployment.scheduleName, + futureDate, + deploymentRetryConfig.deployMaxRetries + ) .flatMap { data => - handleEvent(ScheduledEvent(data, firstSchedule = false)) + handleEvent(ScheduledEvent(data.deployment, firstSchedule = false)) } Some(action) case Right(None) => @@ -318,9 +325,9 @@ class PeriodicProcessService( } if (scheduleActions.forall(_.isEmpty)) { - logger.info(s"No scheduled deployments for periodic process: ${process.id.value}. Deactivating") - deactivateAction(process).flatMap { _ => - markProcessActionExecutionFinished(processScheduleData.process.processActionId) + logger.info(s"No scheduled deployments for periodic process: ${periodicProcessMetadata.id.value}. Deactivating") + deactivateAction(periodicProcessMetadata).flatMap { _ => + markProcessActionExecutionFinished(processScheduleData.periodicProcessMetadata.processActionId) } } else @@ -332,11 +339,11 @@ class PeriodicProcessService( for { _ <- scheduledProcessesRepository.markFinished(deployment.id) currentState <- scheduledProcessesRepository.findProcessData(deployment.id) - } yield handleEvent(FinishedEvent(currentState, state)) + } yield handleEvent(FinishedEvent(currentState.deployment, state)) } private def handleFailedDeployment( - deployment: PeriodicProcessDeployment[_], + deployment: PeriodicProcessDeployment, state: Option[StatusDetails] ): RepositoryAction[Unit] = { def calculateNextRetryAt = now().plus(deploymentRetryConfig.deployRetryPenalize.toMillis, ChronoUnit.MILLIS) @@ -359,7 +366,7 @@ class PeriodicProcessService( for { _ <- scheduledProcessesRepository.markFailedOnDeployWithStatus(deployment.id, status, retriesLeft, nextRetryAt) currentState <- scheduledProcessesRepository.findProcessData(deployment.id) - } yield handleEvent(FailedOnDeployEvent(currentState, state)) + } yield handleEvent(FailedOnDeployEvent(currentState.deployment, state)) } private def markFailedAction( @@ -370,23 +377,26 @@ class PeriodicProcessService( for { _ <- scheduledProcessesRepository.markFailed(deployment.id) currentState <- scheduledProcessesRepository.findProcessData(deployment.id) - } yield handleEvent(FailedOnRunEvent(currentState, state)) + } yield handleEvent(FailedOnRunEvent(currentState.deployment, state)) } def deactivate(processName: ProcessName): Future[Iterable[DeploymentId]] = for { activeSchedules <- getLatestDeploymentsForActiveSchedules(processName) (runningDeploymentsForSchedules, _) <- synchronizeDeploymentsStates(processName, activeSchedules) - _ <- activeSchedules.groupedByPeriodicProcess.map(p => deactivateAction(p.process)).sequence.runWithCallbacks + _ <- activeSchedules.groupedByPeriodicProcess + .map(p => deactivateAction(p.periodicProcessMetadata)) + .sequence + .runWithCallbacks } yield runningDeploymentsForSchedules.map(deployment => DeploymentId(deployment.toString)) - private def deactivateAction(process: PeriodicProcess[_]): RepositoryAction[Callback] = { + private def deactivateAction(process: PeriodicProcessMetadata): RepositoryAction[Callback] = { logger.info(s"Deactivate periodic process id: ${process.id.value}") for { _ <- scheduledProcessesRepository.markInactive(process.id) // we want to delete jars only after we successfully mark process as inactive. It's better to leave jar garbage than // have process without jar - } yield () => jarManager.deleteJar(process.deploymentData.jarFileName) + } yield () => jarManager.deleteJar(process.jarFileName) } private def markProcessActionExecutionFinished( @@ -399,8 +409,17 @@ class PeriodicProcessService( .map(_ => ()) } - def deploy(deployment: PeriodicProcessDeployment[CanonicalProcess]): Future[Unit] = { + def deploy(deploymentWithCanonicalProcess: PeriodicProcessDeploymentWithFullProcess): Future[Unit] = { // TODO: set status before deployment? + val deployment = deploymentWithCanonicalProcess.deployment + val periodicProcessMetadata = deployment.periodicProcessMetadata + val deploymentWithJarData = DeploymentWithJarData( + processName = periodicProcessMetadata.processName, + versionId = periodicProcessMetadata.versionId, + process = deploymentWithCanonicalProcess.process, + inputConfigDuringExecutionJson = deploymentWithCanonicalProcess.inputConfigDuringExecutionJson, + jarFileName = periodicProcessMetadata.jarFileName + ) val id = deployment.id val deploymentData = DeploymentData( DeploymentId(id.toString), @@ -412,10 +431,14 @@ class PeriodicProcessService( AdditionalComponentConfigsForRuntimeExtractor.getRequiredAdditionalConfigsForRuntime(configsFromProvider) ) ) - val deploymentWithJarData = deployment.periodicProcess.deploymentData val deploymentAction = for { _ <- Future.successful( - logger.info("Deploying scenario {} for deployment id {}", deploymentWithJarData.processVersion, id) + logger.info( + "Deploying scenario name={} versionId={} for deployment id {}", + periodicProcessMetadata.processName, + periodicProcessMetadata.versionId, + id + ) ) enrichedProcessConfig <- processConfigEnricher.onDeploy( ProcessConfigEnricher.DeployData( @@ -431,12 +454,17 @@ class PeriodicProcessService( } yield externalDeploymentId deploymentAction .flatMap { externalDeploymentId => - logger.info("Scenario has been deployed {} for deployment id {}", deploymentWithJarData.processVersion, id) + logger.info( + "Scenario has been deployed name={} versionId={} for deployment id {}", + periodicProcessMetadata.processName, + periodicProcessMetadata.versionId, + id + ) // TODO: add externalDeploymentId?? scheduledProcessesRepository .markDeployed(id) .flatMap(_ => scheduledProcessesRepository.findProcessData(id)) - .flatMap(afterChange => handleEvent(DeployedEvent(afterChange, externalDeploymentId))) + .flatMap(afterChange => handleEvent(DeployedEvent(afterChange.deployment, externalDeploymentId))) .run } // We can recover since deployment actor watches only future completion. @@ -483,7 +511,7 @@ class PeriodicProcessService( deployment.createdAt, deployment.runAt, deployment.state.status, - scheduleData.process.active, + scheduleData.periodicProcessMetadata.active, runtimeStatuses.getStatus(deployment.id) ) } @@ -531,7 +559,7 @@ class PeriodicProcessService( } private def scheduledExecutionStatusAndDateFinished( - entity: PeriodicProcessDeployment[Unit], + entity: PeriodicProcessDeployment, ): Option[FinishedScheduledExecutionMetadata] = { for { status <- entity.state.status match { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala index 267f6316fbf..0e1a90aa5dd 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala @@ -4,11 +4,15 @@ import cats.Monad import com.github.tminglei.slickpg.ExPostgresProfile import com.typesafe.scalalogging.LazyLogging import io.circe.parser.decode -import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId -import pl.touk.nussknacker.engine.api.process.ProcessName +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.management.periodic._ +import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.{ + createPeriodicProcessMetadata, + createPeriodicProcessWithoutJson, + prepareScheduleProperty +} import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.management.periodic.model._ import slick.dbio.{DBIOAction, Effect, NoStream} @@ -21,14 +25,36 @@ import scala.language.higherKinds object PeriodicProcessesRepository { + def createPeriodicProcessDeploymentWithFullProcess( + processEntity: PeriodicProcessEntity, + processDeploymentEntity: PeriodicProcessDeploymentEntity + ): PeriodicProcessDeploymentWithFullProcess = { + val process = createPeriodicProcessWithJson(processEntity) + PeriodicProcessDeploymentWithFullProcess( + PeriodicProcessesRepository.createPeriodicProcessDeploymentFromProcess(process, processDeploymentEntity), + process.deploymentData.process, + process.deploymentData.inputConfigDuringExecutionJson, + ) + } + def createPeriodicProcessDeployment( processEntity: PeriodicProcessEntity, processDeploymentEntity: PeriodicProcessDeploymentEntity - ): PeriodicProcessDeployment[CanonicalProcess] = { + ): PeriodicProcessDeployment = { val process = createPeriodicProcessWithJson(processEntity) + createPeriodicProcessDeploymentFromProcess( + process, + processDeploymentEntity + ) + } + + def createPeriodicProcessDeploymentFromProcess( + periodicProcess: PeriodicProcess[CanonicalProcess], + processDeploymentEntity: PeriodicProcessDeploymentEntity + ): PeriodicProcessDeployment = { PeriodicProcessDeployment( processDeploymentEntity.id, - process, + createPeriodicProcessMetadata(periodicProcess), processDeploymentEntity.createdAt, processDeploymentEntity.runAt, ScheduleName(processDeploymentEntity.scheduleName), @@ -49,12 +75,12 @@ object PeriodicProcessesRepository { } def createPeriodicProcessWithJson(processEntity: PeriodicProcessEntity): PeriodicProcess[CanonicalProcess] = { - val processVersion = createProcessVersion(processEntity) - val scheduleProperty = prepareScheduleProperty(processEntity) + val scheduleProperty = prepareScheduleProperty(processEntity.scheduleProperty) PeriodicProcess( processEntity.id, model.DeploymentWithJarData( - processVersion = processVersion, + processName = processEntity.processName, + versionId = processEntity.processVersionId, inputConfigDuringExecutionJson = processEntity.inputConfigDuringExecutionJson, jarFileName = processEntity.jarFileName, process = processEntity.processJson.getOrElse( @@ -69,12 +95,12 @@ object PeriodicProcessesRepository { } def createPeriodicProcessWithoutJson(processEntity: PeriodicProcessEntity): PeriodicProcess[Unit] = { - val processVersion = createProcessVersion(processEntity) - val scheduleProperty = prepareScheduleProperty(processEntity) + val scheduleProperty = prepareScheduleProperty(processEntity.scheduleProperty) PeriodicProcess( processEntity.id, model.DeploymentWithJarData( - processVersion = processVersion, + processName = processEntity.processName, + versionId = processEntity.processVersionId, inputConfigDuringExecutionJson = processEntity.inputConfigDuringExecutionJson, jarFileName = processEntity.jarFileName, process = () @@ -86,14 +112,20 @@ object PeriodicProcessesRepository { ) } - private def prepareScheduleProperty(processEntity: PeriodicProcessEntity) = { - val scheduleProperty = decode[ScheduleProperty](processEntity.scheduleProperty) - .fold(e => throw new IllegalArgumentException(e), identity) - scheduleProperty + def createPeriodicProcessMetadata(process: PeriodicProcess[_]): PeriodicProcessMetadata = { + PeriodicProcessMetadata( + id = process.id, + processName = process.deploymentData.processName, + versionId = process.deploymentData.versionId, + jarFileName = process.deploymentData.jarFileName, + scheduleProperty = process.scheduleProperty, + active = process.active, + processActionId = process.processActionId + ) } - private def createProcessVersion(processEntity: PeriodicProcessEntity): ProcessVersion = { - ProcessVersion.empty.copy(versionId = processEntity.processVersionId, processName = processEntity.processName) + def prepareScheduleProperty(scheduleProperty: String) = { + decode[ScheduleProperty](scheduleProperty).fold(e => throw new IllegalArgumentException(e), identity) } } @@ -133,15 +165,15 @@ trait PeriodicProcessesRepository { deploymentsPerScheduleMaxCount: Int ): Action[SchedulesState] - def findToBeDeployed: Action[Seq[PeriodicProcessDeployment[CanonicalProcess]]] + def findToBeDeployed: Action[Seq[PeriodicProcessDeploymentWithFullProcess]] - def findToBeRetried: Action[Seq[PeriodicProcessDeployment[CanonicalProcess]]] + def findToBeRetried: Action[Seq[PeriodicProcessDeploymentWithFullProcess]] def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus] ): Action[SchedulesState] - def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment[CanonicalProcess]] + def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeploymentWithFullProcess] def findProcessData(processName: ProcessName): Action[Seq[PeriodicProcess[CanonicalProcess]]] @@ -163,7 +195,7 @@ trait PeriodicProcessesRepository { scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Action[PeriodicProcessDeployment[CanonicalProcess]] + ): Action[PeriodicProcessDeploymentWithFullProcess] } @@ -192,9 +224,13 @@ class SlickPeriodicProcessesRepository( ): Action[SchedulesState] = { PeriodicProcessesWithoutJson .filter(_.processName === scenarioName) + .map(extractPeriodicProcessMetadataColumns) .join(PeriodicProcessDeployments) - .on(_.id === _.periodicProcessId) + .on(_._1 === _.periodicProcessId) .result + .map(_.map { case (periodicProcessMetadataColumnValues, deploymentEntity) => + (createPeriodicProcessMetadataFromColumnValues(periodicProcessMetadataColumnValues), deploymentEntity) + }) .map(toSchedulesState) } @@ -205,8 +241,8 @@ class SlickPeriodicProcessesRepository( ): Action[PeriodicProcess[CanonicalProcess]] = { val processEntity = PeriodicProcessEntity( id = PeriodicProcessId(-1), - processName = deploymentWithJarData.processVersion.processName, - processVersionId = deploymentWithJarData.processVersion.versionId, + processName = deploymentWithJarData.processName, + processVersionId = deploymentWithJarData.versionId, processingType = processingType, processJson = Some(deploymentWithJarData.process), inputConfigDuringExecutionJson = deploymentWithJarData.inputConfigDuringExecutionJson, @@ -222,30 +258,32 @@ class SlickPeriodicProcessesRepository( private def now(): LocalDateTime = LocalDateTime.now(clock) - override def findToBeDeployed: Action[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = + override def findToBeDeployed: Action[Seq[PeriodicProcessDeploymentWithFullProcess]] = activePeriodicProcessWithDeploymentQuery .filter { case (_, d) => d.runAt <= now() && d.status === (PeriodicProcessDeploymentStatus.Scheduled: PeriodicProcessDeploymentStatus) } .result - .map(_.map((PeriodicProcessesRepository.createPeriodicProcessDeployment _).tupled)) + .map(_.map((PeriodicProcessesRepository.createPeriodicProcessDeploymentWithFullProcess _).tupled)) - override def findToBeRetried: Action[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = + override def findToBeRetried: Action[Seq[PeriodicProcessDeploymentWithFullProcess]] = activePeriodicProcessWithDeploymentQuery .filter { case (_, d) => d.nextRetryAt <= now() && d.status === (PeriodicProcessDeploymentStatus.RetryingDeploy: PeriodicProcessDeploymentStatus) } .result - .map(_.map((PeriodicProcessesRepository.createPeriodicProcessDeployment _).tupled)) + .map(_.map((PeriodicProcessesRepository.createPeriodicProcessDeploymentWithFullProcess _).tupled)) - override def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment[CanonicalProcess]] = { + override def findProcessData( + id: PeriodicProcessDeploymentId + ): Action[PeriodicProcessDeploymentWithFullProcess] = { (PeriodicProcessesWithJson join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) .filter { case (_, deployment) => deployment.id === id } .result .head - .map((PeriodicProcessesRepository.createPeriodicProcessDeployment _).tupled) + .map((PeriodicProcessesRepository.createPeriodicProcessDeploymentWithFullProcess _).tupled) } override def findProcessData(processName: ProcessName): Action[Seq[PeriodicProcess[CanonicalProcess]]] = { @@ -347,7 +385,7 @@ class SlickPeriodicProcessesRepository( private def getLatestDeploymentsForEachSchedulePostgres( periodicProcessesQuery: Query[PeriodicProcessesTable, PeriodicProcessEntity, Seq], deploymentsPerScheduleMaxCount: Int - ): Action[Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)]] = { + ): Action[Seq[(PeriodicProcessMetadata, PeriodicProcessDeploymentEntity)]] = { // To effectively limit deployments to given count for each schedule in one query, we use window functions in slick import ExPostgresProfile.api._ import com.github.tminglei.slickpg.window.PgWindowFuncSupport.WindowFunctions._ @@ -369,10 +407,49 @@ class SlickPeriodicProcessesRepository( } .subquery .filter(_._1 <= deploymentsPerScheduleMaxCount.longValue()) - .map { case (_, process, deployment) => - (process, deployment) - } + .map { case (_, process, deployment) => (extractPeriodicProcessMetadataColumns(process), deployment) } .result + .map(_.map { case (periodicProcessMetadataColumnValues, deploymentEntity) => + (createPeriodicProcessMetadataFromColumnValues(periodicProcessMetadataColumnValues), deploymentEntity) + }) + } + + private def extractPeriodicProcessMetadataColumns( + periodicProcessesTable: PeriodicProcessesTable, + ): ( + Rep[PeriodicProcessId], + Rep[ProcessName], + Rep[VersionId], + Rep[String], + Rep[String], + Rep[Boolean], + Rep[Option[ProcessActionId]] + ) = + ( + periodicProcessesTable.id, + periodicProcessesTable.processName, + periodicProcessesTable.processVersionId, + periodicProcessesTable.jarFileName, + periodicProcessesTable.scheduleProperty, + periodicProcessesTable.active, + periodicProcessesTable.processActionId, + ) + + private def createPeriodicProcessMetadataFromColumnValues( + columnValues: (PeriodicProcessId, ProcessName, VersionId, String, String, Boolean, Option[ProcessActionId]) + ): PeriodicProcessMetadata = { + columnValues match { + case (id, processName, versionId, jarFileName, scheduleProperty, active, processActionId) => + PeriodicProcessMetadata( + id, + processName, + versionId, + jarFileName, + prepareScheduleProperty(scheduleProperty), + active, + processActionId + ) + } } // This variant of method is much less optimal than postgres one. It is highly recommended to use postgres with periodics @@ -381,7 +458,7 @@ class SlickPeriodicProcessesRepository( private def getLatestDeploymentsForEachScheduleJdbcGeneric( periodicProcessesQuery: Query[PeriodicProcessesTable, PeriodicProcessEntity, Seq], deploymentsPerScheduleMaxCount: Int - ): Action[Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)]] = { + ): Action[Seq[(PeriodicProcessMetadata, PeriodicProcessDeploymentEntity)]] = { // It is debug instead of warn to not bloast logs when e.g. for some reasons is used hsql under the hood logger.debug( "WARN: Using not optimized version of getLatestDeploymentsForEachSchedule that not uses window functions" @@ -399,6 +476,11 @@ class SlickPeriodicProcessesRepository( .map(_.map((process, _))) }) .map(_.flatten) + .map(_.map { case (entity, scheduleName) => + val process = createPeriodicProcessWithoutJson(entity) + val metadata = createPeriodicProcessMetadata(process) + (metadata, scheduleName) + }) deploymentsForSchedules <- DBIO .sequence(schedulesForProcesses.map { case (process, scheduleName) => @@ -421,7 +503,7 @@ class SlickPeriodicProcessesRepository( scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Action[PeriodicProcessDeployment[CanonicalProcess]] = { + ): Action[PeriodicProcessDeploymentWithFullProcess] = { val deploymentEntity = PeriodicProcessDeploymentEntity( id = PeriodicProcessDeploymentId(-1), periodicProcessId = id, @@ -452,21 +534,22 @@ class SlickPeriodicProcessesRepository( join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) } - private def toSchedulesState(list: Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)]): SchedulesState = { + private def toSchedulesState( + list: Seq[(PeriodicProcessMetadata, PeriodicProcessDeploymentEntity)] + ): SchedulesState = { SchedulesState( list - .map { case (process, deployment) => - val scheduleId = ScheduleId(process.id, ScheduleName(deployment.scheduleName)) - val scheduleDataWithoutDeployment = - (scheduleId, PeriodicProcessesRepository.createPeriodicProcessWithoutJson(process)) + .map { case (periodicProcessMetadata, deployment) => + val scheduleId = ScheduleId(periodicProcessMetadata.id, ScheduleName(deployment.scheduleName)) + val scheduleData = (scheduleId, periodicProcessMetadata) val scheduleDeployment = ScheduleDeploymentData(deployment) - (scheduleDataWithoutDeployment, scheduleDeployment) + (scheduleData, scheduleDeployment) } .toList .toGroupedMap .toList - .map { case ((scheduleId, process), deployments) => - scheduleId -> ScheduleData(process, deployments) + .map { case ((scheduleId, periodicProcessMetadata), deployments) => + scheduleId -> ScheduleData(periodicProcessMetadata, deployments) } .toMap ) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkJarManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkJarManager.scala index 297d21a1d7c..a9b9f6a6bca 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkJarManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkJarManager.scala @@ -4,6 +4,7 @@ import com.typesafe.scalalogging.LazyLogging import org.apache.flink.api.common.JobID import pl.touk.nussknacker.engine.{BaseModelData, newdeployment} import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData @@ -55,7 +56,8 @@ private[periodic] class FlinkJarManager( logger.info(s"Prepare deployment for scenario: $processVersion") copyJarToLocalDir(processVersion).map { jarFileName => DeploymentWithJarData( - processVersion = processVersion, + processName = processVersion.processName, + versionId = processVersion.versionId, process = canonicalProcess, inputConfigDuringExecutionJson = inputConfigDuringExecution.serialized, jarFileName = jarFileName @@ -77,14 +79,14 @@ private[periodic] class FlinkJarManager( deploymentWithJarData: DeploymentWithJarData[CanonicalProcess], deploymentData: DeploymentData ): Future[Option[ExternalDeploymentId]] = { - val processVersion = deploymentWithJarData.processVersion logger.info( - s"Deploying scenario ${processVersion.processName}, version id: ${processVersion.versionId} and jar: ${deploymentWithJarData.jarFileName}" + s"Deploying scenario ${deploymentWithJarData.processName}, version id: ${deploymentWithJarData.versionId} and jar: ${deploymentWithJarData.jarFileName}" ) val jarFile = jarsDir.resolve(deploymentWithJarData.jarFileName).toFile val args = FlinkDeploymentManager.prepareProgramArgs( deploymentWithJarData.inputConfigDuringExecutionJson, - processVersion, + ProcessVersion.empty + .copy(processName = deploymentWithJarData.processName, versionId = deploymentWithJarData.versionId), deploymentData, deploymentWithJarData.process ) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/DeploymentWithJarData.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/DeploymentWithJarData.scala index be290b90e9a..edacf2729c9 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/DeploymentWithJarData.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/DeploymentWithJarData.scala @@ -1,9 +1,10 @@ package pl.touk.nussknacker.engine.management.periodic.model -import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} case class DeploymentWithJarData[ProcessRep]( - processVersion: ProcessVersion, + processName: ProcessName, + versionId: VersionId, process: ProcessRep, inputConfigDuringExecutionJson: String, jarFileName: String diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala index effee642b22..ba4d312cb92 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala @@ -16,6 +16,4 @@ case class PeriodicProcess[ProcessRep]( active: Boolean, createdAt: LocalDateTime, processActionId: Option[ProcessActionId] -) { - val processVersion: ProcessVersion = deploymentData.processVersion -} +) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala index cca749d2036..ebbe7afcbd7 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala @@ -1,15 +1,22 @@ package pl.touk.nussknacker.engine.management.periodic.model -import pl.touk.nussknacker.engine.management.periodic.{MultipleScheduleProperty, SingleScheduleProperty} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.management.periodic.{MultipleScheduleProperty, SingleScheduleProperty} import slick.lifted.MappedTo import java.time.{Clock, LocalDateTime} +case class PeriodicProcessDeploymentWithFullProcess( + deployment: PeriodicProcessDeployment, + process: CanonicalProcess, + inputConfigDuringExecutionJson: String, +) + // TODO: We should separate schedules concept from deployments - fully switch to ScheduleData and ScheduleDeploymentData -case class PeriodicProcessDeployment[ProcessRep]( +case class PeriodicProcessDeployment( id: PeriodicProcessDeploymentId, - periodicProcess: PeriodicProcess[ProcessRep], + periodicProcessMetadata: PeriodicProcessMetadata, createdAt: LocalDateTime, runAt: LocalDateTime, scheduleName: ScheduleName, @@ -19,7 +26,7 @@ case class PeriodicProcessDeployment[ProcessRep]( ) { def nextRunAt(clock: Clock): Either[String, Option[LocalDateTime]] = - (periodicProcess.scheduleProperty, scheduleName.value) match { + (periodicProcessMetadata.scheduleProperty, scheduleName.value) match { case (MultipleScheduleProperty(schedules), Some(name)) => schedules.get(name).toRight(s"Failed to find schedule: $scheduleName").flatMap(_.nextRunAt(clock)) case (e: SingleScheduleProperty, None) => e.nextRunAt(clock) @@ -27,7 +34,7 @@ case class PeriodicProcessDeployment[ProcessRep]( } def display: String = - s"${periodicProcess.processVersion} with scheduleName=${scheduleName.display} and deploymentId=$id" + s"${periodicProcessMetadata.processName} with scheduleName=${scheduleName.display} and deploymentId=$id" } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala index ff8aeaac666..f5fb1ce63cc 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala @@ -1,6 +1,8 @@ package pl.touk.nussknacker.engine.management.periodic.model -import pl.touk.nussknacker.engine.api.process.ProcessName +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} +import pl.touk.nussknacker.engine.management.periodic.ScheduleProperty import pl.touk.nussknacker.engine.management.periodic.db.{PeriodicProcessDeploymentEntity, PeriodicProcessesRepository} import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap @@ -19,14 +21,15 @@ case class SchedulesState(schedules: Map[ScheduleId, ScheduleData]) { def isEmpty: Boolean = schedules.isEmpty def groupByProcessName: Map[ProcessName, SchedulesState] = - schedules.groupBy(_._2.process.processVersion.processName).mapValuesNow(SchedulesState) + schedules.groupBy(_._2.periodicProcessMetadata.processName).mapValuesNow(SchedulesState) lazy val groupedByPeriodicProcess: List[PeriodicProcessScheduleData] = - schedules.toList.groupBy(_._2.process).toList.map { case (periodicProcess, groupedSchedules) => - val deploymentsForSchedules = groupedSchedules.flatMap { case (scheduleId, scheduleData) => - scheduleData.latestDeployments.map(_.toFullDeploymentData(periodicProcess, scheduleId.scheduleName)) - } - PeriodicProcessScheduleData(periodicProcess, deploymentsForSchedules) + schedules.toList.groupBy(_._2.periodicProcessMetadata).toList.map { + case (periodicProcessMetadata, groupedSchedules) => + val deploymentsForSchedules = groupedSchedules.flatMap { case (scheduleId, scheduleData) => + scheduleData.latestDeployments.map(_.toFullDeploymentData(periodicProcessMetadata, scheduleId.scheduleName)) + } + PeriodicProcessScheduleData(periodicProcessMetadata, deploymentsForSchedules) } } @@ -35,7 +38,20 @@ case class SchedulesState(schedules: Map[ScheduleId, ScheduleData]) { // For most operations it will contain only one latest deployment but for purpose of statuses of historical deployments // it has list instead of one element. // This structure should contain SingleScheduleProperty as well. See note above -case class ScheduleData(process: PeriodicProcess[Unit], latestDeployments: List[ScheduleDeploymentData]) +case class ScheduleData( + periodicProcessMetadata: PeriodicProcessMetadata, + latestDeployments: List[ScheduleDeploymentData] +) + +case class PeriodicProcessMetadata( + id: PeriodicProcessId, + processName: ProcessName, + versionId: VersionId, + jarFileName: String, + scheduleProperty: ScheduleProperty, + active: Boolean, + processActionId: Option[ProcessActionId], +) // To identify schedule we need scheduleName - None for SingleScheduleProperty and Some(key) for MultipleScheduleProperty keys // Also we need PeriodicProcessId to distinguish between active schedules and some inactive from the past for the same PeriodicProcessId @@ -53,10 +69,19 @@ case class ScheduleDeploymentData( ) { def toFullDeploymentData( - process: PeriodicProcess[Unit], + periodicProcessMetadata: PeriodicProcessMetadata, scheduleName: ScheduleName - ): PeriodicProcessDeployment[Unit] = - PeriodicProcessDeployment(id, process, createdAt, runAt, scheduleName, retriesLeft, nextRetryAt, state) + ): PeriodicProcessDeployment = + PeriodicProcessDeployment( + id, + periodicProcessMetadata, + createdAt, + runAt, + scheduleName, + retriesLeft, + nextRetryAt, + state + ) def display = s"deploymentId=$id" @@ -80,14 +105,14 @@ object ScheduleDeploymentData { // These below are temporary structures, see notice next to SchedulesState case class PeriodicProcessScheduleData( - process: PeriodicProcess[Unit], - deployments: List[PeriodicProcessDeployment[Unit]] + periodicProcessMetadata: PeriodicProcessMetadata, + deployments: List[PeriodicProcessDeployment] ) { - def existsDeployment(predicate: PeriodicProcessDeployment[Unit] => Boolean): Boolean = deployments.exists(predicate) + def existsDeployment(predicate: PeriodicProcessDeployment => Boolean): Boolean = deployments.exists(predicate) def display: String = { val deploymentsForSchedules = deployments.map(_.display) - s"processName=${process.processVersion.processName}, deploymentsForSchedules=$deploymentsForSchedules" + s"processName=${periodicProcessMetadata.processName}, deploymentsForSchedules=$deploymentsForSchedules" } } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/AdditionalDeploymentDataProvider.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/AdditionalDeploymentDataProvider.scala index 1871cf05518..6883383c4f9 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/AdditionalDeploymentDataProvider.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/AdditionalDeploymentDataProvider.scala @@ -7,13 +7,13 @@ import java.time.format.DateTimeFormatter trait AdditionalDeploymentDataProvider { - def prepareAdditionalData(runDetails: PeriodicProcessDeployment[CanonicalProcess]): Map[String, String] + def prepareAdditionalData(runDetails: PeriodicProcessDeployment): Map[String, String] } object DefaultAdditionalDeploymentDataProvider extends AdditionalDeploymentDataProvider { - override def prepareAdditionalData(runDetails: PeriodicProcessDeployment[CanonicalProcess]): Map[String, String] = { + override def prepareAdditionalData(runDetails: PeriodicProcessDeployment): Map[String, String] = { Map( "deploymentId" -> runDetails.id.value.toString, "runAt" -> runDetails.runAt.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/PeriodicProcessListener.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/PeriodicProcessListener.scala index 73a092e9c57..ae0405905fe 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/PeriodicProcessListener.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/PeriodicProcessListener.scala @@ -21,29 +21,28 @@ trait PeriodicProcessListenerFactory { } sealed trait PeriodicProcessEvent { - val deployment: PeriodicProcessDeployment[CanonicalProcess] + val deployment: PeriodicProcessDeployment } case class DeployedEvent( - deployment: PeriodicProcessDeployment[CanonicalProcess], + deployment: PeriodicProcessDeployment, externalDeploymentId: Option[ExternalDeploymentId] ) extends PeriodicProcessEvent -case class FinishedEvent(deployment: PeriodicProcessDeployment[CanonicalProcess], processState: Option[StatusDetails]) +case class FinishedEvent(deployment: PeriodicProcessDeployment, processState: Option[StatusDetails]) extends PeriodicProcessEvent case class FailedOnDeployEvent( - deployment: PeriodicProcessDeployment[CanonicalProcess], + deployment: PeriodicProcessDeployment, processState: Option[StatusDetails] ) extends PeriodicProcessEvent case class FailedOnRunEvent( - deployment: PeriodicProcessDeployment[CanonicalProcess], + deployment: PeriodicProcessDeployment, processState: Option[StatusDetails] ) extends PeriodicProcessEvent -case class ScheduledEvent(deployment: PeriodicProcessDeployment[CanonicalProcess], firstSchedule: Boolean) - extends PeriodicProcessEvent +case class ScheduledEvent(deployment: PeriodicProcessDeployment, firstSchedule: Boolean) extends PeriodicProcessEvent object EmptyListener extends EmptyListener diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/ProcessConfigEnricher.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/ProcessConfigEnricher.scala index 1cdf3177953..ab269f6fd2b 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/ProcessConfigEnricher.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/ProcessConfigEnricher.scala @@ -46,7 +46,7 @@ object ProcessConfigEnricher { case class DeployData( canonicalProcess: CanonicalProcess, inputConfigDuringExecutionJson: String, - deployment: PeriodicProcessDeployment[CanonicalProcess] + deployment: PeriodicProcessDeployment ) extends ProcessConfigEnricherInputData case class EnrichedProcessConfig(inputConfigDuringExecutionJson: String) diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActorTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActorTest.scala index a32745a2b53..7036cb8236b 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActorTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActorTest.scala @@ -8,7 +8,10 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.management.periodic.DeploymentActor.CheckToBeDeployed -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment +import pl.touk.nussknacker.engine.management.periodic.model.{ + PeriodicProcessDeployment, + PeriodicProcessDeploymentWithFullProcess +} import scala.concurrent.Future import scala.concurrent.duration._ @@ -33,11 +36,11 @@ class DeploymentActorTest extends AnyFunSuite with TestKitBase with Matchers wit } private def shouldFindToBeDeployedScenarios( - result: Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]] + result: Future[Seq[PeriodicProcessDeploymentWithFullProcess]] ): Unit = { val probe = TestProbe() var counter = 0 - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = { + def findToBeDeployed: Future[Seq[PeriodicProcessDeploymentWithFullProcess]] = { counter += 1 probe.ref ! s"invoked $counter" result @@ -54,14 +57,14 @@ class DeploymentActorTest extends AnyFunSuite with TestKitBase with Matchers wit } test("should deploy found scenario") { - val probe = TestProbe() - val waitingDeployment = PeriodicProcessDeploymentGen() - var toBeDeployed: Seq[PeriodicProcessDeployment[CanonicalProcess]] = Seq(waitingDeployment) - var actor: ActorRef = null - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = { + val probe = TestProbe() + val waitingDeployment = PeriodicProcessDeploymentGen() + var toBeDeployed: Seq[PeriodicProcessDeploymentWithFullProcess] = Seq(waitingDeployment) + var actor: ActorRef = null + def findToBeDeployed: Future[Seq[PeriodicProcessDeploymentWithFullProcess]] = { Future.successful(toBeDeployed) } - def deploy(deployment: PeriodicProcessDeployment[CanonicalProcess]): Future[Unit] = { + def deploy(deployment: PeriodicProcessDeploymentWithFullProcess): Future[Unit] = { probe.ref ! deployment // Simulate periodic check for waiting scenarios while deploying a scenario. actor ! CheckToBeDeployed diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala index 63d37875d1c..3a189b5e28f 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala @@ -18,7 +18,8 @@ class JarManagerStub extends JarManager { ): Future[DeploymentWithJarData[CanonicalProcess]] = { Future.successful( model.DeploymentWithJarData( - processVersion = processVersion, + processName = processVersion.processName, + versionId = processVersion.versionId, process = canonicalProcess, inputConfigDuringExecutionJson = "", jarFileName = "" diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessDeploymentGen.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessDeploymentGen.scala index 31b847143f6..3a90adf12a4 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessDeploymentGen.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessDeploymentGen.scala @@ -1,13 +1,7 @@ package pl.touk.nussknacker.engine.management.periodic -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.{ - PeriodicProcessDeployment, - PeriodicProcessDeploymentId, - PeriodicProcessDeploymentState, - PeriodicProcessDeploymentStatus, - ScheduleName -} +import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessMetadata +import pl.touk.nussknacker.engine.management.periodic.model._ import java.time.LocalDateTime @@ -15,21 +9,27 @@ object PeriodicProcessDeploymentGen { val now: LocalDateTime = LocalDateTime.now() - def apply(): PeriodicProcessDeployment[CanonicalProcess] = { - PeriodicProcessDeployment( - id = PeriodicProcessDeploymentId(42), - periodicProcess = PeriodicProcessGen(), - createdAt = now.minusMinutes(10), - runAt = now, - scheduleName = ScheduleName(None), - retriesLeft = 0, - nextRetryAt = None, - state = PeriodicProcessDeploymentState( - deployedAt = None, - completedAt = None, - status = PeriodicProcessDeploymentStatus.Scheduled, - ) + def apply(): PeriodicProcessDeploymentWithFullProcess = { + val periodicProcess = PeriodicProcessGen() + PeriodicProcessDeploymentWithFullProcess( + PeriodicProcessDeployment( + id = PeriodicProcessDeploymentId(42), + periodicProcessMetadata = createPeriodicProcessMetadata(periodicProcess), + createdAt = now.minusMinutes(10), + runAt = now, + scheduleName = ScheduleName(None), + retriesLeft = 0, + nextRetryAt = None, + state = PeriodicProcessDeploymentState( + deployedAt = None, + completedAt = None, + status = PeriodicProcessDeploymentStatus.Scheduled, + ) + ), + periodicProcess.deploymentData.process, + periodicProcess.deploymentData.inputConfigDuringExecutionJson, ) + } } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessGen.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessGen.scala index 1ddf34889a1..8cec5d4b5c1 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessGen.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessGen.scala @@ -14,7 +14,8 @@ object PeriodicProcessGen { PeriodicProcess( id = PeriodicProcessId(42), deploymentData = DeploymentWithJarData( - processVersion = ProcessVersion.empty, + processName = ProcessVersion.empty.processName, + versionId = ProcessVersion.empty.versionId, process = buildCanonicalProcess(), inputConfigDuringExecutionJson = "{}", jarFileName = "jar-file-name.jar" diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala index 0fd79cd3c06..d2e6e7ac314 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -201,7 +201,7 @@ class PeriodicProcessServiceIntegrationTest stateAfterSchedule should have size 1 val afterSchedule = stateAfterSchedule.firstScheduleData - afterSchedule.process.processVersion.processName shouldBe processName + afterSchedule.periodicProcessMetadata.processName shouldBe processName afterSchedule.latestDeployments.head.state shouldBe PeriodicProcessDeploymentState( None, None, @@ -214,9 +214,9 @@ class PeriodicProcessServiceIntegrationTest val allToDeploy = service.findToBeDeployed.futureValue allToDeploy.map( - _.periodicProcess.processVersion.processName + _.deployment.periodicProcessMetadata.processName ) should contain only (processName, every30MinutesProcessName) - val toDeploy = allToDeploy.find(_.periodicProcess.processVersion.processName == processName).value + val toDeploy = allToDeploy.find(_.deployment.periodicProcessMetadata.processName == processName).value service.deploy(toDeploy).futureValue otherProcessingTypeService.deploy(otherProcessingTypeService.findToBeDeployed.futureValue.loneElement).futureValue @@ -237,7 +237,9 @@ class PeriodicProcessServiceIntegrationTest service.handleFinished.futureValue val toDeployAfterFinish = service.findToBeDeployed.futureValue - toDeployAfterFinish.map(_.periodicProcess.processVersion.processName) should contain only every30MinutesProcessName + toDeployAfterFinish.map( + _.deployment.periodicProcessMetadata.processName + ) should contain only every30MinutesProcessName service.deactivate(processName).futureValue service.getLatestDeploymentsForActiveSchedules(processName).futureValue shouldBe empty val inactiveStates = service @@ -294,9 +296,9 @@ class PeriodicProcessServiceIntegrationTest service.deploy(toDeploy).futureValue val toBeRetried :: Nil = service.findToBeDeployed.futureValue.toList - toBeRetried.state.status shouldBe PeriodicProcessDeploymentStatus.RetryingDeploy - toBeRetried.retriesLeft shouldBe 1 - toBeRetried.nextRetryAt.isDefined shouldBe true + toBeRetried.deployment.state.status shouldBe PeriodicProcessDeploymentStatus.RetryingDeploy + toBeRetried.deployment.retriesLeft shouldBe 1 + toBeRetried.deployment.nextRetryAt.isDefined shouldBe true service.deploy(toBeRetried).futureValue service.findToBeDeployed.futureValue.toList shouldBe Nil @@ -372,12 +374,12 @@ class PeriodicProcessServiceIntegrationTest val allToDeploy = service.findToBeDeployed.futureValue allToDeploy should have length 4 - val toDeploy = allToDeploy.filter(_.periodicProcess.processVersion.processName == processName) + val toDeploy = allToDeploy.filter(_.deployment.periodicProcessMetadata.processName == processName) toDeploy should have length 2 - toDeploy.head.runAt shouldBe localTime(expectedScheduleTime.plus(5, ChronoUnit.MINUTES)) - toDeploy.head.scheduleName.value shouldBe Some(scheduleMinute5) - toDeploy.last.runAt shouldBe localTime(expectedScheduleTime.plus(10, ChronoUnit.MINUTES)) - toDeploy.last.scheduleName.value shouldBe Some(scheduleMinute10) + toDeploy.head.deployment.runAt shouldBe localTime(expectedScheduleTime.plus(5, ChronoUnit.MINUTES)) + toDeploy.head.deployment.scheduleName.value shouldBe Some(scheduleMinute5) + toDeploy.last.deployment.runAt shouldBe localTime(expectedScheduleTime.plus(10, ChronoUnit.MINUTES)) + toDeploy.last.deployment.scheduleName.value shouldBe Some(scheduleMinute10) service.deactivate(processName).futureValue service.getLatestDeploymentsForActiveSchedules(processName).futureValue shouldBe empty @@ -414,19 +416,19 @@ class PeriodicProcessServiceIntegrationTest val toDeploy = service.findToBeDeployed.futureValue toDeploy should have length 2 - val deployment = toDeploy.find(_.scheduleName.value.contains(firstSchedule)).value + val deployment = toDeploy.find(_.deployment.scheduleName.value.contains(firstSchedule)).value service.deploy(deployment) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deployment.deployment.id)) val toDeployAfterDeploy = service.findToBeDeployed.futureValue toDeployAfterDeploy should have length 0 - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deployment.deployment.id)) service.handleFinished.futureValue val toDeployAfterFinish = service.findToBeDeployed.futureValue toDeployAfterFinish should have length 1 - toDeployAfterFinish.head.scheduleName.value.value shouldBe secondSchedule + toDeployAfterFinish.head.deployment.scheduleName.value.value shouldBe secondSchedule val activities = service.getScenarioActivitiesSpecificToPeriodicProcess(processIdWithName).futureValue val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] @@ -487,7 +489,7 @@ class PeriodicProcessServiceIntegrationTest currentTime = timeToTriggerSchedule1 val toDeployOnSchedule1 = service.findToBeDeployed.futureValue.loneElement - toDeployOnSchedule1.scheduleName.value.value shouldBe schedule1 + toDeployOnSchedule1.deployment.scheduleName.value.value shouldBe schedule1 service.deploy(toDeployOnSchedule1).futureValue val stateAfterSchedule1Deploy = service.getLatestDeploymentsForActiveSchedules(processName).futureValue @@ -520,7 +522,7 @@ class PeriodicProcessServiceIntegrationTest currentTime = timeToTriggerSchedule2 val toDeployOnSchedule2 = service.findToBeDeployed.futureValue.loneElement - toDeployOnSchedule2.scheduleName.value.value shouldBe schedule2 + toDeployOnSchedule2.deployment.scheduleName.value.value shouldBe schedule2 service.deploy(toDeployOnSchedule2).futureValue val stateAfterSchedule2Deploy = service.getLatestDeploymentsForActiveSchedules(processName).futureValue @@ -607,7 +609,7 @@ class PeriodicProcessServiceIntegrationTest toDeploy should have length 1 val deployment = toDeploy.head service.deploy(deployment).futureValue - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deployment.deployment.id)) tryWithFailedListener { () => service.deactivate(processName) @@ -637,7 +639,7 @@ class PeriodicProcessServiceIntegrationTest val deployment = toDeploy.head service.deploy(deployment).futureValue - f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deployment.deployment.id)) // this one is cyclically called by RescheduleActor service.handleFinished.futureValue diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala index 7da7f65421a..e713f02b3d9 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala @@ -15,7 +15,10 @@ import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.PeriodicProcessStatus -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessDeployment +import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.{ + createPeriodicProcessDeployment, + createPeriodicProcessDeploymentWithFullProcess +} import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.management.periodic.model.{PeriodicProcessDeployment, PeriodicProcessDeploymentStatus} import pl.touk.nussknacker.engine.management.periodic.service.ProcessConfigEnricher.EnrichedProcessConfig @@ -85,7 +88,7 @@ class PeriodicProcessServiceTest additionalDeploymentDataProvider = new AdditionalDeploymentDataProvider { override def prepareAdditionalData( - runDetails: PeriodicProcessDeployment[CanonicalProcess] + runDetails: PeriodicProcessDeployment ): Map[String, String] = additionalData + ("runId" -> runDetails.id.value.toString) @@ -138,7 +141,7 @@ class PeriodicProcessServiceTest PeriodicProcessDeploymentStatus.Scheduled, deployMaxRetries = 0 ) - fWithNoRetries.periodicProcessService.findToBeDeployed.futureValue.map(_.id) shouldBe List(scheduledId1) + fWithNoRetries.periodicProcessService.findToBeDeployed.futureValue.map(_.deployment.id) shouldBe List(scheduledId1) val fWithRetries = new Fixture val failedId2 = fWithRetries.repository.addActiveProcess( @@ -151,7 +154,10 @@ class PeriodicProcessServiceTest PeriodicProcessDeploymentStatus.Scheduled, deployMaxRetries = 1 ) - fWithRetries.periodicProcessService.findToBeDeployed.futureValue.map(_.id) shouldBe List(scheduledId2, failedId2) + fWithRetries.periodicProcessService.findToBeDeployed.futureValue.map(_.deployment.id) shouldBe List( + scheduledId2, + failedId2 + ) } test("findToBeDeployed - should not return scenarios with different processing type") { @@ -360,7 +366,7 @@ class PeriodicProcessServiceTest test("deploy - should deploy and mark as so") { val f = new Fixture f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) - val toSchedule = createPeriodicProcessDeployment( + val toSchedule = createPeriodicProcessDeploymentWithFullProcess( f.repository.processEntities.loneElement, f.repository.deploymentEntities.loneElement ) @@ -382,7 +388,7 @@ class PeriodicProcessServiceTest val f = new Fixture f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) f.jarManagerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) - val toSchedule = createPeriodicProcessDeployment( + val toSchedule = createPeriodicProcessDeploymentWithFullProcess( f.repository.processEntities.loneElement, f.repository.deploymentEntities.loneElement ) diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala index d16786058ea..ad453417f4c 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala @@ -11,7 +11,7 @@ import pl.touk.nussknacker.engine.management.periodic.db.InMemPeriodicProcessesR DeploymentIdSequence, ProcessIdSequence } -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessDeployment +import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessDeploymentWithFullProcess import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.management.periodic.model._ @@ -139,8 +139,8 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP val id = PeriodicProcessId(Random.nextLong()) val periodicProcess = PeriodicProcessEntity( id = id, - processName = deploymentWithJarData.processVersion.processName, - processVersionId = deploymentWithJarData.processVersion.versionId, + processName = deploymentWithJarData.processName, + processVersionId = deploymentWithJarData.versionId, processingType = processingType, processJson = Some(deploymentWithJarData.process), inputConfigDuringExecutionJson = deploymentWithJarData.inputConfigDuringExecutionJson, @@ -199,25 +199,29 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP .take(deploymentsPerScheduleMaxCount) .map(ScheduleDeploymentData(_)) .toList - scheduleId -> ScheduleData(PeriodicProcessesRepository.createPeriodicProcessWithoutJson(process), ds) + scheduleId -> ScheduleData( + PeriodicProcessesRepository + .createPeriodicProcessMetadata(PeriodicProcessesRepository.createPeriodicProcessWithoutJson(process)), + ds + ) } } yield deploymentGroupedByScheduleName).toMap) - override def findToBeDeployed: Seq[PeriodicProcessDeployment[CanonicalProcess]] = { + override def findToBeDeployed: Seq[PeriodicProcessDeploymentWithFullProcess] = { val scheduled = findActive(PeriodicProcessDeploymentStatus.Scheduled) readyToRun(scheduled) } - override def findToBeRetried: Action[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = { - val toBeRetried = findActive(PeriodicProcessDeploymentStatus.FailedOnDeploy).filter(_.retriesLeft > 0) + override def findToBeRetried: Action[Seq[PeriodicProcessDeploymentWithFullProcess]] = { + val toBeRetried = findActive(PeriodicProcessDeploymentStatus.FailedOnDeploy).filter(_.deployment.retriesLeft > 0) readyToRun(toBeRetried) } - override def findProcessData(id: PeriodicProcessDeploymentId): PeriodicProcessDeployment[CanonicalProcess] = + override def findProcessData(id: PeriodicProcessDeploymentId): PeriodicProcessDeploymentWithFullProcess = (for { d <- deploymentEntities if d.id == id p <- processEntities if p.id == d.periodicProcessId - } yield createPeriodicProcessDeployment(p, d)).head + } yield createPeriodicProcessDeploymentWithFullProcess(p, d)).head override def findProcessData(processName: ProcessName): Seq[PeriodicProcess[CanonicalProcess]] = processEntities(processName) @@ -267,7 +271,7 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): PeriodicProcessDeployment[CanonicalProcess] = { + ): PeriodicProcessDeploymentWithFullProcess = { val deploymentEntity = PeriodicProcessDeploymentEntity( id = PeriodicProcessDeploymentId(Random.nextLong()), periodicProcessId = id, @@ -281,7 +285,7 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP status = PeriodicProcessDeploymentStatus.Scheduled ) deploymentEntities += deploymentEntity - createPeriodicProcessDeployment(processEntities.find(_.id == id).head, deploymentEntity) + createPeriodicProcessDeploymentWithFullProcess(processEntities.find(_.id == id).head, deploymentEntity) } private def update( @@ -294,24 +298,24 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP } } - private def findActive(status: PeriodicProcessDeploymentStatus): Seq[PeriodicProcessDeployment[CanonicalProcess]] = + private def findActive(status: PeriodicProcessDeploymentStatus): Seq[PeriodicProcessDeploymentWithFullProcess] = findActive( Seq(status) ) private def findActive( statusList: Seq[PeriodicProcessDeploymentStatus] - ): Seq[PeriodicProcessDeployment[CanonicalProcess]] = + ): Seq[PeriodicProcessDeploymentWithFullProcess] = (for { p <- processEntities if p.active && p.processingType == processingType d <- deploymentEntities if d.periodicProcessId == p.id && statusList.contains(d.status) - } yield createPeriodicProcessDeployment(p, d)).toSeq + } yield createPeriodicProcessDeploymentWithFullProcess(p, d)).toSeq private def readyToRun( - deployments: Seq[PeriodicProcessDeployment[CanonicalProcess]] - ): Seq[PeriodicProcessDeployment[CanonicalProcess]] = { + deployments: Seq[PeriodicProcessDeploymentWithFullProcess] + ): Seq[PeriodicProcessDeploymentWithFullProcess] = { val now = LocalDateTime.now() - deployments.filter(d => d.runAt.isBefore(now) || d.runAt.isEqual(now)) + deployments.filter(d => d.deployment.runAt.isBefore(now) || d.deployment.runAt.isEqual(now)) } } From 5b5e01097476baf522d5e1df6b9ebaf104185571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Mon, 9 Dec 2024 17:49:05 +0100 Subject: [PATCH 02/16] ggg --- ...inkPeriodicDeploymentManagerProvider.scala | 5 +- .../periodic/PeriodicDeploymentManager.scala | 38 ++-- .../periodic/PeriodicProcessService.scala | 14 +- .../db/PeriodicProcessesRepository.scala | 56 +++++- ...cProcessesRepositoryCachingDecorator.scala | 179 ++++++++++++++++++ ...riodicProcessesCachingRepositoryTest.scala | 111 +++++++++++ .../db/InMemPeriodicProcessesRepository.scala | 49 ++++- 7 files changed, 415 insertions(+), 37 deletions(-) create mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala create mode 100644 engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesCachingRepositoryTest.scala diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala index bd64c7e8bd8..11f7f8c7d36 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala @@ -38,7 +38,7 @@ class FlinkPeriodicDeploymentManagerProvider extends DeploymentManagerProvider w modelData: BaseModelData, dependencies: DeploymentManagerDependencies, config: Config, - scenarioStateCacheTTL: Option[FiniteDuration] + scenarioStateCacheTTL: Option[FiniteDuration], ): ValidatedNel[String, DeploymentManager] = { logger.info("Creating FlinkPeriodic scenario manager") delegate.createDeploymentManager(modelData, dependencies, config, scenarioStateCacheTTL).map { @@ -58,7 +58,8 @@ class FlinkPeriodicDeploymentManagerProvider extends DeploymentManagerProvider w modelData = modelData, EmptyPeriodicProcessListenerFactory, DefaultAdditionalDeploymentDataProvider, - dependencies + dependencies, + scenarioStateCacheTTL, ) } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala index e7eedd55b13..da3e550c218 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala @@ -13,6 +13,7 @@ import pl.touk.nussknacker.engine.management.periodic.Utils.{createActorWithRetr import pl.touk.nussknacker.engine.management.periodic.db.{ DbInitializer, PeriodicProcessesRepository, + PeriodicProcessesRepositoryCachingDecorator, SlickPeriodicProcessesRepository } import pl.touk.nussknacker.engine.management.periodic.flink.FlinkJarManager @@ -26,6 +27,7 @@ import slick.jdbc import slick.jdbc.JdbcProfile import java.time.Clock +import scala.concurrent.duration.FiniteDuration import scala.concurrent.{ExecutionContext, Future} object PeriodicDeploymentManager { @@ -40,7 +42,8 @@ object PeriodicDeploymentManager { modelData: BaseModelData, listenerFactory: PeriodicProcessListenerFactory, additionalDeploymentDataProvider: AdditionalDeploymentDataProvider, - dependencies: DeploymentManagerDependencies + dependencies: DeploymentManagerDependencies, + scenarioStateCacheTTL: Option[FiniteDuration], ): PeriodicDeploymentManager = { import dependencies._ @@ -49,13 +52,17 @@ object PeriodicDeploymentManager { val (db: jdbc.JdbcBackend.DatabaseDef, dbProfile: JdbcProfile) = DbInitializer.init(periodicBatchConfig.db) val scheduledProcessesRepository = new SlickPeriodicProcessesRepository(db, dbProfile, clock, periodicBatchConfig.processingType) + val repositoryWithCaching = scenarioStateCacheTTL match { + case Some(ttl) => new PeriodicProcessesRepositoryCachingDecorator(scheduledProcessesRepository, ttl) + case None => scheduledProcessesRepository + } val jarManager = FlinkJarManager(flinkConfig, periodicBatchConfig, modelData) val listener = listenerFactory.create(originalConfig) val processConfigEnricher = processConfigEnricherFactory(originalConfig) val service = new PeriodicProcessService( delegate, jarManager, - scheduledProcessesRepository, + repositoryWithCaching, listener, additionalDeploymentDataProvider, periodicBatchConfig.deploymentRetry, @@ -263,17 +270,20 @@ class PeriodicDeploymentManager private[periodic] ( } // TODO: Why we don't allow running not scheduled scenario? Maybe we can try to schedule it? - private def instantSchedule(processName: ProcessName): OptionT[Future, Unit] = for { - // schedule for immediate run - processDeployment <- OptionT( - service - .getLatestDeploymentsForActiveSchedules(processName) - .map(_.groupedByPeriodicProcess.headOption.flatMap(_.deployments.headOption)) - ) - processDeploymentWithProcessJson <- OptionT.liftF( - repository.findProcessData(processDeployment.id).run - ) - _ <- OptionT.liftF(service.deploy(processDeploymentWithProcessJson)) - } yield () + private def instantSchedule(processName: ProcessName): OptionT[Future, Unit] = { + implicit val freshnessPolicy: DataFreshnessPolicy = DataFreshnessPolicy.Fresh + for { + // schedule for immediate run + processDeployment <- OptionT( + service + .getLatestDeploymentsForActiveSchedules(processName) + .map(_.groupedByPeriodicProcess.headOption.flatMap(_.deployments.headOption)) + ) + processDeploymentWithProcessJson <- OptionT.liftF( + repository.findProcessData(processDeployment.id).run + ) + _ <- OptionT.liftF(service.deploy(processDeploymentWithProcessJson)) + } yield () + } } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala index 2e8ed1569fe..e8bfd534993 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala @@ -28,7 +28,6 @@ import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeplo import pl.touk.nussknacker.engine.management.periodic.model._ import pl.touk.nussknacker.engine.management.periodic.service._ import pl.touk.nussknacker.engine.management.periodic.util.DeterministicUUIDFromLong -import pl.touk.nussknacker.engine.management.periodic.util.DeterministicUUIDFromLong.longUUID import pl.touk.nussknacker.engine.util.AdditionalComponentConfigsForRuntimeExtractor import java.time.chrono.ChronoLocalDateTime @@ -386,7 +385,8 @@ class PeriodicProcessService( } yield handleEvent(FailedOnRunEvent(currentState.deployment, state)) } - def deactivate(processName: ProcessName): Future[Iterable[DeploymentId]] = + def deactivate(processName: ProcessName): Future[Iterable[DeploymentId]] = { + implicit val freshnessPolicy: DataFreshnessPolicy = DataFreshnessPolicy.Fresh for { activeSchedules <- getLatestDeploymentsForActiveSchedules(processName) (runningDeploymentsForSchedules, _) <- synchronizeDeploymentsStates(processName, activeSchedules) @@ -395,6 +395,7 @@ class PeriodicProcessService( .sequence .runWithCallbacks } yield runningDeploymentsForSchedules.map(deployment => DeploymentId(deployment.toString)) + } private def deactivateAction(process: PeriodicProcessMetadata): RepositoryAction[Callback] = { logger.info(s"Deactivate periodic process id: ${process.id.value}") @@ -507,7 +508,7 @@ class PeriodicProcessService( private def mergeStatusWithDeployments( name: ProcessName, runtimeStatuses: List[StatusDetails] - ): Future[StatusDetails] = { + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[StatusDetails] = { def toDeploymentStatuses(schedulesState: SchedulesState) = schedulesState.schedules.toList .flatMap { case (scheduleId, scheduleData) => scheduleData.latestDeployments.map { deployment => @@ -540,21 +541,20 @@ class PeriodicProcessService( def getLatestDeploymentsForActiveSchedules( processName: ProcessName, deploymentsPerScheduleMaxCount: Int = 1 - ): Future[SchedulesState] = - scheduledProcessesRepository.getLatestDeploymentsForActiveSchedules(processName, deploymentsPerScheduleMaxCount).run + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = + scheduledProcessesRepository.getLatestDeploymentsForActiveSchedules(processName, deploymentsPerScheduleMaxCount) def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - ): Future[SchedulesState] = + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = scheduledProcessesRepository .getLatestDeploymentsForLatestInactiveSchedules( processName, inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount ) - .run implicit class RuntimeStatusesExt(runtimeStatuses: List[StatusDetails]) { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala index 390b2501539..f75f79099ab 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala @@ -4,7 +4,7 @@ import cats.Monad import com.github.tminglei.slickpg.ExPostgresProfile import com.typesafe.scalalogging.LazyLogging import io.circe.parser.decode -import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId} import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.management.periodic._ @@ -19,7 +19,7 @@ import slick.dbio.{DBIOAction, Effect, NoStream} import slick.jdbc.PostgresProfile.api._ import slick.jdbc.{JdbcBackend, JdbcProfile} -import java.time.{Clock, Instant, LocalDateTime, ZoneId} +import java.time.{Clock, LocalDateTime} import scala.concurrent.{ExecutionContext, Future} import scala.language.higherKinds @@ -157,13 +157,22 @@ trait PeriodicProcessesRepository { def getLatestDeploymentsForActiveSchedules( processName: ProcessName, deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] + + def getLatestDeploymentsForActiveSchedules( + deploymentsPerScheduleMaxCount: Int + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] + + def getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] def findToBeDeployed: Action[Seq[PeriodicProcessDeploymentWithFullProcess]] @@ -232,6 +241,7 @@ class SlickPeriodicProcessesRepository( (createPeriodicProcessMetadataFromColumnValues(periodicProcessMetadataColumnValues), deploymentEntity) }) .map(toSchedulesState) + .map(_.getOrElse(scenarioName, SchedulesState(Map.empty))) } override def create( @@ -345,33 +355,55 @@ class SlickPeriodicProcessesRepository( getLatestDeploymentsForEachSchedule( processesHavingDeploymentsWithMatchingStatus, deploymentsPerScheduleMaxCount = 1 - ) + ).map(_.values.headOption.getOrElse(SchedulesState(Map.empty))) } override def getLatestDeploymentsForActiveSchedules( processName: ProcessName, deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] = { + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { val activeProcessesQuery = PeriodicProcessesWithoutJson.filter(p => p.processName === processName && p.active) getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount) + .map(_.getOrElse(processName, SchedulesState(Map.empty))) + .run + } + + override def getLatestDeploymentsForActiveSchedules( + deploymentsPerScheduleMaxCount: Int + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = { + val activeProcessesQuery = PeriodicProcessesWithoutJson.filter(_.active) + getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount).run } override def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] = { + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { val filteredProcessesQuery = PeriodicProcessesWithoutJson .filter(p => p.processName === processName && !p.active) .sortBy(_.createdAt.desc) .take(inactiveProcessesMaxCount) getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount) + .map(_.getOrElse(processName, SchedulesState(Map.empty))) + .run + } + + override def getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = { + val filteredProcessesQuery = PeriodicProcessesWithoutJson + .filter(!_.active) + .sortBy(_.createdAt.desc) + .take(inactiveProcessesMaxCount) + getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount).run } private def getLatestDeploymentsForEachSchedule( periodicProcessesQuery: Query[PeriodicProcessesTable, PeriodicProcessEntity, Seq], deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] = { + ): Action[Map[ProcessName, SchedulesState]] = { val filteredPeriodicProcessQuery = periodicProcessesQuery.filter(p => p.processingType === processingType) val latestDeploymentsForSchedules = profile match { case _: ExPostgresProfile => @@ -536,6 +568,14 @@ class SlickPeriodicProcessesRepository( private def toSchedulesState( list: Seq[(PeriodicProcessMetadata, PeriodicProcessDeploymentEntity)] + ): Map[ProcessName, SchedulesState] = { + list + .groupBy(_._1.processName) + .map { case (processName, list) => processName -> toSchedulesStateForSinglePeriodicProcess(list) } + } + + private def toSchedulesStateForSinglePeriodicProcess( + list: Seq[(PeriodicProcessMetadata, PeriodicProcessDeploymentEntity)] ): SchedulesState = { SchedulesState( list diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala new file mode 100644 index 00000000000..1553f37848e --- /dev/null +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala @@ -0,0 +1,179 @@ +package pl.touk.nussknacker.engine.management.periodic.db + +import cats.Monad +import com.github.benmanes.caffeine.cache.{AsyncCache, Caffeine} +import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId} +import pl.touk.nussknacker.engine.api.process.ProcessName +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.management.periodic._ +import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.management.periodic.model._ + +import java.time.LocalDateTime +import scala.compat.java8.FutureConverters.{CompletionStageOps, FutureOps} +import scala.concurrent.duration.{DurationInt, FiniteDuration} +import scala.concurrent.{ExecutionContext, Future} + +class PeriodicProcessesRepositoryCachingDecorator( + val underlying: PeriodicProcessesRepository, + cacheTTL: FiniteDuration, +)(implicit ec: ExecutionContext) + extends PeriodicProcessesRepository { + + private val activeSchedulesCache: AsyncCache[Int, Map[ProcessName, SchedulesState]] = + Caffeine + .newBuilder() + .expireAfterWrite(java.time.Duration.ofMillis(cacheTTL.toMillis)) + .buildAsync[Int, Map[ProcessName, SchedulesState]] + + private val inactiveSchedulesCache: AsyncCache[(Int, Int), Map[ProcessName, SchedulesState]] = + Caffeine + .newBuilder() + .expireAfterWrite(java.time.Duration.ofMillis((10 seconds).toMillis)) + .buildAsync[(Int, Int), Map[ProcessName, SchedulesState]] + + type Action[T] = underlying.Action[T] + + override implicit def monad: Monad[Action] = underlying.monad + + override def run[T](action: Action[T]): Future[T] = underlying.run(action) + + override def getLatestDeploymentsForActiveSchedules( + processName: ProcessName, + deploymentsPerScheduleMaxCount: Int + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { + fetchActiveSchedules(deploymentsPerScheduleMaxCount) + .map(_.getOrElse(processName, SchedulesState(Map.empty))) + } + + override def getLatestDeploymentsForActiveSchedules( + deploymentsPerScheduleMaxCount: Int + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = { + fetchActiveSchedules(deploymentsPerScheduleMaxCount) + } + + override def getLatestDeploymentsForLatestInactiveSchedules( + processName: ProcessName, + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { + fetchInactiveSchedules(inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount) + .map(_.getOrElse(processName, SchedulesState(Map.empty))) + } + + override def getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = { + fetchInactiveSchedules(inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount) + } + + private def fetchActiveSchedules(deploymentsPerScheduleMaxCount: Int)( + implicit freshnessPolicy: DataFreshnessPolicy + ) = { + def fetchAndUpdateCache(): Future[Map[ProcessName, SchedulesState]] = { + val resultFuture = underlying.getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount) + activeSchedulesCache.put(deploymentsPerScheduleMaxCount, resultFuture.toJava.toCompletableFuture) + resultFuture + } + freshnessPolicy match { + case DataFreshnessPolicy.Fresh => + fetchAndUpdateCache() + case DataFreshnessPolicy.CanBeCached => + Option(activeSchedulesCache.getIfPresent(deploymentsPerScheduleMaxCount)) + .map(_.toScala) + .getOrElse(fetchAndUpdateCache()) + } + } + + private def fetchInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + )(implicit freshnessPolicy: DataFreshnessPolicy) = { + def fetchAndUpdateCache(): Future[Map[ProcessName, SchedulesState]] = { + val resultFuture = underlying.getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount, + deploymentsPerScheduleMaxCount + ) + inactiveSchedulesCache.put( + (inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount), + resultFuture.toJava.toCompletableFuture + ) + resultFuture + } + freshnessPolicy match { + case DataFreshnessPolicy.Fresh => + fetchAndUpdateCache() + case DataFreshnessPolicy.CanBeCached => + Option(inactiveSchedulesCache.getIfPresent(inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount)) + .map(_.toScala) + .getOrElse(fetchAndUpdateCache()) + } + + Option(inactiveSchedulesCache.getIfPresent(inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount)) + .map(_.toScala) + .getOrElse(fetchAndUpdateCache()) + } + + // Methods below are not using caching mechanism + + override def getSchedulesState( + scenarioName: ProcessName + ): Action[SchedulesState] = + underlying.getSchedulesState(scenarioName) + + override def create( + deploymentWithJarData: DeploymentWithJarData[CanonicalProcess], + scheduleProperty: ScheduleProperty, + processActionId: ProcessActionId + ): Action[PeriodicProcess[CanonicalProcess]] = + underlying.create(deploymentWithJarData, scheduleProperty, processActionId) + + override def schedule( + id: PeriodicProcessId, + scheduleName: ScheduleName, + runAt: LocalDateTime, + deployMaxRetries: Int + ): Action[PeriodicProcessDeploymentWithFullProcess] = + underlying.schedule(id, scheduleName, runAt, deployMaxRetries) + + override def findToBeDeployed: Action[Seq[PeriodicProcessDeploymentWithFullProcess]] = + underlying.findToBeDeployed + + override def findToBeRetried: Action[Seq[PeriodicProcessDeploymentWithFullProcess]] = + underlying.findToBeRetried + + override def findProcessData( + id: PeriodicProcessDeploymentId + ): Action[PeriodicProcessDeploymentWithFullProcess] = + underlying.findProcessData(id) + + override def findProcessData(processName: ProcessName): Action[Seq[PeriodicProcess[CanonicalProcess]]] = + underlying.findProcessData(processName) + + override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus] + ): Action[SchedulesState] = + underlying.findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus(expectedDeploymentStatuses) + + override def markDeployed(id: PeriodicProcessDeploymentId): Action[Unit] = + underlying.markDeployed(id) + + override def markFailed(id: PeriodicProcessDeploymentId): Action[Unit] = + underlying.markFailed(id) + + override def markFinished(id: PeriodicProcessDeploymentId): Action[Unit] = + underlying.markFinished(id) + + override def markFailedOnDeployWithStatus( + id: PeriodicProcessDeploymentId, + status: PeriodicProcessDeploymentStatus, + retriesLeft: Int, + retryAt: Option[LocalDateTime] + ): Action[Unit] = + underlying.markFailedOnDeployWithStatus(id, status, retriesLeft, retryAt) + + override def markInactive(processId: PeriodicProcessId): Action[Unit] = + underlying.markInactive(processId) + +} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesCachingRepositoryTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesCachingRepositoryTest.scala new file mode 100644 index 00000000000..2505f83efd5 --- /dev/null +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesCachingRepositoryTest.scala @@ -0,0 +1,111 @@ +package pl.touk.nussknacker.engine.management.periodic + +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatest.prop.TableDrivenPropertyChecks +import org.scalatest.{Inside, OptionValues} +import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus +import pl.touk.nussknacker.engine.api.process.ProcessName +import pl.touk.nussknacker.engine.deployment.DeploymentData +import pl.touk.nussknacker.engine.management.periodic.db.InMemPeriodicProcessesRepository.getLatestDeploymentQueryCount +import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepositoryCachingDecorator +import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.management.periodic.service.{ + DefaultAdditionalDeploymentDataProvider, + EmptyListener, + ProcessConfigEnricher +} +import pl.touk.nussknacker.test.PatientScalaFutures + +import java.time.Clock +import java.util.UUID +import scala.concurrent.duration.DurationInt + +class PeriodicProcessesCachingRepositoryTest + extends AnyFunSuite + with Matchers + with ScalaFutures + with OptionValues + with Inside + with TableDrivenPropertyChecks + with PatientScalaFutures { + + protected implicit val freshnessPolicy: DataFreshnessPolicy = DataFreshnessPolicy.Fresh + + import scala.concurrent.ExecutionContext.Implicits.global + + private def processName(n: Int) = ProcessName(s"test$n") + + class Fixture(executionConfig: PeriodicExecutionConfig = PeriodicExecutionConfig()) { + val repository = new db.InMemPeriodicProcessesRepository(processingType = "testProcessingType") + val delegateDeploymentManagerStub = new DeploymentManagerStub + val jarManagerStub = new JarManagerStub + val preparedDeploymentData = DeploymentData.withDeploymentId(UUID.randomUUID().toString) + + val periodicProcessService = new PeriodicProcessService( + delegateDeploymentManager = delegateDeploymentManagerStub, + jarManager = jarManagerStub, + scheduledProcessesRepository = new PeriodicProcessesRepositoryCachingDecorator(repository, 10 seconds), + periodicProcessListener = EmptyListener, + additionalDeploymentDataProvider = DefaultAdditionalDeploymentDataProvider, + deploymentRetryConfig = DeploymentRetryConfig(), + executionConfig = executionConfig, + processConfigEnricher = ProcessConfigEnricher.identity, + clock = Clock.systemDefaultZone(), + new ProcessingTypeActionServiceStub, + Map.empty + ) + + val periodicDeploymentManager = new PeriodicDeploymentManager( + delegate = delegateDeploymentManagerStub, + service = periodicProcessService, + repository = repository, + schedulePropertyExtractor = CronSchedulePropertyExtractor(), + toClose = () => () + ) + + } + + test("getStatusDetails - should perform 2 db queries for N periodic processes when using cache") { + val f = new Fixture + val n = 10 + + for (i <- 1 to n) { + val deploymentId = f.repository.addActiveProcess(processName(i), PeriodicProcessDeploymentStatus.Deployed) + f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deploymentId)) + } + + getLatestDeploymentQueryCount.set(0) + + implicit val freshnessPolicy: DataFreshnessPolicy = DataFreshnessPolicy.CanBeCached + + for (i <- 1 to n) { + f.periodicProcessService.getStatusDetails(processName(i)).futureValue + } + + getLatestDeploymentQueryCount.get() shouldEqual 2 + } + + test("getStatusDetails - should perform 2*N db queries for N periodic processes when not using cache") { + val f = new Fixture + val n = 10 + + for (i <- 1 to n) { + val deploymentId = f.repository.addActiveProcess(processName(i), PeriodicProcessDeploymentStatus.Deployed) + f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deploymentId)) + } + + getLatestDeploymentQueryCount.set(0) + + implicit val freshnessPolicy: DataFreshnessPolicy = DataFreshnessPolicy.Fresh + + for (i <- 1 to n) { + f.periodicProcessService.getStatusDetails(processName(i)).futureValue + } + + getLatestDeploymentQueryCount.get() shouldEqual 2 * n + } + +} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala index ad453417f4c..d62c1d4ec4c 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala @@ -2,14 +2,15 @@ package pl.touk.nussknacker.engine.management.periodic.db import cats.{Id, Monad} import io.circe.syntax.EncoderOps -import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId} import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.management.periodic._ import pl.touk.nussknacker.engine.management.periodic.db.InMemPeriodicProcessesRepository.{ DeploymentIdSequence, - ProcessIdSequence + ProcessIdSequence, + getLatestDeploymentQueryCount } import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessDeploymentWithFullProcess import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus @@ -26,6 +27,8 @@ import scala.util.Random object InMemPeriodicProcessesRepository { private val ProcessIdSequence = new AtomicLong(0) private val DeploymentIdSequence = new AtomicLong(0) + + val getLatestDeploymentQueryCount = new AtomicLong(0) } class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicProcessesRepository { @@ -167,22 +170,50 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP override def getLatestDeploymentsForActiveSchedules( processName: ProcessName, deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] = + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { + getLatestDeploymentQueryCount.incrementAndGet() getLatestDeploymentsForPeriodicProcesses( processEntities(processName).filter(_.active), deploymentsPerScheduleMaxCount - ) + ).run + } + + override def getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount: Int)( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[Map[ProcessName, SchedulesState]] = { + getLatestDeploymentQueryCount.incrementAndGet() + allProcessEntities.map { case (processName, list) => + processName -> getLatestDeploymentsForPeriodicProcesses( + list.filter(_.active), + deploymentsPerScheduleMaxCount + ) + } + }.run override def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] = { + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { + getLatestDeploymentQueryCount.incrementAndGet() val filteredProcesses = processEntities(processName).filterNot(_.active).sortBy(_.createdAt).takeRight(inactiveProcessesMaxCount) - getLatestDeploymentsForPeriodicProcesses(filteredProcesses, deploymentsPerScheduleMaxCount) + getLatestDeploymentsForPeriodicProcesses(filteredProcesses, deploymentsPerScheduleMaxCount).run } + override def getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = { + getLatestDeploymentQueryCount.incrementAndGet() + allProcessEntities.map { case (processName, list) => + processName -> getLatestDeploymentsForPeriodicProcesses( + list.filterNot(_.active).sortBy(_.createdAt).takeRight(inactiveProcessesMaxCount), + deploymentsPerScheduleMaxCount + ) + } + }.run + private def getLatestDeploymentsForPeriodicProcesses( processes: Seq[PeriodicProcessEntity], deploymentsPerScheduleMaxCount: Int @@ -228,6 +259,12 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP .filter(_.active) .map(PeriodicProcessesRepository.createPeriodicProcessWithJson) + private def allProcessEntities: Map[ProcessName, Seq[PeriodicProcessEntity]] = + processEntities + .filter(process => process.processingType == processingType) + .toSeq + .groupBy(_.processName) + private def processEntities(processName: ProcessName): Seq[PeriodicProcessEntity] = processEntities .filter(process => process.processName == processName && process.processingType == processingType) From 0fab80bfb5653afe45f9d8bff48e2781d4f88408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Fri, 13 Dec 2024 11:48:12 +0100 Subject: [PATCH 03/16] merge fixes --- .../periodic/PeriodicDeploymentManager.scala | 1 + .../periodic/PeriodicProcessService.scala | 5 ++- .../db/PeriodicProcessesRepository.scala | 34 ++++++++++------ ...cProcessesRepositoryCachingDecorator.scala | 20 +++++----- .../model/PeriodicProcessDeployment.scala | 2 +- .../periodic/model/SchedulesState.scala | 4 +- .../management/periodic/JarManagerStub.scala | 5 +-- ...eriodicProcessServiceIntegrationTest.scala | 40 +++++++++---------- .../db/InMemPeriodicProcessesRepository.scala | 5 ++- 9 files changed, 65 insertions(+), 51 deletions(-) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala index ee9248a480a..13678c7e651 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala @@ -27,6 +27,7 @@ import slick.jdbc import slick.jdbc.JdbcProfile import java.time.{Clock, Instant} +import scala.concurrent.duration.FiniteDuration import scala.concurrent.{ExecutionContext, Future} object PeriodicDeploymentManager { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala index 5c506144946..967274a9d34 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala @@ -90,7 +90,8 @@ class PeriodicProcessService( scenarioActivityId = ScenarioActivityId(DeterministicUUIDFromLong.longUUID(deployment.id.value)), user = ScenarioUser.internalNuUser, date = metadata.dateDeployed.getOrElse(metadata.dateFinished), - scenarioVersionId = Some(ScenarioVersionId.from(deployment.periodicProcess.processVersion.versionId)), + scenarioVersionId = + Some(ScenarioVersionId.from(deployment.periodicProcess.deploymentData.processVersion.versionId)), scheduledExecutionStatus = metadata.status, dateFinished = metadata.dateFinished, scheduleName = deployment.scheduleName.display, @@ -203,7 +204,7 @@ class PeriodicProcessService( toDeploy: PeriodicProcessDeployment[WithCanonicalProcess] ): Future[Option[PeriodicProcessDeployment[WithCanonicalProcess]]] = { delegateDeploymentManager - .getProcessStates(toDeploy.periodicProcess.processVersion.processName)(DataFreshnessPolicy.Fresh) + .getProcessStates(toDeploy.periodicProcess.deploymentData.processVersion.processName)(DataFreshnessPolicy.Fresh) .map( _.value .map(_.status) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala index 2f14b48128c..0f7c169a8bf 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala @@ -5,9 +5,10 @@ import com.github.tminglei.slickpg.ExPostgresProfile import com.typesafe.scalalogging.LazyLogging import io.circe.parser.decode import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId} import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.management.periodic._ +import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessWithoutJson import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.{ WithCanonicalProcess, WithoutCanonicalProcess @@ -210,7 +211,7 @@ class SlickPeriodicProcessesRepository( .on(_.id === _.periodicProcessId) .filterOpt(afterOpt)((entities, after) => entities._2.completedAt > after) .result - .map(toSchedulesState) + .map(toSchedulesStateForSinglePeriodicProcess) } override def create( @@ -386,7 +387,7 @@ class SlickPeriodicProcessesRepository( private def getLatestDeploymentsForEachSchedulePostgres( periodicProcessesQuery: Query[PeriodicProcessWithoutJson, PeriodicProcessEntityWithoutJson, Seq], deploymentsPerScheduleMaxCount: Int - ): Action[Seq[(PeriodicProcessMetadata, PeriodicProcessDeploymentEntity)]] = { + ): Action[Seq[(PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity)]] = { // To effectively limit deployments to given count for each schedule in one query, we use window functions in slick import ExPostgresProfile.api._ import com.github.tminglei.slickpg.window.PgWindowFuncSupport.WindowFunctions._ @@ -420,7 +421,7 @@ class SlickPeriodicProcessesRepository( private def getLatestDeploymentsForEachScheduleJdbcGeneric( periodicProcessesQuery: Query[PeriodicProcessWithoutJson, PeriodicProcessEntityWithoutJson, Seq], deploymentsPerScheduleMaxCount: Int - ): Action[Seq[(PeriodicProcessMetadata, PeriodicProcessDeploymentEntity)]] = { + ): Action[Seq[(PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity)]] = { // It is debug instead of warn to not bloast logs when e.g. for some reasons is used hsql under the hood logger.debug( "WARN: Using not optimized version of getLatestDeploymentsForEachSchedule that not uses window functions" @@ -491,21 +492,30 @@ class SlickPeriodicProcessesRepository( join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) } - private def toSchedulesState(list: Seq[(PeriodicProcessEntity, PeriodicProcessDeploymentEntity)]): SchedulesState = { + private def toSchedulesState( + list: Seq[(PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity)] + ): Map[ProcessName, SchedulesState] = { + list + .groupBy(_._1.processName) + .map { case (processName, list) => processName -> toSchedulesStateForSinglePeriodicProcess(list) } + } + + private def toSchedulesStateForSinglePeriodicProcess( + list: Seq[(PeriodicProcessEntityWithoutJson, PeriodicProcessDeploymentEntity)] + ): SchedulesState = { SchedulesState( list - .map { case (process, deployment) => - val scheduleId = ScheduleId(process.id, ScheduleName(deployment.scheduleName)) - val scheduleDataWithoutDeployment = - (scheduleId, PeriodicProcessesRepository.createPeriodicProcessWithoutJson(process)) + .map { case (periodicProcessMetadata, deployment) => + val scheduleId = ScheduleId(periodicProcessMetadata.id, ScheduleName(deployment.scheduleName)) + val scheduleData = (scheduleId, periodicProcessMetadata) val scheduleDeployment = ScheduleDeploymentData(deployment) - (scheduleDataWithoutDeployment, scheduleDeployment) + (scheduleData, scheduleDeployment) } .toList .toGroupedMap .toList - .map { case ((scheduleId, process), deployments) => - scheduleId -> ScheduleData(process, deployments) + .map { case ((scheduleId, processEntity), deployments) => + scheduleId -> ScheduleData(createPeriodicProcessWithoutJson(processEntity), deployments) } .toMap ) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala index 1553f37848e..0cb88f29721 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala @@ -6,6 +6,7 @@ import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessAc import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.management.periodic._ +import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.management.periodic.model._ @@ -118,15 +119,16 @@ class PeriodicProcessesRepositoryCachingDecorator( // Methods below are not using caching mechanism override def getSchedulesState( - scenarioName: ProcessName + scenarioName: ProcessName, + after: Option[LocalDateTime] ): Action[SchedulesState] = - underlying.getSchedulesState(scenarioName) + underlying.getSchedulesState(scenarioName, after) override def create( - deploymentWithJarData: DeploymentWithJarData[CanonicalProcess], + deploymentWithJarData: WithCanonicalProcess, scheduleProperty: ScheduleProperty, processActionId: ProcessActionId - ): Action[PeriodicProcess[CanonicalProcess]] = + ): Action[PeriodicProcess[WithCanonicalProcess]] = underlying.create(deploymentWithJarData, scheduleProperty, processActionId) override def schedule( @@ -134,21 +136,21 @@ class PeriodicProcessesRepositoryCachingDecorator( scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Action[PeriodicProcessDeploymentWithFullProcess] = + ): Action[PeriodicProcessDeployment[WithCanonicalProcess]] = underlying.schedule(id, scheduleName, runAt, deployMaxRetries) - override def findToBeDeployed: Action[Seq[PeriodicProcessDeploymentWithFullProcess]] = + override def findToBeDeployed: Action[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] = underlying.findToBeDeployed - override def findToBeRetried: Action[Seq[PeriodicProcessDeploymentWithFullProcess]] = + override def findToBeRetried: Action[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] = underlying.findToBeRetried override def findProcessData( id: PeriodicProcessDeploymentId - ): Action[PeriodicProcessDeploymentWithFullProcess] = + ): Action[PeriodicProcessDeployment[WithCanonicalProcess]] = underlying.findProcessData(id) - override def findProcessData(processName: ProcessName): Action[Seq[PeriodicProcess[CanonicalProcess]]] = + override def findProcessData(processName: ProcessName): Action[Seq[PeriodicProcess[WithCanonicalProcess]]] = underlying.findProcessData(processName) override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala index 88f620eff9e..aecd38dc2f0 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala @@ -27,7 +27,7 @@ case class PeriodicProcessDeployment[DeploymentData <: DeploymentWithJarData]( } def display: String = - s"${periodicProcess.processVersion} with scheduleName=${scheduleName.display} and deploymentId=$id" + s"${periodicProcess.deploymentData.processVersion} with scheduleName=${scheduleName.display} and deploymentId=$id" } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala index ca697fe27d9..40f8447b722 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala @@ -20,7 +20,7 @@ case class SchedulesState(schedules: Map[ScheduleId, ScheduleData]) { def isEmpty: Boolean = schedules.isEmpty def groupByProcessName: Map[ProcessName, SchedulesState] = - schedules.groupBy(_._2.process.processVersion.processName).mapValuesNow(SchedulesState) + schedules.groupBy(_._2.process.deploymentData.processVersion.processName).mapValuesNow(SchedulesState) lazy val groupedByPeriodicProcess: List[PeriodicProcessScheduleData] = schedules.toList.groupBy(_._2.process).toList.map { case (periodicProcess, groupedSchedules) => @@ -92,7 +92,7 @@ case class PeriodicProcessScheduleData( def display: String = { val deploymentsForSchedules = deployments.map(_.display) - s"processName=${process.processVersion.processName}, deploymentsForSchedules=$deploymentsForSchedules" + s"processName=${process.deploymentData.processVersion.processName}, deploymentsForSchedules=$deploymentsForSchedules" } } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala index 9da6b1f35c4..9edc7ad949d 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala @@ -18,11 +18,10 @@ class JarManagerStub extends JarManager { ): Future[DeploymentWithJarData.WithCanonicalProcess] = { Future.successful( model.DeploymentWithJarData.WithCanonicalProcess( - processName = processVersion.processName, - versionId = processVersion.versionId, + processVersion = processVersion, + jarFileName = "", process = canonicalProcess, inputConfigDuringExecutionJson = "", - jarFileName = "" ) ) } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala index ac267ca06ea..4b164d09a2d 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -202,7 +202,7 @@ class PeriodicProcessServiceIntegrationTest stateAfterSchedule should have size 1 val afterSchedule = stateAfterSchedule.firstScheduleData - afterSchedule.periodicProcessMetadata.processName shouldBe processName + afterSchedule.process.deploymentData.processVersion.processName shouldBe processName afterSchedule.latestDeployments.head.state shouldBe PeriodicProcessDeploymentState( None, None, @@ -215,9 +215,9 @@ class PeriodicProcessServiceIntegrationTest val allToDeploy = service.findToBeDeployed.futureValue allToDeploy.map( - _.deployment.periodicProcessMetadata.processName + _.periodicProcess.deploymentData.processVersion.processName ) should contain only (processName, every30MinutesProcessName) - val toDeploy = allToDeploy.find(_.deployment.periodicProcessMetadata.processName == processName).value + val toDeploy = allToDeploy.find(_.periodicProcess.deploymentData.processVersion.processName == processName).value service.deploy(toDeploy).futureValue otherProcessingTypeService.deploy(otherProcessingTypeService.findToBeDeployed.futureValue.loneElement).futureValue @@ -239,7 +239,7 @@ class PeriodicProcessServiceIntegrationTest val toDeployAfterFinish = service.findToBeDeployed.futureValue toDeployAfterFinish.map( - _.deployment.periodicProcessMetadata.processName + _.periodicProcess.deploymentData.processVersion.processName ) should contain only every30MinutesProcessName service.deactivate(processName).futureValue service.getLatestDeploymentsForActiveSchedules(processName).futureValue shouldBe empty @@ -297,9 +297,9 @@ class PeriodicProcessServiceIntegrationTest service.deploy(toDeploy).futureValue val toBeRetried :: Nil = service.findToBeDeployed.futureValue.toList - toBeRetried.deployment.state.status shouldBe PeriodicProcessDeploymentStatus.RetryingDeploy - toBeRetried.deployment.retriesLeft shouldBe 1 - toBeRetried.deployment.nextRetryAt.isDefined shouldBe true + toBeRetried.state.status shouldBe PeriodicProcessDeploymentStatus.RetryingDeploy + toBeRetried.retriesLeft shouldBe 1 + toBeRetried.nextRetryAt.isDefined shouldBe true service.deploy(toBeRetried).futureValue service.findToBeDeployed.futureValue.toList shouldBe Nil @@ -375,12 +375,12 @@ class PeriodicProcessServiceIntegrationTest val allToDeploy = service.findToBeDeployed.futureValue allToDeploy should have length 4 - val toDeploy = allToDeploy.filter(_.deployment.periodicProcessMetadata.processName == processName) + val toDeploy = allToDeploy.filter(_.periodicProcess.deploymentData.processVersion.processName == processName) toDeploy should have length 2 - toDeploy.head.deployment.runAt shouldBe localTime(expectedScheduleTime.plus(5, ChronoUnit.MINUTES)) - toDeploy.head.deployment.scheduleName.value shouldBe Some(scheduleMinute5) - toDeploy.last.deployment.runAt shouldBe localTime(expectedScheduleTime.plus(10, ChronoUnit.MINUTES)) - toDeploy.last.deployment.scheduleName.value shouldBe Some(scheduleMinute10) + toDeploy.head.runAt shouldBe localTime(expectedScheduleTime.plus(5, ChronoUnit.MINUTES)) + toDeploy.head.scheduleName.value shouldBe Some(scheduleMinute5) + toDeploy.last.runAt shouldBe localTime(expectedScheduleTime.plus(10, ChronoUnit.MINUTES)) + toDeploy.last.scheduleName.value shouldBe Some(scheduleMinute10) service.deactivate(processName).futureValue service.getLatestDeploymentsForActiveSchedules(processName).futureValue shouldBe empty @@ -417,19 +417,19 @@ class PeriodicProcessServiceIntegrationTest val toDeploy = service.findToBeDeployed.futureValue toDeploy should have length 2 - val deployment = toDeploy.find(_.deployment.scheduleName.value.contains(firstSchedule)).value + val deployment = toDeploy.find(_.scheduleName.value.contains(firstSchedule)).value service.deploy(deployment) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deployment.deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deployment.id)) val toDeployAfterDeploy = service.findToBeDeployed.futureValue toDeployAfterDeploy should have length 0 - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deployment.deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deployment.id)) service.handleFinished.futureValue val toDeployAfterFinish = service.findToBeDeployed.futureValue toDeployAfterFinish should have length 1 - toDeployAfterFinish.head.deployment.scheduleName.value.value shouldBe secondSchedule + toDeployAfterFinish.head.scheduleName.value.value shouldBe secondSchedule val firstActivity = eventually { val result = service.getScenarioActivitiesSpecificToPeriodicProcess(processIdWithName, None).futureValue @@ -492,7 +492,7 @@ class PeriodicProcessServiceIntegrationTest currentTime = timeToTriggerSchedule1 val toDeployOnSchedule1 = service.findToBeDeployed.futureValue.loneElement - toDeployOnSchedule1.deployment.scheduleName.value.value shouldBe schedule1 + toDeployOnSchedule1.scheduleName.value.value shouldBe schedule1 service.deploy(toDeployOnSchedule1).futureValue val stateAfterSchedule1Deploy = service.getLatestDeploymentsForActiveSchedules(processName).futureValue @@ -525,7 +525,7 @@ class PeriodicProcessServiceIntegrationTest currentTime = timeToTriggerSchedule2 val toDeployOnSchedule2 = service.findToBeDeployed.futureValue.loneElement - toDeployOnSchedule2.deployment.scheduleName.value.value shouldBe schedule2 + toDeployOnSchedule2.scheduleName.value.value shouldBe schedule2 service.deploy(toDeployOnSchedule2).futureValue val stateAfterSchedule2Deploy = service.getLatestDeploymentsForActiveSchedules(processName).futureValue @@ -612,7 +612,7 @@ class PeriodicProcessServiceIntegrationTest toDeploy should have length 1 val deployment = toDeploy.head service.deploy(deployment).futureValue - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deployment.deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deployment.id)) tryWithFailedListener { () => service.deactivate(processName) @@ -642,7 +642,7 @@ class PeriodicProcessServiceIntegrationTest val deployment = toDeploy.head service.deploy(deployment).futureValue - f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deployment.deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deployment.id)) // this one is cyclically called by RescheduleActor service.handleFinished.futureValue diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala index 8167ce78c85..7c8b5218ea2 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala @@ -2,14 +2,15 @@ package pl.touk.nussknacker.engine.management.periodic.db import cats.{Id, Monad} import io.circe.syntax.EncoderOps -import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId} import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.management.periodic._ import pl.touk.nussknacker.engine.management.periodic.db.InMemPeriodicProcessesRepository.{ DeploymentIdSequence, - ProcessIdSequence + ProcessIdSequence, + getLatestDeploymentQueryCount } import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessDeployment import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess From 8e0bf70628e53c6cde3791db0450e7de58d76abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Fri, 13 Dec 2024 12:00:41 +0100 Subject: [PATCH 04/16] qs --- .../periodic/PeriodicProcessService.scala | 5 +- ...cProcessesRepositoryCachingDecorator.scala | 56 +++++++++---------- .../periodic/model/PeriodicProcess.scala | 4 +- .../model/PeriodicProcessDeployment.scala | 2 +- .../periodic/model/SchedulesState.scala | 4 +- ...eriodicProcessServiceIntegrationTest.scala | 10 ++-- 6 files changed, 41 insertions(+), 40 deletions(-) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala index 967274a9d34..5c506144946 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala @@ -90,8 +90,7 @@ class PeriodicProcessService( scenarioActivityId = ScenarioActivityId(DeterministicUUIDFromLong.longUUID(deployment.id.value)), user = ScenarioUser.internalNuUser, date = metadata.dateDeployed.getOrElse(metadata.dateFinished), - scenarioVersionId = - Some(ScenarioVersionId.from(deployment.periodicProcess.deploymentData.processVersion.versionId)), + scenarioVersionId = Some(ScenarioVersionId.from(deployment.periodicProcess.processVersion.versionId)), scheduledExecutionStatus = metadata.status, dateFinished = metadata.dateFinished, scheduleName = deployment.scheduleName.display, @@ -204,7 +203,7 @@ class PeriodicProcessService( toDeploy: PeriodicProcessDeployment[WithCanonicalProcess] ): Future[Option[PeriodicProcessDeployment[WithCanonicalProcess]]] = { delegateDeploymentManager - .getProcessStates(toDeploy.periodicProcess.deploymentData.processVersion.processName)(DataFreshnessPolicy.Fresh) + .getProcessStates(toDeploy.periodicProcess.processVersion.processName)(DataFreshnessPolicy.Fresh) .map( _.value .map(_.status) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala index 0cb88f29721..643cb55d87f 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala @@ -4,7 +4,6 @@ import cats.Monad import com.github.benmanes.caffeine.cache.{AsyncCache, Caffeine} import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId} import pl.touk.nussknacker.engine.api.process.ProcessName -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.management.periodic._ import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus @@ -12,7 +11,7 @@ import pl.touk.nussknacker.engine.management.periodic.model._ import java.time.LocalDateTime import scala.compat.java8.FutureConverters.{CompletionStageOps, FutureOps} -import scala.concurrent.duration.{DurationInt, FiniteDuration} +import scala.concurrent.duration.FiniteDuration import scala.concurrent.{ExecutionContext, Future} class PeriodicProcessesRepositoryCachingDecorator( @@ -30,7 +29,7 @@ class PeriodicProcessesRepositoryCachingDecorator( private val inactiveSchedulesCache: AsyncCache[(Int, Int), Map[ProcessName, SchedulesState]] = Caffeine .newBuilder() - .expireAfterWrite(java.time.Duration.ofMillis((10 seconds).toMillis)) + .expireAfterWrite(java.time.Duration.ofMillis(cacheTTL.toMillis)) .buildAsync[(Int, Int), Map[ProcessName, SchedulesState]] type Action[T] = underlying.Action[T] @@ -71,47 +70,48 @@ class PeriodicProcessesRepositoryCachingDecorator( private def fetchActiveSchedules(deploymentsPerScheduleMaxCount: Int)( implicit freshnessPolicy: DataFreshnessPolicy - ) = { - def fetchAndUpdateCache(): Future[Map[ProcessName, SchedulesState]] = { - val resultFuture = underlying.getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount) - activeSchedulesCache.put(deploymentsPerScheduleMaxCount, resultFuture.toJava.toCompletableFuture) - resultFuture - } - freshnessPolicy match { - case DataFreshnessPolicy.Fresh => - fetchAndUpdateCache() - case DataFreshnessPolicy.CanBeCached => - Option(activeSchedulesCache.getIfPresent(deploymentsPerScheduleMaxCount)) - .map(_.toScala) - .getOrElse(fetchAndUpdateCache()) - } - } + ): Future[Map[ProcessName, SchedulesState]] = + fetchUsingCache[Int]( + cache = activeSchedulesCache, + key = deploymentsPerScheduleMaxCount, + fetch = deploymentsPerScheduleMaxCount => + underlying.getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount) + ) private def fetchInactiveSchedules( inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int, + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = + fetchUsingCache[(Int, Int)]( + cache = inactiveSchedulesCache, + key = (inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount), + fetch = key => + underlying.getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount = key._1, + deploymentsPerScheduleMaxCount = key._2 + ) + ) + + private def fetchUsingCache[KEY]( + cache: AsyncCache[KEY, Map[ProcessName, SchedulesState]], + key: KEY, + fetch: KEY => Future[Map[ProcessName, SchedulesState]], )(implicit freshnessPolicy: DataFreshnessPolicy) = { def fetchAndUpdateCache(): Future[Map[ProcessName, SchedulesState]] = { - val resultFuture = underlying.getLatestDeploymentsForLatestInactiveSchedules( - inactiveProcessesMaxCount, - deploymentsPerScheduleMaxCount - ) - inactiveSchedulesCache.put( - (inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount), - resultFuture.toJava.toCompletableFuture - ) + val resultFuture = fetch(key) + cache.put(key, resultFuture.toJava.toCompletableFuture) resultFuture } freshnessPolicy match { case DataFreshnessPolicy.Fresh => fetchAndUpdateCache() case DataFreshnessPolicy.CanBeCached => - Option(inactiveSchedulesCache.getIfPresent(inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount)) + Option(cache.getIfPresent(key)) .map(_.toScala) .getOrElse(fetchAndUpdateCache()) } - Option(inactiveSchedulesCache.getIfPresent(inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount)) + Option(cache.getIfPresent(key)) .map(_.toScala) .getOrElse(fetchAndUpdateCache()) } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala index 7f4db356785..e89deab2320 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala @@ -16,4 +16,6 @@ case class PeriodicProcess[DeploymentData <: DeploymentWithJarData]( active: Boolean, createdAt: LocalDateTime, processActionId: Option[ProcessActionId] -) +) { + val processVersion: ProcessVersion = deploymentData.processVersion +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala index aecd38dc2f0..88f620eff9e 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala @@ -27,7 +27,7 @@ case class PeriodicProcessDeployment[DeploymentData <: DeploymentWithJarData]( } def display: String = - s"${periodicProcess.deploymentData.processVersion} with scheduleName=${scheduleName.display} and deploymentId=$id" + s"${periodicProcess.processVersion} with scheduleName=${scheduleName.display} and deploymentId=$id" } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala index 40f8447b722..ca697fe27d9 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala @@ -20,7 +20,7 @@ case class SchedulesState(schedules: Map[ScheduleId, ScheduleData]) { def isEmpty: Boolean = schedules.isEmpty def groupByProcessName: Map[ProcessName, SchedulesState] = - schedules.groupBy(_._2.process.deploymentData.processVersion.processName).mapValuesNow(SchedulesState) + schedules.groupBy(_._2.process.processVersion.processName).mapValuesNow(SchedulesState) lazy val groupedByPeriodicProcess: List[PeriodicProcessScheduleData] = schedules.toList.groupBy(_._2.process).toList.map { case (periodicProcess, groupedSchedules) => @@ -92,7 +92,7 @@ case class PeriodicProcessScheduleData( def display: String = { val deploymentsForSchedules = deployments.map(_.display) - s"processName=${process.deploymentData.processVersion.processName}, deploymentsForSchedules=$deploymentsForSchedules" + s"processName=${process.processVersion.processName}, deploymentsForSchedules=$deploymentsForSchedules" } } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala index 4b164d09a2d..18767b855d1 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -202,7 +202,7 @@ class PeriodicProcessServiceIntegrationTest stateAfterSchedule should have size 1 val afterSchedule = stateAfterSchedule.firstScheduleData - afterSchedule.process.deploymentData.processVersion.processName shouldBe processName + afterSchedule.process.processVersion.processName shouldBe processName afterSchedule.latestDeployments.head.state shouldBe PeriodicProcessDeploymentState( None, None, @@ -215,9 +215,9 @@ class PeriodicProcessServiceIntegrationTest val allToDeploy = service.findToBeDeployed.futureValue allToDeploy.map( - _.periodicProcess.deploymentData.processVersion.processName + _.periodicProcess.processVersion.processName ) should contain only (processName, every30MinutesProcessName) - val toDeploy = allToDeploy.find(_.periodicProcess.deploymentData.processVersion.processName == processName).value + val toDeploy = allToDeploy.find(_.periodicProcess.processVersion.processName == processName).value service.deploy(toDeploy).futureValue otherProcessingTypeService.deploy(otherProcessingTypeService.findToBeDeployed.futureValue.loneElement).futureValue @@ -239,7 +239,7 @@ class PeriodicProcessServiceIntegrationTest val toDeployAfterFinish = service.findToBeDeployed.futureValue toDeployAfterFinish.map( - _.periodicProcess.deploymentData.processVersion.processName + _.periodicProcess.processVersion.processName ) should contain only every30MinutesProcessName service.deactivate(processName).futureValue service.getLatestDeploymentsForActiveSchedules(processName).futureValue shouldBe empty @@ -375,7 +375,7 @@ class PeriodicProcessServiceIntegrationTest val allToDeploy = service.findToBeDeployed.futureValue allToDeploy should have length 4 - val toDeploy = allToDeploy.filter(_.periodicProcess.deploymentData.processVersion.processName == processName) + val toDeploy = allToDeploy.filter(_.periodicProcess.processVersion.processName == processName) toDeploy should have length 2 toDeploy.head.runAt shouldBe localTime(expectedScheduleTime.plus(5, ChronoUnit.MINUTES)) toDeploy.head.scheduleName.value shouldBe Some(scheduleMinute5) From a380f15ebb50b2c23df4cbed03bc51ab676678b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Fri, 13 Dec 2024 12:04:39 +0100 Subject: [PATCH 05/16] qs --- .../nussknacker/engine/management/periodic/JarManager.scala | 1 - .../management/periodic/PeriodicDeploymentManager.scala | 2 +- .../engine/management/periodic/JarManagerStub.scala | 2 +- .../periodic/PeriodicProcessServiceIntegrationTest.scala | 4 +--- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala index a604e9265f6..bf5cab7a4f7 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala @@ -1,7 +1,6 @@ package pl.touk.nussknacker.engine.management.periodic import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala index 13678c7e651..d1679f61ba2 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala @@ -43,7 +43,7 @@ object PeriodicDeploymentManager { listenerFactory: PeriodicProcessListenerFactory, additionalDeploymentDataProvider: AdditionalDeploymentDataProvider, dependencies: DeploymentManagerDependencies, - scenarioStateCacheTTL: Option[FiniteDuration], + scenarioStateCacheTTL: Option[FiniteDuration] ): PeriodicDeploymentManager = { import dependencies._ diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala index 9edc7ad949d..4ded31dd0d1 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala @@ -19,9 +19,9 @@ class JarManagerStub extends JarManager { Future.successful( model.DeploymentWithJarData.WithCanonicalProcess( processVersion = processVersion, - jarFileName = "", process = canonicalProcess, inputConfigDuringExecutionJson = "", + jarFileName = "" ) ) } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala index 18767b855d1..e8b05eb53c8 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -238,9 +238,7 @@ class PeriodicProcessServiceIntegrationTest service.handleFinished.futureValue val toDeployAfterFinish = service.findToBeDeployed.futureValue - toDeployAfterFinish.map( - _.periodicProcess.processVersion.processName - ) should contain only every30MinutesProcessName + toDeployAfterFinish.map(_.periodicProcess.processVersion.processName) should contain only every30MinutesProcessName service.deactivate(processName).futureValue service.getLatestDeploymentsForActiveSchedules(processName).futureValue shouldBe empty val inactiveStates = service From a50b53ed6a58da189de4bbdca31ff25494c9b713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Fri, 13 Dec 2024 12:38:39 +0100 Subject: [PATCH 06/16] Scala 2.12 fix --- .../db/PeriodicProcessesRepositoryCachingDecorator.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala index 643cb55d87f..cbf4f42ef5a 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala @@ -23,12 +23,14 @@ class PeriodicProcessesRepositoryCachingDecorator( private val activeSchedulesCache: AsyncCache[Int, Map[ProcessName, SchedulesState]] = Caffeine .newBuilder() + .asInstanceOf[Caffeine[Int, Map[ProcessName, SchedulesState]]] .expireAfterWrite(java.time.Duration.ofMillis(cacheTTL.toMillis)) .buildAsync[Int, Map[ProcessName, SchedulesState]] private val inactiveSchedulesCache: AsyncCache[(Int, Int), Map[ProcessName, SchedulesState]] = Caffeine .newBuilder() + .asInstanceOf[Caffeine[(Int, Int), Map[ProcessName, SchedulesState]]] .expireAfterWrite(java.time.Duration.ofMillis(cacheTTL.toMillis)) .buildAsync[(Int, Int), Map[ProcessName, SchedulesState]] From c9f72a7c12f1f35d39269bde4d0238fa6aaa608d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Mon, 16 Dec 2024 15:26:04 +0100 Subject: [PATCH 07/16] qs --- .../api/deployment/DeploymentManager.scala | 8 ++ .../ui/process/ProcessStateProvider.scala | 2 +- ...inkPeriodicDeploymentManagerProvider.scala | 2 +- .../periodic/PeriodicDeploymentManager.scala | 9 ++- .../periodic/PeriodicProcessService.scala | 67 +++++++++++++++- .../periodic/DeploymentManagerStub.scala | 45 +++++++---- .../PeriodicDeploymentManagerTest.scala | 26 +++---- ...eriodicProcessServiceIntegrationTest.scala | 14 ++-- .../periodic/PeriodicProcessServiceTest.scala | 22 ++++-- ...riodicProcessesCachingRepositoryTest.scala | 4 +- .../engine/management/FlinkRestManager.scala | 76 ++++++++++++------- ...nkStreamingDeploymentManagerProvider.scala | 64 ++++++++++++++-- 12 files changed, 258 insertions(+), 81 deletions(-) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala index 602048fd3a2..24e18a1e346 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala @@ -109,6 +109,14 @@ trait ManagerSpecificScenarioActivitiesStoredByManager { self: DeploymentManager } +trait StateQueryForAllScenariosSupported { self: DeploymentManager => + + def getProcessesStates()( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] + +} + sealed trait DeploymentSynchronisationSupport trait DeploymentSynchronisationSupported extends DeploymentSynchronisationSupport { diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/ProcessStateProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/ProcessStateProvider.scala index ab8e63f8e14..e2f718bf40f 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/ProcessStateProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/ProcessStateProvider.scala @@ -7,7 +7,7 @@ import pl.touk.nussknacker.restmodel.scenariodetails.ScenarioWithDetails import pl.touk.nussknacker.ui.process.repository.ScenarioWithDetailsEntity import pl.touk.nussknacker.ui.security.api.LoggedUser -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.Future import scala.language.higherKinds trait ProcessStateProvider { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala index 11f7f8c7d36..a000282f114 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala @@ -41,7 +41,7 @@ class FlinkPeriodicDeploymentManagerProvider extends DeploymentManagerProvider w scenarioStateCacheTTL: Option[FiniteDuration], ): ValidatedNel[String, DeploymentManager] = { logger.info("Creating FlinkPeriodic scenario manager") - delegate.createDeploymentManager(modelData, dependencies, config, scenarioStateCacheTTL).map { + delegate.createDeploymentManagerWithCapabilities(modelData, dependencies, config, scenarioStateCacheTTL).map { delegateDeploymentManager => import net.ceedubs.ficus.Ficus._ import net.ceedubs.ficus.readers.ArbitraryTypeReader._ diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala index d1679f61ba2..0072f0ee0db 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala @@ -33,7 +33,7 @@ import scala.concurrent.{ExecutionContext, Future} object PeriodicDeploymentManager { def apply( - delegate: DeploymentManager, + delegate: DeploymentManager with StateQueryForAllScenariosSupported, schedulePropertyExtractorFactory: SchedulePropertyExtractorFactory, processConfigEnricherFactory: ProcessConfigEnricherFactory, periodicBatchConfig: PeriodicBatchConfig, @@ -113,6 +113,7 @@ class PeriodicDeploymentManager private[periodic] ( toClose: () => Unit )(implicit val ec: ExecutionContext) extends DeploymentManager + with StateQueryForAllScenariosSupported with ManagerSpecificScenarioActivitiesStoredByManager with LazyLogging { @@ -205,6 +206,12 @@ class PeriodicDeploymentManager private[periodic] ( service.getStatusDetails(name).map(_.map(List(_))) } + override def getProcessesStates()( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = { + service.getStatusDetails().map(_.map(_.map { case (k, v) => (k, List(v)) })) + } + override def resolve( idWithName: ProcessIdWithName, statusDetailsList: List[StatusDetails], diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala index 5c506144946..ddcce8247c9 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala @@ -41,7 +41,7 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.control.NonFatal class PeriodicProcessService( - delegateDeploymentManager: DeploymentManager, + delegateDeploymentManager: DeploymentManager with StateQueryForAllScenariosSupported, jarManager: JarManager, scheduledProcessesRepository: PeriodicProcessesRepository, periodicProcessListener: PeriodicProcessListener, @@ -492,6 +492,18 @@ class PeriodicProcessService( } } + def getStatusDetails()( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[Map[ProcessName, StatusDetails]]] = { + delegateDeploymentManager.getProcessesStates().flatMap { statusesWithFreshness => + mergeStatusWithDeployments(statusesWithFreshness.value).map { statusDetails => + statusesWithFreshness.map(_.flatMap { case (name, _) => + statusDetails.get(name).map((name, _)) + }) + } + } + } + private def mergeStatusWithDeployments( name: ProcessName, runtimeStatuses: List[StatusDetails] @@ -525,12 +537,55 @@ class PeriodicProcessService( } } + private def mergeStatusWithDeployments( + runtimeStatuses: Map[ProcessName, List[StatusDetails]] + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, StatusDetails]] = { + def toDeploymentStatuses(processName: ProcessName, schedulesState: SchedulesState) = + schedulesState.schedules.toList + .flatMap { case (scheduleId, scheduleData) => + scheduleData.latestDeployments.map { deployment => + DeploymentStatus( + deployment.id, + scheduleId, + deployment.createdAt, + deployment.runAt, + deployment.state.status, + scheduleData.process.active, + runtimeStatuses.getOrElse(processName, List.empty).getStatus(deployment.id) + ) + } + } + .sorted(DeploymentStatus.ordering.reverse) + + for { + activeSchedules <- getLatestDeploymentsForActiveSchedules(MaxDeploymentsStatus) + inactiveSchedules <- getLatestDeploymentsForLatestInactiveSchedules(MaxDeploymentsStatus, MaxDeploymentsStatus) + } yield { + val allProcessNames = activeSchedules.keySet ++ inactiveSchedules.keySet + allProcessNames.map { processName => + val activeSchedulesForProcess = activeSchedules.getOrElse(processName, SchedulesState(Map.empty)) + val inactiveSchedulesForProcess = inactiveSchedules.getOrElse(processName, SchedulesState(Map.empty)) + val status = PeriodicProcessStatus( + toDeploymentStatuses(processName, activeSchedulesForProcess), + toDeploymentStatuses(processName, inactiveSchedulesForProcess) + ) + val mergedStatus = status.mergedStatusDetails.copy(status = status) + (processName, mergedStatus) + }.toMap + } + } + def getLatestDeploymentsForActiveSchedules( processName: ProcessName, deploymentsPerScheduleMaxCount: Int = 1 )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = scheduledProcessesRepository.getLatestDeploymentsForActiveSchedules(processName, deploymentsPerScheduleMaxCount) + def getLatestDeploymentsForActiveSchedules( + deploymentsPerScheduleMaxCount: Int + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = + scheduledProcessesRepository.getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount) + def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, @@ -543,6 +598,16 @@ class PeriodicProcessService( deploymentsPerScheduleMaxCount ) + def getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int + )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = + scheduledProcessesRepository + .getLatestDeploymentsForLatestInactiveSchedules( + inactiveProcessesMaxCount, + deploymentsPerScheduleMaxCount + ) + implicit class RuntimeStatusesExt(runtimeStatuses: List[StatusDetails]) { private val runtimeStatusesMap = runtimeStatuses.flatMap(status => status.deploymentId.map(_ -> status)).toMap diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala index c2e8a4edb4f..343e047da39 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala @@ -9,20 +9,29 @@ import pl.touk.nussknacker.engine.testing.StubbingCommands import scala.concurrent.Future -class DeploymentManagerStub extends BaseDeploymentManager with StubbingCommands { - - var jobStatus: Option[StatusDetails] = None - - def setStateStatus(status: StateStatus, deploymentIdOpt: Option[PeriodicProcessDeploymentId]): Unit = { - jobStatus = Some( - StatusDetails( - deploymentId = deploymentIdOpt.map(pdid => DeploymentId(pdid.toString)), - externalDeploymentId = Some(ExternalDeploymentId("1")), - status = status, - version = None, - startTime = None, - attributes = None, - errors = Nil +class DeploymentManagerStub + extends BaseDeploymentManager + with StateQueryForAllScenariosSupported + with StubbingCommands { + + var jobStatus: Map[ProcessName, List[StatusDetails]] = Map.empty + + def setStateStatus( + processName: ProcessName, + status: StateStatus, + deploymentIdOpt: Option[PeriodicProcessDeploymentId] + ): Unit = { + jobStatus = Map( + processName -> List( + StatusDetails( + deploymentId = deploymentIdOpt.map(pdid => DeploymentId(pdid.toString)), + externalDeploymentId = Some(ExternalDeploymentId("1")), + status = status, + version = None, + startTime = None, + attributes = None, + errors = Nil + ) ) ) } @@ -47,7 +56,13 @@ class DeploymentManagerStub extends BaseDeploymentManager with StubbingCommands override def getProcessStates( name: ProcessName )(implicit freshnessPolicy: DataFreshnessPolicy): Future[WithDataFreshnessStatus[List[StatusDetails]]] = { - Future.successful(WithDataFreshnessStatus.fresh(jobStatus.toList)) + Future.successful(WithDataFreshnessStatus.fresh(jobStatus.get(name).toList.flatten)) + } + + override def getProcessesStates()( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = { + Future.successful(WithDataFreshnessStatus.fresh(jobStatus)) } override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = NoDeploymentSynchronisationSupport diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala index bae299e53fd..8fa44cb2b7c 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala @@ -151,7 +151,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be scheduled when scenario scheduled and job finished on Flink") { val f = new Fixture val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Finished, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails statusDetails.status shouldBe a[ScheduledStatus] @@ -169,7 +169,7 @@ class PeriodicDeploymentManagerTest PeriodicProcessDeploymentStatus.Finished, LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC) ) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Finished, Some(deploymentId)) f.periodicProcessService.deactivate(processName).futureValue val state = @@ -185,7 +185,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be running when scenario deployed and job running on Flink") { val f = new Fixture val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Running, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails statusDetails.status shouldBe SimpleStateStatus.Running @@ -197,7 +197,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be waiting for reschedule if job finished on Flink but scenario is still deployed") { val f = new Fixture val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Finished, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails statusDetails.status shouldBe WaitingForScheduleStatus @@ -295,7 +295,7 @@ class PeriodicDeploymentManagerTest test("should get status of failed job") { val f = new Fixture val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, ProblemStateStatus.Failed, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails statusDetails.status shouldBe ProblemStateStatus.Failed @@ -307,7 +307,7 @@ class PeriodicDeploymentManagerTest test("should redeploy failed scenario") { val f = new Fixture val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, ProblemStateStatus.Failed, Some(deploymentId)) val statusDetailsBeforeRedeploy = f.getMergedStatusDetails statusDetailsBeforeRedeploy.status shouldBe ProblemStateStatus.Failed f.getAllowedActions( @@ -374,7 +374,7 @@ class PeriodicDeploymentManagerTest test("should redeploy running scenario") { val f = new Fixture val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Running, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails f.getAllowedActions(statusDetails, processVersion.versionId, None, Some(processVersion.versionId)) shouldBe List( ScenarioActionName.Cancel @@ -401,7 +401,7 @@ class PeriodicDeploymentManagerTest test("should redeploy finished scenario") { val f = new Fixture val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Finished, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails f.getAllowedActions(statusDetails, processVersion.versionId, None, Some(processVersion.versionId)) shouldBe List( ScenarioActionName.Cancel @@ -428,7 +428,7 @@ class PeriodicDeploymentManagerTest test("should cancel failed job after RescheduleActor handles finished") { val f = new Fixture val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, ProblemStateStatus.Failed, Some(deploymentId)) // this one is cyclically called by RescheduleActor f.periodicProcessService.handleFinished.futureValue @@ -448,7 +448,7 @@ class PeriodicDeploymentManagerTest test("should reschedule failed job after RescheduleActor handles finished when configured") { val f = new Fixture(executionConfig = PeriodicExecutionConfig(rescheduleOnFailure = true)) val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, ProblemStateStatus.Failed, Some(deploymentId)) // this one is cyclically called by RescheduleActor f.periodicProcessService.handleFinished.futureValue @@ -464,7 +464,7 @@ class PeriodicDeploymentManagerTest test("should cancel failed job before RescheduleActor handles finished") { val f = new Fixture val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, ProblemStateStatus.Failed, Some(deploymentId)) f.periodicDeploymentManager.processCommand(DMCancelScenarioCommand(processName, User("test", "Tester"))).futureValue @@ -476,13 +476,13 @@ class PeriodicDeploymentManagerTest test("should cancel failed scenario after disappeared from Flink console") { val f = new Fixture val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, ProblemStateStatus.Failed, Some(deploymentId)) // this one is cyclically called by RescheduleActor f.periodicProcessService.handleFinished.futureValue // after some time Flink stops returning job status - f.delegateDeploymentManagerStub.jobStatus = None + f.delegateDeploymentManagerStub.jobStatus = Map.empty f.getMergedStatusDetails.status shouldEqual ProblemStateStatus.Failed f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala index e8b05eb53c8..060e35aa575 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -234,7 +234,11 @@ class PeriodicProcessServiceIntegrationTest ) afterDeployDeployment.runAt shouldBe localTime(expectedScheduleTime) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(afterDeployDeployment.id)) + f.delegateDeploymentManagerStub.setStateStatus( + processName, + SimpleStateStatus.Finished, + Some(afterDeployDeployment.id) + ) service.handleFinished.futureValue val toDeployAfterFinish = service.findToBeDeployed.futureValue @@ -417,12 +421,12 @@ class PeriodicProcessServiceIntegrationTest val deployment = toDeploy.find(_.scheduleName.value.contains(firstSchedule)).value service.deploy(deployment) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Running, Some(deployment.id)) val toDeployAfterDeploy = service.findToBeDeployed.futureValue toDeployAfterDeploy should have length 0 - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Finished, Some(deployment.id)) service.handleFinished.futureValue val toDeployAfterFinish = service.findToBeDeployed.futureValue @@ -610,7 +614,7 @@ class PeriodicProcessServiceIntegrationTest toDeploy should have length 1 val deployment = toDeploy.head service.deploy(deployment).futureValue - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Finished, Some(deployment.id)) tryWithFailedListener { () => service.deactivate(processName) @@ -640,7 +644,7 @@ class PeriodicProcessServiceIntegrationTest val deployment = toDeploy.head service.deploy(deployment).futureValue - f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deployment.id)) + f.delegateDeploymentManagerStub.setStateStatus(processName, ProblemStateStatus.Failed, Some(deployment.id)) // this one is cyclically called by RescheduleActor service.handleFinished.futureValue diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala index d35b65a0593..2b085e09943 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala @@ -220,7 +220,7 @@ class PeriodicProcessServiceTest PeriodicProcessDeploymentStatus.Deployed, processActionId = Some(processActionId) ) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Finished, Some(deploymentId)) f.periodicProcessService.handleFinished.futureValue @@ -237,7 +237,7 @@ class PeriodicProcessServiceTest val finished :: scheduled :: Nil = f.repository.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList f.events.toList shouldBe List( - FinishedEvent(finished, f.delegateDeploymentManagerStub.jobStatus), + FinishedEvent(finished, f.delegateDeploymentManagerStub.jobStatus.get(processName).flatMap(_.headOption)), ScheduledEvent(scheduled, firstSchedule = false) ) } @@ -251,7 +251,7 @@ class PeriodicProcessServiceTest PeriodicProcessDeploymentStatus.Deployed, processActionId = Some(processActionId) ) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.DuringDeploy, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.DuringDeploy, Some(deploymentId)) f.periodicProcessService.handleFinished.futureValue @@ -268,7 +268,7 @@ class PeriodicProcessServiceTest scheduleProperty = cronInPast, processActionId = Some(processActionId) ) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, SimpleStateStatus.Finished, Some(deploymentId)) f.periodicProcessService.handleFinished.futureValue @@ -280,7 +280,10 @@ class PeriodicProcessServiceTest // TODO: active should be false val event = createPeriodicProcessDeployment(processEntity.copy(active = true), f.repository.deploymentEntities.loneElement) - f.events.loneElement shouldBe FinishedEvent(event, f.delegateDeploymentManagerStub.jobStatus) + f.events.loneElement shouldBe FinishedEvent( + event, + f.delegateDeploymentManagerStub.jobStatus.get(processName).flatMap(_.headOption) + ) } test("handleFinished - should not deactivate process if there is future schedule") { @@ -345,7 +348,7 @@ class PeriodicProcessServiceTest test("handleFinished - should mark as failed for failed Flink job") { val f = new Fixture val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName, ProblemStateStatus.Failed, Some(deploymentId)) f.periodicProcessService.handleFinished.futureValue @@ -355,7 +358,12 @@ class PeriodicProcessServiceTest f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed val expectedDetails = createPeriodicProcessDeployment(processEntity, f.repository.deploymentEntities.head) - f.events.toList shouldBe List(FailedOnRunEvent(expectedDetails, f.delegateDeploymentManagerStub.jobStatus)) + f.events.toList shouldBe List( + FailedOnRunEvent( + expectedDetails, + f.delegateDeploymentManagerStub.jobStatus.get(processName).flatMap(_.headOption) + ) + ) } test("deploy - should deploy and mark as so") { diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesCachingRepositoryTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesCachingRepositoryTest.scala index 2505f83efd5..a6239641e23 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesCachingRepositoryTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesCachingRepositoryTest.scala @@ -74,7 +74,7 @@ class PeriodicProcessesCachingRepositoryTest for (i <- 1 to n) { val deploymentId = f.repository.addActiveProcess(processName(i), PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName(i), SimpleStateStatus.Running, Some(deploymentId)) } getLatestDeploymentQueryCount.set(0) @@ -94,7 +94,7 @@ class PeriodicProcessesCachingRepositoryTest for (i <- 1 to n) { val deploymentId = f.repository.addActiveProcess(processName(i), PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deploymentId)) + f.delegateDeploymentManagerStub.setStateStatus(processName(i), SimpleStateStatus.Running, Some(deploymentId)) } getLatestDeploymentQueryCount.set(0) diff --git a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala index fae0d547781..f6670501ba0 100644 --- a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala +++ b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala @@ -5,12 +5,12 @@ import org.apache.flink.api.common.{JobID, JobStatus} import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus -import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, ProcessName, VersionId} +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.{DeploymentId, ExternalDeploymentId} import pl.touk.nussknacker.engine.management.FlinkRestManager.ParsedJobConfig import pl.touk.nussknacker.engine.management.rest.FlinkClient -import pl.touk.nussknacker.engine.management.rest.flinkRestModel.BaseJobStatusCounts +import pl.touk.nussknacker.engine.management.rest.flinkRestModel.{BaseJobStatusCounts, JobOverview} import pl.touk.nussknacker.engine.{BaseModelData, DeploymentManagerDependencies, newdeployment} import scala.concurrent.Future @@ -22,6 +22,7 @@ class FlinkRestManager( dependencies: DeploymentManagerDependencies, mainClassName: String ) extends FlinkDeploymentManager(modelData, dependencies, config.shouldVerifyBeforeDeploy, mainClassName) + with StateQueryForAllScenariosSupported with LazyLogging { import dependencies._ @@ -33,39 +34,56 @@ class FlinkRestManager( override def getProcessStates( name: ProcessName )(implicit freshnessPolicy: DataFreshnessPolicy): Future[WithDataFreshnessStatus[List[StatusDetails]]] = { - val preparedName = modelData.namingStrategy.prepareName(name.value) + getProcessesStates() + .map(_.map(_.getOrElse(name, List.empty))) + } + override def getProcessesStates()( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = { client .getJobsOverviews() - .flatMap(result => - Future - .sequence( - result.value - .filter(_.name == preparedName) - .map(job => - withParsedJobConfig(job.jid, name).map { jobConfig => - // TODO: return error when there's no correct version in process - // currently we're rather lax on this, so that this change is backward-compatible - // we log debug here for now, since it's invoked v. often - if (jobConfig.isEmpty) { - logger.debug(s"No correct job details in deployed scenario: ${job.name}") - } - StatusDetails( - SimpleStateStatus.fromDeploymentStatus(toDeploymentStatus(job.state, job.tasks)), - jobConfig.flatMap(_.deploymentId), - Some(ExternalDeploymentId(job.jid)), - version = jobConfig.map(_.version), - startTime = Some(job.`start-time`), - attributes = Option.empty, - errors = List.empty - ) - } - ) - ) + .flatMap { result => + statusDetailsFrom(result.value) .map(WithDataFreshnessStatus(_, cached = result.cached)) // TODO: How to do it nicer? - ) + } } + private def statusDetailsFrom( + jobOverviews: List[JobOverview] + ): Future[Map[ProcessName, List[StatusDetails]]] = Future + .sequence { + jobOverviews + .groupBy(_.name) + .flatMap { case (name, jobs) => + modelData.namingStrategy.decodeName(name).map(decoded => (ProcessName(decoded), jobs)) + } + .map { case (name, jobs) => + val statusDetails = jobs.map { job => + withParsedJobConfig(job.jid, name).map { jobConfig => + // TODO: return error when there's no correct version in process + // currently we're rather lax on this, so that this change is backward-compatible + // we log debug here for now, since it's invoked v. often + if (jobConfig.isEmpty) { + logger.debug(s"No correct job details in deployed scenario: ${job.name}") + } + StatusDetails( + SimpleStateStatus.fromDeploymentStatus(toDeploymentStatus(job.state, job.tasks)), + jobConfig.flatMap(_.deploymentId), + Some(ExternalDeploymentId(job.jid)), + version = jobConfig.map(_.version), + startTime = Some(job.`start-time`), + attributes = Option.empty, + errors = List.empty + ) + } + } + Future.sequence(statusDetails).map((name, _)) + } + + } + .map(_.toMap) + override val deploymentSynchronisationSupport: DeploymentSynchronisationSupport = new DeploymentSynchronisationSupported { diff --git a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala index fbdc8b5df8a..8bbedec92c3 100644 --- a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala +++ b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala @@ -6,12 +6,14 @@ import pl.touk.nussknacker.engine._ import pl.touk.nussknacker.engine.api.StreamMetaData import pl.touk.nussknacker.engine.api.component.ScenarioPropertyConfig import pl.touk.nussknacker.engine.api.definition._ -import pl.touk.nussknacker.engine.api.deployment.DeploymentManager +import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.deployment.cache.CachingProcessStateDeploymentManager -import pl.touk.nussknacker.engine.deployment.EngineSetupName +import pl.touk.nussknacker.engine.api.process.{ProcessIdWithName, ProcessName, VersionId} +import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, EngineSetupName} import pl.touk.nussknacker.engine.management.FlinkConfig.RestUrlPath import pl.touk.nussknacker.engine.management.rest.FlinkClient +import scala.concurrent.Future import scala.concurrent.duration.FiniteDuration import scala.util.Try @@ -27,13 +29,63 @@ class FlinkStreamingDeploymentManagerProvider extends DeploymentManagerProvider deploymentConfig: Config, scenarioStateCacheTTL: Option[FiniteDuration] ): ValidatedNel[String, DeploymentManager] = { + createDeploymentManagerWithCapabilities(modelData, dependencies, deploymentConfig, scenarioStateCacheTTL) + } + + def createDeploymentManagerWithCapabilities( + modelData: BaseModelData, + dependencies: DeploymentManagerDependencies, + deploymentConfig: Config, + scenarioStateCacheTTL: Option[FiniteDuration] + ): ValidatedNel[String, DeploymentManager with StateQueryForAllScenariosSupported] = { import dependencies._ val flinkConfig = deploymentConfig.rootAs[FlinkConfig] FlinkClient.create(flinkConfig, scenarioStateCacheTTL).map { client => - CachingProcessStateDeploymentManager.wrapWithCachingIfNeeded( - new FlinkStreamingRestManager(client, flinkConfig, modelData, dependencies), - scenarioStateCacheTTL - ) + val underlying = new FlinkStreamingRestManager(client, flinkConfig, modelData, dependencies) + val withCaching = CachingProcessStateDeploymentManager.wrapWithCachingIfNeeded(underlying, scenarioStateCacheTTL) + new DeploymentManager with StateQueryForAllScenariosSupported { + + override def getProcessesStates()( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = + underlying.getProcessesStates() + + override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = + withCaching.deploymentSynchronisationSupport + + override def processCommand[Result](command: DMScenarioCommand[Result]): Future[Result] = + withCaching.processCommand(command) + + override def getProcessStates(name: ProcessName)( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[List[StatusDetails]]] = + withCaching.getProcessStates(name) + + override def resolve( + idWithName: ProcessIdWithName, + statusDetails: List[StatusDetails], + lastStateAction: Option[ProcessAction], + latestVersionId: VersionId, + deployedVersionId: Option[VersionId], + currentlyPresentedVersionId: Option[VersionId] + ): Future[ProcessState] = withCaching.resolve( + idWithName, + statusDetails, + lastStateAction, + latestVersionId, + deployedVersionId, + currentlyPresentedVersionId + ) + + override def processStateDefinitionManager: ProcessStateDefinitionManager = + withCaching.processStateDefinitionManager + + override def customActionsDefinitions: List[CustomActionDefinition] = + withCaching.customActionsDefinitions + + override def close(): Unit = + withCaching.close() + } } } From 5c9de75afa641819fdcbb403d7187e75c53664af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Tue, 17 Dec 2024 12:06:18 +0100 Subject: [PATCH 08/16] remove cache --- .../api/deployment/DeploymentManager.scala | 20 ++ .../deployment/DeploymentService.scala | 86 ++++++-- .../util/WithDataFreshnessStatusUtils.scala | 16 ++ ...inkPeriodicDeploymentManagerProvider.scala | 3 +- .../periodic/PeriodicDeploymentManager.scala | 10 +- ...cProcessesRepositoryCachingDecorator.scala | 183 ------------------ .../periodic/DeploymentManagerStub.scala | 24 +++ ...la => PeriodicProcessesFetchingTest.scala} | 36 ++-- 8 files changed, 148 insertions(+), 230 deletions(-) create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/util/WithDataFreshnessStatusUtils.scala delete mode 100644 engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala rename engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/{PeriodicProcessesCachingRepositoryTest.scala => PeriodicProcessesFetchingTest.scala} (80%) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala index 24e18a1e346..f2d0c5da9e0 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala @@ -59,6 +59,26 @@ trait DeploymentManager extends AutoCloseable { ): Future[WithDataFreshnessStatus[ProcessState]] = { for { statusDetailsWithFreshness <- getProcessStates(idWithName.name) + stateWithFreshness <- getProcessState( + idWithName, + lastStateAction, + latestVersionId, + deployedVersionId, + currentlyPresentedVersionId, + statusDetailsWithFreshness, + ) + } yield stateWithFreshness + } + + final def getProcessState( + idWithName: ProcessIdWithName, + lastStateAction: Option[ProcessAction], + latestVersionId: VersionId, + deployedVersionId: Option[VersionId], + currentlyPresentedVersionId: Option[VersionId], + statusDetailsWithFreshness: WithDataFreshnessStatus[List[StatusDetails]], + ): Future[WithDataFreshnessStatus[ProcessState]] = { + for { stateWithFreshness <- resolve( idWithName, statusDetailsWithFreshness.value, diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala index 596b887f541..1e496cd0f80 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala @@ -26,7 +26,7 @@ import pl.touk.nussknacker.ui.api.{DeploymentCommentSettings, ListenerApiUser} import pl.touk.nussknacker.ui.listener.ProcessChangeEvent.{OnActionExecutionFinished, OnActionFailed, OnActionSuccess} import pl.touk.nussknacker.ui.listener.{ProcessChangeListener, User => ListenerUser} import pl.touk.nussknacker.ui.process.ProcessStateProvider -import pl.touk.nussknacker.ui.process.ScenarioWithDetailsConversions._ +import pl.touk.nussknacker.ui.process.ScenarioWithDetailsConversions.Ops import pl.touk.nussknacker.ui.process.deployment.LoggedUserConversions.LoggedUserOps import pl.touk.nussknacker.ui.process.exception.{DeployingInvalidScenarioError, ProcessIllegalAction} import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataProvider @@ -34,6 +34,7 @@ import pl.touk.nussknacker.ui.process.repository.ProcessDBQueryRepository.Proces import pl.touk.nussknacker.ui.process.repository._ import pl.touk.nussknacker.ui.security.api.{AdminUser, LoggedUser, NussknackerInternalUser} import pl.touk.nussknacker.ui.util.FutureUtils._ +import pl.touk.nussknacker.ui.util.WithDataFreshnessStatusUtils.WithDataFreshnessStatusOps import pl.touk.nussknacker.ui.validation.{CustomActionValidator, UIProcessValidator} import pl.touk.nussknacker.ui.{BadRequestError, NotFoundError} import slick.dbio.{DBIO, DBIOAction} @@ -169,9 +170,9 @@ class DeploymentService( } /** - * Common validations and operations for a command execution. - * @return gathered data for further command execution - */ + * Common validations and operations for a command execution. + * @return gathered data for further command execution + */ private def prepareCommandContextWithAction[PS: ScenarioShapeFetchStrategy]( processId: ProcessIdWithName, actionName: ScenarioActionName, @@ -203,7 +204,7 @@ class DeploymentService( _ = checkIfCanPerformActionOnScenario(actionName, processDetails) // 1.7. check if action is allowed for current state inProgressActionNames <- actionRepository.getInProgressActionNames(processDetails.processId) - processState <- getProcessState(processDetails, inProgressActionNames, None) + processState <- getProcessState(processDetails, inProgressActionNames, None, None) _ = checkIfCanPerformActionInState(actionName, processDetails, processState) // 1.8. create new action, action is started with "in progress" state, the whole command execution can take some time actionId <- actionRepository.addInProgressAction( @@ -437,7 +438,7 @@ class DeploymentService( processDetailsOpt <- processRepository.fetchLatestProcessDetailsForProcessId[Unit](processIdWithName.id) processDetails <- existsOrFail(processDetailsOpt, ProcessNotFoundError(processIdWithName.name)) inProgressActionNames <- actionRepository.getInProgressActionNames(processDetails.processId) - result <- getProcessState(processDetails, inProgressActionNames, currentlyPresentedVersionId) + result <- getProcessState(processDetails, inProgressActionNames, currentlyPresentedVersionId, None) } yield result) } @@ -446,7 +447,7 @@ class DeploymentService( )(implicit user: LoggedUser, freshnessPolicy: DataFreshnessPolicy): Future[ProcessState] = { dbioRunner.run(for { inProgressActionNames <- actionRepository.getInProgressActionNames(processDetails.processId) - result <- getProcessState(processDetails, inProgressActionNames, None) + result <- getProcessState(processDetails, inProgressActionNames, None, None) } yield result) } @@ -454,17 +455,24 @@ class DeploymentService( implicit user: LoggedUser, freshnessPolicy: DataFreshnessPolicy ): Future[F[ScenarioWithDetails]] = { + val scenarios = processTraverse.toList dbioRunner.run( for { - actionsInProgress <- getInProgressActionTypesForProcessTraverse(processTraverse) + actionsInProgress <- getInProgressActionTypesForScenarios(scenarios) + prefetchedStates <- DBIO.from(getPrefetchedStatesForDeploymentManagers(scenarios)) processesWithState <- processTraverse .map { case process if process.isFragment => DBIO.successful(process) case process => + val prefetched = + prefetchedStates + .get(process.processingType) + .flatMap(_.get(process.name)) getProcessState( process.toEntity, actionsInProgress.getOrElse(process.processIdUnsafe, Set.empty), None, + prefetched, ).map(state => process.copy(state = Some(state))) } .sequence[DB, ScenarioWithDetails] @@ -472,12 +480,36 @@ class DeploymentService( ) } + private def getPrefetchedStatesForDeploymentManagers( + scenarios: List[ScenarioWithDetails], + )( + implicit user: LoggedUser, + freshnessPolicy: DataFreshnessPolicy + ): Future[Map[ProcessingType, WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]]] = { + val allProcessingTypes = scenarios.map(_.processingType).toSet + val deploymentManagersByProcessingTypes = + allProcessingTypes.flatMap(pt => dispatcher.deploymentManager(pt).map(dm => (pt, dm))) + val prefetchedStatesByProcessingTypes = Future + .sequence( + deploymentManagersByProcessingTypes.flatMap { case (processingType, manager) => + manager match { + case dm: StateQueryForAllScenariosSupported => + Some(dm.getProcessesStates().map(states => (processingType, states))) + case _ => + None + } + } + ) + .map(_.toMap) + prefetchedStatesByProcessingTypes + } + // This is optimisation tweak. We want to reduce number of calls for in progress action types. So for >1 scenarios // we do one call for all in progress action types for all scenarios - private def getInProgressActionTypesForProcessTraverse[F[_]: Traverse]( - processTraverse: F[ScenarioWithDetails] + private def getInProgressActionTypesForScenarios( + scenarios: List[ScenarioWithDetails] ): DB[Map[ProcessId, Set[ScenarioActionName]]] = { - processTraverse.toList match { + scenarios match { case Nil => DBIO.successful(Map.empty) case head :: Nil => actionRepository @@ -493,6 +525,7 @@ class DeploymentService( processDetails: ScenarioWithDetailsEntity[_], inProgressActionNames: Set[ScenarioActionName], currentlyPresentedVersionId: Option[VersionId], + prefetchedStatusDetailsWithFreshness: Option[WithDataFreshnessStatus[List[StatusDetails]]], )(implicit freshnessPolicy: DataFreshnessPolicy, user: LoggedUser): DB[ProcessState] = { val processVersionId = processDetails.processVersionId val deployedVersionId = processDetails.lastDeployedAction.map(_.processVersionId) @@ -535,6 +568,7 @@ class DeploymentService( processVersionId, deployedVersionId, currentlyPresentedVersionId, + prefetchedStatusDetailsWithFreshness, ) ) .map { statusWithFreshness => @@ -627,18 +661,32 @@ class DeploymentService( latestVersionId: VersionId, deployedVersionId: Option[VersionId], currentlyPresentedVersionId: Option[VersionId], + prefetchedStatusDetailsWithFreshness: Option[WithDataFreshnessStatus[List[StatusDetails]]], )( implicit freshnessPolicy: DataFreshnessPolicy ): Future[WithDataFreshnessStatus[ProcessState]] = { - val state = deploymentManager - .getProcessState( - processIdWithName, - lastStateAction, - latestVersionId, - deployedVersionId, - currentlyPresentedVersionId - ) + val state = (prefetchedStatusDetailsWithFreshness match { + case Some(statusDetailsWithFreshness) => + deploymentManager + .getProcessState( + processIdWithName, + lastStateAction, + latestVersionId, + deployedVersionId, + currentlyPresentedVersionId, + statusDetailsWithFreshness, + ) + case None => + deploymentManager + .getProcessState( + processIdWithName, + lastStateAction, + latestVersionId, + deployedVersionId, + currentlyPresentedVersionId, + ) + }) .recover { case NonFatal(e) => logger.warn(s"Failed to get status of ${processIdWithName.name}: ${e.getMessage}", e) failedToGetProcessState(latestVersionId) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/WithDataFreshnessStatusUtils.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/WithDataFreshnessStatusUtils.scala new file mode 100644 index 00000000000..9023930cf08 --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/WithDataFreshnessStatusUtils.scala @@ -0,0 +1,16 @@ +package pl.touk.nussknacker.ui.util + +import pl.touk.nussknacker.engine.api.deployment.WithDataFreshnessStatus + +object WithDataFreshnessStatusUtils { + + implicit class WithDataFreshnessStatusOps[K, V](scenarioActivity: WithDataFreshnessStatus[Map[K, V]]) { + + def get(k: K): Option[WithDataFreshnessStatus[V]] = scenarioActivity.map(_.get(k)) match { + case WithDataFreshnessStatus(Some(value), cached) => Some(WithDataFreshnessStatus(value, cached)) + case WithDataFreshnessStatus(None, _) => None + } + + } + +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala index a000282f114..9ce983104cc 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala @@ -58,8 +58,7 @@ class FlinkPeriodicDeploymentManagerProvider extends DeploymentManagerProvider w modelData = modelData, EmptyPeriodicProcessListenerFactory, DefaultAdditionalDeploymentDataProvider, - dependencies, - scenarioStateCacheTTL, + dependencies ) } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala index 0072f0ee0db..b2256ce083c 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala @@ -13,7 +13,6 @@ import pl.touk.nussknacker.engine.management.periodic.Utils.{createActorWithRetr import pl.touk.nussknacker.engine.management.periodic.db.{ DbInitializer, PeriodicProcessesRepository, - PeriodicProcessesRepositoryCachingDecorator, SlickPeriodicProcessesRepository } import pl.touk.nussknacker.engine.management.periodic.flink.FlinkJarManager @@ -42,8 +41,7 @@ object PeriodicDeploymentManager { modelData: BaseModelData, listenerFactory: PeriodicProcessListenerFactory, additionalDeploymentDataProvider: AdditionalDeploymentDataProvider, - dependencies: DeploymentManagerDependencies, - scenarioStateCacheTTL: Option[FiniteDuration] + dependencies: DeploymentManagerDependencies ): PeriodicDeploymentManager = { import dependencies._ @@ -52,17 +50,13 @@ object PeriodicDeploymentManager { val (db: jdbc.JdbcBackend.DatabaseDef, dbProfile: JdbcProfile) = DbInitializer.init(periodicBatchConfig.db) val scheduledProcessesRepository = new SlickPeriodicProcessesRepository(db, dbProfile, clock, periodicBatchConfig.processingType) - val repositoryWithCaching = scenarioStateCacheTTL match { - case Some(ttl) => new PeriodicProcessesRepositoryCachingDecorator(scheduledProcessesRepository, ttl) - case None => scheduledProcessesRepository - } val jarManager = FlinkJarManager(flinkConfig, periodicBatchConfig, modelData) val listener = listenerFactory.create(originalConfig) val processConfigEnricher = processConfigEnricherFactory(originalConfig) val service = new PeriodicProcessService( delegate, jarManager, - repositoryWithCaching, + scheduledProcessesRepository, listener, additionalDeploymentDataProvider, periodicBatchConfig.deploymentRetry, diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala deleted file mode 100644 index cbf4f42ef5a..00000000000 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepositoryCachingDecorator.scala +++ /dev/null @@ -1,183 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic.db - -import cats.Monad -import com.github.benmanes.caffeine.cache.{AsyncCache, Caffeine} -import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId} -import pl.touk.nussknacker.engine.api.process.ProcessName -import pl.touk.nussknacker.engine.management.periodic._ -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData.WithCanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model._ - -import java.time.LocalDateTime -import scala.compat.java8.FutureConverters.{CompletionStageOps, FutureOps} -import scala.concurrent.duration.FiniteDuration -import scala.concurrent.{ExecutionContext, Future} - -class PeriodicProcessesRepositoryCachingDecorator( - val underlying: PeriodicProcessesRepository, - cacheTTL: FiniteDuration, -)(implicit ec: ExecutionContext) - extends PeriodicProcessesRepository { - - private val activeSchedulesCache: AsyncCache[Int, Map[ProcessName, SchedulesState]] = - Caffeine - .newBuilder() - .asInstanceOf[Caffeine[Int, Map[ProcessName, SchedulesState]]] - .expireAfterWrite(java.time.Duration.ofMillis(cacheTTL.toMillis)) - .buildAsync[Int, Map[ProcessName, SchedulesState]] - - private val inactiveSchedulesCache: AsyncCache[(Int, Int), Map[ProcessName, SchedulesState]] = - Caffeine - .newBuilder() - .asInstanceOf[Caffeine[(Int, Int), Map[ProcessName, SchedulesState]]] - .expireAfterWrite(java.time.Duration.ofMillis(cacheTTL.toMillis)) - .buildAsync[(Int, Int), Map[ProcessName, SchedulesState]] - - type Action[T] = underlying.Action[T] - - override implicit def monad: Monad[Action] = underlying.monad - - override def run[T](action: Action[T]): Future[T] = underlying.run(action) - - override def getLatestDeploymentsForActiveSchedules( - processName: ProcessName, - deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { - fetchActiveSchedules(deploymentsPerScheduleMaxCount) - .map(_.getOrElse(processName, SchedulesState(Map.empty))) - } - - override def getLatestDeploymentsForActiveSchedules( - deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = { - fetchActiveSchedules(deploymentsPerScheduleMaxCount) - } - - override def getLatestDeploymentsForLatestInactiveSchedules( - processName: ProcessName, - inactiveProcessesMaxCount: Int, - deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { - fetchInactiveSchedules(inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount) - .map(_.getOrElse(processName, SchedulesState(Map.empty))) - } - - override def getLatestDeploymentsForLatestInactiveSchedules( - inactiveProcessesMaxCount: Int, - deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = { - fetchInactiveSchedules(inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount) - } - - private def fetchActiveSchedules(deploymentsPerScheduleMaxCount: Int)( - implicit freshnessPolicy: DataFreshnessPolicy - ): Future[Map[ProcessName, SchedulesState]] = - fetchUsingCache[Int]( - cache = activeSchedulesCache, - key = deploymentsPerScheduleMaxCount, - fetch = deploymentsPerScheduleMaxCount => - underlying.getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount) - ) - - private def fetchInactiveSchedules( - inactiveProcessesMaxCount: Int, - deploymentsPerScheduleMaxCount: Int, - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = - fetchUsingCache[(Int, Int)]( - cache = inactiveSchedulesCache, - key = (inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount), - fetch = key => - underlying.getLatestDeploymentsForLatestInactiveSchedules( - inactiveProcessesMaxCount = key._1, - deploymentsPerScheduleMaxCount = key._2 - ) - ) - - private def fetchUsingCache[KEY]( - cache: AsyncCache[KEY, Map[ProcessName, SchedulesState]], - key: KEY, - fetch: KEY => Future[Map[ProcessName, SchedulesState]], - )(implicit freshnessPolicy: DataFreshnessPolicy) = { - def fetchAndUpdateCache(): Future[Map[ProcessName, SchedulesState]] = { - val resultFuture = fetch(key) - cache.put(key, resultFuture.toJava.toCompletableFuture) - resultFuture - } - freshnessPolicy match { - case DataFreshnessPolicy.Fresh => - fetchAndUpdateCache() - case DataFreshnessPolicy.CanBeCached => - Option(cache.getIfPresent(key)) - .map(_.toScala) - .getOrElse(fetchAndUpdateCache()) - } - - Option(cache.getIfPresent(key)) - .map(_.toScala) - .getOrElse(fetchAndUpdateCache()) - } - - // Methods below are not using caching mechanism - - override def getSchedulesState( - scenarioName: ProcessName, - after: Option[LocalDateTime] - ): Action[SchedulesState] = - underlying.getSchedulesState(scenarioName, after) - - override def create( - deploymentWithJarData: WithCanonicalProcess, - scheduleProperty: ScheduleProperty, - processActionId: ProcessActionId - ): Action[PeriodicProcess[WithCanonicalProcess]] = - underlying.create(deploymentWithJarData, scheduleProperty, processActionId) - - override def schedule( - id: PeriodicProcessId, - scheduleName: ScheduleName, - runAt: LocalDateTime, - deployMaxRetries: Int - ): Action[PeriodicProcessDeployment[WithCanonicalProcess]] = - underlying.schedule(id, scheduleName, runAt, deployMaxRetries) - - override def findToBeDeployed: Action[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] = - underlying.findToBeDeployed - - override def findToBeRetried: Action[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] = - underlying.findToBeRetried - - override def findProcessData( - id: PeriodicProcessDeploymentId - ): Action[PeriodicProcessDeployment[WithCanonicalProcess]] = - underlying.findProcessData(id) - - override def findProcessData(processName: ProcessName): Action[Seq[PeriodicProcess[WithCanonicalProcess]]] = - underlying.findProcessData(processName) - - override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( - expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus] - ): Action[SchedulesState] = - underlying.findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus(expectedDeploymentStatuses) - - override def markDeployed(id: PeriodicProcessDeploymentId): Action[Unit] = - underlying.markDeployed(id) - - override def markFailed(id: PeriodicProcessDeploymentId): Action[Unit] = - underlying.markFailed(id) - - override def markFinished(id: PeriodicProcessDeploymentId): Action[Unit] = - underlying.markFinished(id) - - override def markFailedOnDeployWithStatus( - id: PeriodicProcessDeploymentId, - status: PeriodicProcessDeploymentStatus, - retriesLeft: Int, - retryAt: Option[LocalDateTime] - ): Action[Unit] = - underlying.markFailedOnDeployWithStatus(id, status, retriesLeft, retryAt) - - override def markInactive(processId: PeriodicProcessId): Action[Unit] = - underlying.markInactive(processId) - -} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala index 343e047da39..f5a08c9e561 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala @@ -16,6 +16,30 @@ class DeploymentManagerStub var jobStatus: Map[ProcessName, List[StatusDetails]] = Map.empty + def setEmptyStateStatus(): Unit = { + jobStatus = Map.empty + } + + def addStateStatus( + processName: ProcessName, + status: StateStatus, + deploymentIdOpt: Option[PeriodicProcessDeploymentId] + ): Unit = { + jobStatus = jobStatus ++ Map( + processName -> List( + StatusDetails( + deploymentId = deploymentIdOpt.map(pdid => DeploymentId(pdid.toString)), + externalDeploymentId = Some(ExternalDeploymentId("1")), + status = status, + version = None, + startTime = None, + attributes = None, + errors = Nil + ) + ) + ) + } + def setStateStatus( processName: ProcessName, status: StateStatus, diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesCachingRepositoryTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala similarity index 80% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesCachingRepositoryTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala index a6239641e23..d1d7dc42437 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesCachingRepositoryTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala @@ -10,20 +10,14 @@ import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.deployment.DeploymentData import pl.touk.nussknacker.engine.management.periodic.db.InMemPeriodicProcessesRepository.getLatestDeploymentQueryCount -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepositoryCachingDecorator import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.service.{ - DefaultAdditionalDeploymentDataProvider, - EmptyListener, - ProcessConfigEnricher -} +import pl.touk.nussknacker.engine.management.periodic.service.{DefaultAdditionalDeploymentDataProvider, EmptyListener, ProcessConfigEnricher} import pl.touk.nussknacker.test.PatientScalaFutures import java.time.Clock import java.util.UUID -import scala.concurrent.duration.DurationInt -class PeriodicProcessesCachingRepositoryTest +class PeriodicProcessesFetchingTest extends AnyFunSuite with Matchers with ScalaFutures @@ -47,7 +41,7 @@ class PeriodicProcessesCachingRepositoryTest val periodicProcessService = new PeriodicProcessService( delegateDeploymentManager = delegateDeploymentManagerStub, jarManager = jarManagerStub, - scheduledProcessesRepository = new PeriodicProcessesRepositoryCachingDecorator(repository, 10 seconds), + scheduledProcessesRepository = repository, periodicProcessListener = EmptyListener, additionalDeploymentDataProvider = DefaultAdditionalDeploymentDataProvider, deploymentRetryConfig = DeploymentRetryConfig(), @@ -68,13 +62,17 @@ class PeriodicProcessesCachingRepositoryTest } - test("getStatusDetails - should perform 2 db queries for N periodic processes when using cache") { + test( + "getStatusDetails - should perform 2*N db queries for N periodic processes when fetching statuses individually" + ) { val f = new Fixture val n = 10 + f.delegateDeploymentManagerStub.setEmptyStateStatus() + for (i <- 1 to n) { val deploymentId = f.repository.addActiveProcess(processName(i), PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(processName(i), SimpleStateStatus.Running, Some(deploymentId)) + f.delegateDeploymentManagerStub.addStateStatus(processName(i), SimpleStateStatus.Running, Some(deploymentId)) } getLatestDeploymentQueryCount.set(0) @@ -85,27 +83,29 @@ class PeriodicProcessesCachingRepositoryTest f.periodicProcessService.getStatusDetails(processName(i)).futureValue } - getLatestDeploymentQueryCount.get() shouldEqual 2 + getLatestDeploymentQueryCount.get() shouldEqual 2 * n } - test("getStatusDetails - should perform 2*N db queries for N periodic processes when not using cache") { + test("getStatusDetails - should perform 2 db queries for N periodic processes when fetching all at once") { val f = new Fixture val n = 10 + f.delegateDeploymentManagerStub.setEmptyStateStatus() + for (i <- 1 to n) { val deploymentId = f.repository.addActiveProcess(processName(i), PeriodicProcessDeploymentStatus.Deployed) - f.delegateDeploymentManagerStub.setStateStatus(processName(i), SimpleStateStatus.Running, Some(deploymentId)) + f.delegateDeploymentManagerStub.addStateStatus(processName(i), SimpleStateStatus.Running, Some(deploymentId)) } getLatestDeploymentQueryCount.set(0) implicit val freshnessPolicy: DataFreshnessPolicy = DataFreshnessPolicy.Fresh - for (i <- 1 to n) { - f.periodicProcessService.getStatusDetails(processName(i)).futureValue - } + val statuses = f.periodicProcessService.getStatusDetails().futureValue.value - getLatestDeploymentQueryCount.get() shouldEqual 2 * n + statuses.size shouldEqual n + + getLatestDeploymentQueryCount.get() shouldEqual 2 } } From b5d9679023b9f4b0b404919395e4d7cfcd3d3b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Tue, 17 Dec 2024 18:53:00 +0100 Subject: [PATCH 09/16] add recover --- .../deployment/DeploymentService.scala | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala index ecf0632e65d..24cfbd453c8 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala @@ -464,15 +464,12 @@ class DeploymentService( .map { case process if process.isFragment => DBIO.successful(process) case process => - val prefetched = - prefetchedStates - .get(process.processingType) - .flatMap(_.get(process.name)) + val prefetchedStatesForProcess = prefetchedStates.get(process.processingType).flatMap(_.get(process.name)) getProcessState( process.toEntity, actionsInProgress.getOrElse(process.processIdUnsafe, Set.empty), None, - prefetched, + prefetchedStatesForProcess, ).map(state => process.copy(state = Some(state))) } .sequence[DB, ScenarioWithDetails] @@ -490,17 +487,27 @@ class DeploymentService( val deploymentManagersByProcessingTypes = allProcessingTypes.flatMap(pt => dispatcher.deploymentManager(pt).map(dm => (pt, dm))) val prefetchedStatesByProcessingTypes = Future - .sequence( - deploymentManagersByProcessingTypes.flatMap { case (processingType, manager) => + .sequence { + deploymentManagersByProcessingTypes.map { case (processingType, manager) => manager match { case dm: StateQueryForAllScenariosSupported => - Some(dm.getProcessesStates().map(states => (processingType, states))) + dm + .getProcessesStates() + .map(states => Some((processingType, states))) + .recover { case NonFatal(e) => + logger.warn( + s"Failed to get statuses of all scenarios in deployment manager for $processingType: ${e.getMessage}", + e + ) + None + } case _ => - None + Future.successful(None) } } - ) - .map(_.toMap) + } + .map(_.flatten.toMap) + prefetchedStatesByProcessingTypes } From 19808b731296d2f4d8707c76c5c21fc150b44de7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Tue, 17 Dec 2024 23:18:02 +0100 Subject: [PATCH 10/16] qs --- .../periodic/PeriodicProcessesFetchingTest.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala index d1d7dc42437..2ed2ff48a1a 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala @@ -11,7 +11,11 @@ import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.deployment.DeploymentData import pl.touk.nussknacker.engine.management.periodic.db.InMemPeriodicProcessesRepository.getLatestDeploymentQueryCount import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.service.{DefaultAdditionalDeploymentDataProvider, EmptyListener, ProcessConfigEnricher} +import pl.touk.nussknacker.engine.management.periodic.service.{ + DefaultAdditionalDeploymentDataProvider, + EmptyListener, + ProcessConfigEnricher +} import pl.touk.nussknacker.test.PatientScalaFutures import java.time.Clock @@ -46,6 +50,7 @@ class PeriodicProcessesFetchingTest additionalDeploymentDataProvider = DefaultAdditionalDeploymentDataProvider, deploymentRetryConfig = DeploymentRetryConfig(), executionConfig = executionConfig, + maxFetchedPeriodicScenarioActivities = Some(200), processConfigEnricher = ProcessConfigEnricher.identity, clock = Clock.systemDefaultZone(), new ProcessingTypeActionServiceStub, From 30540eab628a835a355eaecdfdb2cbb117c3f58c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Wed, 18 Dec 2024 13:34:18 +0100 Subject: [PATCH 11/16] qs --- .../api/deployment/DeploymentManager.scala | 22 ++--- .../deployment/DeploymentService.scala | 87 ++++++++--------- .../periodic/PeriodicProcessService.scala | 22 ++--- .../db/PeriodicProcessesRepository.scala | 24 +++-- .../db/InMemPeriodicProcessesRepository.scala | 23 +++-- .../engine/management/FlinkRestManager.scala | 5 +- ...nkStreamingDeploymentManagerProvider.scala | 93 ++++++++++--------- 7 files changed, 141 insertions(+), 135 deletions(-) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala index f2d0c5da9e0..b8f610245a1 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala @@ -59,7 +59,7 @@ trait DeploymentManager extends AutoCloseable { ): Future[WithDataFreshnessStatus[ProcessState]] = { for { statusDetailsWithFreshness <- getProcessStates(idWithName.name) - stateWithFreshness <- getProcessState( + stateWithFreshness <- resolvePrefetchedProcessState( idWithName, lastStateAction, latestVersionId, @@ -70,7 +70,7 @@ trait DeploymentManager extends AutoCloseable { } yield stateWithFreshness } - final def getProcessState( + final def resolvePrefetchedProcessState( idWithName: ProcessIdWithName, lastStateAction: Option[ProcessAction], latestVersionId: VersionId, @@ -78,16 +78,14 @@ trait DeploymentManager extends AutoCloseable { currentlyPresentedVersionId: Option[VersionId], statusDetailsWithFreshness: WithDataFreshnessStatus[List[StatusDetails]], ): Future[WithDataFreshnessStatus[ProcessState]] = { - for { - stateWithFreshness <- resolve( - idWithName, - statusDetailsWithFreshness.value, - lastStateAction, - latestVersionId, - deployedVersionId, - currentlyPresentedVersionId, - ).map(state => statusDetailsWithFreshness.map(_ => state)) - } yield stateWithFreshness + resolve( + idWithName, + statusDetailsWithFreshness.value, + lastStateAction, + latestVersionId, + deployedVersionId, + currentlyPresentedVersionId, + ).map(state => statusDetailsWithFreshness.map(_ => state)) } /** diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala index 24cfbd453c8..6afe777928a 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala @@ -459,17 +459,19 @@ class DeploymentService( dbioRunner.run( for { actionsInProgress <- getInProgressActionTypesForScenarios(scenarios) - prefetchedStates <- DBIO.from(getPrefetchedStatesForDeploymentManagers(scenarios)) + // DeploymentManager's may support fetching state of all scenarios at once (StateQueryForAllScenariosSupported capability) + // For those DM's we prefetch the state and then use it for resolving individual scenarios. + prefetchedStates <- DBIO.from(getPrefetchedStatesForSupportedManagers(scenarios)) processesWithState <- processTraverse .map { case process if process.isFragment => DBIO.successful(process) case process => - val prefetchedStatesForProcess = prefetchedStates.get(process.processingType).flatMap(_.get(process.name)) + val prefetchedStateForProcess = prefetchedStates.get(process.processingType).flatMap(_.get(process.name)) getProcessState( process.toEntity, actionsInProgress.getOrElse(process.processIdUnsafe, Set.empty), None, - prefetchedStatesForProcess, + prefetchedStateForProcess, ).map(state => process.copy(state = Some(state))) } .sequence[DB, ScenarioWithDetails] @@ -477,7 +479,7 @@ class DeploymentService( ) } - private def getPrefetchedStatesForDeploymentManagers( + private def getPrefetchedStatesForSupportedManagers( scenarios: List[ScenarioWithDetails], )( implicit user: LoggedUser, @@ -490,19 +492,8 @@ class DeploymentService( .sequence { deploymentManagersByProcessingTypes.map { case (processingType, manager) => manager match { - case dm: StateQueryForAllScenariosSupported => - dm - .getProcessesStates() - .map(states => Some((processingType, states))) - .recover { case NonFatal(e) => - logger.warn( - s"Failed to get statuses of all scenarios in deployment manager for $processingType: ${e.getMessage}", - e - ) - None - } - case _ => - Future.successful(None) + case dm: StateQueryForAllScenariosSupported => getProcessesStates(processingType, dm) + case _ => Future.successful(None) } } } @@ -511,6 +502,21 @@ class DeploymentService( prefetchedStatesByProcessingTypes } + private def getProcessesStates(processingType: ProcessingType, manager: StateQueryForAllScenariosSupported)( + implicit freshnessPolicy: DataFreshnessPolicy, + ): Future[Option[(ProcessingType, WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]])]] = { + manager + .getProcessesStates() + .map(states => Some((processingType, states))) + .recover { case NonFatal(e) => + logger.warn( + s"Failed to get statuses of all scenarios in deployment manager for $processingType: ${e.getMessage}", + e + ) + None + } + } + // This is optimisation tweak. We want to reduce number of calls for in progress action types. So for >1 scenarios // we do one call for all in progress action types for all scenarios private def getInProgressActionTypesForScenarios( @@ -668,36 +674,33 @@ class DeploymentService( latestVersionId: VersionId, deployedVersionId: Option[VersionId], currentlyPresentedVersionId: Option[VersionId], - prefetchedStatusDetailsWithFreshness: Option[WithDataFreshnessStatus[List[StatusDetails]]], + prefetchedStateOpt: Option[WithDataFreshnessStatus[List[StatusDetails]]], )( implicit freshnessPolicy: DataFreshnessPolicy ): Future[WithDataFreshnessStatus[ProcessState]] = { - val state = (prefetchedStatusDetailsWithFreshness match { - case Some(statusDetailsWithFreshness) => - deploymentManager - .getProcessState( - processIdWithName, - lastStateAction, - latestVersionId, - deployedVersionId, - currentlyPresentedVersionId, - statusDetailsWithFreshness, - ) + val state = (prefetchedStateOpt match { + case Some(prefetchedState) => + deploymentManager.resolvePrefetchedProcessState( + processIdWithName, + lastStateAction, + latestVersionId, + deployedVersionId, + currentlyPresentedVersionId, + prefetchedState, + ) case None => - deploymentManager - .getProcessState( - processIdWithName, - lastStateAction, - latestVersionId, - deployedVersionId, - currentlyPresentedVersionId, - ) - }) - .recover { case NonFatal(e) => - logger.warn(s"Failed to get status of ${processIdWithName.name}: ${e.getMessage}", e) - failedToGetProcessState(latestVersionId) - } + deploymentManager.getProcessState( + processIdWithName, + lastStateAction, + latestVersionId, + deployedVersionId, + currentlyPresentedVersionId, + ) + }).recover { case NonFatal(e) => + logger.warn(s"Failed to get status of ${processIdWithName.name}: ${e.getMessage}", e) + failedToGetProcessState(latestVersionId) + } scenarioStateTimeout .map { timeout => diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala index 882060ac3b0..7719183091a 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala @@ -398,14 +398,12 @@ class PeriodicProcessService( } yield handleEvent(FailedOnRunEvent(currentState, state)) } - def deactivate(processName: ProcessName): Future[Iterable[DeploymentId]] = { - implicit val freshnessPolicy: DataFreshnessPolicy = DataFreshnessPolicy.Fresh + def deactivate(processName: ProcessName): Future[Iterable[DeploymentId]] = for { activeSchedules <- getLatestDeploymentsForActiveSchedules(processName) (runningDeploymentsForSchedules, _) <- synchronizeDeploymentsStates(processName, activeSchedules) _ <- activeSchedules.groupedByPeriodicProcess.map(p => deactivateAction(p.process)).sequence.runWithCallbacks } yield runningDeploymentsForSchedules.map(deployment => DeploymentId(deployment.toString)) - } private def deactivateAction(process: PeriodicProcess[WithoutCanonicalProcess]): RepositoryAction[Callback] = { logger.info(s"Deactivate periodic process id: ${process.id.value}") @@ -512,7 +510,7 @@ class PeriodicProcessService( private def mergeStatusWithDeployments( name: ProcessName, runtimeStatuses: List[StatusDetails] - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[StatusDetails] = { + ): Future[StatusDetails] = { def toDeploymentStatuses(schedulesState: SchedulesState) = schedulesState.schedules.toList .flatMap { case (scheduleId, scheduleData) => scheduleData.latestDeployments.map { deployment => @@ -544,7 +542,7 @@ class PeriodicProcessService( private def mergeStatusWithDeployments( runtimeStatuses: Map[ProcessName, List[StatusDetails]] - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, StatusDetails]] = { + ): Future[Map[ProcessName, StatusDetails]] = { def toDeploymentStatuses(processName: ProcessName, schedulesState: SchedulesState) = schedulesState.schedules.toList .flatMap { case (scheduleId, scheduleData) => @@ -583,35 +581,37 @@ class PeriodicProcessService( def getLatestDeploymentsForActiveSchedules( processName: ProcessName, deploymentsPerScheduleMaxCount: Int = 1 - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = - scheduledProcessesRepository.getLatestDeploymentsForActiveSchedules(processName, deploymentsPerScheduleMaxCount) + ): Future[SchedulesState] = + scheduledProcessesRepository.getLatestDeploymentsForActiveSchedules(processName, deploymentsPerScheduleMaxCount).run def getLatestDeploymentsForActiveSchedules( deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = - scheduledProcessesRepository.getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount) + ): Future[Map[ProcessName, SchedulesState]] = + scheduledProcessesRepository.getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount).run def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = + ): Future[SchedulesState] = scheduledProcessesRepository .getLatestDeploymentsForLatestInactiveSchedules( processName, inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount ) + .run def getLatestDeploymentsForLatestInactiveSchedules( inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = + ): Future[Map[ProcessName, SchedulesState]] = scheduledProcessesRepository .getLatestDeploymentsForLatestInactiveSchedules( inactiveProcessesMaxCount, deploymentsPerScheduleMaxCount ) + .run implicit class RuntimeStatusesExt(runtimeStatuses: List[StatusDetails]) { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala index 0f7c169a8bf..8ad8ce0bb45 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala @@ -5,7 +5,7 @@ import com.github.tminglei.slickpg.ExPostgresProfile import com.typesafe.scalalogging.LazyLogging import io.circe.parser.decode import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId} +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.management.periodic._ import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessWithoutJson @@ -130,22 +130,22 @@ trait PeriodicProcessesRepository { def getLatestDeploymentsForActiveSchedules( processName: ProcessName, deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] + ): Action[SchedulesState] def getLatestDeploymentsForActiveSchedules( deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] + ): Action[Map[ProcessName, SchedulesState]] def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] + ): Action[SchedulesState] def getLatestDeploymentsForLatestInactiveSchedules( inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] + ): Action[Map[ProcessName, SchedulesState]] def findToBeDeployed: Action[Seq[PeriodicProcessDeployment[WithCanonicalProcess]]] @@ -331,43 +331,41 @@ class SlickPeriodicProcessesRepository( override def getLatestDeploymentsForActiveSchedules( processName: ProcessName, deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { + ): Action[SchedulesState] = { val activeProcessesQuery = PeriodicProcessesWithoutJson.filter(p => p.processName === processName && p.active) getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount) .map(_.getOrElse(processName, SchedulesState(Map.empty))) - .run } override def getLatestDeploymentsForActiveSchedules( deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = { + ): Action[Map[ProcessName, SchedulesState]] = { val activeProcessesQuery = PeriodicProcessesWithoutJson.filter(_.active) - getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount).run + getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount) } override def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { + ): Action[SchedulesState] = { val filteredProcessesQuery = PeriodicProcessesWithoutJson .filter(p => p.processName === processName && !p.active) .sortBy(_.createdAt.desc) .take(inactiveProcessesMaxCount) getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount) .map(_.getOrElse(processName, SchedulesState(Map.empty))) - .run } override def getLatestDeploymentsForLatestInactiveSchedules( inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = { + ): Action[Map[ProcessName, SchedulesState]] = { val filteredProcessesQuery = PeriodicProcessesWithoutJson .filter(!_.active) .sortBy(_.createdAt.desc) .take(inactiveProcessesMaxCount) - getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount).run + getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount) } private def getLatestDeploymentsForEachSchedule( diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala index 7c8b5218ea2..2ebb103de28 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala @@ -2,10 +2,9 @@ package pl.touk.nussknacker.engine.management.periodic.db import cats.{Id, Monad} import io.circe.syntax.EncoderOps -import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId} +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.build.ScenarioBuilder -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.management.periodic._ import pl.touk.nussknacker.engine.management.periodic.db.InMemPeriodicProcessesRepository.{ DeploymentIdSequence, @@ -170,17 +169,17 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP override def getLatestDeploymentsForActiveSchedules( processName: ProcessName, deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { + ): Action[SchedulesState] = { getLatestDeploymentQueryCount.incrementAndGet() getLatestDeploymentsForPeriodicProcesses( processEntities(processName).filter(_.active), deploymentsPerScheduleMaxCount - ).run + ) } - override def getLatestDeploymentsForActiveSchedules(deploymentsPerScheduleMaxCount: Int)( - implicit freshnessPolicy: DataFreshnessPolicy - ): Future[Map[ProcessName, SchedulesState]] = { + override def getLatestDeploymentsForActiveSchedules( + deploymentsPerScheduleMaxCount: Int + ): Action[Map[ProcessName, SchedulesState]] = { getLatestDeploymentQueryCount.incrementAndGet() allProcessEntities.map { case (processName, list) => processName -> getLatestDeploymentsForPeriodicProcesses( @@ -188,23 +187,23 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP deploymentsPerScheduleMaxCount ) } - }.run + } override def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[SchedulesState] = { + ): Action[SchedulesState] = { getLatestDeploymentQueryCount.incrementAndGet() val filteredProcesses = processEntities(processName).filterNot(_.active).sortBy(_.createdAt).takeRight(inactiveProcessesMaxCount) - getLatestDeploymentsForPeriodicProcesses(filteredProcesses, deploymentsPerScheduleMaxCount).run + getLatestDeploymentsForPeriodicProcesses(filteredProcesses, deploymentsPerScheduleMaxCount) } override def getLatestDeploymentsForLatestInactiveSchedules( inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int - )(implicit freshnessPolicy: DataFreshnessPolicy): Future[Map[ProcessName, SchedulesState]] = { + ): Action[Map[ProcessName, SchedulesState]] = { getLatestDeploymentQueryCount.incrementAndGet() allProcessEntities.map { case (processName, list) => processName -> getLatestDeploymentsForPeriodicProcesses( @@ -212,7 +211,7 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP deploymentsPerScheduleMaxCount ) } - }.run + } private def getLatestDeploymentsForPeriodicProcesses( processes: Seq[PeriodicProcessEntity], diff --git a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala index f6670501ba0..51b6e0d1337 100644 --- a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala +++ b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala @@ -44,8 +44,9 @@ class FlinkRestManager( client .getJobsOverviews() .flatMap { result => - statusDetailsFrom(result.value) - .map(WithDataFreshnessStatus(_, cached = result.cached)) // TODO: How to do it nicer? + statusDetailsFrom(result.value).map( + WithDataFreshnessStatus(_, cached = result.cached) + ) // TODO: How to do it nicer? } } diff --git a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala index 8bbedec92c3..c37d2d4c2f5 100644 --- a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala +++ b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala @@ -43,49 +43,56 @@ class FlinkStreamingDeploymentManagerProvider extends DeploymentManagerProvider FlinkClient.create(flinkConfig, scenarioStateCacheTTL).map { client => val underlying = new FlinkStreamingRestManager(client, flinkConfig, modelData, dependencies) val withCaching = CachingProcessStateDeploymentManager.wrapWithCachingIfNeeded(underlying, scenarioStateCacheTTL) - new DeploymentManager with StateQueryForAllScenariosSupported { - - override def getProcessesStates()( - implicit freshnessPolicy: DataFreshnessPolicy - ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = - underlying.getProcessesStates() - - override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = - withCaching.deploymentSynchronisationSupport - - override def processCommand[Result](command: DMScenarioCommand[Result]): Future[Result] = - withCaching.processCommand(command) - - override def getProcessStates(name: ProcessName)( - implicit freshnessPolicy: DataFreshnessPolicy - ): Future[WithDataFreshnessStatus[List[StatusDetails]]] = - withCaching.getProcessStates(name) - - override def resolve( - idWithName: ProcessIdWithName, - statusDetails: List[StatusDetails], - lastStateAction: Option[ProcessAction], - latestVersionId: VersionId, - deployedVersionId: Option[VersionId], - currentlyPresentedVersionId: Option[VersionId] - ): Future[ProcessState] = withCaching.resolve( - idWithName, - statusDetails, - lastStateAction, - latestVersionId, - deployedVersionId, - currentlyPresentedVersionId - ) - - override def processStateDefinitionManager: ProcessStateDefinitionManager = - withCaching.processStateDefinitionManager - - override def customActionsDefinitions: List[CustomActionDefinition] = - withCaching.customActionsDefinitions - - override def close(): Unit = - withCaching.close() - } + deploymentManagerWithStateQueryForAllScenariosSupported(withCaching, underlying) + } + } + + private def deploymentManagerWithStateQueryForAllScenariosSupported( + underlying: DeploymentManager, + withStateQueryForAllScenariosSupported: StateQueryForAllScenariosSupported + ): DeploymentManager with StateQueryForAllScenariosSupported = { + new DeploymentManager with StateQueryForAllScenariosSupported { + + override def getProcessesStates()( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = + withStateQueryForAllScenariosSupported.getProcessesStates() + + override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = + underlying.deploymentSynchronisationSupport + + override def processCommand[Result](command: DMScenarioCommand[Result]): Future[Result] = + underlying.processCommand(command) + + override def getProcessStates(name: ProcessName)( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[List[StatusDetails]]] = + underlying.getProcessStates(name) + + override def resolve( + idWithName: ProcessIdWithName, + statusDetails: List[StatusDetails], + lastStateAction: Option[ProcessAction], + latestVersionId: VersionId, + deployedVersionId: Option[VersionId], + currentlyPresentedVersionId: Option[VersionId] + ): Future[ProcessState] = underlying.resolve( + idWithName, + statusDetails, + lastStateAction, + latestVersionId, + deployedVersionId, + currentlyPresentedVersionId + ) + + override def processStateDefinitionManager: ProcessStateDefinitionManager = + underlying.processStateDefinitionManager + + override def customActionsDefinitions: List[CustomActionDefinition] = + underlying.customActionsDefinitions + + override def close(): Unit = + underlying.close() } } From 55ff8daef6dcf3219d1aa81a04c5b2c5988abf61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Wed, 18 Dec 2024 13:36:47 +0100 Subject: [PATCH 12/16] qs --- .../periodic/PeriodicDeploymentManager.scala | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala index 7820a3c9ccd..141438291ca 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala @@ -26,7 +26,6 @@ import slick.jdbc import slick.jdbc.JdbcProfile import java.time.{Clock, Instant} -import scala.concurrent.duration.FiniteDuration import scala.concurrent.{ExecutionContext, Future} object PeriodicDeploymentManager { @@ -275,20 +274,17 @@ class PeriodicDeploymentManager private[periodic] ( } // TODO: Why we don't allow running not scheduled scenario? Maybe we can try to schedule it? - private def instantSchedule(processName: ProcessName): OptionT[Future, Unit] = { - implicit val freshnessPolicy: DataFreshnessPolicy = DataFreshnessPolicy.Fresh - for { - // schedule for immediate run - processDeployment <- OptionT( - service - .getLatestDeploymentsForActiveSchedules(processName) - .map(_.groupedByPeriodicProcess.headOption.flatMap(_.deployments.headOption)) - ) - processDeploymentWithProcessJson <- OptionT.liftF( - repository.findProcessData(processDeployment.id).run - ) - _ <- OptionT.liftF(service.deploy(processDeploymentWithProcessJson)) - } yield () - } + private def instantSchedule(processName: ProcessName): OptionT[Future, Unit] = for { + // schedule for immediate run + processDeployment <- OptionT( + service + .getLatestDeploymentsForActiveSchedules(processName) + .map(_.groupedByPeriodicProcess.headOption.flatMap(_.deployments.headOption)) + ) + processDeploymentWithProcessJson <- OptionT.liftF( + repository.findProcessData(processDeployment.id).run + ) + _ <- OptionT.liftF(service.deploy(processDeploymentWithProcessJson)) + } yield () } From 6c336988762414bb4605ddbc40772b293bd47b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Wed, 18 Dec 2024 13:50:18 +0100 Subject: [PATCH 13/16] qs --- .../ui/process/deployment/DeploymentService.scala | 5 +++-- .../periodic/db/PeriodicProcessesRepository.scala | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala index 6afe777928a..fbb9282eb22 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala @@ -466,12 +466,13 @@ class DeploymentService( .map { case process if process.isFragment => DBIO.successful(process) case process => - val prefetchedStateForProcess = prefetchedStates.get(process.processingType).flatMap(_.get(process.name)) + val prefetchedStateForProcessOpt = + prefetchedStates.get(process.processingType).flatMap(_.get(process.name)) getProcessState( process.toEntity, actionsInProgress.getOrElse(process.processIdUnsafe, Set.empty), None, - prefetchedStateForProcess, + prefetchedStateForProcessOpt, ).map(state => process.copy(state = Some(state))) } .sequence[DB, ScenarioWithDetails] diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala index 8ad8ce0bb45..9906c609bc4 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala @@ -503,17 +503,17 @@ class SlickPeriodicProcessesRepository( ): SchedulesState = { SchedulesState( list - .map { case (periodicProcessMetadata, deployment) => - val scheduleId = ScheduleId(periodicProcessMetadata.id, ScheduleName(deployment.scheduleName)) - val scheduleData = (scheduleId, periodicProcessMetadata) + .map { case (process, deployment) => + val scheduleId = ScheduleId(process.id, ScheduleName(deployment.scheduleName)) + val scheduleData = (scheduleId, process) val scheduleDeployment = ScheduleDeploymentData(deployment) (scheduleData, scheduleDeployment) } .toList .toGroupedMap .toList - .map { case ((scheduleId, processEntity), deployments) => - scheduleId -> ScheduleData(createPeriodicProcessWithoutJson(processEntity), deployments) + .map { case ((scheduleId, process), deployments) => + scheduleId -> ScheduleData(createPeriodicProcessWithoutJson(process), deployments) } .toMap ) From 59d7ac811162f80747e33967281383e0bd25d147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Thu, 19 Dec 2024 13:29:14 +0100 Subject: [PATCH 14/16] review changes part 1 --- .../api/deployment/DeploymentManager.scala | 35 ++++------ ...CachingProcessStateDeploymentManager.scala | 10 ++- .../testing/DeploymentManagerStub.scala | 2 + .../util/WithDataFreshnessStatusUtils.scala | 31 +++++++++ ...ingProcessStateDeploymentManagerSpec.scala | 26 +++++-- .../deployment/DeploymentService.scala | 59 +++++++++------- .../InvalidDeploymentManagerStub.scala | 2 + .../util/WithDataFreshnessStatusUtils.scala | 16 ----- .../test/mock/MockDeploymentManager.scala | 2 + ...DevelopmentDeploymentManagerProvider.scala | 1 + .../MockableDeploymentManagerProvider.scala | 2 + .../periodic/PeriodicDeploymentManager.scala | 12 ++-- .../periodic/PeriodicProcessService.scala | 33 ++++++--- .../periodic/DeploymentManagerStub.scala | 21 +++--- .../PeriodicProcessesFetchingTest.scala | 6 +- .../engine/management/FlinkRestManager.scala | 67 +++++++++++-------- ...nkStreamingDeploymentManagerProvider.scala | 60 ++--------------- .../embedded/EmbeddedDeploymentManager.scala | 2 + .../k8s/manager/K8sDeploymentManager.scala | 2 + 19 files changed, 204 insertions(+), 185 deletions(-) create mode 100644 designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/util/WithDataFreshnessStatusUtils.scala delete mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/util/WithDataFreshnessStatusUtils.scala diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala index b8f610245a1..ce7aeba9d43 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/DeploymentManager.scala @@ -4,6 +4,7 @@ import pl.touk.nussknacker.engine.api.deployment.inconsistency.InconsistentState import pl.touk.nussknacker.engine.api.process.{ProcessIdWithName, ProcessName, VersionId} import pl.touk.nussknacker.engine.deployment.CustomActionDefinition import pl.touk.nussknacker.engine.newdeployment +import pl.touk.nussknacker.engine.util.WithDataFreshnessStatusUtils.WithDataFreshnessStatusOps import java.time.Instant import scala.concurrent.ExecutionContext.Implicits._ @@ -46,6 +47,8 @@ trait DeploymentManager extends AutoCloseable { def deploymentSynchronisationSupport: DeploymentSynchronisationSupport + def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport + def processCommand[Result](command: DMScenarioCommand[Result]): Future[Result] final def getProcessState( @@ -59,35 +62,17 @@ trait DeploymentManager extends AutoCloseable { ): Future[WithDataFreshnessStatus[ProcessState]] = { for { statusDetailsWithFreshness <- getProcessStates(idWithName.name) - stateWithFreshness <- resolvePrefetchedProcessState( + stateWithFreshness <- resolve( idWithName, + statusDetailsWithFreshness.value, lastStateAction, latestVersionId, deployedVersionId, currentlyPresentedVersionId, - statusDetailsWithFreshness, - ) + ).map(statusDetailsWithFreshness.withValue) } yield stateWithFreshness } - final def resolvePrefetchedProcessState( - idWithName: ProcessIdWithName, - lastStateAction: Option[ProcessAction], - latestVersionId: VersionId, - deployedVersionId: Option[VersionId], - currentlyPresentedVersionId: Option[VersionId], - statusDetailsWithFreshness: WithDataFreshnessStatus[List[StatusDetails]], - ): Future[WithDataFreshnessStatus[ProcessState]] = { - resolve( - idWithName, - statusDetailsWithFreshness.value, - lastStateAction, - latestVersionId, - deployedVersionId, - currentlyPresentedVersionId, - ).map(state => statusDetailsWithFreshness.map(_ => state)) - } - /** * We provide a special wrapper called WithDataFreshnessStatus to ensure that fetched data is restored * from the cache or not. If you use any kind of cache in your DM implementation please wrap result data @@ -127,14 +112,18 @@ trait ManagerSpecificScenarioActivitiesStoredByManager { self: DeploymentManager } -trait StateQueryForAllScenariosSupported { self: DeploymentManager => +sealed trait StateQueryForAllScenariosSupport + +trait StateQueryForAllScenariosSupported extends StateQueryForAllScenariosSupport { - def getProcessesStates()( + def getAllProcessesStates()( implicit freshnessPolicy: DataFreshnessPolicy ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] } +case object NoStateQueryForAllScenariosSupport extends StateQueryForAllScenariosSupport + sealed trait DeploymentSynchronisationSupport trait DeploymentSynchronisationSupported extends DeploymentSynchronisationSupport { diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/cache/CachingProcessStateDeploymentManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/cache/CachingProcessStateDeploymentManager.scala index 378ec214aa7..5807c745e93 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/cache/CachingProcessStateDeploymentManager.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/cache/CachingProcessStateDeploymentManager.scala @@ -15,7 +15,8 @@ import scala.concurrent.duration._ class CachingProcessStateDeploymentManager( delegate: DeploymentManager, cacheTTL: FiniteDuration, - override val deploymentSynchronisationSupport: DeploymentSynchronisationSupport + override val deploymentSynchronisationSupport: DeploymentSynchronisationSupport, + override val stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport, ) extends DeploymentManager { private val cache: AsyncCache[ProcessName, List[StatusDetails]] = Caffeine @@ -81,7 +82,12 @@ object CachingProcessStateDeploymentManager extends LazyLogging { scenarioStateCacheTTL .map { cacheTTL => logger.debug(s"Wrapping DeploymentManager: $delegate with caching mechanism with TTL: $cacheTTL") - new CachingProcessStateDeploymentManager(delegate, cacheTTL, delegate.deploymentSynchronisationSupport) + new CachingProcessStateDeploymentManager( + delegate, + cacheTTL, + delegate.deploymentSynchronisationSupport, + delegate.stateQueryForAllScenariosSupport + ) } .getOrElse { logger.debug(s"Skipping ProcessState caching for DeploymentManager: $delegate") diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/testing/DeploymentManagerStub.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/testing/DeploymentManagerStub.scala index 5f483981534..08b86b1a9fa 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/testing/DeploymentManagerStub.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/testing/DeploymentManagerStub.scala @@ -62,6 +62,8 @@ class DeploymentManagerStub extends BaseDeploymentManager with StubbingCommands override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = NoDeploymentSynchronisationSupport + override def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport = NoStateQueryForAllScenariosSupport + override def close(): Unit = {} } diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/util/WithDataFreshnessStatusUtils.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/util/WithDataFreshnessStatusUtils.scala new file mode 100644 index 00000000000..76172afe86b --- /dev/null +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/util/WithDataFreshnessStatusUtils.scala @@ -0,0 +1,31 @@ +package pl.touk.nussknacker.engine.util + +import pl.touk.nussknacker.engine.api.deployment.WithDataFreshnessStatus + +object WithDataFreshnessStatusUtils { + + implicit class WithDataFreshnessStatusMapOps[K, V](withDataFreshnessStatus: WithDataFreshnessStatus[Map[K, V]]) { + + def get(k: K): Option[WithDataFreshnessStatus[V]] = withDataFreshnessStatus.map(_.get(k)) match { + case WithDataFreshnessStatus(Some(value), cached) => Some(WithDataFreshnessStatus(value, cached)) + case WithDataFreshnessStatus(None, _) => None + } + + def getOrElse(k: K, orElse: V): WithDataFreshnessStatus[V] = { + withDataFreshnessStatus.map(_.get(k)) match { + case WithDataFreshnessStatus(Some(value), cached) => WithDataFreshnessStatus(value, cached) + case WithDataFreshnessStatus(None, cached) => WithDataFreshnessStatus(orElse, cached) + } + } + + } + + implicit class WithDataFreshnessStatusOps[A, B](scenarioActivity: WithDataFreshnessStatus[A]) { + + def withValue(v: B): WithDataFreshnessStatus[B] = { + scenarioActivity.map(_ => v) + } + + } + +} diff --git a/designer/deployment-manager-api/src/test/scala/pl/touk/nussknacker/engine/api/deployment/cache/CachingProcessStateDeploymentManagerSpec.scala b/designer/deployment-manager-api/src/test/scala/pl/touk/nussknacker/engine/api/deployment/cache/CachingProcessStateDeploymentManagerSpec.scala index 99633fddfc7..172d141e792 100644 --- a/designer/deployment-manager-api/src/test/scala/pl/touk/nussknacker/engine/api/deployment/cache/CachingProcessStateDeploymentManagerSpec.scala +++ b/designer/deployment-manager-api/src/test/scala/pl/touk/nussknacker/engine/api/deployment/cache/CachingProcessStateDeploymentManagerSpec.scala @@ -7,8 +7,8 @@ import org.scalatest.OptionValues import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatestplus.mockito.MockitoSugar -import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.deployment.ExternalDeploymentId import pl.touk.nussknacker.test.PatientScalaFutures @@ -26,8 +26,12 @@ class CachingProcessStateDeploymentManagerSpec test("should ask delegate for a fresh state each time") { val delegate = prepareDMReturningRandomStates - val cachingManager = - new CachingProcessStateDeploymentManager(delegate, 10 seconds, NoDeploymentSynchronisationSupport) + val cachingManager = new CachingProcessStateDeploymentManager( + delegate, + 10 seconds, + NoDeploymentSynchronisationSupport, + NoStateQueryForAllScenariosSupport + ) val results = List( cachingManager.getProcessStatesDeploymentIdNow(DataFreshnessPolicy.Fresh), @@ -41,8 +45,12 @@ class CachingProcessStateDeploymentManagerSpec test("should cache state for DataFreshnessPolicy.CanBeCached") { val delegate = prepareDMReturningRandomStates - val cachingManager = - new CachingProcessStateDeploymentManager(delegate, 10 seconds, NoDeploymentSynchronisationSupport) + val cachingManager = new CachingProcessStateDeploymentManager( + delegate, + 10 seconds, + NoDeploymentSynchronisationSupport, + NoStateQueryForAllScenariosSupport + ) val firstInvocation = cachingManager.getProcessStatesDeploymentIdNow(DataFreshnessPolicy.CanBeCached) firstInvocation.cached shouldBe false @@ -55,8 +63,12 @@ class CachingProcessStateDeploymentManagerSpec test("should reuse state updated by DataFreshnessPolicy.Fresh during reading with DataFreshnessPolicy.CanBeCached") { val delegate = prepareDMReturningRandomStates - val cachingManager = - new CachingProcessStateDeploymentManager(delegate, 10 seconds, NoDeploymentSynchronisationSupport) + val cachingManager = new CachingProcessStateDeploymentManager( + delegate, + 10 seconds, + NoDeploymentSynchronisationSupport, + NoStateQueryForAllScenariosSupport + ) val resultForFresh = cachingManager.getProcessStatesDeploymentIdNow(DataFreshnessPolicy.Fresh) resultForFresh.cached shouldBe false diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala index fbb9282eb22..4ce2a16a398 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala @@ -21,6 +21,10 @@ import pl.touk.nussknacker.engine.api.process._ import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment._ import pl.touk.nussknacker.engine.util.AdditionalComponentConfigsForRuntimeExtractor +import pl.touk.nussknacker.engine.util.WithDataFreshnessStatusUtils.{ + WithDataFreshnessStatusMapOps, + WithDataFreshnessStatusOps +} import pl.touk.nussknacker.restmodel.scenariodetails.ScenarioWithDetails import pl.touk.nussknacker.ui.api.{DeploymentCommentSettings, ListenerApiUser} import pl.touk.nussknacker.ui.listener.ProcessChangeEvent.{OnActionExecutionFinished, OnActionFailed, OnActionSuccess} @@ -34,7 +38,6 @@ import pl.touk.nussknacker.ui.process.repository.ProcessDBQueryRepository.Proces import pl.touk.nussknacker.ui.process.repository._ import pl.touk.nussknacker.ui.security.api.{AdminUser, LoggedUser, NussknackerInternalUser} import pl.touk.nussknacker.ui.util.FutureUtils._ -import pl.touk.nussknacker.ui.util.WithDataFreshnessStatusUtils.WithDataFreshnessStatusOps import pl.touk.nussknacker.ui.validation.{CustomActionValidator, UIProcessValidator} import pl.touk.nussknacker.ui.{BadRequestError, NotFoundError} import slick.dbio.{DBIO, DBIOAction} @@ -459,9 +462,7 @@ class DeploymentService( dbioRunner.run( for { actionsInProgress <- getInProgressActionTypesForScenarios(scenarios) - // DeploymentManager's may support fetching state of all scenarios at once (StateQueryForAllScenariosSupported capability) - // For those DM's we prefetch the state and then use it for resolving individual scenarios. - prefetchedStates <- DBIO.from(getPrefetchedStatesForSupportedManagers(scenarios)) + prefetchedStates <- DBIO.from(getPrefetchedStatesForSupportedManagers(scenarios)) processesWithState <- processTraverse .map { case process if process.isFragment => DBIO.successful(process) @@ -480,6 +481,10 @@ class DeploymentService( ) } + // DeploymentManager's may support fetching state of all scenarios at once + // State is prefetched only when: + // - DM has capability StateQueryForAllScenariosSupported + // - the query is about more than one scenario handled by that DM private def getPrefetchedStatesForSupportedManagers( scenarios: List[ScenarioWithDetails], )( @@ -487,27 +492,33 @@ class DeploymentService( freshnessPolicy: DataFreshnessPolicy ): Future[Map[ProcessingType, WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]]] = { val allProcessingTypes = scenarios.map(_.processingType).toSet - val deploymentManagersByProcessingTypes = - allProcessingTypes.flatMap(pt => dispatcher.deploymentManager(pt).map(dm => (pt, dm))) - val prefetchedStatesByProcessingTypes = Future + val numberOfScenariosByProcessingType = + allProcessingTypes + .map(processingType => (processingType, scenarios.count(_.processingType == processingType))) + .toMap + val processingTypesWithMoreThanOneScenario = numberOfScenariosByProcessingType.filter(_._2 > 1).keys + + Future .sequence { - deploymentManagersByProcessingTypes.map { case (processingType, manager) => - manager match { - case dm: StateQueryForAllScenariosSupported => getProcessesStates(processingType, dm) - case _ => Future.successful(None) - } + processingTypesWithMoreThanOneScenario.map { processingType => + (for { + manager <- dispatcher.deploymentManager(processingType) + managerWithCapability <- manager.stateQueryForAllScenariosSupport match { + case supported: StateQueryForAllScenariosSupported => Some(supported) + case NoStateQueryForAllScenariosSupport => None + } + } yield getProcessesStates(processingType, managerWithCapability)) + .getOrElse(Future.successful(None)) } } .map(_.flatten.toMap) - - prefetchedStatesByProcessingTypes } private def getProcessesStates(processingType: ProcessingType, manager: StateQueryForAllScenariosSupported)( implicit freshnessPolicy: DataFreshnessPolicy, ): Future[Option[(ProcessingType, WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]])]] = { manager - .getProcessesStates() + .getAllProcessesStates() .map(states => Some((processingType, states))) .recover { case NonFatal(e) => logger.warn( @@ -682,14 +693,16 @@ class DeploymentService( val state = (prefetchedStateOpt match { case Some(prefetchedState) => - deploymentManager.resolvePrefetchedProcessState( - processIdWithName, - lastStateAction, - latestVersionId, - deployedVersionId, - currentlyPresentedVersionId, - prefetchedState, - ) + deploymentManager + .resolve( + processIdWithName, + prefetchedState.value, + lastStateAction, + latestVersionId, + deployedVersionId, + currentlyPresentedVersionId, + ) + .map(prefetchedState.withValue) case None => deploymentManager.getProcessState( processIdWithName, diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/InvalidDeploymentManagerStub.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/InvalidDeploymentManagerStub.scala index 82149948176..37b31a16494 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/InvalidDeploymentManagerStub.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/processingtype/InvalidDeploymentManagerStub.scala @@ -55,5 +55,7 @@ object InvalidDeploymentManagerStub extends DeploymentManager { override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = NoDeploymentSynchronisationSupport + override def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport = NoStateQueryForAllScenariosSupport + override def close(): Unit = () } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/WithDataFreshnessStatusUtils.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/WithDataFreshnessStatusUtils.scala deleted file mode 100644 index 9023930cf08..00000000000 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/util/WithDataFreshnessStatusUtils.scala +++ /dev/null @@ -1,16 +0,0 @@ -package pl.touk.nussknacker.ui.util - -import pl.touk.nussknacker.engine.api.deployment.WithDataFreshnessStatus - -object WithDataFreshnessStatusUtils { - - implicit class WithDataFreshnessStatusOps[K, V](scenarioActivity: WithDataFreshnessStatus[Map[K, V]]) { - - def get(k: K): Option[WithDataFreshnessStatus[V]] = scenarioActivity.map(_.get(k)) match { - case WithDataFreshnessStatus(Some(value), cached) => Some(WithDataFreshnessStatus(value, cached)) - case WithDataFreshnessStatus(None, _) => None - } - - } - -} diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala index dcd25483e6d..728d3188696 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala @@ -298,6 +298,8 @@ class MockDeploymentManager( override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = NoDeploymentSynchronisationSupport + override def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport = NoStateQueryForAllScenariosSupport + } class MockManagerProvider(deploymentManager: DeploymentManager = new MockDeploymentManager()) diff --git a/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/DevelopmentDeploymentManagerProvider.scala b/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/DevelopmentDeploymentManagerProvider.scala index fe9518f9f26..ab972d49afe 100644 --- a/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/DevelopmentDeploymentManagerProvider.scala +++ b/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/DevelopmentDeploymentManagerProvider.scala @@ -228,6 +228,7 @@ class DevelopmentDeploymentManager(actorSystem: ActorSystem, modelData: BaseMode override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = NoDeploymentSynchronisationSupport + override def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport = NoStateQueryForAllScenariosSupport } class DevelopmentDeploymentManagerProvider extends DeploymentManagerProvider { diff --git a/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/MockableDeploymentManagerProvider.scala b/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/MockableDeploymentManagerProvider.scala index 4154b8d7030..6ccc6299038 100644 --- a/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/MockableDeploymentManagerProvider.scala +++ b/engine/development/deploymentManager/src/main/scala/pl/touk/nussknacker/development/manager/MockableDeploymentManagerProvider.scala @@ -144,6 +144,8 @@ object MockableDeploymentManagerProvider { override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = NoDeploymentSynchronisationSupport + override def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport = NoStateQueryForAllScenariosSupport + override def managerSpecificScenarioActivities( processIdWithName: ProcessIdWithName, after: Option[Instant], diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala index 141438291ca..b17a6d18aae 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala @@ -31,7 +31,7 @@ import scala.concurrent.{ExecutionContext, Future} object PeriodicDeploymentManager { def apply( - delegate: DeploymentManager with StateQueryForAllScenariosSupported, + delegate: DeploymentManager, schedulePropertyExtractorFactory: SchedulePropertyExtractorFactory, processConfigEnricherFactory: ProcessConfigEnricherFactory, periodicBatchConfig: PeriodicBatchConfig, @@ -107,7 +107,6 @@ class PeriodicDeploymentManager private[periodic] ( toClose: () => Unit )(implicit val ec: ExecutionContext) extends DeploymentManager - with StateQueryForAllScenariosSupported with ManagerSpecificScenarioActivitiesStoredByManager with LazyLogging { @@ -194,18 +193,15 @@ class PeriodicDeploymentManager private[periodic] ( } } + override def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport = + service.stateQueryForAllScenariosSupport + override def getProcessStates( name: ProcessName )(implicit freshnessPolicy: DataFreshnessPolicy): Future[WithDataFreshnessStatus[List[StatusDetails]]] = { service.getStatusDetails(name).map(_.map(List(_))) } - override def getProcessesStates()( - implicit freshnessPolicy: DataFreshnessPolicy - ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = { - service.getStatusDetails().map(_.map(_.map { case (k, v) => (k, List(v)) })) - } - override def resolve( idWithName: ProcessIdWithName, statusDetailsList: List[StatusDetails], diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala index 7719183091a..ac4ab19cbd5 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala @@ -41,7 +41,7 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.control.NonFatal class PeriodicProcessService( - delegateDeploymentManager: DeploymentManager with StateQueryForAllScenariosSupported, + delegateDeploymentManager: DeploymentManager, jarManager: JarManager, scheduledProcessesRepository: PeriodicProcessesRepository, periodicProcessListener: PeriodicProcessListener, @@ -495,17 +495,28 @@ class PeriodicProcessService( } } - def getStatusDetails()( - implicit freshnessPolicy: DataFreshnessPolicy - ): Future[WithDataFreshnessStatus[Map[ProcessName, StatusDetails]]] = { - delegateDeploymentManager.getProcessesStates().flatMap { statusesWithFreshness => - mergeStatusWithDeployments(statusesWithFreshness.value).map { statusDetails => - statusesWithFreshness.map(_.flatMap { case (name, _) => - statusDetails.get(name).map((name, _)) - }) - } + def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport = + delegateDeploymentManager.stateQueryForAllScenariosSupport match { + case supported: StateQueryForAllScenariosSupported => + new StateQueryForAllScenariosSupported { + + override def getAllProcessesStates()( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = { + supported.getAllProcessesStates().flatMap { statusesWithFreshness => + mergeStatusWithDeployments(statusesWithFreshness.value).map { statusDetails => + statusesWithFreshness.map(_.flatMap { case (name, _) => + statusDetails.get(name).map(statusDetails => (name, List(statusDetails))) + }) + } + } + } + + } + + case NoStateQueryForAllScenariosSupport => + NoStateQueryForAllScenariosSupport } - } private def mergeStatusWithDeployments( name: ProcessName, diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala index f5a08c9e561..e424331f837 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala @@ -9,10 +9,7 @@ import pl.touk.nussknacker.engine.testing.StubbingCommands import scala.concurrent.Future -class DeploymentManagerStub - extends BaseDeploymentManager - with StateQueryForAllScenariosSupported - with StubbingCommands { +class DeploymentManagerStub extends BaseDeploymentManager with StubbingCommands { var jobStatus: Map[ProcessName, List[StatusDetails]] = Map.empty @@ -83,12 +80,16 @@ class DeploymentManagerStub Future.successful(WithDataFreshnessStatus.fresh(jobStatus.get(name).toList.flatten)) } - override def getProcessesStates()( - implicit freshnessPolicy: DataFreshnessPolicy - ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = { - Future.successful(WithDataFreshnessStatus.fresh(jobStatus)) - } - override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = NoDeploymentSynchronisationSupport + override def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport = + new StateQueryForAllScenariosSupported { + + override def getAllProcessesStates()( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = + Future.successful(WithDataFreshnessStatus.fresh(jobStatus)) + + } + } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala index 2ed2ff48a1a..a5278f94586 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessesFetchingTest.scala @@ -106,7 +106,11 @@ class PeriodicProcessesFetchingTest implicit val freshnessPolicy: DataFreshnessPolicy = DataFreshnessPolicy.Fresh - val statuses = f.periodicProcessService.getStatusDetails().futureValue.value + val statuses = f.periodicProcessService.stateQueryForAllScenariosSupport + .asInstanceOf[StateQueryForAllScenariosSupported] + .getAllProcessesStates() + .futureValue + .value statuses.size shouldEqual n diff --git a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala index 51b6e0d1337..3fef55f145e 100644 --- a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala +++ b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkRestManager.scala @@ -11,6 +11,10 @@ import pl.touk.nussknacker.engine.deployment.{DeploymentId, ExternalDeploymentId import pl.touk.nussknacker.engine.management.FlinkRestManager.ParsedJobConfig import pl.touk.nussknacker.engine.management.rest.FlinkClient import pl.touk.nussknacker.engine.management.rest.flinkRestModel.{BaseJobStatusCounts, JobOverview} +import pl.touk.nussknacker.engine.util.WithDataFreshnessStatusUtils.{ + WithDataFreshnessStatusMapOps, + WithDataFreshnessStatusOps +} import pl.touk.nussknacker.engine.{BaseModelData, DeploymentManagerDependencies, newdeployment} import scala.concurrent.Future @@ -22,7 +26,6 @@ class FlinkRestManager( dependencies: DeploymentManagerDependencies, mainClassName: String ) extends FlinkDeploymentManager(modelData, dependencies, config.shouldVerifyBeforeDeploy, mainClassName) - with StateQueryForAllScenariosSupported with LazyLogging { import dependencies._ @@ -34,23 +37,53 @@ class FlinkRestManager( override def getProcessStates( name: ProcessName )(implicit freshnessPolicy: DataFreshnessPolicy): Future[WithDataFreshnessStatus[List[StatusDetails]]] = { - getProcessesStates() - .map(_.map(_.getOrElse(name, List.empty))) + getAllProcessesStatesFromFlink().map(_.getOrElse(name, List.empty)) } - override def getProcessesStates()( + override val deploymentSynchronisationSupport: DeploymentSynchronisationSupport = + new DeploymentSynchronisationSupported { + + override def getDeploymentStatusesToUpdate( + deploymentIdsToCheck: Set[newdeployment.DeploymentId] + ): Future[Map[newdeployment.DeploymentId, DeploymentStatus]] = { + Future + .sequence( + deploymentIdsToCheck.toSeq + .map { deploymentId => + client + .getJobDetails(toJobId(deploymentId)) + .map(_.map { jobDetails => + deploymentId -> toDeploymentStatus(jobDetails.state, jobDetails.`status-counts`) + }) + } + ) + .map(_.flatten.toMap) + } + + } + + override def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport = + new StateQueryForAllScenariosSupported { + + override def getAllProcessesStates()( + implicit freshnessPolicy: DataFreshnessPolicy + ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = getAllProcessesStatesFromFlink() + + } + + private def getAllProcessesStatesFromFlink()( implicit freshnessPolicy: DataFreshnessPolicy ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = { client .getJobsOverviews() .flatMap { result => - statusDetailsFrom(result.value).map( + statusDetailsFromJobOverviews(result.value).map( WithDataFreshnessStatus(_, cached = result.cached) ) // TODO: How to do it nicer? } } - private def statusDetailsFrom( + private def statusDetailsFromJobOverviews( jobOverviews: List[JobOverview] ): Future[Map[ProcessName, List[StatusDetails]]] = Future .sequence { @@ -85,28 +118,6 @@ class FlinkRestManager( } .map(_.toMap) - override val deploymentSynchronisationSupport: DeploymentSynchronisationSupport = - new DeploymentSynchronisationSupported { - - override def getDeploymentStatusesToUpdate( - deploymentIdsToCheck: Set[newdeployment.DeploymentId] - ): Future[Map[newdeployment.DeploymentId, DeploymentStatus]] = { - Future - .sequence( - deploymentIdsToCheck.toSeq - .map { deploymentId => - client - .getJobDetails(toJobId(deploymentId)) - .map(_.map { jobDetails => - deploymentId -> toDeploymentStatus(jobDetails.state, jobDetails.`status-counts`) - }) - } - ) - .map(_.flatten.toMap) - } - - } - // NOTE: Flink <1.10 compatibility - protected to make it easier to work with Flink 1.9, JobStatus changed package, so we use String in case class protected def toDeploymentStatus(jobState: String, jobStatusCounts: BaseJobStatusCounts): DeploymentStatus = { toJobStatus(jobState) match { diff --git a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala index c37d2d4c2f5..fdc3eef46ac 100644 --- a/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala +++ b/engine/flink/management/src/main/scala/pl/touk/nussknacker/engine/management/FlinkStreamingDeploymentManagerProvider.scala @@ -8,12 +8,10 @@ import pl.touk.nussknacker.engine.api.component.ScenarioPropertyConfig import pl.touk.nussknacker.engine.api.definition._ import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.deployment.cache.CachingProcessStateDeploymentManager -import pl.touk.nussknacker.engine.api.process.{ProcessIdWithName, ProcessName, VersionId} -import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, EngineSetupName} +import pl.touk.nussknacker.engine.deployment.EngineSetupName import pl.touk.nussknacker.engine.management.FlinkConfig.RestUrlPath import pl.touk.nussknacker.engine.management.rest.FlinkClient -import scala.concurrent.Future import scala.concurrent.duration.FiniteDuration import scala.util.Try @@ -37,62 +35,12 @@ class FlinkStreamingDeploymentManagerProvider extends DeploymentManagerProvider dependencies: DeploymentManagerDependencies, deploymentConfig: Config, scenarioStateCacheTTL: Option[FiniteDuration] - ): ValidatedNel[String, DeploymentManager with StateQueryForAllScenariosSupported] = { + ): ValidatedNel[String, DeploymentManager] = { import dependencies._ val flinkConfig = deploymentConfig.rootAs[FlinkConfig] FlinkClient.create(flinkConfig, scenarioStateCacheTTL).map { client => - val underlying = new FlinkStreamingRestManager(client, flinkConfig, modelData, dependencies) - val withCaching = CachingProcessStateDeploymentManager.wrapWithCachingIfNeeded(underlying, scenarioStateCacheTTL) - deploymentManagerWithStateQueryForAllScenariosSupported(withCaching, underlying) - } - } - - private def deploymentManagerWithStateQueryForAllScenariosSupported( - underlying: DeploymentManager, - withStateQueryForAllScenariosSupported: StateQueryForAllScenariosSupported - ): DeploymentManager with StateQueryForAllScenariosSupported = { - new DeploymentManager with StateQueryForAllScenariosSupported { - - override def getProcessesStates()( - implicit freshnessPolicy: DataFreshnessPolicy - ): Future[WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]]] = - withStateQueryForAllScenariosSupported.getProcessesStates() - - override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = - underlying.deploymentSynchronisationSupport - - override def processCommand[Result](command: DMScenarioCommand[Result]): Future[Result] = - underlying.processCommand(command) - - override def getProcessStates(name: ProcessName)( - implicit freshnessPolicy: DataFreshnessPolicy - ): Future[WithDataFreshnessStatus[List[StatusDetails]]] = - underlying.getProcessStates(name) - - override def resolve( - idWithName: ProcessIdWithName, - statusDetails: List[StatusDetails], - lastStateAction: Option[ProcessAction], - latestVersionId: VersionId, - deployedVersionId: Option[VersionId], - currentlyPresentedVersionId: Option[VersionId] - ): Future[ProcessState] = underlying.resolve( - idWithName, - statusDetails, - lastStateAction, - latestVersionId, - deployedVersionId, - currentlyPresentedVersionId - ) - - override def processStateDefinitionManager: ProcessStateDefinitionManager = - underlying.processStateDefinitionManager - - override def customActionsDefinitions: List[CustomActionDefinition] = - underlying.customActionsDefinitions - - override def close(): Unit = - underlying.close() + val underlying = new FlinkStreamingRestManager(client, flinkConfig, modelData, dependencies) + CachingProcessStateDeploymentManager.wrapWithCachingIfNeeded(underlying, scenarioStateCacheTTL) } } diff --git a/engine/lite/embeddedDeploymentManager/src/main/scala/pl/touk/nussknacker/engine/embedded/EmbeddedDeploymentManager.scala b/engine/lite/embeddedDeploymentManager/src/main/scala/pl/touk/nussknacker/engine/embedded/EmbeddedDeploymentManager.scala index 76862770f0d..ca3839d100f 100644 --- a/engine/lite/embeddedDeploymentManager/src/main/scala/pl/touk/nussknacker/engine/embedded/EmbeddedDeploymentManager.scala +++ b/engine/lite/embeddedDeploymentManager/src/main/scala/pl/touk/nussknacker/engine/embedded/EmbeddedDeploymentManager.scala @@ -255,6 +255,8 @@ class EmbeddedDeploymentManager( } + override def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport = NoStateQueryForAllScenariosSupport + override def processStateDefinitionManager: ProcessStateDefinitionManager = EmbeddedProcessStateDefinitionManager override def close(): Unit = { diff --git a/engine/lite/k8sDeploymentManager/src/main/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManager.scala b/engine/lite/k8sDeploymentManager/src/main/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManager.scala index 85e68aa9e04..a479ce85402 100644 --- a/engine/lite/k8sDeploymentManager/src/main/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManager.scala +++ b/engine/lite/k8sDeploymentManager/src/main/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManager.scala @@ -387,6 +387,8 @@ class K8sDeploymentManager( // for each scenario in this case and where store the deploymentId override def deploymentSynchronisationSupport: DeploymentSynchronisationSupport = NoDeploymentSynchronisationSupport + override def stateQueryForAllScenariosSupport: StateQueryForAllScenariosSupport = NoStateQueryForAllScenariosSupport + } object K8sDeploymentManager { From 84c28ec1c3f9e3dec7be946d9a8a107cd9cae4ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Thu, 19 Dec 2024 13:50:38 +0100 Subject: [PATCH 15/16] review changes part 2 --- .../deployment/DeploymentService.scala | 136 +++++++++++------- 1 file changed, 87 insertions(+), 49 deletions(-) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala index 4ce2a16a398..be38d1ec39f 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala @@ -207,7 +207,7 @@ class DeploymentService( _ = checkIfCanPerformActionOnScenario(actionName, processDetails) // 1.7. check if action is allowed for current state inProgressActionNames <- actionRepository.getInProgressActionNames(processDetails.processId) - processState <- getProcessState(processDetails, inProgressActionNames, None, None) + processState <- getProcessStateFetchingStatusFromManager(processDetails, inProgressActionNames, None) _ = checkIfCanPerformActionInState(actionName, processDetails, processState) // 1.8. create new action, action is started with "in progress" state, the whole command execution can take some time actionId <- actionRepository.addInProgressAction( @@ -441,7 +441,11 @@ class DeploymentService( processDetailsOpt <- processRepository.fetchLatestProcessDetailsForProcessId[Unit](processIdWithName.id) processDetails <- existsOrFail(processDetailsOpt, ProcessNotFoundError(processIdWithName.name)) inProgressActionNames <- actionRepository.getInProgressActionNames(processDetails.processId) - result <- getProcessState(processDetails, inProgressActionNames, currentlyPresentedVersionId, None) + result <- getProcessStateFetchingStatusFromManager( + processDetails, + inProgressActionNames, + currentlyPresentedVersionId + ) } yield result) } @@ -450,7 +454,7 @@ class DeploymentService( )(implicit user: LoggedUser, freshnessPolicy: DataFreshnessPolicy): Future[ProcessState] = { dbioRunner.run(for { inProgressActionNames <- actionRepository.getInProgressActionNames(processDetails.processId) - result <- getProcessState(processDetails, inProgressActionNames, None, None) + result <- getProcessStateFetchingStatusFromManager(processDetails, inProgressActionNames, None) } yield result) } @@ -467,14 +471,26 @@ class DeploymentService( .map { case process if process.isFragment => DBIO.successful(process) case process => - val prefetchedStateForProcessOpt = - prefetchedStates.get(process.processingType).flatMap(_.get(process.name)) - getProcessState( - process.toEntity, - actionsInProgress.getOrElse(process.processIdUnsafe, Set.empty), - None, - prefetchedStateForProcessOpt, - ).map(state => process.copy(state = Some(state))) + val prefetchedStatusDetails = for { + prefetchedStatusDetailsForProcessingTypes <- prefetchedStates.get(process.processingType) + prefetchedStatusDetails <- prefetchedStatusDetailsForProcessingTypes.get(process.name) + } yield prefetchedStatusDetails + prefetchedStatusDetails match { + case Some(prefetchedStatusDetails) => + getProcessStateUsingPrefetchedStatus( + process.toEntity, + actionsInProgress.getOrElse(process.processIdUnsafe, Set.empty), + None, + prefetchedStatusDetails, + ).map(state => process.copy(state = Some(state))) + case None => + getProcessStateFetchingStatusFromManager( + process.toEntity, + actionsInProgress.getOrElse(process.processIdUnsafe, Set.empty), + None, + ).map(state => process.copy(state = Some(state))) + } + } .sequence[DB, ScenarioWithDetails] } yield processesWithState @@ -546,12 +562,57 @@ class DeploymentService( } } - private def getProcessState( + private def getProcessStateFetchingStatusFromManager( processDetails: ScenarioWithDetailsEntity[_], inProgressActionNames: Set[ScenarioActionName], currentlyPresentedVersionId: Option[VersionId], - prefetchedStatusDetailsWithFreshness: Option[WithDataFreshnessStatus[List[StatusDetails]]], )(implicit freshnessPolicy: DataFreshnessPolicy, user: LoggedUser): DB[ProcessState] = { + getProcessState( + processDetails, + inProgressActionNames, + currentlyPresentedVersionId, + manager => + getStateFromDeploymentManager( + manager, + processDetails.idWithName, + processDetails.lastStateAction, + processDetails.processVersionId, + processDetails.lastDeployedAction.map(_.processVersionId), + currentlyPresentedVersionId, + ) + ) + } + + private def getProcessStateUsingPrefetchedStatus( + processDetails: ScenarioWithDetailsEntity[_], + inProgressActionNames: Set[ScenarioActionName], + currentlyPresentedVersionId: Option[VersionId], + prefetchedStatusDetails: WithDataFreshnessStatus[List[StatusDetails]], + )(implicit user: LoggedUser): DB[ProcessState] = { + getProcessState( + processDetails, + inProgressActionNames, + currentlyPresentedVersionId, + manager => + manager + .resolve( + processDetails.idWithName, + prefetchedStatusDetails.value, + processDetails.lastStateAction, + processDetails.processVersionId, + processDetails.lastDeployedAction.map(_.processVersionId), + currentlyPresentedVersionId, + ) + .map(prefetchedStatusDetails.withValue) + ) + } + + private def getProcessState( + processDetails: ScenarioWithDetailsEntity[_], + inProgressActionNames: Set[ScenarioActionName], + currentlyPresentedVersionId: Option[VersionId], + fetchState: DeploymentManager => Future[WithDataFreshnessStatus[ProcessState]], + )(implicit user: LoggedUser): DB[ProcessState] = { val processVersionId = processDetails.processVersionId val deployedVersionId = processDetails.lastDeployedAction.map(_.processVersionId) dispatcher @@ -585,17 +646,7 @@ class DeploymentService( processDetails.lastStateAction match { case Some(_) => DBIOAction - .from( - getStateFromDeploymentManager( - manager, - processDetails.idWithName, - processDetails.lastStateAction, - processVersionId, - deployedVersionId, - currentlyPresentedVersionId, - prefetchedStatusDetailsWithFreshness, - ) - ) + .from(fetchState(manager)) .map { statusWithFreshness => logger.debug( s"Status for: '${processDetails.name}' is: ${statusWithFreshness.value.status}, cached: ${statusWithFreshness.cached}, last status action: ${processDetails.lastStateAction @@ -686,35 +737,22 @@ class DeploymentService( latestVersionId: VersionId, deployedVersionId: Option[VersionId], currentlyPresentedVersionId: Option[VersionId], - prefetchedStateOpt: Option[WithDataFreshnessStatus[List[StatusDetails]]], )( implicit freshnessPolicy: DataFreshnessPolicy ): Future[WithDataFreshnessStatus[ProcessState]] = { - val state = (prefetchedStateOpt match { - case Some(prefetchedState) => - deploymentManager - .resolve( - processIdWithName, - prefetchedState.value, - lastStateAction, - latestVersionId, - deployedVersionId, - currentlyPresentedVersionId, - ) - .map(prefetchedState.withValue) - case None => - deploymentManager.getProcessState( - processIdWithName, - lastStateAction, - latestVersionId, - deployedVersionId, - currentlyPresentedVersionId, - ) - }).recover { case NonFatal(e) => - logger.warn(s"Failed to get status of ${processIdWithName.name}: ${e.getMessage}", e) - failedToGetProcessState(latestVersionId) - } + val state = deploymentManager + .getProcessState( + processIdWithName, + lastStateAction, + latestVersionId, + deployedVersionId, + currentlyPresentedVersionId, + ) + .recover { case NonFatal(e) => + logger.warn(s"Failed to get status of ${processIdWithName.name}: ${e.getMessage}", e) + failedToGetProcessState(latestVersionId) + } scenarioStateTimeout .map { timeout => From dbadf80fc728c22eff565f367907aea4789a6986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Goworko?= Date: Thu, 19 Dec 2024 14:02:52 +0100 Subject: [PATCH 16/16] review changes part 3 --- .../nussknacker/ui/process/deployment/DeploymentService.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala index be38d1ec39f..b8b27bbaed3 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/deployment/DeploymentService.scala @@ -523,14 +523,14 @@ class DeploymentService( case supported: StateQueryForAllScenariosSupported => Some(supported) case NoStateQueryForAllScenariosSupport => None } - } yield getProcessesStates(processingType, managerWithCapability)) + } yield getAllProcessesStates(processingType, managerWithCapability)) .getOrElse(Future.successful(None)) } } .map(_.flatten.toMap) } - private def getProcessesStates(processingType: ProcessingType, manager: StateQueryForAllScenariosSupported)( + private def getAllProcessesStates(processingType: ProcessingType, manager: StateQueryForAllScenariosSupported)( implicit freshnessPolicy: DataFreshnessPolicy, ): Future[Option[(ProcessingType, WithDataFreshnessStatus[Map[ProcessName, List[StatusDetails]]])]] = { manager