diff --git a/src/main/scala/swiss/dasch/api/MaintenanceEndpoints.scala b/src/main/scala/swiss/dasch/api/MaintenanceEndpoints.scala index 7da880e8..a81b8983 100644 --- a/src/main/scala/swiss/dasch/api/MaintenanceEndpoints.scala +++ b/src/main/scala/swiss/dasch/api/MaintenanceEndpoints.scala @@ -10,13 +10,21 @@ import sttp.tapir.Codec import sttp.tapir.CodecFormat.TextPlain import sttp.tapir.ztapir.* import swiss.dasch.domain.ProjectShortcode -import zio.ZLayer import zio.json.{DeriveJsonCodec, JsonCodec} import zio.schema.{DeriveSchema, Schema} +import zio.ZLayer + +final case class MappingEntry(internalFilename: String, originalFilename: String) + +object MappingEntry { + given codec: JsonCodec[MappingEntry] = DeriveJsonCodec.gen[MappingEntry] + given schema: Schema[MappingEntry] = DeriveSchema.gen[MappingEntry] +} enum ActionName { - case UpdateAssetMetadata extends ActionName - case ImportProjectsToDb extends ActionName + case ApplyTopLeftCorrection extends ActionName + case UpdateAssetMetadata extends ActionName + case ImportProjectsToDb extends ActionName } object ActionName { @@ -59,7 +67,24 @@ final case class MaintenanceEndpoints(base: BaseEndpoints) { .tag(maintenance) .description("Authorization: admin scope required.") - val endpoints = List(postMaintenanceActionEndpoint) + val needsTopLeftCorrectionEndpoint = base.secureEndpoint.get + .in(maintenance / "needs-top-left-correction") + .out(stringBody) + .out(statusCode(StatusCode.Accepted)) + .tag(maintenance) + .description("Authorization: admin scope required.") + + val wasTopLeftCorrectionAppliedEndpoint = base.secureEndpoint.get + .in(maintenance / "was-top-left-correction-applied") + .out(stringBody) + .out(statusCode(StatusCode.Accepted)) + .tag(maintenance) + .description("Authorization: admin scope required.") + + val endpoints = List( + postMaintenanceActionEndpoint, + needsTopLeftCorrectionEndpoint, + ) } object MaintenanceEndpoints { diff --git a/src/main/scala/swiss/dasch/api/MaintenanceEndpointsHandler.scala b/src/main/scala/swiss/dasch/api/MaintenanceEndpointsHandler.scala index 36890b89..d6e3b2f7 100644 --- a/src/main/scala/swiss/dasch/api/MaintenanceEndpointsHandler.scala +++ b/src/main/scala/swiss/dasch/api/MaintenanceEndpointsHandler.scala @@ -6,7 +6,7 @@ package swiss.dasch.api import sttp.tapir.ztapir.ZServerEndpoint -import swiss.dasch.api.ActionName.{ImportProjectsToDb, UpdateAssetMetadata} +import swiss.dasch.api.ActionName.{ApplyTopLeftCorrection, ImportProjectsToDb, UpdateAssetMetadata} import swiss.dasch.domain.* import zio.{ZIO, ZLayer} @@ -34,13 +34,43 @@ final case class MaintenanceEndpointsHandler( .mapError(ApiProblem.InternalServerError(_)) _ <- ZIO.logDebug(s"Maintenance endpoint called $action, $shortcodes, $paths") _ <- action match { - case UpdateAssetMetadata => maintenanceActions.updateAssetMetadata(paths).forkDaemon.logError - case ImportProjectsToDb => maintenanceActions.importProjectsToDb().forkDaemon.logError + case UpdateAssetMetadata => maintenanceActions.updateAssetMetadata(paths).forkDaemon.logError + case ApplyTopLeftCorrection => maintenanceActions.applyTopLeftCorrections(paths).forkDaemon.logError + case ImportProjectsToDb => maintenanceActions.importProjectsToDb().forkDaemon.logError } } yield s"work in progress for projects ${paths.map(_.shortcode).mkString(", ")} (for details see logs)" }) - val endpoints: List[ZServerEndpoint[Any, Any]] = List(postMaintenanceEndpoint) + val needsTopLeftCorrectionEndpoint: ZServerEndpoint[Any, Any] = + maintenanceEndpoints.needsTopLeftCorrectionEndpoint + .serverLogic(userSession => + _ => + authorizationHandler.ensureAdminScope(userSession) *> + maintenanceActions + .createNeedsTopLeftCorrectionReport() + .forkDaemon + .logError + .as("work in progress"), + ) + + val wasTopLeftCorrectionAppliedEndpoint: ZServerEndpoint[Any, Any] = + maintenanceEndpoints.wasTopLeftCorrectionAppliedEndpoint + .serverLogic(userSession => + _ => + authorizationHandler.ensureAdminScope(userSession) *> + maintenanceActions + .createWasTopLeftCorrectionAppliedReport() + .forkDaemon + .logError + .as("work in progress"), + ) + + val endpoints: List[ZServerEndpoint[Any, Any]] = + List( + postMaintenanceEndpoint, + needsTopLeftCorrectionEndpoint, + wasTopLeftCorrectionAppliedEndpoint, + ) } object MaintenanceEndpointsHandler { diff --git a/src/main/scala/swiss/dasch/domain/MaintenanceActions.scala b/src/main/scala/swiss/dasch/domain/MaintenanceActions.scala index df56c5b9..47cd25d4 100644 --- a/src/main/scala/swiss/dasch/domain/MaintenanceActions.scala +++ b/src/main/scala/swiss/dasch/domain/MaintenanceActions.scala @@ -8,12 +8,27 @@ package swiss.dasch.domain import swiss.dasch.api.ActionName import swiss.dasch.domain import swiss.dasch.domain.AugmentedPath.* +import swiss.dasch.domain.AugmentedPath.Conversions.given_Conversion_AugmentedPath_Path +import swiss.dasch.domain.FileFilters.isJpeg2000 import zio.* +import zio.json.interop.refined.* +import zio.json.{DeriveJsonCodec, JsonCodec, JsonEncoder} import zio.nio.file -import zio.nio.file.Path +import zio.nio.file.{Files, Path} +import zio.stream.{ZSink, ZStream} + +import java.io.IOException trait MaintenanceActions { def updateAssetMetadata(projects: Iterable[ProjectFolder]): Task[Unit] + def createNeedsTopLeftCorrectionReport(): Task[Unit] + def createWasTopLeftCorrectionAppliedReport(): Task[Unit] + def applyTopLeftCorrections(projectPath: ProjectFolder): Task[Int] + final def applyTopLeftCorrections(projectPath: Iterable[ProjectFolder]): Task[Int] = + ZIO + .foreach(projectPath)(applyTopLeftCorrections) + .map(_.sum) + .tap(sum => ZIO.logInfo(s"Finished ${ActionName.ApplyTopLeftCorrection} for $sum files")) def importProjectsToDb(): Task[Unit] } @@ -63,12 +78,114 @@ final case class MaintenanceActionsLive( } yield () } + private def saveReport[A]( + tmpDir: Path, + name: String, + report: A, + )(implicit encoder: JsonEncoder[A]): Task[Unit] = + Files.createDirectories(tmpDir / "reports") *> + Files.deleteIfExists(tmpDir / "reports" / s"$name.json") *> + Files.createFile(tmpDir / "reports" / s"$name.json") *> + storageService.saveJsonFile(tmpDir / "reports" / s"$name.json", report) + + override def createNeedsTopLeftCorrectionReport(): Task[Unit] = + for { + _ <- ZIO.logInfo(s"Checking for top left correction") + tmpDir <- storageService.getTempFolder() + projects <- projectService.listAllProjects() + _ <- + ZIO + .foreach(projects)(prj => + Files + .walk(prj.path) + .mapZIOPar(8)(imageService.needsTopLeftCorrection) + .filter(identity) + .runHead + .map(_.map(_ => prj.shortcode)), + ) + .map(_.flatten) + .map(_.map(_.toString)) + .flatMap(saveReport(tmpDir, "needsTopLeftCorrection", _)) + .zipLeft(ZIO.logInfo(s"Created needsTopLeftCorrection.json")) + } yield () + + case class ReportAsset(id: AssetId, dimensions: Dimensions) + object ReportAsset { + given codec: JsonCodec[ReportAsset] = DeriveJsonCodec.gen[ReportAsset] + } + case class ProjectWithBakFiles(id: ProjectShortcode, assetIds: Chunk[ReportAsset]) + object ProjectWithBakFiles { + given codec: JsonCodec[ProjectWithBakFiles] = DeriveJsonCodec.gen[ProjectWithBakFiles] + } + case class ProjectsWithBakfilesReport(projects: Chunk[ProjectWithBakFiles]) + object ProjectsWithBakfilesReport { + given codec: JsonCodec[ProjectsWithBakfilesReport] = DeriveJsonCodec.gen[ProjectsWithBakfilesReport] + } + + override def createWasTopLeftCorrectionAppliedReport(): Task[Unit] = + for { + _ <- ZIO.logInfo(s"Checking where top left correction was applied") + tmpDir <- storageService.getTempFolder() + projects <- projectService.listAllProjects() + assetsWithBak <- + ZIO + .foreach(projects) { prj => + Files + .walk(prj.path) + .flatMapPar(8)(hasBeenTopLeftTransformed) + .runCollect + .map { assetIdDimensions => + ProjectWithBakFiles( + prj.shortcode, + assetIdDimensions.map { case (id: AssetId, dim: Dimensions) => ReportAsset(id, dim) }, + ) + } + } + report = ProjectsWithBakfilesReport(assetsWithBak.filter(_.assetIds.nonEmpty)) + _ <- saveReport(tmpDir, "wasTopLeftCorrectionApplied", report) + _ <- ZIO.logInfo(s"Created wasTopLeftCorrectionApplied.json") + } yield () + + private def hasBeenTopLeftTransformed(path: Path): ZStream[Any, Throwable, (AssetId, Dimensions)] = { + val zioTask: ZIO[Any, Option[Throwable], (AssetId, Dimensions)] = for { + // must be a .bak file + bakFile <- ZIO.succeed(path).whenZIO(FileFilters.isBakFile(path)).some + // must have an AssetId + assetId <- ZIO.fromOption(AssetId.fromPath(bakFile)) + // must have a corresponding Jpeg2000 derivative + bakFilename = bakFile.filename.toString + derivativeFilename = bakFilename.substring(0, bakFilename.length - ".bak".length) + derivative = JpxDerivativeFile.unsafeFrom(path.parent.head / derivativeFilename) + _ <- ZIO.fail(None).whenZIO(FileFilters.isJpeg2000(derivative.file).negate.asSomeError) + // get the dimensions + dimensions <- imageService.getDimensions(derivative).asSomeError + } yield (assetId, dimensions) + + ZStream.fromZIOOption( + zioTask + // None.type errors are just a sign that the path should be ignored. Some.type errors are real errors. + .tapSomeError { case Some(e) => ZIO.logError(s"Error while processing $path: $e") } + // We have logged real errors above, from here on out ignore all errors so that the stream can continue. + .orElseFail(None), + ) + } + + override def applyTopLeftCorrections(projectPath: ProjectFolder): Task[Int] = + ZIO.logInfo(s"Starting top left corrections in ${projectPath.path}") *> + findJpeg2000Files(projectPath) + .mapZIOPar(8)(imageService.applyTopLeftCorrection) + .map(_.map(_ => 1).getOrElse(0)) + .run(ZSink.sum) + .tap(sum => ZIO.logInfo(s"Top left corrections applied for $sum files in ${projectPath.path}")) + + private def findJpeg2000Files(projectPath: ProjectFolder) = StorageService.findInPath(projectPath.path, isJpeg2000) + override def importProjectsToDb(): Task[Unit] = for { prjFolders <- projectService.listAllProjects() _ <- ZIO.foreachDiscard(prjFolders.map(_.shortcode))(projectService.addProjectToDb) } yield () -} +} object MaintenanceActionsLive { val layer = ZLayer.derive[MaintenanceActionsLive] } diff --git a/src/test/scala/swiss/dasch/api/MaintenanceEndpointsSpec.scala b/src/test/scala/swiss/dasch/api/MaintenanceEndpointsSpec.scala new file mode 100644 index 00000000..cd27a11d --- /dev/null +++ b/src/test/scala/swiss/dasch/api/MaintenanceEndpointsSpec.scala @@ -0,0 +1,79 @@ +/* + * Copyright © 2021 - 2025 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package swiss.dasch.api + +import sttp.tapir.server.ziohttp.{ZioHttpInterpreter, ZioHttpServerOptions} +import swiss.dasch.domain.* +import swiss.dasch.domain.Exif.Image.OrientationValue +import swiss.dasch.infrastructure.CommandExecutorMock +import swiss.dasch.test.SpecConstants.* +import swiss.dasch.test.SpecConfigurations +import swiss.dasch.util.TestUtils +import zio.* +import zio.http.* +import zio.nio.file +import zio.nio.file.Files +import zio.test.* + +object MaintenanceEndpointsSpec extends ZIOSpecDefault { + + private def awaitTrue[R, E](awaitThis: ZIO[R, E, Boolean], timeout: Duration = 3.seconds): ZIO[R, E, Boolean] = + awaitThis.repeatUntil(identity).timeout(timeout).map(_.getOrElse(false)) + + private def executeRequest(request: Request) = for { + app <- ZIO.serviceWith[MaintenanceEndpointsHandler](handler => + ZioHttpInterpreter(ZioHttpServerOptions.default).toHttp(handler.endpoints), + ) + response <- app.runZIO(request).logError + } yield response + + private def loadReport(name: String) = + StorageService.getTempFolder().flatMap { tmpDir => + val report = tmpDir / "reports" / name + awaitTrue(Files.exists(report)) *> StorageService.loadJsonFile[Chunk[String]](report) + } + + private val needsTopleftCorrectionSuite = + suite("/maintenance/needs-top-left-correction should")( + test("should return 204 and create a report") { + val request = Request + .get(URL(Path.root / "maintenance" / "needs-top-left-correction")) + .addHeader(Header.Authorization.name, "Bearer fakeToken") + for { + _ <- SipiClientMock.setOrientation(OrientationValue.Rotate270CW) + response <- executeRequest(request) + projects <- loadReport("needsTopLeftCorrection.json") + status = response.status + } yield { + assertTrue(status == Status.Accepted, projects == Chunk("0001")) + } + }, + ) @@ TestAspect.withLiveClock + + val spec = suite("MaintenanceEndpoint")(needsTopleftCorrectionSuite) + .provide( + AssetInfoServiceLive.layer, + AuthServiceLive.layer, + AuthorizationHandlerLive.layer, + BaseEndpoints.layer, + CommandExecutorMock.layer, + FileChecksumServiceLive.layer, + MaintenanceActionsLive.layer, + MaintenanceEndpoints.layer, + MaintenanceEndpointsHandler.layer, + MimeTypeGuesser.layer, + MovingImageService.layer, + OtherFilesService.layer, + ProjectService.layer, + ProjectRepositoryLive.layer, + SipiClientMock.layer, + SpecConfigurations.jwtConfigDisableAuthLayer, + SpecConfigurations.storageConfigLayer, + StillImageService.layer, + StorageServiceLive.layer, + TestUtils.testDbLayerWithEmptyDb, + ) +}