diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d3043d719c..8b165233d6 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -46,6 +46,12 @@ object Dependencies { val zioPrelude = "dev.zio" %% "zio-prelude" % ZioPreludeVersion val zioSttp = "com.softwaremill.sttp.client3" %% "zio" % "3.9.0" + // refined + val refined = Seq( + "eu.timepit" %% "refined" % "0.11.0", + "dev.zio" %% "zio-json-interop-refined" % "0.6.2" + ) + // zio-test and friends val zioTest = "dev.zio" %% "zio-test" % ZioVersion val zioTestSbt = "dev.zio" %% "zio-test-sbt" % ZioVersion @@ -135,7 +141,7 @@ object Dependencies { val webapiTestDependencies = Seq(zioTest, zioTestSbt, zioMock, wiremock).map(_ % Test) - val webapiDependencies = Seq( + val webapiDependencies = refined ++ Seq( pekkoActor, pekkoHttp, pekkoHttpCors, diff --git a/webapi/src/main/scala/dsp/valueobjects/Project.scala b/webapi/src/main/scala/dsp/valueobjects/Project.scala index 8614589ad1..d2a149e615 100644 --- a/webapi/src/main/scala/dsp/valueobjects/Project.scala +++ b/webapi/src/main/scala/dsp/valueobjects/Project.scala @@ -52,6 +52,9 @@ object Project { implicit val encoder: JsonEncoder[Shortcode] = JsonEncoder[String].contramap((shortcode: Shortcode) => shortcode.value) + def unsafeFrom(str: String) = make(str) + .getOrElse(throw new IllegalArgumentException(s"Invalid project shortcode: $str")) + def make(value: String): Validation[ValidationException, Shortcode] = if (value.isEmpty) { Validation.fail(ValidationException(ProjectErrorMessages.ShortcodeMissing)) diff --git a/webapi/src/main/scala/org/knora/webapi/slice/admin/api/model/MaintenanceRequests.scala b/webapi/src/main/scala/org/knora/webapi/slice/admin/api/model/MaintenanceRequests.scala new file mode 100644 index 0000000000..0b46d10235 --- /dev/null +++ b/webapi/src/main/scala/org/knora/webapi/slice/admin/api/model/MaintenanceRequests.scala @@ -0,0 +1,55 @@ +/* + * Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.slice.admin.api.model + +import eu.timepit.refined.api.Refined +import eu.timepit.refined.api.RefinedTypeOps +import eu.timepit.refined.numeric.Positive +import eu.timepit.refined.refineV +import eu.timepit.refined.string.MatchesRegex +import zio.Chunk +import zio.json.DeriveJsonCodec +import zio.json.JsonCodec +import zio.json.interop.refined._ + +import dsp.valueobjects.Project.Shortcode + +object MaintenanceRequests { + + type AssetId = String Refined MatchesRegex["^[a-zA-Z0-9-_]{4,}$"] + object AssetId extends RefinedTypeOps[AssetId, String] { + implicit val codec: JsonCodec[AssetId] = JsonCodec[String].transformOrFail(AssetId.from, _.toString) + } + + final case class Dimensions(width: Int Refined Positive, height: Int Refined Positive) + + object Dimensions { + def from(width: Int, height: Int): Either[String, Dimensions] = for { + widthRefined <- refineV[Positive](width) + heightRefined <- refineV[Positive](height) + } yield Dimensions(widthRefined, heightRefined) + + implicit val codec: JsonCodec[Dimensions] = DeriveJsonCodec.gen[Dimensions] + } + + case class ReportAsset(id: AssetId, dimensions: Dimensions) + + object ReportAsset { + implicit val codec: JsonCodec[ReportAsset] = DeriveJsonCodec.gen[ReportAsset] + } + + case class ProjectWithBakFiles(id: Shortcode, assetIds: Chunk[ReportAsset]) + + object ProjectWithBakFiles { + implicit val codec: JsonCodec[ProjectWithBakFiles] = DeriveJsonCodec.gen[ProjectWithBakFiles] + } + + case class ProjectsWithBakfilesReport(projects: Chunk[ProjectWithBakFiles]) + + object ProjectsWithBakfilesReport { + implicit val codec: JsonCodec[ProjectsWithBakfilesReport] = DeriveJsonCodec.gen[ProjectsWithBakfilesReport] + } +} diff --git a/webapi/src/main/scala/org/knora/webapi/slice/admin/api/service/MaintenanceService.scala b/webapi/src/main/scala/org/knora/webapi/slice/admin/api/service/MaintenanceService.scala new file mode 100644 index 0000000000..afecb5a00b --- /dev/null +++ b/webapi/src/main/scala/org/knora/webapi/slice/admin/api/service/MaintenanceService.scala @@ -0,0 +1,145 @@ +/* + * Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.slice.admin.api.service + +import zio.Task +import zio.ZIO +import zio.ZLayer +import zio.macros.accessible +import zio.stream.ZStream + +import dsp.valueobjects.Project.Shortcode +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.AssetId +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.Dimensions +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.ProjectsWithBakfilesReport +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests.ReportAsset +import org.knora.webapi.slice.admin.domain.model.KnoraProject +import org.knora.webapi.slice.admin.domain.service.KnoraProjectRepo +import org.knora.webapi.slice.admin.domain.service.ProjectADMService +import org.knora.webapi.slice.common.repo.service.PredicateObjectMapper +import org.knora.webapi.slice.resourceinfo.domain.InternalIri +import org.knora.webapi.store.triplestore.api.TriplestoreService +import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Select +import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Update + +@accessible +trait MaintenanceService { + def fixTopLeftDimensions(report: ProjectsWithBakfilesReport): Task[Unit] + + def doNothing: Task[Unit] +} + +final case class MaintenanceServiceLive( + projectRepo: KnoraProjectRepo, + triplestoreService: TriplestoreService, + mapper: PredicateObjectMapper +) extends MaintenanceService { + override def doNothing: Task[Unit] = ZIO.unit + override def fixTopLeftDimensions(report: ProjectsWithBakfilesReport): Task[Unit] = + ZStream + .fromIterable(report.projects) + .flatMap { project => + ZStream + .fromIterable(project.assetIds) + .flatMapPar(5)(assetId => + ZStream.fromZIOOption( + fixAsset(project.id, assetId) + // None.type errors are just a sign that the assetId should be ignored. Some.type errors are real errors. + .tapSomeError { case Some(e) => ZIO.logError(s"Error while processing ${project.id}, $assetId: $e") } + // We have logged real errors above, from here on out ignore all errors so that the stream can continue. + .orElseFail(None) + ) + ) + } + .runDrain + + private def fixAsset(shortcode: Shortcode, asset: ReportAsset): ZIO[Any, Option[Throwable], Unit] = + for { + project <- projectRepo.findByShortcode(shortcode).some + stillImageFileValueIri <- checkDimensions(project, asset) + _ <- transposeImageDimensions(project, stillImageFileValueIri) + } yield () + + private def checkDimensions( + project: KnoraProject, + asset: ReportAsset + ): ZIO[Any, Option[Throwable], InternalIri] = + for { + res <- getDimensionAndStillImageValueIri(project, asset) + (dim, iri) = res + _ <- ZIO.when(dim == asset.dimensions)(ZIO.fail(None)) + } yield iri + + private def getDimensionAndStillImageValueIri( + project: KnoraProject, + asset: ReportAsset + ): ZIO[Any, Option[Throwable], (Dimensions, InternalIri)] = + for { + result <- triplestoreService.query(checkDimensionsQuery(project, asset.id)).asSomeError + rowMap <- ZIO.fromOption(result.results.bindings.headOption.map(_.rowMap)) + iri <- ZIO.fromOption(rowMap.get("valueIri")).map(InternalIri) + width <- ZIO.fromOption(rowMap.get("dimX").flatMap(_.toIntOption)) + height <- ZIO.fromOption(rowMap.get("dimY").flatMap(_.toIntOption)) + dim <- ZIO.fromOption(Dimensions.from(width, height).toOption) + } yield (dim, iri) + + private def checkDimensionsQuery(project: KnoraProject, assetId: AssetId) = { + val projectGraph = ProjectADMService.projectDataNamedGraphV2(project) + Select(s""" + |PREFIX rdfs: + |PREFIX knora-base: + | + |SELECT ?valueIri ?dimX ?dimY + | FROM <${projectGraph.value}> + |WHERE { + | ?valueIri a knora-base:StillImageFileValue . + | ?valueIri knora-base:internalFilename ?filename . + | FILTER (strstarts(str(?filename), "$assetId")) + | ?valueIri knora-base:dimX ?dimX . + | ?valueIri knora-base:dimY ?dimY . + |} + |""".stripMargin) + } + + private def transposeImageDimensions( + project: KnoraProject, + stillImageFileValueIri: InternalIri + ): ZIO[Any, Option[Throwable], Unit] = + triplestoreService.query(transposeUpdate(project, stillImageFileValueIri)).asSomeError + + private def transposeUpdate(project: KnoraProject, stillImageFileValueIri: InternalIri) = { + val projectGraph = ProjectADMService.projectDataNamedGraphV2(project) + Update( + s""" + |PREFIX rdfs: + |PREFIX knora-base: + | + |WITH <${projectGraph.value}> + |DELETE + |{ + | ?r knora-base:dimX ?oldX . + | ?r knora-base:dimY ?oldY . + |} + |INSERT + |{ + | ?r knora-base:dimX ?oldY . + | ?r knora-base:dimY ?oldX . + |} + |WHERE + |{ + | BIND (<${stillImageFileValueIri.value}> AS ?r) + | ?r knora-base:dimX ?oldX . + | ?r knora-base:dimY ?oldY . + |} + |""".stripMargin + ) + } +} + +object MaintenanceServiceLive { + + val layer = ZLayer.derive[MaintenanceServiceLive] +} diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemory.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemory.scala index 492b9d81b2..685949cdb3 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemory.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/api/TriplestoreServiceInMemory.scala @@ -4,12 +4,7 @@ */ package org.knora.webapi.store.triplestore.api -import org.apache.jena.query.Dataset -import org.apache.jena.query.QueryExecution -import org.apache.jena.query.QueryExecutionFactory -import org.apache.jena.query.QuerySolution -import org.apache.jena.query.ReadWrite -import org.apache.jena.query.ResultSet +import org.apache.jena.query._ import org.apache.jena.rdf.model.Model import org.apache.jena.rdf.model.ModelFactory import org.apache.jena.tdb2.TDB2Factory @@ -19,9 +14,11 @@ import zio.Ref import zio.Scope import zio.Task import zio.UIO +import zio.ULayer import zio.URIO import zio.ZIO import zio.ZLayer +import zio.macros.accessible import java.nio.charset.StandardCharsets import java.nio.file.Path @@ -33,15 +30,7 @@ import org.knora.webapi.IRI import org.knora.webapi.messages.StringFormatter import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject import org.knora.webapi.messages.store.triplestoremessages.SparqlConstructResponse -import org.knora.webapi.messages.util.rdf.QuadFormat -import org.knora.webapi.messages.util.rdf.RdfFeatureFactory -import org.knora.webapi.messages.util.rdf.RdfFormatUtil -import org.knora.webapi.messages.util.rdf.RdfStringSource -import org.knora.webapi.messages.util.rdf.SparqlSelectResult -import org.knora.webapi.messages.util.rdf.SparqlSelectResultBody -import org.knora.webapi.messages.util.rdf.SparqlSelectResultHeader -import org.knora.webapi.messages.util.rdf.Turtle -import org.knora.webapi.messages.util.rdf.VariableResultsRow +import org.knora.webapi.messages.util.rdf._ import org.knora.webapi.messages.util.rdf.jenaimpl.JenaFormatUtil import org.knora.webapi.slice.resourceinfo.domain.InternalIri import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Ask @@ -59,8 +48,14 @@ import org.knora.webapi.util.ZScopedJavaIoStreams.byteArrayOutputStream import org.knora.webapi.util.ZScopedJavaIoStreams.fileInputStream import org.knora.webapi.util.ZScopedJavaIoStreams.fileOutputStream +@accessible +trait TestTripleStore extends TriplestoreService { + def setDataset(ds: Dataset): UIO[Unit] +} + final case class TriplestoreServiceInMemory(datasetRef: Ref[Dataset], implicit val sf: StringFormatter) - extends TriplestoreService { + extends TriplestoreService + with TestTripleStore { private val rdfFormatUtil: RdfFormatUtil = RdfFeatureFactory.getRdfFormatUtil() override def query(query: Select): Task[SparqlSelectResult] = { @@ -240,6 +235,9 @@ final case class TriplestoreServiceInMemory(datasetRef: Ref[Dataset], implicit v override def uploadRepository(inputFile: Path): Task[Unit] = ZIO.fail(new UnsupportedOperationException("Not implemented in TriplestoreServiceInMemory.")) + + override def setDataset(ds: Dataset): UIO[Unit] = + datasetRef.set(ds) } object TriplestoreServiceInMemory { @@ -252,6 +250,8 @@ object TriplestoreServiceInMemory { */ val createEmptyDataset: UIO[Dataset] = ZIO.succeed(TDB2Factory.createDataset()) - val layer: ZLayer[Ref[Dataset] with StringFormatter, Nothing, TriplestoreService] = + val emptyDatasetRefLayer: ULayer[Ref[Dataset]] = ZLayer.fromZIO(createEmptyDataset.flatMap(Ref.make(_))) + + val layer: ZLayer[Ref[Dataset] with StringFormatter, Nothing, TestTripleStore with TriplestoreService] = ZLayer.fromFunction(TriplestoreServiceInMemory.apply _) } diff --git a/webapi/src/test/scala/org/knora/webapi/TestDataFactory.scala b/webapi/src/test/scala/org/knora/webapi/TestDataFactory.scala index 08b6dc07be..5b67fa92b5 100644 --- a/webapi/src/test/scala/org/knora/webapi/TestDataFactory.scala +++ b/webapi/src/test/scala/org/knora/webapi/TestDataFactory.scala @@ -5,15 +5,32 @@ package org.knora.webapi +import zio.NonEmptyChunk + import dsp.valueobjects.Iri._ import dsp.valueobjects.Project._ import dsp.valueobjects.V2 import org.knora.webapi.messages.admin.responder.projectsmessages.ProjectIdentifierADM._ +import org.knora.webapi.slice.admin.domain.model.KnoraProject +import org.knora.webapi.slice.resourceinfo.domain.InternalIri /** * Helps in creating value objects for tests. */ object TestDataFactory { + + val someProject = KnoraProject( + InternalIri("http://rdfh.ch/projects/0001"), + "shortname", + Shortcode.unsafeFrom("0001"), + None, + NonEmptyChunk(V2.StringLiteralV2("Some description", None)), + List.empty, + None, + true, + false + ) + def projectShortcodeIdentifier(shortcode: String): ShortcodeIdentifier = ShortcodeIdentifier .fromString(shortcode) diff --git a/webapi/src/test/scala/org/knora/webapi/slice/admin/api/service/MaintenanceServiceLiveSpec.scala b/webapi/src/test/scala/org/knora/webapi/slice/admin/api/service/MaintenanceServiceLiveSpec.scala new file mode 100644 index 0000000000..524021fdf8 --- /dev/null +++ b/webapi/src/test/scala/org/knora/webapi/slice/admin/api/service/MaintenanceServiceLiveSpec.scala @@ -0,0 +1,111 @@ +/* + * Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.slice.admin.api.service + +import eu.timepit.refined.auto._ +import zio.Chunk +import zio.ZIO +import zio.test.ZIOSpecDefault +import zio.test.assertCompletes +import zio.test.assertTrue + +import org.knora.webapi.TestDataFactory +import org.knora.webapi.messages.StringFormatter +import org.knora.webapi.slice.admin.api.model.MaintenanceRequests._ +import org.knora.webapi.slice.admin.domain.repo.service.KnoraProjectRepoInMemory +import org.knora.webapi.slice.admin.domain.service.ProjectADMService +import org.knora.webapi.slice.common.repo.service.PredicateObjectMapper +import org.knora.webapi.slice.resourceinfo.domain.IriConverter +import org.knora.webapi.store.triplestore.TestDatasetBuilder +import org.knora.webapi.store.triplestore.api.TestTripleStore +import org.knora.webapi.store.triplestore.api.TriplestoreService +import org.knora.webapi.store.triplestore.api.TriplestoreService.Queries.Select +import org.knora.webapi.store.triplestore.api.TriplestoreServiceInMemory +import org.knora.webapi.store.triplestore.api.TriplestoreServiceInMemory.emptyDatasetRefLayer + +object MaintenanceServiceLiveSpec extends ZIOSpecDefault { + + private val testProject = TestDataFactory.someProject + private val projectDataNamedGraphIri = ProjectADMService.projectDataNamedGraphV2(testProject).value + private val testAssetId = AssetId.unsafeFrom("some-asset-id") + private val expectedDimension = Dimensions(5202, 3602) + private val testReport = ProjectsWithBakfilesReport( + Chunk(ProjectWithBakFiles(testProject.shortcode, Chunk(ReportAsset(testAssetId, expectedDimension)))) + ) + private val testValueIri = "http://rdfh.ch/some-value-iri" + + private def saveStillImageFileValueWithDimensions(width: Int, height: Int) = TestDatasetBuilder + .datasetFromTriG(s""" + | @prefix knora-base: . + | @prefix rdf: . + | @prefix xsd: . + | + | <$projectDataNamedGraphIri> { + | <$testValueIri> a knora-base:StillImageFileValue; + | knora-base:dimX "${width}"^^xsd:integer; + | knora-base:dimY "${height}"^^xsd:integer; + | knora-base:internalFilename "$testAssetId.jp2"^^xsd:string; + | } + |""".stripMargin) + .flatMap(TestTripleStore.setDataset) + + def queryForDim() = for { + rowMap <- TriplestoreService + .query(Select(s""" + |PREFIX knora-base: + | + |SELECT ?width ?height + |FROM <$projectDataNamedGraphIri> + |WHERE { + | <$testValueIri> a knora-base:StillImageFileValue ; + | knora-base:dimX ?width ; + | knora-base:dimY ?height . + |} + |""".stripMargin)) + .map(_.results.bindings.head.rowMap) + width <- ZIO.fromOption(rowMap.get("width").map(_.toInt)) + height <- ZIO.fromOption(rowMap.get("height").map(_.toInt)) + dim <- ZIO.fromEither(Dimensions.from(width, height)) + } yield dim + + val spec = suite("MaintenanceServiceLive")( + test("fixTopLeftDimensions should not fail for an empty report") { + MaintenanceService.fixTopLeftDimensions(ProjectsWithBakfilesReport(Chunk.empty)).as(assertCompletes) + }, + test("fixTopLeftDimensions should not fail if no StillImageFileValue is found") { + MaintenanceService.fixTopLeftDimensions(testReport).as(assertCompletes) + }, + test("fixTopLeftDimensions should transpose dimension for an existing StillImageFileValue") { + for { + // given + _ <- ZIO.serviceWithZIO[KnoraProjectRepoInMemory](_.save(testProject)) + _ <- saveStillImageFileValueWithDimensions(width = expectedDimension.height, height = expectedDimension.width) + // when + _ <- MaintenanceService.fixTopLeftDimensions(testReport) + // then + actualDimension <- queryForDim() + } yield assertTrue(actualDimension == expectedDimension) + }, + test("fixTopLeftDimensions not should transpose dimension for an existing StillImageFileValue") { + for { + // given + _ <- ZIO.serviceWithZIO[KnoraProjectRepoInMemory](_.save(testProject)) + _ <- saveStillImageFileValueWithDimensions(width = expectedDimension.width, height = expectedDimension.height) + // when + _ <- MaintenanceService.fixTopLeftDimensions(testReport) + // then + actualDimension <- queryForDim() + } yield assertTrue(actualDimension == expectedDimension) + } + ).provide( + MaintenanceServiceLive.layer, + KnoraProjectRepoInMemory.layer, + emptyDatasetRefLayer >>> TriplestoreServiceInMemory.layer, + PredicateObjectMapper.layer, + IriConverter.layer, + StringFormatter.test + ) +} diff --git a/webapi/src/test/scala/org/knora/webapi/slice/common/repo/service/AbstractInMemoryCrudRepository.scala b/webapi/src/test/scala/org/knora/webapi/slice/common/repo/service/AbstractInMemoryCrudRepository.scala index 47f6e9326f..0d6ee4be28 100644 --- a/webapi/src/test/scala/org/knora/webapi/slice/common/repo/service/AbstractInMemoryCrudRepository.scala +++ b/webapi/src/test/scala/org/knora/webapi/slice/common/repo/service/AbstractInMemoryCrudRepository.scala @@ -16,7 +16,8 @@ abstract class AbstractInMemoryCrudRepository[Entity, Id](entities: Ref[List[Ent * @param entity The entity to be saved. * @return the saved entity. */ - override def save(entity: Entity): Task[Entity] = entities.update(entity :: _).as(entity) + override def save(entity: Entity): Task[Entity] = + entities.update(entity :: _).as(entity) /** * Deletes a given entity. diff --git a/webapi/src/test/scala/org/knora/webapi/store/triplestore/TestDatasetBuilder.scala b/webapi/src/test/scala/org/knora/webapi/store/triplestore/TestDatasetBuilder.scala index 7a7edb0261..44a5848ba5 100644 --- a/webapi/src/test/scala/org/knora/webapi/store/triplestore/TestDatasetBuilder.scala +++ b/webapi/src/test/scala/org/knora/webapi/store/triplestore/TestDatasetBuilder.scala @@ -5,9 +5,12 @@ package org.knora.webapi.store.triplestore +import org.apache.commons.io.IOUtils import org.apache.jena.query.Dataset import org.apache.jena.query.ReadWrite import org.apache.jena.rdf.model.Model +import org.apache.jena.riot.Lang +import org.apache.jena.riot.RDFDataMgr import zio._ import java.io.StringReader @@ -32,6 +35,14 @@ object TestDatasetBuilder { private def datasetFromTurtle(turtle: String): Task[Dataset] = createEmptyDataset.flatMap(transactionalWrite(readToModel(turtle))) + def datasetFromTriG(trig: String): Task[Dataset] = + for { + ds <- createEmptyDataset + is = IOUtils.toInputStream(trig, "UTF-8") + r: Runnable = () => { RDFDataMgr.read(ds, is, Lang.TRIG) } + _ <- ZIO.attempt(ds.executeWrite(r)) + } yield ds + private def asLayer(ds: Task[Dataset]): TaskLayer[Ref[Dataset]] = ZLayer.fromZIO(ds.flatMap(Ref.make[Dataset](_))) def datasetLayerFromTurtle(turtle: String): TaskLayer[Ref[Dataset]] = asLayer(datasetFromTurtle(turtle))