From 89d1b22ec132863622ae75bbf6d52b98eb9487a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Tue, 19 Dec 2023 10:12:04 +0100 Subject: [PATCH 1/3] Make SttpBackend a dependency for DspIngestClientLive --- .../domain/service/DspIngestClient.scala | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala b/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala index 44bf2e1b4a..f1f6c210e6 100644 --- a/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala +++ b/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala @@ -6,6 +6,7 @@ package org.knora.webapi.slice.admin.domain.service import sttp.capabilities.zio.ZioStreams +import sttp.client3.SttpBackend import sttp.client3.UriContext import sttp.client3.asStreamAlways import sttp.client3.basicRequest @@ -38,27 +39,29 @@ trait DspIngestClient { def importProject(shortcode: Shortcode, fileToImport: Path): Task[Path] } + final case class DspIngestClientLive( jwtService: JwtService, - dspIngestConfig: DspIngestConfig + dspIngestConfig: DspIngestConfig, + sttpBackend: SttpBackend[Task, ZioStreams] ) extends DspIngestClient { private def projectsPath(shortcode: Shortcode) = s"${dspIngestConfig.baseUrl}/projects/${shortcode.value}" + + private val authenticatedRequest = + jwtService.createJwtForDspIngest().map(token => basicRequest.auth.bearer(token.jwtString)) def exportProject(shortcode: Shortcode): ZIO[Scope, Throwable, Path] = for { - token <- jwtService.createJwtForDspIngest() - tempdir <- Files.createTempDirectoryScoped(Some("export"), List.empty) - exportFile = tempdir / "export.zip" - response <- { - val request = basicRequest.auth - .bearer(token.jwtString) - .post(uri"${projectsPath(shortcode)}/export") - .readTimeout(30.minutes) - .response(asStreamAlways(ZioStreams)(_.run(ZSink.fromFile(exportFile.toFile)))) - HttpClientZioBackend.scoped().flatMap(request.send(_)) - } - _ <- ZIO.logInfo(s"Response from ingest :${response.code}") + tempDir <- Files.createTempDirectoryScoped(Some("export"), List.empty) + exportFile = tempDir / "export.zip" + request <- authenticatedRequest.map { + _.post(uri"${projectsPath(shortcode)}/export") + .readTimeout(30.minutes) + .response(asStreamAlways(ZioStreams)(_.run(ZSink.fromFile(exportFile.toFile)))) + } + response <- request.send(backend = sttpBackend) + _ <- ZIO.logInfo(s"Response from ingest :${response.code}") } yield exportFile def importProject(shortcode: Shortcode, fileToImport: Path): Task[Path] = ZIO.scoped { @@ -81,5 +84,5 @@ final case class DspIngestClientLive( } object DspIngestClientLive { - val layer = ZLayer.fromFunction(DspIngestClientLive.apply _) + val layer = HttpClientZioBackend.layer().orDie >>> ZLayer.derive[DspIngestClientLive] } From 38d1b5376afb79443255640c063dd2675adae5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Tue, 19 Dec 2023 10:27:09 +0100 Subject: [PATCH 2/3] Add caching of token as long it is valid --- .../domain/service/DspIngestClient.scala | 29 +++++++++++++++---- .../service/DspIngestClientLiveSpec.scala | 12 +++++--- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala b/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala index f1f6c210e6..1d0a115cee 100644 --- a/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala +++ b/webapi/src/main/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClient.scala @@ -11,8 +11,11 @@ import sttp.client3.UriContext import sttp.client3.asStreamAlways import sttp.client3.basicRequest import sttp.client3.httpclient.zio.HttpClientZioBackend +import zio.Clock +import zio.Ref import zio.Scope import zio.Task +import zio.UIO import zio.ZIO import zio.ZLayer import zio.http.Body @@ -27,9 +30,11 @@ import zio.nio.file.Files import zio.nio.file.Path import zio.stream.ZSink +import java.util.concurrent.TimeUnit import scala.concurrent.duration.DurationInt import org.knora.webapi.config.DspIngestConfig +import org.knora.webapi.routing.Jwt import org.knora.webapi.routing.JwtService import org.knora.webapi.slice.admin.domain.model.KnoraProject.Shortcode @@ -43,13 +48,24 @@ trait DspIngestClient { final case class DspIngestClientLive( jwtService: JwtService, dspIngestConfig: DspIngestConfig, - sttpBackend: SttpBackend[Task, ZioStreams] + sttpBackend: SttpBackend[Task, ZioStreams], + tokenRef: Ref[Option[Jwt]] ) extends DspIngestClient { private def projectsPath(shortcode: Shortcode) = s"${dspIngestConfig.baseUrl}/projects/${shortcode.value}" - - private val authenticatedRequest = - jwtService.createJwtForDspIngest().map(token => basicRequest.auth.bearer(token.jwtString)) + + private val getJwtString: UIO[String] = for { + // check the current token and create a new one if: + // * it is not present + // * it is expired or close to expiring within the next 10 seconds + threshold <- Clock.currentTime(TimeUnit.SECONDS).map(_ - 10) + token <- tokenRef.get.flatMap { + case Some(jwt) if jwt.expiration <= threshold => ZIO.succeed(jwt) + case _ => jwtService.createJwtForDspIngest().tap(jwt => tokenRef.set(Some(jwt))) + } + } yield token.jwtString + + private val authenticatedRequest = getJwtString.map(basicRequest.auth.bearer(_)) def exportProject(shortcode: Shortcode): ZIO[Scope, Throwable, Path] = for { @@ -84,5 +100,8 @@ final case class DspIngestClientLive( } object DspIngestClientLive { - val layer = HttpClientZioBackend.layer().orDie >>> ZLayer.derive[DspIngestClientLive] + val layer = + HttpClientZioBackend.layer().orDie >+> + ZLayer.fromZIO(Ref.make[Option[Jwt]](None)) >>> + ZLayer.derive[DspIngestClientLive] } diff --git a/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala b/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala index b14f2f949c..ec440c9bb8 100644 --- a/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala @@ -47,6 +47,7 @@ object DspIngestClientLiveSpec extends ZIOSpecDefault { suite("DspIngestClientLive")(test("should download a project export") { ZIO.scoped { for { + // given wiremock <- ZIO.service[WireMockServer] _ = wiremock.stubFor( WireMock @@ -59,14 +60,17 @@ object DspIngestClientLiveSpec extends ZIOSpecDefault { .withStatus(200) ) ) - path <- DspIngestClient.exportProject(testProject) - contentIsDownloaded <- Files.readAllBytes(path).map(_.toArray).map(_ sameElements testContent) - // Verify the request is valid - mockJwt <- ZIO.serviceWithZIO[JwtService](_.createJwtForDspIngest()) + mockJwt <- JwtService.createJwtForDspIngest() + + // when + path <- DspIngestClient.exportProject(testProject) + + // then _ = wiremock.verify( postRequestedFor(urlPathEqualTo(expectedPath)) .withHeader("Authorization", equalTo(s"Bearer ${mockJwt.jwtString}")) ) + contentIsDownloaded <- Files.readAllBytes(path).map(_.toArray).map(_ sameElements testContent) } yield assertTrue(contentIsDownloaded) } }).provide( From 5496a1a593d8f80124dec082000a283cf89c5180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Tue, 19 Dec 2023 10:38:32 +0100 Subject: [PATCH 3/3] extract exportProjectSuite --- .../service/DspIngestClientLiveSpec.scala | 61 ++++++++++--------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala b/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala index ec440c9bb8..ac8ea668fe 100644 --- a/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/slice/admin/domain/service/DspIngestClientLiveSpec.scala @@ -43,37 +43,40 @@ object DspIngestClientLiveSpec extends ZIOSpecDefault { private val testProject = Shortcode.unsafeFrom(testShortCodeStr) private val testContent = "testContent".getBytes() private val expectedPath = s"/projects/$testShortCodeStr/export" - override def spec: Spec[TestEnvironment & Scope, Any] = - suite("DspIngestClientLive")(test("should download a project export") { - ZIO.scoped { - for { - // given - wiremock <- ZIO.service[WireMockServer] - _ = wiremock.stubFor( - WireMock - .post(urlPathEqualTo(expectedPath)) - .willReturn( - aResponse() - .withHeader("Content-Type", "application/zip") - .withHeader("Content-Disposition", s"export-$testShortCodeStr.zip") - .withBody(testContent) - .withStatus(200) - ) - ) - mockJwt <- JwtService.createJwtForDspIngest() - // when - path <- DspIngestClient.exportProject(testProject) + private val exportProjectSuite = suite("exportProject")(test("should download a project export") { + ZIO.scoped { + for { + // given + wiremock <- ZIO.service[WireMockServer] + _ = wiremock.stubFor( + WireMock + .post(urlPathEqualTo(expectedPath)) + .willReturn( + aResponse() + .withHeader("Content-Type", "application/zip") + .withHeader("Content-Disposition", s"export-$testShortCodeStr.zip") + .withBody(testContent) + .withStatus(200) + ) + ) + mockJwt <- JwtService.createJwtForDspIngest() + + // when + path <- DspIngestClient.exportProject(testProject) - // then - _ = wiremock.verify( - postRequestedFor(urlPathEqualTo(expectedPath)) - .withHeader("Authorization", equalTo(s"Bearer ${mockJwt.jwtString}")) - ) - contentIsDownloaded <- Files.readAllBytes(path).map(_.toArray).map(_ sameElements testContent) - } yield assertTrue(contentIsDownloaded) - } - }).provide( + // then + _ = wiremock.verify( + postRequestedFor(urlPathEqualTo(expectedPath)) + .withHeader("Authorization", equalTo(s"Bearer ${mockJwt.jwtString}")) + ) + contentIsDownloaded <- Files.readAllBytes(path).map(_.toArray).map(_ sameElements testContent) + } yield assertTrue(contentIsDownloaded) + } + }) + + override def spec: Spec[TestEnvironment & Scope, Any] = + suite("DspIngestClientLive")(exportProjectSuite).provide( DspIngestClientLive.layer, dspIngestConfigLayer, mockJwtServiceLayer,