feat: Add http metrics for gravsearch endpoints (DEV-2936) (#2946)

Co-authored-by: Marcin Procyk <[email protected]>
dasch-swiss · Dec 4, 2023 · 7ca5946 · 7ca5946
1 parent 1c37768
commit 7ca5946
Show file tree

Hide file tree

Showing 41 changed files with 766 additions and 606 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -39,7 +39,7 @@ jobs:
 
   integration-test:
     name: Build and integration-test
-    runs-on: actuated
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         java: [ "17", "21" ]

diff --git a/integration/src/test/scala/org/knora/webapi/R2RSpec.scala b/integration/src/test/scala/org/knora/webapi/R2RSpec.scala
@@ -6,7 +6,11 @@
 package org.knora.webapi
 
 import com.typesafe.scalalogging.Logger
-import org.apache.pekko
+import org.apache.pekko.actor.ActorRef
+import org.apache.pekko.actor.ActorSystem
+import org.apache.pekko.http.scaladsl.model.HttpResponse
+import org.apache.pekko.http.scaladsl.testkit.RouteTestTimeout
+import org.apache.pekko.http.scaladsl.testkit.ScalatestRouteTest
 import org.scalatest.BeforeAndAfterAll
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
@@ -22,22 +26,15 @@ import scala.concurrent.duration.FiniteDuration
 import scala.concurrent.duration.NANOSECONDS
 
 import org.knora.webapi.config.AppConfig
-import org.knora.webapi.core.AppRouter
 import org.knora.webapi.core.AppServer
-import org.knora.webapi.core.LayersTest.DefaultTestEnvironmentWithoutSipi
 import org.knora.webapi.core.TestStartupUtils
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
 import org.knora.webapi.messages.util.rdf._
 import org.knora.webapi.routing.KnoraRouteData
+import org.knora.webapi.routing.UnsafeZioRun
 import org.knora.webapi.util.FileUtil
 import org.knora.webapi.util.LogAspect
 
-import pekko.actor.ActorRef
-import pekko.actor.ActorSystem
-import pekko.http.scaladsl.model.HttpResponse
-import pekko.http.scaladsl.testkit.RouteTestTimeout
-import pekko.http.scaladsl.testkit.ScalatestRouteTest
-
 /**
  * R(oute)2R(esponder) Spec base class. Please, for any new E2E tests, use E2ESpec.
  */
@@ -50,16 +47,15 @@ abstract class R2RSpec
 
   /**
    * The `Environment` that we require to exist at startup.
-   * Can be overriden in specs that need other implementations.
+   * Can be overridden in specs that need other implementations.
    */
   type Environment = core.LayersTest.DefaultTestEnvironmentWithoutSipi
 
   /**
    * The effect layers from which the App is built.
-   * Can be overriden in specs that need other implementations.
+   * Can be overridden in specs that need other implementations.
    */
-  lazy val effectLayers: ULayer[DefaultTestEnvironmentWithoutSipi] =
-    core.LayersTest.integrationTestsWithFusekiTestcontainers(Some(system))
+  lazy val effectLayers: ULayer[Environment] = core.LayersTest.integrationTestsWithFusekiTestcontainers(Some(system))
 
   /**
    * `Bootstrap` will ensure that everything is instantiated when the Runtime is created
@@ -75,33 +71,14 @@ abstract class R2RSpec
     FiniteDuration(appConfig.defaultTimeout.toNanos, NANOSECONDS)
   )
 
-  // An effect for getting stuff out, so that we can pass them
-  // to some legacy code
-  private val routerAndConfig = for {
-    router <- ZIO.service[core.AppRouter]
-    config <- ZIO.service[AppConfig]
-  } yield (router, config)
-
-  /**
-   * Create router and config by unsafe running them.
-   */
-  private val (router: AppRouter, config: AppConfig) =
-    Unsafe.unsafe { implicit u =>
-      runtime.unsafe
-        .run(
-          routerAndConfig
-        )
-        .getOrThrowFiberFailure()
-    }
-
   // main difference to other specs (no own systen and executionContext defined)
   lazy val rdfDataObjects = List.empty[RdfDataObject]
   val log: Logger         = Logger(this.getClass())
-  val appActor: ActorRef  = router.ref
+  val appActor: ActorRef  = UnsafeZioRun.runOrThrow(ZIO.serviceWith[core.AppRouter](_.ref))
 
   // needed by some tests
-  val routeData: KnoraRouteData = KnoraRouteData(system, appActor, config)
-  val appConfig: AppConfig      = config
+  val appConfig: AppConfig      = UnsafeZioRun.runOrThrow(ZIO.service[AppConfig])
+  val routeData: KnoraRouteData = KnoraRouteData(system, appActor, appConfig)
 
   final override def beforeAll(): Unit =
     /* Here we start our app and initialize the repository before each suit runs */

diff --git a/integration/src/test/scala/org/knora/webapi/core/LayersTest.scala b/integration/src/test/scala/org/knora/webapi/core/LayersTest.scala
@@ -46,6 +46,7 @@ import org.knora.webapi.slice.ontology.repo.service.PredicateRepositoryLive
 import org.knora.webapi.slice.resourceinfo.ResourceInfoLayers
 import org.knora.webapi.slice.resourceinfo.api.service.RestResourceInfoService
 import org.knora.webapi.slice.resourceinfo.domain.IriConverter
+import org.knora.webapi.slice.search.api.SearchApiRoutes
 import org.knora.webapi.store.cache.CacheServiceRequestMessageHandler
 import org.knora.webapi.store.cache.CacheServiceRequestMessageHandlerLive
 import org.knora.webapi.store.cache.api.CacheService
@@ -113,6 +114,7 @@ object LayersTest {
       with RestCardinalityService
       with RestPermissionService
       with RestResourceInfoService
+      with SearchApiRoutes
       with SearchResponderV2
       with SipiResponderADM
       with OntologyInferencer
@@ -148,6 +150,7 @@ object LayersTest {
       IriConverter.layer,
       IriService.layer,
       KnoraProjectRepoLive.layer,
+      KnoraResponseRenderer.layer,
       ListsResponderADMLive.layer,
       ListsResponderV2Live.layer,
       MaintenanceEndpoints.layer,
@@ -179,6 +182,7 @@ object LayersTest {
       ResourcesResponderV2Live.layer,
       RestCardinalityServiceLive.layer,
       RestPermissionServiceLive.layer,
+      SearchApiRoutes.layer,
       SearchResponderV2Live.layer,
       SipiResponderADMLive.layer,
       StandoffResponderV2Live.layer,

diff --git a/integration/src/test/scala/org/knora/webapi/e2e/v2/MarkupHeader.scala b/integration/src/test/scala/org/knora/webapi/e2e/v2/MarkupHeader.scala
@@ -9,25 +9,25 @@ import org.apache.pekko
 
 import scala.util.Try
 
-import org.knora.webapi.routing.RouteUtilV2
-
 import pekko.http.scaladsl.model.headers.ModeledCustomHeader
 import pekko.http.scaladsl.model.headers.ModeledCustomHeaderCompanion
 
 /**
- * A custom Pekko HTTP header representing [[RouteUtilV2.MARKUP_HEADER]], which a client can send to specify
+ * A custom Pekko HTTP header representing "x-knora-accept-markup", which a client can send to specify
  * how text markup should be returned in an API response.
  *
  * The definition follows [[https://doc.pekko.io/docs/pekko-http/current/common/http-model.html#custom-headers]].
  */
-final class MarkupHeader(token: String) extends ModeledCustomHeader[MarkupHeader] {
+final class MarkupHeader private (token: String) extends ModeledCustomHeader[MarkupHeader] {
   override def renderInRequests             = true
   override def renderInResponses            = true
   override val companion: MarkupHeader.type = MarkupHeader
   override def value: String                = token
 }
 
 object MarkupHeader extends ModeledCustomHeaderCompanion[MarkupHeader] {
-  override val name: String         = RouteUtilV2.MARKUP_HEADER
+  override val name: String         = "x-knora-accept-markup"
   override def parse(value: String) = Try(new MarkupHeader(value))
+
+  val standoff: MarkupHeader = new MarkupHeader("standoff")
 }
diff --git a/integration/src/test/scala/org/knora/webapi/e2e/v2/ProjectHeader.scala b/integration/src/test/scala/org/knora/webapi/e2e/v2/ProjectHeader.scala
@@ -5,17 +5,13 @@
 
 package org.knora.webapi.e2e.v2
 
-import org.apache.pekko
+import org.apache.pekko.http.scaladsl.model.headers.ModeledCustomHeader
+import org.apache.pekko.http.scaladsl.model.headers.ModeledCustomHeaderCompanion
 
 import scala.util.Try
 
-import org.knora.webapi.routing.RouteUtilV2
-
-import pekko.http.scaladsl.model.headers.ModeledCustomHeader
-import pekko.http.scaladsl.model.headers.ModeledCustomHeaderCompanion
-
 /**
- * A custom Pekko HTTP header representing [[RouteUtilV2.PROJECT_HEADER]], which a client can send to specify
+ * A custom Pekko HTTP header "x-knora-accept-schema", which a client can send to specify
  * a project from which results should be returned.
  *
  * The definition follows [[https://doc.pekko.io/docs/pekko-http/current/common/http-model.html#custom-headers]].
@@ -28,6 +24,6 @@ class ProjectHeader(token: String) extends ModeledCustomHeader[ProjectHeader] {
 }
 
 object ProjectHeader extends ModeledCustomHeaderCompanion[ProjectHeader] {
-  override val name: String         = RouteUtilV2.PROJECT_HEADER
-  override def parse(value: String) = Try(new ProjectHeader(value))
+  override val name: String                             = "x-knora-accept-project"
+  override def parse(value: String): Try[ProjectHeader] = Try(new ProjectHeader(value))
 }
diff --git a/integration/src/test/scala/org/knora/webapi/e2e/v2/ResourcesRouteV2E2ESpec.scala b/integration/src/test/scala/org/knora/webapi/e2e/v2/ResourcesRouteV2E2ESpec.scala
@@ -7,7 +7,13 @@ package org.knora.webapi.e2e.v2
 
 import com.typesafe.config.Config
 import com.typesafe.config.ConfigFactory
-import org.apache.pekko
+import org.apache.pekko.http.scaladsl.model.HttpEntity
+import org.apache.pekko.http.scaladsl.model.HttpResponse
+import org.apache.pekko.http.scaladsl.model.MediaRange
+import org.apache.pekko.http.scaladsl.model.StatusCodes
+import org.apache.pekko.http.scaladsl.model.headers.Accept
+import org.apache.pekko.http.scaladsl.model.headers.BasicHttpCredentials
+import org.apache.pekko.http.scaladsl.unmarshalling.Unmarshal
 import org.xmlunit.builder.DiffBuilder
 import org.xmlunit.builder.Input
 import org.xmlunit.diff.Diff
@@ -41,20 +47,11 @@ import org.knora.webapi.messages.ValuesValidator
 import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject
 import org.knora.webapi.messages.util._
 import org.knora.webapi.messages.util.rdf._
-import org.knora.webapi.routing.RouteUtilV2
 import org.knora.webapi.routing.v2.OntologiesRouteV2
 import org.knora.webapi.sharedtestdata.SharedOntologyTestDataADM
 import org.knora.webapi.sharedtestdata.SharedTestDataADM
 import org.knora.webapi.util._
 
-import pekko.http.scaladsl.model.HttpEntity
-import pekko.http.scaladsl.model.HttpResponse
-import pekko.http.scaladsl.model.MediaRange
-import pekko.http.scaladsl.model.StatusCodes
-import pekko.http.scaladsl.model.headers.Accept
-import pekko.http.scaladsl.model.headers.BasicHttpCredentials
-import pekko.http.scaladsl.unmarshalling.Unmarshal
-
 /**
  * Tests the API v2 resources route.
  */
@@ -221,7 +218,7 @@ class ResourcesRouteV2E2ESpec extends E2ESpec {
 
     "perform a resource request for the book 'Reise ins Heilige Land' using the simple schema (specified by an HTTP header) in JSON-LD" in {
       val request = Get(s"$baseApiUrl/v2/resources/$reiseInsHeiligeLandIriEncoded")
-        .addHeader(new SchemaHeader(RouteUtilV2.SIMPLE_SCHEMA_NAME))
+        .addHeader(SchemaHeader.simple)
       val response: HttpResponse = singleAwaitingRequest(request)
       val responseAsString       = responseToString(response)
       assert(response.status == StatusCodes.OK, responseAsString)
@@ -238,7 +235,7 @@ class ResourcesRouteV2E2ESpec extends E2ESpec {
 
     "perform a resource request for the book 'Reise ins Heilige Land' using the simple schema in Turtle" in {
       val request = Get(s"$baseApiUrl/v2/resources/$reiseInsHeiligeLandIriEncoded")
-        .addHeader(new SchemaHeader(RouteUtilV2.SIMPLE_SCHEMA_NAME))
+        .addHeader(SchemaHeader.simple)
         .addHeader(Accept(RdfMediaTypes.`text/turtle`))
       val response: HttpResponse = singleAwaitingRequest(request)
       val responseAsString       = responseToString(response)
@@ -249,7 +246,7 @@ class ResourcesRouteV2E2ESpec extends E2ESpec {
 
     "perform a resource request for the book 'Reise ins Heilige Land' using the simple schema in RDF/XML" in {
       val request = Get(s"$baseApiUrl/v2/resources/$reiseInsHeiligeLandIriEncoded")
-        .addHeader(new SchemaHeader(RouteUtilV2.SIMPLE_SCHEMA_NAME))
+        .addHeader(SchemaHeader.simple)
         .addHeader(Accept(RdfMediaTypes.`application/rdf+xml`))
       val response: HttpResponse = singleAwaitingRequest(request)
       val responseAsString       = responseToString(response)
@@ -260,7 +257,7 @@ class ResourcesRouteV2E2ESpec extends E2ESpec {
 
     "perform a resource preview request for the book 'Reise ins Heilige Land' using the simple schema (specified by an HTTP header)" in {
       val request = Get(s"$baseApiUrl/v2/resourcespreview/$reiseInsHeiligeLandIriEncoded")
-        .addHeader(new SchemaHeader(RouteUtilV2.SIMPLE_SCHEMA_NAME))
+        .addHeader(SchemaHeader.simple)
       val response: HttpResponse = singleAwaitingRequest(request)
       val responseAsString       = responseToString(response)
       assert(response.status == StatusCodes.OK, responseAsString)
@@ -270,7 +267,7 @@ class ResourcesRouteV2E2ESpec extends E2ESpec {
 
     "perform a resource request for the book 'Reise ins Heilige Land' using the simple schema (specified by a URL parameter)" in {
       val request = Get(
-        s"$baseApiUrl/v2/resources/$reiseInsHeiligeLandIriEncoded?${RouteUtilV2.SCHEMA_PARAM}=${RouteUtilV2.SIMPLE_SCHEMA_NAME}"
+        s"$baseApiUrl/v2/resources/$reiseInsHeiligeLandIriEncoded?schema=simple"
       )
       val response: HttpResponse = singleAwaitingRequest(request)
       val responseAsString       = responseToString(response)
@@ -343,7 +340,7 @@ class ResourcesRouteV2E2ESpec extends E2ESpec {
     "perform a full resource request for a resource with a list value (in the simple schema)" in {
       val iri = URLEncoder.encode("http://rdfh.ch/0001/thing_with_list_value", "UTF-8")
       val request = Get(s"$baseApiUrl/v2/resources/$iri")
-        .addHeader(new SchemaHeader(RouteUtilV2.SIMPLE_SCHEMA_NAME))
+        .addHeader(SchemaHeader.simple)
       val response: HttpResponse = singleAwaitingRequest(request)
       val responseAsString       = responseToString(response)
       assert(response.status == StatusCodes.OK, responseAsString)
@@ -378,7 +375,7 @@ class ResourcesRouteV2E2ESpec extends E2ESpec {
     "perform a full resource request for a resource with a link (in the simple schema)" in {
       val iri = URLEncoder.encode("http://rdfh.ch/0001/0C-0L1kORryKzJAJxxRyRQ", "UTF-8")
       val request = Get(s"$baseApiUrl/v2/resources/$iri")
-        .addHeader(new SchemaHeader(RouteUtilV2.SIMPLE_SCHEMA_NAME))
+        .addHeader(SchemaHeader.simple)
       val response: HttpResponse = singleAwaitingRequest(request)
       val responseAsString       = responseToString(response)
       assert(response.status == StatusCodes.OK, responseAsString)
@@ -413,7 +410,7 @@ class ResourcesRouteV2E2ESpec extends E2ESpec {
     "perform a full resource request for a resource with a Text language (in the simple schema)" in {
       val iri = URLEncoder.encode("http://rdfh.ch/0001/a-thing-with-text-valuesLanguage", "UTF-8")
       val request = Get(s"$baseApiUrl/v2/resources/$iri")
-        .addHeader(new SchemaHeader(RouteUtilV2.SIMPLE_SCHEMA_NAME))
+        .addHeader(SchemaHeader.simple)
       val response: HttpResponse = singleAwaitingRequest(request)
       val responseAsString       = responseToString(response)
       assert(response.status == StatusCodes.OK, responseAsString)
@@ -833,7 +830,7 @@ class ResourcesRouteV2E2ESpec extends E2ESpec {
 
       // Request the newly created resource in the simple schema, and check that it matches the ontology.
       val resourceSimpleGetRequest = Get(s"$baseApiUrl/v2/resources/${URLEncoder.encode(resourceIri, "UTF-8")}")
-        .addHeader(new SchemaHeader(RouteUtilV2.SIMPLE_SCHEMA_NAME)) ~> addCredentials(
+        .addHeader(SchemaHeader.simple) ~> addCredentials(
         BasicHttpCredentials(anythingUserEmail, password)
       )
       val resourceSimpleGetResponse: HttpResponse = singleAwaitingRequest(resourceSimpleGetRequest)
@@ -901,7 +898,7 @@ class ResourcesRouteV2E2ESpec extends E2ESpec {
 
       // Request the newly created resource in the simple schema, and check that it matches the ontology.
       val resourceSimpleGetRequest = Get(s"$baseApiUrl/v2/resources/${URLEncoder.encode(resourceIri, "UTF-8")}")
-        .addHeader(new SchemaHeader(RouteUtilV2.SIMPLE_SCHEMA_NAME)) ~> addCredentials(
+        .addHeader(SchemaHeader.simple) ~> addCredentials(
         BasicHttpCredentials(anythingUserEmail, password)
       )
       val resourceSimpleGetResponse: HttpResponse = singleAwaitingRequest(resourceSimpleGetRequest)
@@ -1748,7 +1745,7 @@ class ResourcesRouteV2E2ESpec extends E2ESpec {
     "read the large text without its markup, and get the markup separately as pages of standoff" ignore { // depends on previous test
       // Get the resource without markup.
       val resourceGetRequest = Get(s"$baseApiUrl/v2/resources/${URLEncoder.encode(hamletResourceIri.get, "UTF-8")}")
-        .addHeader(new MarkupHeader(RouteUtilV2.MARKUP_STANDOFF)) ~> addCredentials(
+        .addHeader(MarkupHeader.standoff) ~> addCredentials(
         BasicHttpCredentials(anythingUserEmail, password)
       )
       val resourceGetResponse: HttpResponse = singleAwaitingRequest(resourceGetRequest)

diff --git a/integration/src/test/scala/org/knora/webapi/e2e/v2/SchemaHeader.scala b/integration/src/test/scala/org/knora/webapi/e2e/v2/SchemaHeader.scala
@@ -5,29 +5,28 @@
 
 package org.knora.webapi.e2e.v2
 
-import org.apache.pekko
+import org.apache.pekko.http.scaladsl.model.headers.ModeledCustomHeader
+import org.apache.pekko.http.scaladsl.model.headers.ModeledCustomHeaderCompanion
 
 import scala.util.Try
 
-import org.knora.webapi.routing.RouteUtilV2
-
-import pekko.http.scaladsl.model.headers.ModeledCustomHeader
-import pekko.http.scaladsl.model.headers.ModeledCustomHeaderCompanion
-
 /**
- * A custom Pekko HTTP header representing [[RouteUtilV2.SCHEMA_HEADER]], which a client can send to specify
- * which ontology schema should be used in an API response.
+ * A custom Pekko HTTP header "x-knora-accept-schema", which a client can send to specify which ontology schema
+ * should be used in an API response.
  *
  * The definition follows [[https://doc.pekko.io/docs/pekko-http/current/common/http-model.html#custom-headers]].
  */
-final class SchemaHeader(token: String) extends ModeledCustomHeader[SchemaHeader] {
+final class SchemaHeader private (token: String) extends ModeledCustomHeader[SchemaHeader] {
   override def renderInRequests             = true
   override def renderInResponses            = true
   override val companion: SchemaHeader.type = SchemaHeader
   override def value: String                = token
 }
 
 object SchemaHeader extends ModeledCustomHeaderCompanion[SchemaHeader] {
-  override val name: String         = RouteUtilV2.SCHEMA_HEADER
+  override val name: String         = "x-knora-accept-schema"
   override def parse(value: String) = Try(new SchemaHeader(value))
+
+  val simple: SchemaHeader  = SchemaHeader("simple")
+  val complex: SchemaHeader = SchemaHeader("complex")
 }