feat: Add generating OpenApi yamls for the admin api (DEV-1812)

.gitignore

@@ -52,3 +52,7 @@ dependencies.txt
 # exclude python virtual environments used for serving docs locally
 env/
 venv/
+
+# exclude generated openapi
+docs/03-endpoints/generated-openapi/*.yml
+

integration/src/test/scala/org/knora/webapi/core/LayersTest.scala

@@ -49,6 +49,7 @@ import org.knora.webapi.slice.resourceinfo.ResourceInfoLayers
 import org.knora.webapi.slice.resourceinfo.api.service.RestResourceInfoService
 import org.knora.webapi.slice.resourceinfo.domain.IriConverter
 import org.knora.webapi.slice.search.api.SearchApiRoutes
+import org.knora.webapi.slice.search.api.SearchEndpoints
 import org.knora.webapi.store.cache.CacheServiceRequestMessageHandler
 import org.knora.webapi.store.cache.CacheServiceRequestMessageHandlerLive
 import org.knora.webapi.store.cache.api.CacheService
@@ -81,6 +82,8 @@ object LayersTest {
   type CommonR0 = ActorSystem with AppConfigurations with JwtService with SipiService with StringFormatter
   type CommonR =
     ApiRoutes
+      with ApiV2Endpoints
+      with AdminApiEndpoints
       with AppRouter
       with Authenticator
       with AuthorizationRestService
@@ -138,6 +141,8 @@ object LayersTest {
   private val commonLayersForAllIntegrationTests =
     ZLayer.makeSome[CommonR0, CommonR](
       AdminApiRoutes.layer,
+      AdminApiEndpoints.layer,
+      ApiV2Endpoints.layer,
       ApiRoutes.layer,
       AppRouter.layer,
       AuthenticatorLive.layer,
@@ -194,6 +199,7 @@ object LayersTest {
       ResourcesResponderV2Live.layer,
       RestCardinalityServiceLive.layer,
       SearchApiRoutes.layer,
+      SearchEndpoints.layer,
       SearchResponderV2Live.layer,
       SipiResponderADMLive.layer,
       StandoffResponderV2Live.layer,

justfile

@@ -0,0 +1,12 @@
+openapiDir := "./docs/03-endpoints/generated-openapi"
+
+# List all recipies
+default:
+    @just --list
+
+# Update the OpenApi yml files by generating these from the tAPIr specs
+alias dog := docs-openapi-generate
+docs-openapi-generate:
+    mkdir -p {{openapiDir}}
+    rm {{openapiDir}}/*.yml >> /dev/null 2>&1 || true
+    sbt "webapi/runMain org.knora.webapi.slice.common.api.DocsGenerator {{openapiDir}}"

webapi/src/main/scala/org/knora/webapi/config/AppConfig.scala

@@ -203,7 +203,7 @@ final case class InstrumentationServerConfig(
 )
 
 object AppConfig {
-  type AppConfigurations = AppConfig & JwtConfig & DspIngestConfig & Triplestore
+  type AppConfigurations = AppConfig & JwtConfig & DspIngestConfig & InstrumentationServerConfig & Triplestore
 
   val descriptor: Config[AppConfig] = deriveConfig[AppConfig].mapKey(toKebabCase)
 
@@ -216,7 +216,10 @@ object AppConfig {
   }
 
   def projectAppConfigurations[R](appConfigLayer: URLayer[R, AppConfig]): URLayer[R, AppConfigurations] =
-    appConfigLayer ++ appConfigLayer.project(_.dspIngest) ++ appConfigLayer.project(_.triplestore) ++
+    appConfigLayer ++
+      appConfigLayer.project(_.dspIngest) ++
+      appConfigLayer.project(_.triplestore) ++
+      appConfigLayer.project(_.instrumentationServerConfig) ++
       appConfigLayer.project { appConfig =>
         val jwtConfig = appConfig.jwt
         val issuerFromConfigOrDefault: Option[String] =

webapi/src/main/scala/org/knora/webapi/core/LayersLive.scala

@@ -10,6 +10,7 @@ import zio.ZLayer
 
 import org.knora.webapi.config.AppConfig
 import org.knora.webapi.config.AppConfig.AppConfigurations
+import org.knora.webapi.config.InstrumentationServerConfig
 import org.knora.webapi.messages.StringFormatter
 import org.knora.webapi.messages.util.*
 import org.knora.webapi.messages.util.search.QueryTraverser
@@ -50,6 +51,7 @@ import org.knora.webapi.slice.resourceinfo.api.service.RestResourceInfoService
 import org.knora.webapi.slice.resourceinfo.api.service.RestResourceInfoServiceLive
 import org.knora.webapi.slice.resourceinfo.domain.IriConverter
 import org.knora.webapi.slice.search.api.SearchApiRoutes
+import org.knora.webapi.slice.search.api.SearchEndpoints
 import org.knora.webapi.store.cache.CacheServiceRequestMessageHandler
 import org.knora.webapi.store.cache.CacheServiceRequestMessageHandlerLive
 import org.knora.webapi.store.cache.api.CacheService
@@ -68,17 +70,18 @@ object LayersLive {
    * The `Environment` that we require to exist at startup.
    */
   type DspEnvironmentLive =
-    ActorSystem & ApiRoutes & AppConfigurations & AppRouter & Authenticator & CacheService &
-      CacheServiceRequestMessageHandler & CardinalityHandler & CardinalityService & ConstructResponseUtilV2 &
-      ConstructTransformer & GravsearchTypeInspectionRunner & GroupsResponderADM & HttpServer &
-      IIIFRequestMessageHandler & InferenceOptimizationService & IriConverter & IriService & JwtService & SipiService &
-      KnoraProjectRepo & ListsResponderADM & ListsResponderV2 & MessageRelay & OntologyCache & OntologyHelpers &
-      OntologyRepo & OntologyResponderV2 & PermissionsResponderADM & PermissionsRestService & PermissionUtilADM & PredicateObjectMapper &
-      ProjectADMRestService & ProjectADMService & ProjectExportService & ProjectExportStorageService &
-      ProjectImportService & ProjectsResponderADM & QueryTraverser & RepositoryUpdater & ResourceUtilV2 &
-      AuthorizationRestService & ResourcesResponderV2 & ResourceUtilV2 & RestCardinalityService & RestResourceInfoService &
-      OntologyInferencer & SearchApiRoutes & SearchResponderV2 & SipiResponderADM & StandoffResponderV2 & StandoffTagUtilV2 & State &
-      StoresResponderADM & StringFormatter & TriplestoreService & UsersResponderADM & ValuesResponderV2
+    ActorSystem & AdminApiEndpoints & ApiRoutes & ApiV2Endpoints & AppConfigurations & AppRouter & Authenticator &
+      AuthorizationRestService & CacheService & CacheServiceRequestMessageHandler & CardinalityHandler &
+      CardinalityService & ConstructResponseUtilV2 & ConstructTransformer & GravsearchTypeInspectionRunner &
+      GroupsResponderADM & HttpServer & IIIFRequestMessageHandler & InferenceOptimizationService &
+      InstrumentationServerConfig & IriConverter & IriService & JwtService & KnoraProjectRepo & ListsResponderADM &
+      ListsResponderV2 & MessageRelay & OntologyCache & OntologyHelpers & OntologyInferencer & OntologyRepo &
+      OntologyResponderV2 & PermissionsResponderADM & PermissionsRestService & PermissionUtilADM &
+      PredicateObjectMapper & ProjectADMRestService & ProjectADMService & ProjectExportService &
+      ProjectExportStorageService & ProjectImportService & ProjectsResponderADM & QueryTraverser & RepositoryUpdater &
+      ResourcesResponderV2 & ResourceUtilV2 & ResourceUtilV2 & RestCardinalityService & RestResourceInfoService &
+      SearchApiRoutes & SearchResponderV2 & SipiResponderADM & SipiService & StandoffResponderV2 & StandoffTagUtilV2 &
+      State & StoresResponderADM & StringFormatter & TriplestoreService & UsersResponderADM & ValuesResponderV2
 
   /**
    * All effect layers needed to provide the `Environment`
@@ -87,6 +90,8 @@ object LayersLive {
     ZLayer.make[DspEnvironmentLive](
       ActorSystem.layer,
       AdminApiRoutes.layer,
+      AdminApiEndpoints.layer,
+      ApiV2Endpoints.layer,
       ApiRoutes.layer,
       AppConfig.layer,
       AppRouter.layer,
@@ -146,6 +151,7 @@ object LayersLive {
       RestCardinalityServiceLive.layer,
       RestResourceInfoServiceLive.layer,
       SearchApiRoutes.layer,
+      SearchEndpoints.layer,
       SearchResponderV2Live.layer,
       SipiResponderADMLive.layer,
       SipiServiceLive.layer,

webapi/src/main/scala/org/knora/webapi/slice/admin/api/AdminApiEndpoints.scala

@@ -0,0 +1,27 @@
+/*
+ * Copyright © 2021 - 2024 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.knora.webapi.slice.admin.api
+
+import sttp.tapir.AnyEndpoint
+import zio.ZLayer
+
+final case class AdminApiEndpoints(
+  maintenanceEndpoints: MaintenanceEndpoints,
+  permissionsEndpoints: PermissionsEndpoints,
+  projectsEndpoints: ProjectsEndpoints,
+  usersEndpoints: UsersEndpoints
+) {
+
+  val endpoints: Seq[AnyEndpoint] =
+    maintenanceEndpoints.endpoints ++
+      permissionsEndpoints.endpoints ++
+      projectsEndpoints.endpoints ++
+      usersEndpoints.endpoints
+}
+
+object AdminApiEndpoints {
+  val layer = ZLayer.derive[AdminApiEndpoints]
+}

webapi/src/main/scala/org/knora/webapi/slice/admin/api/MaintenanceEndpoints.scala

@@ -21,21 +21,21 @@ final case class MaintenanceEndpoints(baseEndpoints: BaseEndpoints) {
   val postMaintenance = baseEndpoints.securedEndpoint.post
     .in(
       maintenanceBase / path[String]
-        .name("Maintenance action name")
-        .description("""
-                       |The name of the maintenance action to be executed. 
+        .name("action-name")
+        .description("""The name of the maintenance action to be executed.
                        |Maintenance actions are executed asynchronously in the background.
                        |""".stripMargin)
         .example("fix-top-left-dimensions")
     )
     .in(
       zioJsonBody[Option[Json]]
-        .description("""
-                       |The optional parameters as json for the maintenance action. 
+        .description("""The optional parameters as json for the maintenance action.
                        |May be required by certain actions.
                        |""".stripMargin)
     )
     .out(statusCode(StatusCode.Accepted))
+
+  val endpoints: Seq[AnyEndpoint] = Seq(postMaintenance).map(_.endpoint)
 }
 
 object MaintenanceEndpoints {

webapi/src/main/scala/org/knora/webapi/slice/admin/api/PermissionsEndpoints.scala

@@ -94,6 +94,20 @@ final case class PermissionsEndpoints(base: BaseEndpoints) extends PermissionsAD
     .description("Update a permission's property")
     .in(sprayJsonBody[ChangePermissionPropertyApiRequestADM])
     .out(sprayJsonBody[PermissionGetResponseADM])
+
+  val endpoints: Seq[AnyEndpoint] = Seq(
+    postPermissionsAp,
+    getPermissionsApByProjectIri,
+    getPermissionsApByProjectAndGroupIri,
+    getPermissionsDoapByProjectIri,
+    getPermissionsByProjectIri,
+    deletePermission,
+    postPermissionsDoap,
+    putPermissionsProjectIriGroup,
+    putPerrmissionsHasPermissions,
+    putPermisssionsResourceClass,
+    putPermissionsProperty
+  ).map(_.endpoint)
 }
 
 object PermissionsEndpoints {

webapi/src/main/scala/org/knora/webapi/slice/admin/api/ProjectsEndpoints.scala

@@ -195,6 +195,35 @@ final case class ProjectsEndpoints(
       .description("Returns all ontologies, data, and configuration belonging to a project identified by the IRI.")
       .tags(tags)
   }
+
+  val endpoints: Seq[AnyEndpoint] =
+    Seq(
+      Public.getAdminProjects,
+      Public.getAdminProjectsByProjectIri,
+      Public.getAdminProjectsByProjectIriRestrictedViewSettings,
+      Public.getAdminProjectsByProjectShortcode,
+      Public.getAdminProjectsByProjectShortcodeRestrictedViewSettings,
+      Public.getAdminProjectsByProjectShortname,
+      Public.getAdminProjectsByProjectShortnameRestrictedViewSettings,
+      Public.getAdminProjectsKeywords,
+      Public.getAdminProjectsKeywordsByProjectIri
+    ) ++ Seq(
+      Secured.deleteAdminProjectsByIri,
+      Secured.getAdminProjectsByIriAllData,
+      Secured.getAdminProjectsByProjectIriAdminMembers,
+      Secured.getAdminProjectsByProjectIriMembers,
+      Secured.getAdminProjectsByProjectShortcodeAdminMembers,
+      Secured.getAdminProjectsByProjectShortcodeMembers,
+      Secured.getAdminProjectsByProjectShortnameAdminMembers,
+      Secured.getAdminProjectsByProjectShortnameMembers,
+      Secured.getAdminProjectsExports,
+      Secured.postAdminProjects,
+      Secured.postAdminProjectsByShortcodeExport,
+      Secured.postAdminProjectsByShortcodeImport,
+      Secured.putAdminProjectsByIri,
+      Secured.setAdminProjectsByProjectIriRestrictedViewSettings,
+      Secured.setAdminProjectsByProjectShortcodeRestrictedViewSettings
+    ).map(_.endpoint)
 }
 
 object ProjectsEndpoints {

webapi/src/main/scala/org/knora/webapi/slice/admin/api/UsersEndpoints.scala

@@ -26,6 +26,7 @@ final case class UsersEndpoints(baseEndpoints: BaseEndpoints) {
     .description("Returns all users.")
     .tags(tags)
 
+  val endpoints: Seq[AnyEndpoint] = Seq(getUsers).map(_.endpoint)
 }
 
 object UsersEndpoints {

webapi/src/main/scala/org/knora/webapi/slice/common/api/ApiV2Endpoints.scala

@@ -0,0 +1,23 @@
+/*
+ * Copyright © 2021 - 2024 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.knora.webapi.slice.common.api
+
+import sttp.tapir.AnyEndpoint
+import zio.ZLayer
+
+import org.knora.webapi.slice.resourceinfo.api.ResourceInfoEndpoints
+import org.knora.webapi.slice.search.api.SearchEndpoints
+
+final case class ApiV2Endpoints(resourceInfoEndpoints: ResourceInfoEndpoints, searchEndpoints: SearchEndpoints) {
+
+  val endpoints: Seq[AnyEndpoint] =
+    resourceInfoEndpoints.endpoints ++
+      searchEndpoints.endpoints
+}
+
+object ApiV2Endpoints {
+  val layer = ZLayer.derive[ApiV2Endpoints]
+}

webapi/src/main/scala/org/knora/webapi/slice/common/api/DocsGenerator.scala

@@ -0,0 +1,93 @@
+/*
+ * Copyright © 2021 - 2024 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.knora.webapi.slice.common.api
+
+import org.apache.pekko.http.scaladsl.model.HttpResponse
+import org.apache.pekko.http.scaladsl.server.RequestContext
+import sttp.apispec.openapi.Server
+import sttp.apispec.openapi.circe.yaml.RichOpenAPI
+import sttp.tapir.AnyEndpoint
+import sttp.tapir.docs.openapi.OpenAPIDocsInterpreter
+import zio.Chunk
+import zio.Task
+import zio.ZIO
+import zio.ZIOAppArgs
+import zio.ZIOAppDefault
+import zio.ZLayer
+import zio.nio.file.Files
+import zio.nio.file.Path
+
+import org.knora.webapi.http.version.BuildInfo
+import org.knora.webapi.messages.admin.responder.usersmessages.UserIdentifierADM
+import org.knora.webapi.messages.v2.routing.authenticationmessages.KnoraCredentialsV2
+import org.knora.webapi.routing.Authenticator
+import org.knora.webapi.slice.admin.api.AdminApiEndpoints
+import org.knora.webapi.slice.admin.api.MaintenanceEndpoints
+import org.knora.webapi.slice.admin.api.PermissionsEndpoints
+import org.knora.webapi.slice.admin.api.ProjectsEndpoints
+import org.knora.webapi.slice.admin.api.UsersEndpoints
+import org.knora.webapi.slice.admin.domain.model.User
+import org.knora.webapi.slice.resourceinfo.api.ResourceInfoEndpoints
+import org.knora.webapi.slice.search.api.SearchEndpoints
+
+final case class DocsNoopAuthenticator() extends Authenticator {
+  override def getUserADM(requestContext: RequestContext): Task[User]                                    = ???
+  override def calculateCookieName(): String                                                             = "KnoraAuthenticationMFYGSLTEMFZWG2BOON3WS43THI2DIMY9"
+  override def getUserADMThroughCredentialsV2(credentials: Option[KnoraCredentialsV2]): Task[User]       = ???
+  override def doLogoutV2(requestContext: RequestContext): Task[HttpResponse]                            = ???
+  override def doLoginV2(credentials: KnoraCredentialsV2.KnoraPasswordCredentialsV2): Task[HttpResponse] = ???
+  override def doAuthenticateV2(requestContext: RequestContext): Task[HttpResponse]                      = ???
+  override def presentLoginFormV2(requestContext: RequestContext): Task[HttpResponse]                    = ???
+  override def authenticateCredentialsV2(credentials: Option[KnoraCredentialsV2]): Task[Boolean]         = ???
+  override def getUserByIdentifier(identifier: UserIdentifierADM): Task[User]                            = ???
+}
+object DocsNoopAuthenticator {
+  val layer = ZLayer.succeed(DocsNoopAuthenticator())
+}
+
+object DocsGenerator extends ZIOAppDefault {
+
+  private val interp: OpenAPIDocsInterpreter = OpenAPIDocsInterpreter()
+  override def run: ZIO[ZIOAppArgs, java.io.IOException, Int] = {
+    for {
+      _              <- ZIO.logInfo("Generating OpenAPI docs")
+      args           <- getArgs
+      adminEndpoints <- ZIO.serviceWith[AdminApiEndpoints](_.endpoints)
+      v2Endpoints    <- ZIO.serviceWith[ApiV2Endpoints](_.endpoints)
+      path            = Path(args.headOption.getOrElse("/tmp"))
+      filesWritten   <- writeToFile(adminEndpoints, path, "maintenance") <*> writeToFile(v2Endpoints, path, "v2")
+      _              <- ZIO.logInfo(s"Wrote $filesWritten")
+    } yield 0
+  }.provideSome[ZIOAppArgs](
+    AdminApiEndpoints.layer,
+    ApiV2Endpoints.layer,
+    BaseEndpoints.layer,
+    DocsNoopAuthenticator.layer,
+    MaintenanceEndpoints.layer,
+    PermissionsEndpoints.layer,
+    ProjectsEndpoints.layer,
+    ResourceInfoEndpoints.layer,
+    SearchEndpoints.layer,
+    UsersEndpoints.layer
+  )
+
+  private def writeToFile(endpoints: Seq[AnyEndpoint], path: Path, name: String) = {
+    val content = interp
+      .toOpenAPI(endpoints, s"${BuildInfo.name}-$name", BuildInfo.version)
+      .servers(
+        List(
+          Server(url = "http://localhost:3333", description = Some("Local development server")),
+          Server(url = "https://api.dasch.swiss", description = Some("Production server"))
+        )
+      )
+    for {
+      _     <- ZIO.logInfo(s"Writing to $path")
+      target = path / s"openapi-$name.yml"
+      _     <- Files.deleteIfExists(target) *> Files.createFile(target)
+      _     <- Files.writeBytes(target, Chunk.fromArray(content.toYaml.getBytes))
+    } yield target
+  }
+}