GH-1135 Add experimental test mode and fix incomplete 'GROUP BY' quer…

…ies in PostgreSQL scenarios (Fix #1135)

reposilite-backend/build.gradle.kts

@@ -132,6 +132,7 @@ dependencies {
     testImplementation("com.konghq:unirest-objectmapper-jackson:$unirest")
 
     val testcontainers = "1.16.3"
+    testImplementation("org.testcontainers:postgresql:$testcontainers")
     testImplementation("org.testcontainers:mariadb:$testcontainers")
     testImplementation("org.testcontainers:testcontainers:$testcontainers")
     testImplementation("org.testcontainers:junit-jupiter:$testcontainers")

reposilite-backend/src/integration/kotlin/com/reposilite/ReposiliteExperimentalIntegrationJunitExtension.kt

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2022 dzikoysk
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.reposilite
+
+import org.junit.jupiter.api.extension.AfterEachCallback
+import org.junit.jupiter.api.extension.BeforeEachCallback
+import org.junit.jupiter.api.extension.Extension
+import org.junit.jupiter.api.extension.ExtensionContext
+import org.testcontainers.containers.PostgreSQLContainer
+import org.testcontainers.junit.jupiter.Container
+import org.testcontainers.junit.jupiter.Testcontainers
+import org.testcontainers.utility.DockerImageName
+
+/**
+ * Integrations used in remote stack:
+ * - PostreSQL
+ * - Local file system
+ */
+@Testcontainers
+internal class ReposiliteExperimentalIntegrationJunitExtension : Extension, BeforeEachCallback, AfterEachCallback {
+
+    private class SpecifiedPostgreSQLContainer(image: String) : PostgreSQLContainer<SpecifiedPostgreSQLContainer>(DockerImageName.parse(image))
+
+    @Container
+    private val postgres = SpecifiedPostgreSQLContainer("postgres:13.6")
+
+    override fun beforeEach(context: ExtensionContext?) {
+        postgres.start()
+
+        context?.also {
+            val instance = it.requiredTestInstance
+            val type = instance::class.java
+
+            type.getField("_extensionInitialized").set(instance, true)
+            type.getField("_database").set(instance, "postgresql ${postgres.host}:${postgres.getMappedPort(5432)} ${postgres.databaseName} ${postgres.username} ${postgres.password}")
+            type.getField("_storageProvider").set(instance, "fs")
+        }
+    }
+
+    override fun afterEach(context: ExtensionContext) {
+        postgres.stop()
+    }
+
+}

reposilite-backend/src/integration/kotlin/com/reposilite/ReposiliteSpecification.kt

@@ -21,6 +21,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
 import com.fasterxml.jackson.module.kotlin.registerKotlinModule
 import com.reposilite.plugin.api.Facade
 import com.reposilite.token.AccessTokenFacade
+import com.reposilite.token.AccessTokenPermission
 import com.reposilite.token.AccessTokenType.PERSISTENT
 import com.reposilite.token.Route
 import com.reposilite.token.RoutePermission
@@ -48,10 +49,14 @@ internal abstract class ReposiliteSpecification : ReposiliteRunner() {
     fun usePredefinedTemporaryAuth(): Pair<String, String> =
         Pair("manager", "manager-secret")
 
-    fun useAuth(name: String, secret: String, routes: Map<String, RoutePermission> = emptyMap()): Pair<String, String> {
+    fun useAuth(name: String, secret: String, permissions: List<AccessTokenPermission> = emptyList(), routes: Map<String, RoutePermission> = emptyMap()): Pair<String, String> {
         val accessTokenFacade = useFacade<AccessTokenFacade>()
         val accessToken = accessTokenFacade.createAccessToken(CreateAccessTokenRequest(PERSISTENT, name, secret)).accessToken
 
+        permissions.forEach {
+            accessTokenFacade.addPermission(accessToken.identifier, it)
+        }
+
         routes.forEach { (route, permission) ->
             accessTokenFacade.addRoute(accessToken.identifier, Route(route, permission))
         }

reposilite-backend/src/integration/kotlin/com/reposilite/statistics/StatisticsIntegrationTest.kt

@@ -20,6 +20,7 @@ package com.reposilite.statistics
 
 import com.reposilite.statistics.api.ResolvedCountResponse
 import com.reposilite.statistics.specification.StatisticsIntegrationSpecification
+import com.reposilite.token.AccessTokenPermission.MANAGER
 import com.reposilite.token.RoutePermission.READ
 import io.javalin.http.HttpCode.UNAUTHORIZED
 import kong.unirest.Unirest.get
@@ -31,10 +32,10 @@ import panda.std.component1
 internal abstract class StatisticsIntegrationTest : StatisticsIntegrationSpecification() {
 
     @Test
-    fun `should return registered amount of endpoint calls`() = runBlocking {
+    fun `should return registered number of endpoint calls`() = runBlocking {
         // given: a route to request and check
         val (identifier) = useResolvedRequest("releases", "com/reposilite.jar", "content")
-        val endpoint = "$base/api/statistics/resolved/1$identifier"
+        val endpoint = "$base/api/statistics/resolved/phrase/1$identifier"
 
         // when: stats service is requested without valid credentials
         val unauthorizedResponse = get(endpoint).asString()
@@ -43,7 +44,7 @@ internal abstract class StatisticsIntegrationTest : StatisticsIntegrationSpecifi
         assertEquals(UNAUTHORIZED.status, unauthorizedResponse.status)
 
         // given: a valid credentials
-        val (name, secret) = useAuth("name", "secret", mapOf(identifier.toString() to READ))
+        val (name, secret) = useAuth("name", "secret", emptyList(),  mapOf(identifier.toString() to READ))
 
         // when: service is requested with valid credentials
         val response = get(endpoint)
@@ -56,4 +57,29 @@ internal abstract class StatisticsIntegrationTest : StatisticsIntegrationSpecifi
         assertEquals(identifier.gav, response.body.requests[0].gav)
     }
 
+    @Test
+    fun `should return unique number of requests`() {
+        // given: a route to request and check
+        val endpoint = "$base/api/statistics/resolved/unique"
+        repeat(10) { useResolvedRequest("releases", "com/reposilite.jar", "content") }
+
+        // when: stats service is requested without valid credentials
+        val unauthorizedResponse = get(endpoint).asString()
+
+        // then: service rejects request
+        assertEquals(UNAUTHORIZED.status, unauthorizedResponse.status)
+
+        // given: a valid credentials
+        val (name, secret) = useAuth("name", "secret", listOf(MANAGER))
+
+        // when: service is requested with valid credentials
+        val response = get(endpoint)
+            .basicAuth(name, secret)
+            .asObject { it.contentAsString.toLong() }
+
+        // then: service responds with valid stats data
+        assertEquals(200, response.status)
+        assertEquals(1, response.body)
+    }
+
 }

reposilite-backend/src/integration/kotlin/com/reposilite/statistics/infrastructure/ExperimentalStatisticsIntegrationTest.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2022 dzikoysk
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.reposilite.statistics.infrastructure
+
+import com.reposilite.ReposiliteExperimentalIntegrationJunitExtension
+import com.reposilite.statistics.StatisticsIntegrationTest
+import org.junit.jupiter.api.extension.ExtendWith
+
+@ExtendWith(ReposiliteExperimentalIntegrationJunitExtension::class)
+internal class ExperimentalStatisticsIntegrationTest : StatisticsIntegrationTest()

reposilite-backend/src/main/kotlin/com/reposilite/statistics/infrastructure/SqlStatisticsRepository.kt

@@ -118,7 +118,7 @@ internal class SqlStatisticsRepository(private val database: Database) : Statist
             IdentifierTable.leftJoin(ResolvedTable, { IdentifierTable.id }, { ResolvedTable.identifierId })
                 .slice(IdentifierTable.gav, resolvedSum)
                 .select(whereCriteria)
-                .groupBy(ResolvedTable.id)
+                .groupBy(ResolvedTable.id, IdentifierTable.gav)
                 .having { resolvedSum greater 0L }
                 .orderBy(resolvedSum, DESC)
                 .limit(limit)
@@ -129,14 +129,16 @@ internal class SqlStatisticsRepository(private val database: Database) : Statist
     override fun countUniqueResolvedRequests(): Long =
         transaction(database) {
             ResolvedTable.selectAll()
-                .groupBy(ResolvedTable.identifierId)
+                .groupBy(ResolvedTable.id, ResolvedTable.identifierId)
                 .count()
         }
 
     override fun countResolvedRequests(): Long =
         transaction(database) {
             with (ResolvedTable.count.sum()) {
-                ResolvedTable.slice(this).selectAll().firstAndMap { it[this] }
+                ResolvedTable.slice(this)
+                    .selectAll()
+                    .firstAndMap { it[this] }
             }
             ?: 0
         }

reposilite-backend/src/main/kotlin/com/reposilite/statistics/infrastructure/StatisticsEndpoint.kt

@@ -21,29 +21,52 @@ import com.reposilite.statistics.StatisticsFacade
 import com.reposilite.statistics.api.ResolvedCountResponse
 import com.reposilite.web.api.ReposiliteRoute
 import com.reposilite.web.api.ReposiliteRoutes
+import com.reposilite.web.http.ErrorResponse
 import com.reposilite.web.routing.RouteMethod.GET
 import io.javalin.openapi.HttpMethod
 import io.javalin.openapi.OpenApi
+import io.javalin.openapi.OpenApiContent
 import io.javalin.openapi.OpenApiParam
+import io.javalin.openapi.OpenApiResponse
+import panda.std.asSuccess
 
 internal class StatisticsEndpoint(private val statisticsFacade: StatisticsFacade) : ReposiliteRoutes() {
 
     @OpenApi(
-        tags = ["Maven"],
-        path = "/api/statistics/resolved/{limit}/{repository}/*",
+        tags = ["Statistics"],
+        path = "/api/statistics/resolved/phrase/{limit}/{repository}/*",
         methods = [HttpMethod.GET],
         pathParams = [
             OpenApiParam(name = "limit", description = "Amount of records to find (Maximum: $MAX_PAGE_SIZE", required = true),
             OpenApiParam(name = "repository", description = "Repository to search in", required = true),
             OpenApiParam(name = "*", description = "Phrase to search for", required = true, allowEmptyValue = true)
         ],
+        responses = [
+            OpenApiResponse("200", content = [ OpenApiContent(from = ResolvedCountResponse::class) ], description = "Aggregated sum of resolved requests with list a list of them all"),
+            OpenApiResponse("401", content = [ OpenApiContent(from = ErrorResponse::class) ], description = "When invalid token is used")
+        ]
     )
-    val findCount = ReposiliteRoute<ResolvedCountResponse>("/api/statistics/resolved/{limit}/{repository}/<gav>", GET) {
+    val findCountByPhrase = ReposiliteRoute<ResolvedCountResponse>("/api/statistics/resolved/phrase/{limit}/{repository}/<gav>", GET) {
         authorized("/${requireParameter("repository")}/${requireParameter("gav")}") {
             response = statisticsFacade.findResolvedRequestsByPhrase(requireParameter("repository"), requireParameter("gav"), 1)
         }
     }
 
-    override val routes = routes(findCount)
+    @OpenApi(
+        tags = ["Statistics"],
+        path = "/api/statistics/resolved/unique",
+        methods = [HttpMethod.GET],
+        responses = [
+            OpenApiResponse("200", content = [ OpenApiContent(from = Long::class) ], description = "Number of all unique requests"),
+            OpenApiResponse("401", content = [ OpenApiContent(from = ErrorResponse::class) ], description = "When non-manager token is used")
+        ]
+    )
+    val findUniqueCount = ReposiliteRoute<Long>("/api/statistics/resolved/unique", GET) {
+        managerOnly {
+            response = statisticsFacade.countUniqueRecords().asSuccess()
+        }
+    }
+
+    override val routes = routes(findCountByPhrase, findUniqueCount)
 
 }

reposilite-backend/src/main/kotlin/com/reposilite/token/infrastructure/AccessTokenApiEndpoints.kt

@@ -25,10 +25,8 @@ import com.reposilite.token.api.CreateAccessTokenResponse
 import com.reposilite.token.api.CreateAccessTokenWithNoNameRequest
 import com.reposilite.web.api.ReposiliteRoute
 import com.reposilite.web.api.ReposiliteRoutes
-import com.reposilite.web.http.notFound
+import com.reposilite.web.http.ErrorResponse
 import com.reposilite.web.http.notFoundError
-import com.reposilite.web.http.unauthorized
-import com.reposilite.web.http.unauthorizedError
 import com.reposilite.web.http.errorResponse
 import com.reposilite.web.routing.RouteMethod.DELETE
 import com.reposilite.web.routing.RouteMethod.GET
@@ -86,17 +84,18 @@ internal class AccessTokenApiEndpoints(private val accessTokenFacade: AccessToke
     )
     val createOrUpdateToken = ReposiliteRoute<CreateAccessTokenResponse>("/api/tokens/{name}", PUT) {
         managerOnly {
-            response = runCatching { ctx.bodyAsClass<CreateAccessTokenWithNoNameRequest>() }.fold(
-                onSuccess = { request ->
-                    accessTokenFacade.createAccessToken(CreateAccessTokenRequest(request.type, requireParameter("name"), request.secret))
-                        .also { (token) -> request.permissions
-                            .mapNotNull { AccessTokenPermission.findByAny(it) }
-                            .forEach { accessTokenFacade.addPermission(token.identifier, it) }
-                        }
-                        .asSuccess()
-                },
-                onFailure = { errorResponse(HttpCode.BAD_REQUEST, "Failed to read body") }
-            )
+            response = panda.std.Result.attempt { ctx.bodyAsClass<CreateAccessTokenWithNoNameRequest>() }
+                .mapErr { ErrorResponse(HttpCode.BAD_REQUEST, "Failed to read body") }
+                .map { request ->
+                    Pair(
+                        accessTokenFacade.createAccessToken(CreateAccessTokenRequest(request.type, requireParameter("name"), request.secret)),
+                        request.permissions.mapNotNull { AccessTokenPermission.findByAny(it) }
+                    )
+                }
+                .peek { (token, permissions) ->
+                    permissions.forEach { accessTokenFacade.addPermission(token.accessToken.identifier, it) }
+                }
+                .map { (response, _) -> response}
         }
     }