fix(gravsearch): Prevent duplicate results

docs/src/paradox/05-internals/design/api-v2/gravsearch.md

@@ -174,16 +174,10 @@ PREFIX knora-api: <http://api.knora.org/ontology/knora-api/simple/v2#>
     }
 ```
 
-The prequery's SELECT clause is built using the member variables defined in `AbstractPrequeryGenerator`.
-State of member variables after transformation of the input query into the prequery:
-
-- `mainResourceVariable`: `QueryVariable(page)`
-- `dependentResourceVariables`: `Set(QueryVariable(book))`
-- `dependentResourceVariablesGroupConcat`: `Set(QueryVariable(book__Concat))`
-- `valueObjectVariables`: `Set(QueryVariable(book__LinkValue), QueryVariable(seqnum))`: `?book` represents the dependent resource and `?book__LinkValue` the link value connecting `?page` and `?book`.
-- `valueObjectVariablesGroupConcat`: `Set(QueryVariable(seqnum__Concat), QueryVariable(book__LinkValue__Concat))`
-
-The resulting SELECT clause of the prequery looks as follows:
+The prequery's SELECT clause is built by
+`NonTriplestoreSpecificGravsearchToPrequeryTransformer.getSelectColumns`,
+based on the variables used in the input query's `CONSTRUCT` clause.
+The resulting SELECT clause looks as follows:
 
 ```sparql
 SELECT DISTINCT
@@ -219,6 +213,11 @@ is unbound, we concatenate an empty string. This is necessary because, in Apache
 triplestores), "If `GROUP_CONCAT` has an unbound value in the list of values to concat, the overall result is 'error'"
 (see [this Jena issue](https://issues.apache.org/jira/browse/JENA-1856)).
 
+If the input query contains a `UNION`, and a variable is bound in one branch
+of the `UNION` and not in another branch, it is possible that the prequery
+will return more than one row per main resource. To deal with this situation,
+`SearchResponderV2` merges rows that contain the same main resource IRI.
+
 ### Main Query
 
 The purpose of the main query is to get all requested information about the main resource, dependent resources, and value objects.
@@ -233,8 +232,17 @@ The classes involved in generating the main query can be found in
 
 The main query is a SPARQL CONSTRUCT query. Its generation is handled by the
 method `GravsearchMainQueryGenerator.createMainQuery`.
-It takes three arguments: `mainResourceIris: Set[IriRef], dependentResourceIris:
-Set[IriRef], valueObjectIris: Set[IRI]`. From the given Iris, statements are
+It takes three arguments:
+`mainResourceIris: Set[IriRef], dependentResourceIris: Set[IriRef], valueObjectIris: Set[IRI]`.
+
+These sets are constructed based on information about variables representing
+dependent resources and value objects in the prequery, which is provided by
+`NonTriplestoreSpecificGravsearchToPrequeryTransformer`:
+
+- `dependentResourceVariablesGroupConcat`: `Set(QueryVariable(book__Concat))`
+- `valueObjectVariablesGroupConcat`: `Set(QueryVariable(seqnum__Concat), QueryVariable(book__LinkValue__Concat))`
+
+From the given Iris, statements are
 generated that ask for complete information on *exactly* these resources and
 values. For any given resource Iri, only the values present in
 `valueObjectIris` are to be queried. This is achieved by using SPARQL's

webapi/src/main/scala/org/knora/webapi/messages/v2/responder/resourcemessages/ResourceMessagesV2.scala

@@ -824,6 +824,17 @@ case class ReadResourcesSequenceV2(resources: Seq[ReadResourceV2],
         )
     }
 
+    private def getOntologiesFromResource(resource: ReadResourceV2): Set[SmartIri] = {
+        val propertyIriOntologies: Set[SmartIri] = resource.values.keySet.map(_.getOntologyFromEntity)
+
+        val valueOntologies: Set[SmartIri] = resource.values.values.flatten.collect {
+            case readLinkValueV2: ReadLinkValueV2 =>
+                readLinkValueV2.valueContent.nestedResource.map(nested => getOntologiesFromResource(nested))
+        }.flatten.flatten.toSet
+
+        propertyIriOntologies ++ valueOntologies + resource.resourceClassIri.getOntologyFromEntity
+    }
+
     // #generateJsonLD
     private def generateJsonLD(targetSchema: ApiV2Schema, settings: SettingsImpl, schemaOptions: Set[SchemaOption]): JsonLDDocument = {
         // #generateJsonLD
@@ -843,14 +854,7 @@ case class ReadResourcesSequenceV2(resources: Seq[ReadResourceV2],
         // Make JSON-LD prefixes for the project-specific ontologies used in the response.
 
         val projectSpecificOntologiesUsed: Set[SmartIri] = resources.flatMap {
-            resource =>
-                val resourceOntology = resource.resourceClassIri.getOntologyFromEntity
-
-                val propertyOntologies = resource.values.keySet.map {
-                    property => property.getOntologyFromEntity
-                }
-
-                propertyOntologies + resourceOntology
+            resource => getOntologiesFromResource(resource)
         }.toSet.filter(!_.isKnoraBuiltInDefinitionIri)
 
         // Make the knora-api prefix for the target schema.

webapi/src/main/scala/org/knora/webapi/responders/v1/ValuesResponderV1.scala

@@ -744,7 +744,7 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde
             // If we're updating a link, findResourceWithValueResult will contain the IRI of the property that points to the
             // knora-base:LinkValue, but we'll need the IRI of the corresponding link property.
             val propertyIri = changeValueRequest.value match {
-                case linkUpdateV1: LinkUpdateV1 => stringFormatter.linkValuePropertyIri2LinkPropertyIri(findResourceWithValueResult.propertyIri)
+                case linkUpdateV1: LinkUpdateV1 => stringFormatter.linkValuePropertyIriToLinkPropertyIri(findResourceWithValueResult.propertyIri)
                 case _ => findResourceWithValueResult.propertyIri
             }
 
@@ -1075,7 +1075,7 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde
                         case (p, o) => p == OntologyConstants.KnoraBase.HasPermissions
                     }.map(_._2).getOrElse(throw InconsistentTriplestoreDataException(s"Value ${deleteValueRequest.valueIri} has no permissions"))
 
-                    val linkPropertyIri = stringFormatter.linkValuePropertyIri2LinkPropertyIri(findResourceWithValueResult.propertyIri)
+                    val linkPropertyIri = stringFormatter.linkValuePropertyIriToLinkPropertyIri(findResourceWithValueResult.propertyIri)
 
                     for {
                         // Get project info

webapi/src/main/scala/org/knora/webapi/responders/v2/ResourcesResponderV2.scala

@@ -1210,6 +1210,7 @@ class ResourcesResponderV2(responderData: ResponderData) extends ResponderWithSt
             apiResponse: ReadResourcesSequenceV2 <- ConstructResponseUtilV2.createApiResponse(
                 mainResourcesAndValueRdfData = mainResourcesAndValueRdfData,
                 orderByResourceIri = resourceIrisDistinct,
+                pageSizeBeforeFiltering = resourceIris.size, // doesn't matter because we're not doing paging
                 mappings = mappingsAsMap,
                 queryStandoff = queryStandoff,
                 versionDate = versionDate,
@@ -1264,6 +1265,7 @@ class ResourcesResponderV2(responderData: ResponderData) extends ResponderWithSt
             apiResponse: ReadResourcesSequenceV2 <- ConstructResponseUtilV2.createApiResponse(
                 mainResourcesAndValueRdfData = mainResourcesAndValueRdfData,
                 orderByResourceIri = resourceIrisDistinct,
+                pageSizeBeforeFiltering = resourceIris.size, // doesn't matter because we're not doing paging
                 mappings = Map.empty[IRI, MappingAndXSLTransformation],
                 queryStandoff = false,
                 versionDate = None,

webapi/src/main/scala/org/knora/webapi/responders/v2/SearchResponderV2.scala

@@ -146,13 +146,13 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
             // _ = println(searchSparql)
 
             prequeryResponseNotMerged: SparqlSelectResponse <- (storeManager ? SparqlSelectRequest(searchSparql)).mapTo[SparqlSelectResponse]
+            // _ = println(prequeryResponseNotMerged)
 
             mainResourceVar = QueryVariable("resource")
 
+            // Merge rows with the same resource IRI.
             prequeryResponse = mergePrequeryResults(prequeryResponseNotMerged, mainResourceVar)
 
-            // _ = println(prequeryResponse)
-
             // a sequence of resource IRIs that match the search criteria
             // attention: no permission checking has been done so far
             resourceIris: Seq[IRI] = prequeryResponse.results.bindings.map {
@@ -240,6 +240,7 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
             apiResponse: ReadResourcesSequenceV2 <- ConstructResponseUtilV2.createApiResponse(
                 mainResourcesAndValueRdfData = mainResourcesAndValueRdfData,
                 orderByResourceIri = resourceIris,
+                pageSizeBeforeFiltering = resourceIris.size,
                 mappings = mappingsAsMap,
                 queryStandoff = queryStandoff,
                 calculateMayHaveMoreResults = true,
@@ -281,7 +282,8 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
 
             // Create a Select prequery
 
-            nonTriplestoreSpecificConstructToSelectTransformer: NonTriplestoreSpecificGravsearchToCountPrequeryGenerator = new NonTriplestoreSpecificGravsearchToCountPrequeryGenerator(
+            nonTriplestoreSpecificConstructToSelectTransformer: NonTriplestoreSpecificGravsearchToCountPrequeryTransformer = new NonTriplestoreSpecificGravsearchToCountPrequeryTransformer(
+                constructClause = inputQuery.constructClause,
                 typeInspectionResult = typeInspectionResult,
                 querySchema = inputQuery.querySchema.getOrElse(throw AssertionException(s"WhereClause has no querySchema"))
             )
@@ -339,7 +341,6 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
                              targetSchema: ApiV2Schema,
                              schemaOptions: Set[SchemaOption],
                              requestingUser: UserADM): Future[ReadResourcesSequenceV2] = {
-        import org.knora.webapi.responders.v2.search.MainQueryResultProcessor
         import org.knora.webapi.responders.v2.search.gravsearch.mainquery.GravsearchMainQueryGenerator
 
         for {
@@ -356,7 +357,8 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
 
             // Create a Select prequery
 
-            nonTriplestoreSpecificConstructToSelectTransformer: NonTriplestoreSpecificGravsearchToPrequeryGenerator = new NonTriplestoreSpecificGravsearchToPrequeryGenerator(
+            nonTriplestoreSpecificConstructToSelectTransformer: NonTriplestoreSpecificGravsearchToPrequeryTransformer = new NonTriplestoreSpecificGravsearchToPrequeryTransformer(
+                constructClause = inputQuery.constructClause,
                 typeInspectionResult = typeInspectionResult,
                 querySchema = inputQuery.querySchema.getOrElse(throw AssertionException(s"WhereClause has no querySchema")),
                 settings = settings
@@ -366,11 +368,14 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
             // TODO: if the ORDER BY criterion is a property whose occurrence is not 1, then the logic does not work correctly
             // TODO: the ORDER BY criterion has to be included in a GROUP BY statement, returning more than one row if property occurs more than once
 
-            nonTriplestoreSpecficPrequery: SelectQuery = QueryTraverser.transformConstructToSelect(
+            nonTriplestoreSpecificPrequery: SelectQuery = QueryTraverser.transformConstructToSelect(
                 inputQuery = inputQuery.copy(whereClause = whereClauseWithoutAnnotations),
                 transformer = nonTriplestoreSpecificConstructToSelectTransformer
             )
 
+            // variable representing the main resources
+            mainResourceVar: QueryVariable = nonTriplestoreSpecificConstructToSelectTransformer.mainResourceVariable
+
             // Convert the non-triplestore-specific query to a triplestore-specific one.
             triplestoreSpecificQueryPatternTransformerSelect: SelectToSelectTransformer = {
                 if (settings.triplestoreType.startsWith("graphdb")) {
@@ -384,17 +389,16 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
 
             // Convert the preprocessed query to a non-triplestore-specific query.
             triplestoreSpecificPrequery = QueryTraverser.transformSelectToSelect(
-                inputQuery = nonTriplestoreSpecficPrequery,
+                inputQuery = nonTriplestoreSpecificPrequery,
                 transformer = triplestoreSpecificQueryPatternTransformerSelect
             )
 
             // _ = println(triplestoreSpecificPrequery.toSparql)
 
             prequeryResponseNotMerged: SparqlSelectResponse <- (storeManager ? SparqlSelectRequest(triplestoreSpecificPrequery.toSparql)).mapTo[SparqlSelectResponse]
+            pageSizeBeforeFiltering: Int = prequeryResponseNotMerged.results.bindings.size
 
-            // variable representing the main resources
-            mainResourceVar: QueryVariable = nonTriplestoreSpecificConstructToSelectTransformer.getMainResourceVariable
-
+            // Merge rows with the same main resource IRI. This could happen if there are unbound variables in a UNION.
             prequeryResponse = mergePrequeryResults(prequeryResponseNotMerged = prequeryResponseNotMerged, mainResourceVar = mainResourceVar)
 
             // a sequence of resource IRIs that match the search criteria
@@ -404,15 +408,21 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
                     resultRow.rowMap(mainResourceVar.variableName)
             }
 
-            queryResultsSeparatedWithFullGraphPattern: ConstructResponseUtilV2.MainResourcesAndValueRdfData <- if (mainResourceIris.nonEmpty) {
+            mainQueryResults: ConstructResponseUtilV2.MainResourcesAndValueRdfData <- if (mainResourceIris.nonEmpty) {
                 // at least one resource matched the prequery
 
                 // get all the IRIs for variables representing dependent resources per main resource
-                val dependentResourceIrisPerMainResource: MainQueryResultProcessor.DependentResourcesPerMainResource = MainQueryResultProcessor.getDependentResourceIrisPerMainResource(prequeryResponse, nonTriplestoreSpecificConstructToSelectTransformer, mainResourceVar)
+                val dependentResourceIrisPerMainResource: GravsearchMainQueryGenerator.DependentResourcesPerMainResource =
+                    GravsearchMainQueryGenerator.getDependentResourceIrisPerMainResource(
+                        prequeryResponse = prequeryResponse,
+                        transformer = nonTriplestoreSpecificConstructToSelectTransformer,
+                        mainResourceVar = mainResourceVar
+                    )
 
                 // collect all variables representing resources
                 val allResourceVariablesFromTypeInspection: Set[QueryVariable] = typeInspectionResult.entities.collect {
-                    case (queryVar: TypeableVariable, nonPropTypeInfo: NonPropertyTypeInfo) if OntologyConstants.KnoraApi.isKnoraApiV2Resource(nonPropTypeInfo.typeIri) => QueryVariable(queryVar.variableName)
+                    case (queryVar: TypeableVariable, nonPropTypeInfo: NonPropertyTypeInfo) if OntologyConstants.KnoraApi.isKnoraApiV2Resource(nonPropTypeInfo.typeIri) =>
+                        QueryVariable(queryVar.variableName)
                 }.toSet
 
                 // the user may have defined IRIs of dependent resources in the input query (type annotations)
@@ -426,7 +436,11 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
                 val allDependentResourceIris: Set[IRI] = dependentResourceIrisPerMainResource.dependentResourcesPerMainResource.values.flatten.toSet ++ dependentResourceIrisFromTypeInspection
 
                 // for each main resource, create a Map of value object variables and their Iris
-                val valueObjectVarsAndIrisPerMainResource: MainQueryResultProcessor.ValueObjectVariablesAndValueObjectIris = MainQueryResultProcessor.getValueObjectVarsAndIrisPerMainResource(prequeryResponse, nonTriplestoreSpecificConstructToSelectTransformer, mainResourceVar)
+                val valueObjectVarsAndIrisPerMainResource: GravsearchMainQueryGenerator.ValueObjectVariablesAndValueObjectIris = GravsearchMainQueryGenerator.getValueObjectVarsAndIrisPerMainResource(
+                    prequeryResponse = prequeryResponse,
+                    transformer = nonTriplestoreSpecificConstructToSelectTransformer,
+                    mainResourceVar = mainResourceVar
+                )
 
                 // collect all value objects IRIs (for all main resources and for all value object variables)
                 val allValueObjectIris: Set[IRI] = valueObjectVarsAndIrisPerMainResource.valueObjectVariablesAndValueObjectIris.values.foldLeft(Set.empty[IRI]) {
@@ -485,7 +499,6 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
                         typeInspectionResult,
                         inputQuery
                     )
-
                 } yield queryResultsFilteredForPermissions.copy(
                     resources = queryResWithFullGraphPatternOnlyRequestedValues
                 )
@@ -501,14 +514,15 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
 
             // If we're querying standoff, get XML-to standoff mappings.
             mappingsAsMap: Map[IRI, MappingAndXSLTransformation] <- if (queryStandoff) {
-                getMappingsFromQueryResultsSeparated(queryResultsSeparatedWithFullGraphPattern.resources, requestingUser)
+                getMappingsFromQueryResultsSeparated(mainQueryResults.resources, requestingUser)
             } else {
                 FastFuture.successful(Map.empty[IRI, MappingAndXSLTransformation])
             }
 
             apiResponse: ReadResourcesSequenceV2 <- ConstructResponseUtilV2.createApiResponse(
-                mainResourcesAndValueRdfData = queryResultsSeparatedWithFullGraphPattern,
+                mainResourcesAndValueRdfData = mainQueryResults,
                 orderByResourceIri = mainResourceIris,
+                pageSizeBeforeFiltering = pageSizeBeforeFiltering,
                 mappings = mappingsAsMap,
                 queryStandoff = queryStandoff,
                 versionDate = None,
@@ -642,6 +656,7 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
                     readResourcesSequence: ReadResourcesSequenceV2 <- ConstructResponseUtilV2.createApiResponse(
                         mainResourcesAndValueRdfData = mainResourcesAndValueRdfData,
                         orderByResourceIri = mainResourceIris,
+                        pageSizeBeforeFiltering = mainResourceIris.size,
                         mappings = mappings,
                         queryStandoff = maybeStandoffMinStartIndex.nonEmpty,
                         versionDate = None,
@@ -765,6 +780,7 @@ class SearchResponderV2(responderData: ResponderData) extends ResponderWithStand
             apiResponse: ReadResourcesSequenceV2 <- ConstructResponseUtilV2.createApiResponse(
                 mainResourcesAndValueRdfData = mainResourcesAndValueRdfData,
                 orderByResourceIri = mainResourceIris.toSeq.sorted,
+                pageSizeBeforeFiltering = mainResourceIris.size,
                 queryStandoff = false,
                 versionDate = None,
                 calculateMayHaveMoreResults = true,

webapi/src/main/scala/org/knora/webapi/responders/v2/StandoffResponderV2.scala

@@ -106,6 +106,7 @@ class StandoffResponderV2(responderData: ResponderData) extends Responder(respon
             readResourcesSequenceV2: ReadResourcesSequenceV2 <- ConstructResponseUtilV2.createApiResponse(
                 mainResourcesAndValueRdfData = mainResourcesAndValueRdfData,
                 orderByResourceIri = Seq(getStandoffRequestV2.resourceIri),
+                pageSizeBeforeFiltering = 1, // doesn't matter because we're not doing paging
                 mappings = Map.empty,
                 queryStandoff = false,
                 calculateMayHaveMoreResults = false,