From 100355886555cb3a5caf4f6245c88fe9d83bd66b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Wed, 6 Mar 2024 12:48:43 +0100 Subject: [PATCH 01/11] refactor: Remove belongsToInstitution property from knora-admin ontology, code and documentation --- docs/02-dsp-ontologies/knora-base.md | 2 -- .../design/domain/domain-entities-and-relations.md | 1 - test_data/project_data/admin-data.ttl | 1 - test_data/upgrade/pr1746.trig | 2 -- .../main/resources/knora-ontologies/knora-admin.ttl | 13 ------------- .../main/resources/knora-ontologies/knora-base.ttl | 2 +- .../knora/webapi/messages/OntologyConstants.scala | 1 - .../src/main/scala/org/knora/webapi/package.scala | 2 +- .../triplestore/upgrade/RepositoryUpdatePlan.scala | 1 + 9 files changed, 3 insertions(+), 22 deletions(-) diff --git a/docs/02-dsp-ontologies/knora-base.md b/docs/02-dsp-ontologies/knora-base.md index e06c6dd759..3eeb2a4bd6 100644 --- a/docs/02-dsp-ontologies/knora-base.md +++ b/docs/02-dsp-ontologies/knora-base.md @@ -39,8 +39,6 @@ In Knora, each item of data belongs to some particular project. Each project usi - `projectDescription` (1-n): A description of the project. -- `belongsToInstitution` (0-1): The `kb:Institution` that the project belongs to. - Ontologies and resources are associated with a project by means of the `kb:attachedToProject` property, as described in [Ontologies](#ontologies) and [Properties of Resource](#properties-of-resource)). Users are associated with a project by means of diff --git a/docs/05-internals/design/domain/domain-entities-and-relations.md b/docs/05-internals/design/domain/domain-entities-and-relations.md index 5259595968..b41dd4f290 100644 --- a/docs/05-internals/design/domain/domain-entities-and-relations.md +++ b/docs/05-internals/design/domain/domain-entities-and-relations.md @@ -101,7 +101,6 @@ erDiagram - User.phone? - Institution? (name, description, website, phone, address, email) - - Project.belongsToInstitution? ### Overview V2 diff --git a/test_data/project_data/admin-data.ttl b/test_data/project_data/admin-data.ttl index 143038c89d..23f119e798 100644 --- a/test_data/project_data/admin-data.ttl +++ b/test_data/project_data/admin-data.ttl @@ -83,7 +83,6 @@ "collection"^^xsd:string ; knora-admin:projectRestrictedViewSize "!512,512"^^xsd:string ; knora-admin:projectRestrictedViewWatermark "true"^^xsd:boolean; - knora-admin:belongsToInstitution ; knora-admin:status "true"^^xsd:boolean ; knora-admin:hasSelfJoinEnabled "false"^^xsd:boolean . diff --git a/test_data/upgrade/pr1746.trig b/test_data/upgrade/pr1746.trig index c25767bea6..2e2c28bcd5 100644 --- a/test_data/upgrade/pr1746.trig +++ b/test_data/upgrade/pr1746.trig @@ -24,8 +24,6 @@ knora-admin:projectRestrictedViewSize "!512,512"^^xsd:string ; - knora-admin:belongsToInstitution ; - knora-admin:status "true"^^xsd:boolean ; knora-admin:hasSelfJoinEnabled "false"^^xsd:boolean . diff --git a/webapi/src/main/resources/knora-ontologies/knora-admin.ttl b/webapi/src/main/resources/knora-ontologies/knora-admin.ttl index 3e767a3e72..2eace8ad31 100644 --- a/webapi/src/main/resources/knora-ontologies/knora-admin.ttl +++ b/webapi/src/main/resources/knora-ontologies/knora-admin.ttl @@ -27,16 +27,6 @@ ################################################################# -### http://www.knora.org/ontology/knora-admin#belongsToInstitution - -:belongsToInstitution - rdf:type owl:ObjectProperty ; - rdfs:subPropertyOf knora-base:objectCannotBeMarkedAsDeleted ; - rdfs:comment "Indicates which Institution a project belongs to."@en ; - knora-base:subjectClassConstraint :knoraProject ; - knora-base:objectClassConstraint :Institution . - - ### http://www.knora.org/ontology/knora-admin#belongsToProject :belongsToProject @@ -465,9 +455,6 @@ [ rdf:type owl:Restriction ; owl:onProperty :projectRestrictedViewWatermark ; owl:maxCardinality "1"^^xsd:nonNegativeInteger ], - [ rdf:type owl:Restriction ; - owl:onProperty :belongsToInstitution ; - owl:maxCardinality "1"^^xsd:nonNegativeInteger ], [ rdf:type owl:Restriction ; owl:onProperty :status ; owl:cardinality "1"^^xsd:nonNegativeInteger ], diff --git a/webapi/src/main/resources/knora-ontologies/knora-base.ttl b/webapi/src/main/resources/knora-ontologies/knora-base.ttl index a9657aa5eb..3b341827d8 100644 --- a/webapi/src/main/resources/knora-ontologies/knora-base.ttl +++ b/webapi/src/main/resources/knora-ontologies/knora-base.ttl @@ -19,7 +19,7 @@ rdf:type owl:Ontology ; rdfs:label "The Knora base ontology"@en ; :attachedToProject knora-admin:SystemProject ; - :ontologyVersion "knora-base v28" . + :ontologyVersion "knora-base v29" . ################################################################# diff --git a/webapi/src/main/scala/org/knora/webapi/messages/OntologyConstants.scala b/webapi/src/main/scala/org/knora/webapi/messages/OntologyConstants.scala index 7a2a54be7c..556a49b1fb 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/OntologyConstants.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/OntologyConstants.scala @@ -478,7 +478,6 @@ object OntologyConstants { val ProjectLogo: IRI = KnoraAdminPrefixExpansion + "projectLogo" val ProjectRestrictedViewSize: IRI = KnoraAdminPrefixExpansion + "projectRestrictedViewSize" val ProjectRestrictedViewWatermark: IRI = KnoraAdminPrefixExpansion + "projectRestrictedViewWatermark" - val BelongsToInstitution: IRI = KnoraAdminPrefixExpansion + "belongsToInstitution" val HasSelfJoinEnabled: IRI = KnoraAdminPrefixExpansion + "hasSelfJoinEnabled" /* Group */ diff --git a/webapi/src/main/scala/org/knora/webapi/package.scala b/webapi/src/main/scala/org/knora/webapi/package.scala index a340373862..bad29dc045 100644 --- a/webapi/src/main/scala/org/knora/webapi/package.scala +++ b/webapi/src/main/scala/org/knora/webapi/package.scala @@ -11,7 +11,7 @@ package object webapi { * The version of `knora-base` and of the other built-in ontologies that this version of Knora requires. * Must be the same as the object of `knora-base:ontologyVersion` in the `knora-base` ontology being used. */ - val KnoraBaseVersion: String = "knora-base v28" + val KnoraBaseVersion: String = "knora-base v29" /** * `IRI` is a synonym for `String`, used to improve code readability. diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala index f302f153e6..d80df5550b 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala @@ -63,6 +63,7 @@ object RepositoryUpdatePlan { PluginForKnoraBaseVersion(versionNumber = 26, plugin = new MigrateOnlyBuiltInGraphs), // PR 3003 PluginForKnoraBaseVersion(versionNumber = 27, plugin = new MigrateOnlyBuiltInGraphs), // PR 3026 PluginForKnoraBaseVersion(versionNumber = 28, plugin = new MigrateOnlyBuiltInGraphs), // PR 3038 + PluginForKnoraBaseVersion(versionNumber = 29, plugin = new MigrateOnlyBuiltInGraphs), // KEEP IT ON THE BOTTOM // From "versionNumber = 6" don't use prBasedVersionString! ) From 66105eb2ccfa3f236cafd373fde26deaa204e26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Mon, 11 Mar 2024 11:53:50 +0100 Subject: [PATCH 02/11] Remove Institution class and related properties --- .../domain/domain-entities-and-relations.md | 5 -- test_data/project_data/admin-data.ttl | 5 -- ...001__remove_Institution_and_related.sparql | 9 +++ .../knora-ontologies/knora-admin.ttl | 78 ------------------- .../webapi/messages/OntologyConstants.scala | 8 -- .../domain/IriTestConstants.scala | 1 - 6 files changed, 9 insertions(+), 97 deletions(-) create mode 100644 webapi/src/main/resources/db/0001__remove_Institution_and_related.sparql diff --git a/docs/05-internals/design/domain/domain-entities-and-relations.md b/docs/05-internals/design/domain/domain-entities-and-relations.md index b41dd4f290..92179f252e 100644 --- a/docs/05-internals/design/domain/domain-entities-and-relations.md +++ b/docs/05-internals/design/domain/domain-entities-and-relations.md @@ -97,11 +97,6 @@ erDiagram ``` -!!! danger "Unclear/Unexpected Stuff" - - - User.phone? - - Institution? (name, description, website, phone, address, email) - ### Overview V2 ```mermaid diff --git a/test_data/project_data/admin-data.ttl b/test_data/project_data/admin-data.ttl index 23f119e798..be4139deab 100644 --- a/test_data/project_data/admin-data.ttl +++ b/test_data/project_data/admin-data.ttl @@ -140,11 +140,6 @@ knora-admin:hasSelfJoinEnabled "false"^^xsd:boolean . - - rdf:type knora-admin:Institution ; - knora-admin:institutionName "Digital Humanities Lab, Basel" . - - ########################################################## # # INCUNABULA PROJECT / USERS / GROUPS diff --git a/webapi/src/main/resources/db/0001__remove_Institution_and_related.sparql b/webapi/src/main/resources/db/0001__remove_Institution_and_related.sparql new file mode 100644 index 0000000000..a91b4b5b63 --- /dev/null +++ b/webapi/src/main/resources/db/0001__remove_Institution_and_related.sparql @@ -0,0 +1,9 @@ +PREFIX rdf: +PREFIX knora-admin: + +DELETE WHERE { + GRAPH { + ?s a knora-admin:Institution ; + ?p ?o . + } +} diff --git a/webapi/src/main/resources/knora-ontologies/knora-admin.ttl b/webapi/src/main/resources/knora-ontologies/knora-admin.ttl index 2eace8ad31..5e291d0e58 100644 --- a/webapi/src/main/resources/knora-ontologies/knora-admin.ttl +++ b/webapi/src/main/resources/knora-ontologies/knora-admin.ttl @@ -86,16 +86,6 @@ # ################################################################# - -### http://www.knora.org/ontology/knora-admin#address - -:address - rdf:type owl:DatatypeProperty ; - rdfs:subPropertyOf knora-base:objectCannotBeMarkedAsDeleted ; - knora-base:objectDatatypeConstraint xsd:string ; - rdfs:comment "The address of a person or an institution"@en . - - ### http://www.knora.org/ontology/knora-admin#groupName :groupName @@ -129,37 +119,6 @@ knora-base:objectDatatypeConstraint xsd:boolean . -### http://www.knora.org/ontology/knora-admin#institutionName - -:institutionName - rdf:type owl:DatatypeProperty ; - rdfs:subPropertyOf knora-base:objectCannotBeMarkedAsDeleted ; - rdfs:comment "The institutions's name."@en ; - rdfs:subPropertyOf ; - knora-base:subjectClassConstraint :Institution ; - knora-base:objectDatatypeConstraint xsd:string . - - -### http://www.knora.org/ontology/knora-admin#institutionDescription - -:institutionDescription - rdf:type owl:DatatypeProperty ; - rdfs:subPropertyOf knora-base:objectCannotBeMarkedAsDeleted ; - rdfs:comment "A description of an institution."@en ; - knora-base:subjectClassConstraint :Institution ; - knora-base:objectDatatypeConstraint xsd:string . - - -### http://www.knora.org/ontology/knora-admin#institutionWebsite - -:institutionWebsite - rdf:type owl:DatatypeProperty ; - rdfs:subPropertyOf knora-base:objectCannotBeMarkedAsDeleted ; - rdfs:comment "The URL of a web site."@en ; - knora-base:subjectClassConstraint :Institution ; - knora-base:objectDatatypeConstraint xsd:anyURI . - - ### http://www.knora.org/ontology/knora-admin#isActiveUser :isInSystemAdminGroup @@ -199,16 +158,6 @@ knora-base:subjectClassConstraint :User ; knora-base:objectDatatypeConstraint xsd:string . - -### http://www.knora.org/ontology/knora-admin#phone - -:phone - rdf:type owl:DatatypeProperty ; - rdfs:subPropertyOf knora-base:objectCannotBeMarkedAsDeleted ; - rdfs:comment "The phone number of a person, institution, etc."@en ; - knora-base:objectDatatypeConstraint xsd:string . - - ### http://www.knora.org/ontology/knora-admin#preferredLanguage :preferredLanguage @@ -335,33 +284,6 @@ # ################################################################# - -### http://www.knora.org/ontology/knora-admin#Institution - -:Institution - rdf:type owl:Class ; - rdfs:subClassOf foaf:Organization, - [ rdf:type owl:Restriction ; - owl:onProperty :institutionName ; - owl:cardinality "1"^^xsd:nonNegativeInteger ], - [ rdf:type owl:Restriction ; - owl:onProperty :institutionDescription ; - owl:maxCardinality "1"^^xsd:nonNegativeInteger ], - [ rdf:type owl:Restriction ; - owl:onProperty :institutionWebsite ; - owl:minCardinality "0"^^xsd:nonNegativeInteger ], - [ rdf:type owl:Restriction ; - owl:onProperty :phone ; - owl:minCardinality "0"^^xsd:nonNegativeInteger ], - [ rdf:type owl:Restriction ; - owl:onProperty :address ; - owl:minCardinality "0"^^xsd:nonNegativeInteger ], - [ rdf:type owl:Restriction ; - owl:onProperty :email ; - owl:minCardinality "0"^^xsd:nonNegativeInteger ] ; - rdfs:comment "An institution that has data in Knora."@en . - - ### http://www.knora.org/ontology/knora-admin#User :User diff --git a/webapi/src/main/scala/org/knora/webapi/messages/OntologyConstants.scala b/webapi/src/main/scala/org/knora/webapi/messages/OntologyConstants.scala index 556a49b1fb..fb4946dd02 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/OntologyConstants.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/OntologyConstants.scala @@ -457,7 +457,6 @@ object OntologyConstants { val GivenName: IRI = KnoraAdminPrefixExpansion + "givenName" val FamilyName: IRI = KnoraAdminPrefixExpansion + "familyName" val Password: IRI = KnoraAdminPrefixExpansion + "password" - val Address: IRI = KnoraAdminPrefixExpansion + "address" val UsersActiveProject: IRI = KnoraAdminPrefixExpansion + "currentproject" val PreferredLanguage: IRI = KnoraAdminPrefixExpansion + "preferredLanguage" val IsInProject: IRI = KnoraAdminPrefixExpansion + "isInProject" @@ -503,13 +502,6 @@ object OntologyConstants { ProjectAdmin, ) - /* Institution */ - val Institution: IRI = KnoraAdminPrefixExpansion + "Institution" - val InstitutionDescription: IRI = KnoraAdminPrefixExpansion + "institutionDescription" - val InstitutionName: IRI = KnoraAdminPrefixExpansion + "institutionName" - val InstitutionWebsite: IRI = KnoraAdminPrefixExpansion + "institutionWebsite" - val Phone: IRI = KnoraAdminPrefixExpansion + "phone" - /* Permissions */ val Permission: IRI = KnoraAdminPrefixExpansion + "Permission" val AdministrativePermission: IRI = KnoraAdminPrefixExpansion + "AdministrativePermission" diff --git a/webapi/src/test/scala/org/knora/webapi/slice/resourceinfo/domain/IriTestConstants.scala b/webapi/src/test/scala/org/knora/webapi/slice/resourceinfo/domain/IriTestConstants.scala index eaf345576d..26feb8a413 100644 --- a/webapi/src/test/scala/org/knora/webapi/slice/resourceinfo/domain/IriTestConstants.scala +++ b/webapi/src/test/scala/org/knora/webapi/slice/resourceinfo/domain/IriTestConstants.scala @@ -67,7 +67,6 @@ object IriTestConstants { object Class { val AdministrativePermission: InternalIri = makeEntity(KnoraAdmin.Ontology, "AdministrativePermission") - val Institution: InternalIri = makeEntity(KnoraAdmin.Ontology, "Institution") val Permission: InternalIri = makeEntity(KnoraAdmin.Ontology, "Permission") } } From f2ea9ffbf2ca8475082b191dfedffa08a0ddaaf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Tue, 12 Mar 2024 12:58:42 +0100 Subject: [PATCH 03/11] wip use plugin for removal of existing values --- .../upgrade/RepositoryUpdatePlan.scala | 2 +- .../upgrade/plugins/UpgradePluginPR3110.scala | 52 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala index d80df5550b..f9fb9437c8 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala @@ -63,7 +63,7 @@ object RepositoryUpdatePlan { PluginForKnoraBaseVersion(versionNumber = 26, plugin = new MigrateOnlyBuiltInGraphs), // PR 3003 PluginForKnoraBaseVersion(versionNumber = 27, plugin = new MigrateOnlyBuiltInGraphs), // PR 3026 PluginForKnoraBaseVersion(versionNumber = 28, plugin = new MigrateOnlyBuiltInGraphs), // PR 3038 - PluginForKnoraBaseVersion(versionNumber = 29, plugin = new MigrateOnlyBuiltInGraphs), + PluginForKnoraBaseVersion(versionNumber = 29, plugin = new UpgradePluginPR3110()), // KEEP IT ON THE BOTTOM // From "versionNumber = 6" don't use prBasedVersionString! ) diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala new file mode 100644 index 0000000000..f3cff41e00 --- /dev/null +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala @@ -0,0 +1,52 @@ +/* + * Copyright © 2021 - 2024 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.store.triplestore.upgrade.plugins + +import org.apache.jena.query.QueryExecutionFactory +import org.eclipse.rdf4j.sparqlbuilder.core.SparqlBuilder.`var` as variable +import org.eclipse.rdf4j.sparqlbuilder.core.query.* +import org.eclipse.rdf4j.sparqlbuilder.rdf.Rdf + +import org.knora.webapi.messages.util.rdf.JenaModel +import org.knora.webapi.messages.util.rdf.RdfModel +import org.knora.webapi.slice.admin.AdminConstants +import org.knora.webapi.slice.admin.repo.rdf.Vocabulary +import org.knora.webapi.store.triplestore.upgrade.GraphsForMigration +import org.knora.webapi.store.triplestore.upgrade.MigrateSpecificGraphs +import org.knora.webapi.store.triplestore.upgrade.UpgradePlugin + +/** + * Transforms a repository for Knora PR 2094. + * Transforms incorrect value of valueHasUri from node to string type adding also + * missing datatype ^^ + */ +class UpgradePluginPR3110() extends UpgradePlugin { + + override def graphsForMigration: GraphsForMigration = + MigrateSpecificGraphs.from(AdminConstants.adminDataNamedGraph) + + override def transform(model: RdfModel): Unit = { + val dataset = model.asInstanceOf[JenaModel].getDataset + dataset.begin() + + try { + var (s, p, o) = (variable("s"), variable("p"), variable("o")) + val query = Queries + .MODIFY() + .prefix(Vocabulary.KnoraAdmin.NS) + .`with`(Vocabulary.NamedGraphs.knoraAdminIri) + .delete( + s.isA(Rdf.iri("http://www.knora.org/ontology/knora-admin#Institution")) + .andHas(p, o), + ) + + val qExec = QueryExecutionFactory.create(query.getQueryString, dataset) + qExec.close() + } finally { + dataset.commit() + } + } +} From 2719825f8f5cc1f20e250854972a9f243e081944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Wed, 13 Mar 2024 11:39:38 +0100 Subject: [PATCH 04/11] Extract base plugin for Sparql queries --- project/LocalSettings.scala | 2 +- .../upgrade/RepositoryUpdatePlan.scala | 1 + .../upgrade/RepositoryUpdater.scala | 2 +- .../plugins/AbstractSparqlUpdatePlugin.scala | 20 ++++++++++ .../upgrade/plugins/UpgradePluginPR3110.scala | 40 +++++++------------ 5 files changed, 38 insertions(+), 27 deletions(-) create mode 100644 webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala diff --git a/project/LocalSettings.scala b/project/LocalSettings.scala index de132b5689..c5068a7b8e 100644 --- a/project/LocalSettings.scala +++ b/project/LocalSettings.scala @@ -8,6 +8,6 @@ import Keys.* object LocalSettings { val localScalacOptions: Seq[SettingsDefinition] = Seq( - // scalacOptions -= "-Xfatal-warnings" +// scalacOptions -= "-Xfatal-warnings", ) } diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala index f9fb9437c8..5a1569095c 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala @@ -63,6 +63,7 @@ object RepositoryUpdatePlan { PluginForKnoraBaseVersion(versionNumber = 26, plugin = new MigrateOnlyBuiltInGraphs), // PR 3003 PluginForKnoraBaseVersion(versionNumber = 27, plugin = new MigrateOnlyBuiltInGraphs), // PR 3026 PluginForKnoraBaseVersion(versionNumber = 28, plugin = new MigrateOnlyBuiltInGraphs), // PR 3038 + PluginForKnoraBaseVersion(versionNumber = 29, plugin = new MigrateOnlyBuiltInGraphs), PluginForKnoraBaseVersion(versionNumber = 29, plugin = new UpgradePluginPR3110()), // KEEP IT ON THE BOTTOM // From "versionNumber = 6" don't use prBasedVersionString! diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdater.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdater.scala index 5bf5795a4c..9b8c0ac866 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdater.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdater.scala @@ -204,7 +204,7 @@ object RepositoryUpdater { graphs = pluginsForNeededUpdates .map(_.plugin.graphsForMigration) .reduce(_ merge _) - _ <- ZIO.logInfo("Downloading repository file...") + _ <- ZIO.logInfo(s"Downloading .$graphs repository file..") _ <- triplestoreService.downloadRepository(graphsBeforeMigrationFile, graphs) // Run the transformations to produce an output file. diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala new file mode 100644 index 0000000000..da3eac5f74 --- /dev/null +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala @@ -0,0 +1,20 @@ +package org.knora.webapi.store.triplestore.upgrade.plugins + +import org.apache.jena.update.UpdateExecutionFactory +import org.apache.jena.update.UpdateFactory +import org.eclipse.rdf4j.sparqlbuilder.core.query.ModifyQuery +import org.knora.webapi.messages.util.rdf.JenaModel +import org.knora.webapi.messages.util.rdf.RdfModel +import org.knora.webapi.store.triplestore.upgrade.UpgradePlugin + +abstract class AbstractSparqlUpdatePlugin extends UpgradePlugin { + + def getQuery: ModifyQuery + + override def transform(model: RdfModel): Unit = { + val dataset = model.asInstanceOf[JenaModel].getDataset + val update = UpdateFactory.create(getQuery.getQueryString) + val qExec = UpdateExecutionFactory.create(update, dataset) + qExec.execute() + } +} diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala index f3cff41e00..be0916a479 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala @@ -5,48 +5,38 @@ package org.knora.webapi.store.triplestore.upgrade.plugins -import org.apache.jena.query.QueryExecutionFactory +import org.eclipse.rdf4j.model.vocabulary.RDF import org.eclipse.rdf4j.sparqlbuilder.core.SparqlBuilder.`var` as variable +import org.eclipse.rdf4j.sparqlbuilder.core.query.ModifyQuery import org.eclipse.rdf4j.sparqlbuilder.core.query.* import org.eclipse.rdf4j.sparqlbuilder.rdf.Rdf -import org.knora.webapi.messages.util.rdf.JenaModel -import org.knora.webapi.messages.util.rdf.RdfModel +import org.knora.webapi.messages.OntologyConstants.KnoraAdmin.KnoraAdminPrefixExpansion import org.knora.webapi.slice.admin.AdminConstants import org.knora.webapi.slice.admin.repo.rdf.Vocabulary import org.knora.webapi.store.triplestore.upgrade.GraphsForMigration import org.knora.webapi.store.triplestore.upgrade.MigrateSpecificGraphs -import org.knora.webapi.store.triplestore.upgrade.UpgradePlugin /** * Transforms a repository for Knora PR 2094. * Transforms incorrect value of valueHasUri from node to string type adding also * missing datatype ^^ */ -class UpgradePluginPR3110() extends UpgradePlugin { +class UpgradePluginPR3110() extends AbstractSparqlUpdatePlugin { override def graphsForMigration: GraphsForMigration = MigrateSpecificGraphs.from(AdminConstants.adminDataNamedGraph) - override def transform(model: RdfModel): Unit = { - val dataset = model.asInstanceOf[JenaModel].getDataset - dataset.begin() - - try { - var (s, p, o) = (variable("s"), variable("p"), variable("o")) - val query = Queries - .MODIFY() - .prefix(Vocabulary.KnoraAdmin.NS) - .`with`(Vocabulary.NamedGraphs.knoraAdminIri) - .delete( - s.isA(Rdf.iri("http://www.knora.org/ontology/knora-admin#Institution")) - .andHas(p, o), - ) - - val qExec = QueryExecutionFactory.create(query.getQueryString, dataset) - qExec.close() - } finally { - dataset.commit() - } + override def getQuery: ModifyQuery = { + var (s, p, o) = (variable("s"), variable("p"), variable("o")) + Queries + .MODIFY() + .prefix(Vocabulary.KnoraAdmin.NS, RDF.NS) + .delete(s.has(p, o)) + .where( + s.isA(Rdf.iri(KnoraAdminPrefixExpansion, "Institution")) + .andHas(p, o) + .from(Vocabulary.NamedGraphs.knoraAdminIri), + ) } } From 5a96d95396c72a24619decfb4323ee77ae59ea92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Wed, 13 Mar 2024 12:05:38 +0100 Subject: [PATCH 05/11] header & fmt --- .../upgrade/plugins/AbstractSparqlUpdatePlugin.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala index da3eac5f74..7380d62f49 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala @@ -1,8 +1,14 @@ +/* + * Copyright © 2021 - 2024 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + package org.knora.webapi.store.triplestore.upgrade.plugins import org.apache.jena.update.UpdateExecutionFactory import org.apache.jena.update.UpdateFactory import org.eclipse.rdf4j.sparqlbuilder.core.query.ModifyQuery + import org.knora.webapi.messages.util.rdf.JenaModel import org.knora.webapi.messages.util.rdf.RdfModel import org.knora.webapi.store.triplestore.upgrade.UpgradePlugin @@ -13,8 +19,8 @@ abstract class AbstractSparqlUpdatePlugin extends UpgradePlugin { override def transform(model: RdfModel): Unit = { val dataset = model.asInstanceOf[JenaModel].getDataset - val update = UpdateFactory.create(getQuery.getQueryString) - val qExec = UpdateExecutionFactory.create(update, dataset) + val update = UpdateFactory.create(getQuery.getQueryString) + val qExec = UpdateExecutionFactory.create(update, dataset) qExec.execute() } } From bcf545b9f76ddf3b4d0529ef57410b247982be10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Wed, 13 Mar 2024 12:06:27 +0100 Subject: [PATCH 06/11] rm unused query --- .../db/0001__remove_Institution_and_related.sparql | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 webapi/src/main/resources/db/0001__remove_Institution_and_related.sparql diff --git a/webapi/src/main/resources/db/0001__remove_Institution_and_related.sparql b/webapi/src/main/resources/db/0001__remove_Institution_and_related.sparql deleted file mode 100644 index a91b4b5b63..0000000000 --- a/webapi/src/main/resources/db/0001__remove_Institution_and_related.sparql +++ /dev/null @@ -1,9 +0,0 @@ -PREFIX rdf: -PREFIX knora-admin: - -DELETE WHERE { - GRAPH { - ?s a knora-admin:Institution ; - ?p ?o . - } -} From c3001f912732b8fac6fb2116f4949f4c5eb6602c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Wed, 13 Mar 2024 12:07:26 +0100 Subject: [PATCH 07/11] fixup --- project/LocalSettings.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/LocalSettings.scala b/project/LocalSettings.scala index c5068a7b8e..de132b5689 100644 --- a/project/LocalSettings.scala +++ b/project/LocalSettings.scala @@ -8,6 +8,6 @@ import Keys.* object LocalSettings { val localScalacOptions: Seq[SettingsDefinition] = Seq( -// scalacOptions -= "-Xfatal-warnings", + // scalacOptions -= "-Xfatal-warnings" ) } From 960a2759560a84e3cc5c796dfc27c4d4fabf36bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Wed, 13 Mar 2024 12:19:22 +0100 Subject: [PATCH 08/11] Fix scaladoc for upgrade plugin --- .../triplestore/upgrade/plugins/UpgradePluginPR3110.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala index be0916a479..88dd009d81 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala @@ -18,9 +18,8 @@ import org.knora.webapi.store.triplestore.upgrade.GraphsForMigration import org.knora.webapi.store.triplestore.upgrade.MigrateSpecificGraphs /** - * Transforms a repository for Knora PR 2094. - * Transforms incorrect value of valueHasUri from node to string type adding also - * missing datatype ^^ + * After removing `knora-admin:Institution` class and its properties from the knora-admin ontology this cleans up the DB. + * Removes all `knora-admin:Institution` classes from the `knora-admin` named graph. */ class UpgradePluginPR3110() extends AbstractSparqlUpdatePlugin { From 3c86d8475f05e2250fbc45cad9040a3a6d0aa5c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Wed, 13 Mar 2024 16:49:04 +0100 Subject: [PATCH 09/11] Remove belongsToInstitution triples as well --- .../plugins/AbstractSparqlUpdatePlugin.scala | 19 ++++++++++++++----- .../upgrade/plugins/UpgradePluginPR3110.scala | 18 +++++++++++++++--- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala index 7380d62f49..f499b7d3b7 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/AbstractSparqlUpdatePlugin.scala @@ -5,22 +5,31 @@ package org.knora.webapi.store.triplestore.upgrade.plugins +import com.typesafe.scalalogging.Logger +import org.apache.jena.query.Dataset import org.apache.jena.update.UpdateExecutionFactory import org.apache.jena.update.UpdateFactory import org.eclipse.rdf4j.sparqlbuilder.core.query.ModifyQuery -import org.knora.webapi.messages.util.rdf.JenaModel -import org.knora.webapi.messages.util.rdf.RdfModel +import org.knora.webapi.messages.util.rdf.* import org.knora.webapi.store.triplestore.upgrade.UpgradePlugin abstract class AbstractSparqlUpdatePlugin extends UpgradePlugin { - def getQuery: ModifyQuery + private val log: Logger = Logger(this.getClass) + + def getQueries: List[ModifyQuery] override def transform(model: RdfModel): Unit = { val dataset = model.asInstanceOf[JenaModel].getDataset - val update = UpdateFactory.create(getQuery.getQueryString) - val qExec = UpdateExecutionFactory.create(update, dataset) + getQueries.foreach(execute(dataset, _)) + } + + private def execute(dataset: Dataset, query: ModifyQuery) = { + val queryString = query.getQueryString + log.info(s"Executing SPARQL update: $queryString") + val update = UpdateFactory.create(queryString) + val qExec = UpdateExecutionFactory.create(update, dataset) qExec.execute() } } diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala index 88dd009d81..b961c52463 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3110.scala @@ -21,13 +21,13 @@ import org.knora.webapi.store.triplestore.upgrade.MigrateSpecificGraphs * After removing `knora-admin:Institution` class and its properties from the knora-admin ontology this cleans up the DB. * Removes all `knora-admin:Institution` classes from the `knora-admin` named graph. */ -class UpgradePluginPR3110() extends AbstractSparqlUpdatePlugin { +class UpgradePluginPR3110 extends AbstractSparqlUpdatePlugin { override def graphsForMigration: GraphsForMigration = MigrateSpecificGraphs.from(AdminConstants.adminDataNamedGraph) - override def getQuery: ModifyQuery = { - var (s, p, o) = (variable("s"), variable("p"), variable("o")) + private val removeAllInstitutions: ModifyQuery = { + val (s, p, o) = (variable("s"), variable("p"), variable("o")) Queries .MODIFY() .prefix(Vocabulary.KnoraAdmin.NS, RDF.NS) @@ -38,4 +38,16 @@ class UpgradePluginPR3110() extends AbstractSparqlUpdatePlugin { .from(Vocabulary.NamedGraphs.knoraAdminIri), ) } + + private val removeAllBelongsToInstitutionTriples: ModifyQuery = { + val (s, o) = (variable("s"), variable("o")) + val belongsToInstitution = Rdf.iri(KnoraAdminPrefixExpansion, "belongsToInstitution") + Queries + .MODIFY() + .prefix(Vocabulary.KnoraAdmin.NS, RDF.NS) + .delete(s.has(belongsToInstitution, o)) + .where(s.has(belongsToInstitution, o).from(Vocabulary.NamedGraphs.knoraAdminIri)) + } + + override def getQueries: List[ModifyQuery] = List(removeAllInstitutions, removeAllBelongsToInstitutionTriples) } From 998e921b0111dc192d1a5978b3cc52abed896b47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Wed, 13 Mar 2024 16:51:42 +0100 Subject: [PATCH 10/11] remove needless plugin --- .../webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala index 5a1569095c..f9fb9437c8 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala @@ -63,7 +63,6 @@ object RepositoryUpdatePlan { PluginForKnoraBaseVersion(versionNumber = 26, plugin = new MigrateOnlyBuiltInGraphs), // PR 3003 PluginForKnoraBaseVersion(versionNumber = 27, plugin = new MigrateOnlyBuiltInGraphs), // PR 3026 PluginForKnoraBaseVersion(versionNumber = 28, plugin = new MigrateOnlyBuiltInGraphs), // PR 3038 - PluginForKnoraBaseVersion(versionNumber = 29, plugin = new MigrateOnlyBuiltInGraphs), PluginForKnoraBaseVersion(versionNumber = 29, plugin = new UpgradePluginPR3110()), // KEEP IT ON THE BOTTOM // From "versionNumber = 6" don't use prBasedVersionString! From a5267506b9877202af4321b78057dbba7730a617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Kleinb=C3=B6lting?= Date: Wed, 13 Mar 2024 16:59:00 +0100 Subject: [PATCH 11/11] fix: Add database migration which removes invalid triples from the database --- .../resources/knora-ontologies/knora-base.ttl | 2 +- .../main/scala/org/knora/webapi/package.scala | 2 +- .../upgrade/RepositoryUpdatePlan.scala | 3 ++ .../upgrade/plugins/UpgradePluginPR3111.scala | 39 +++++++++++++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3111.scala diff --git a/webapi/src/main/resources/knora-ontologies/knora-base.ttl b/webapi/src/main/resources/knora-ontologies/knora-base.ttl index 3b341827d8..910c3492c6 100644 --- a/webapi/src/main/resources/knora-ontologies/knora-base.ttl +++ b/webapi/src/main/resources/knora-ontologies/knora-base.ttl @@ -19,7 +19,7 @@ rdf:type owl:Ontology ; rdfs:label "The Knora base ontology"@en ; :attachedToProject knora-admin:SystemProject ; - :ontologyVersion "knora-base v29" . + :ontologyVersion "knora-base v30" . ################################################################# diff --git a/webapi/src/main/scala/org/knora/webapi/package.scala b/webapi/src/main/scala/org/knora/webapi/package.scala index bad29dc045..3c51e25537 100644 --- a/webapi/src/main/scala/org/knora/webapi/package.scala +++ b/webapi/src/main/scala/org/knora/webapi/package.scala @@ -11,7 +11,7 @@ package object webapi { * The version of `knora-base` and of the other built-in ontologies that this version of Knora requires. * Must be the same as the object of `knora-base:ontologyVersion` in the `knora-base` ontology being used. */ - val KnoraBaseVersion: String = "knora-base v29" + val KnoraBaseVersion: String = "knora-base v30" /** * `IRI` is a synonym for `String`, used to improve code readability. diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala index f9fb9437c8..408a77b0e7 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/RepositoryUpdatePlan.scala @@ -8,6 +8,7 @@ package org.knora.webapi.store.triplestore.upgrade import com.typesafe.scalalogging.Logger import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject +import org.knora.webapi.store.triplestore.upgrade.plugins.UpgradePluginPR3110 import org.knora.webapi.store.triplestore.upgrade.plugins.* /** @@ -64,6 +65,8 @@ object RepositoryUpdatePlan { PluginForKnoraBaseVersion(versionNumber = 27, plugin = new MigrateOnlyBuiltInGraphs), // PR 3026 PluginForKnoraBaseVersion(versionNumber = 28, plugin = new MigrateOnlyBuiltInGraphs), // PR 3038 PluginForKnoraBaseVersion(versionNumber = 29, plugin = new UpgradePluginPR3110()), + PluginForKnoraBaseVersion(versionNumber = 30, plugin = new UpgradePluginPR3110()), + PluginForKnoraBaseVersion(versionNumber = 30, plugin = new UpgradePluginPR3111()), // KEEP IT ON THE BOTTOM // From "versionNumber = 6" don't use prBasedVersionString! ) diff --git a/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3111.scala b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3111.scala new file mode 100644 index 0000000000..9999e5031f --- /dev/null +++ b/webapi/src/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR3111.scala @@ -0,0 +1,39 @@ +/* + * Copyright © 2021 - 2024 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.knora.webapi.store.triplestore.upgrade.plugins + +import org.eclipse.rdf4j.sparqlbuilder.core.SparqlBuilder.`var` as variable +import org.eclipse.rdf4j.sparqlbuilder.core.query.ModifyQuery +import org.eclipse.rdf4j.sparqlbuilder.core.query.* +import org.eclipse.rdf4j.sparqlbuilder.rdf.Rdf + +import org.knora.webapi.slice.admin.AdminConstants +import org.knora.webapi.slice.admin.repo.rdf.Vocabulary +import org.knora.webapi.store.triplestore.upgrade.GraphsForMigration +import org.knora.webapi.store.triplestore.upgrade.MigrateSpecificGraphs + +/** + * Certain restricted views have a watermark that is not a boolean. This plugin removes the invalid watermark triples. + */ +class UpgradePluginPR3111 extends AbstractSparqlUpdatePlugin { + + override def graphsForMigration: GraphsForMigration = + MigrateSpecificGraphs.from(AdminConstants.adminDataNamedGraph) + + private val removeInvalidRestrictedViewWatermarkTriples: ModifyQuery = { + val invalidTriple = variable("s").has( + Vocabulary.KnoraAdmin.projectRestrictedViewWatermark, + Rdf.literalOf("path_to_image"), + ) + Queries + .MODIFY() + .prefix(Vocabulary.KnoraAdmin.NS) + .delete(invalidTriple) + .where(invalidTriple.from(Vocabulary.NamedGraphs.knoraAdminIri)) + } + + override def getQueries: List[ModifyQuery] = List(removeInvalidRestrictedViewWatermarkTriples) +}