-
Notifications
You must be signed in to change notification settings - Fork 18
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix!: add upgrade plugin that fixes invalid date serialisations (#2081)
* feat: add plugin to upgrade plan * write upgrade script * change gitignore so that not all .trig files are excluded * bump upgrade plugin version * test data for upgrade plugin * finish tests * add make target for uploading the ls trig file without loading it first * make label updating twirl template more robust * fix comment * chore: streamline ls-test make targets
- Loading branch information
1 parent
09688f5
commit 3a0902e
Showing
9 changed files
with
175 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -36,7 +36,7 @@ sipi/test | |
| *.bak | ||
| *.rdb | ||
| .sbtrc | ||
| *.trig | ||
| /*.trig | ||
|
|
||
| dependencies.txt | ||
| /client-test-data.zip | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> . | ||
| @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> . | ||
| @prefix xsd: <http://www.w3.org/2001/XMLSchema#> . | ||
| @prefix knora-base: <http://www.knora.org/ontology/knora-base#> . | ||
|
|
||
| # resources with old date serialization | ||
|
|
||
| <http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9ma> | ||
| a knora-base:Resource ; | ||
| knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ; | ||
| knora-base:attachedToProject <http://rdfh.ch/projects/0001> ; | ||
| knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ; | ||
| knora-base:creationDate "2019-11-29T10:00:00.673298+00:00"^^xsd:dateTime ; | ||
| rdfs:label "a thing" ; | ||
| knora-base:isDeleted false . | ||
|
|
||
| <http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mb> | ||
| a knora-base:Resource ; | ||
| knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ; | ||
| knora-base:attachedToProject <http://rdfh.ch/projects/0001> ; | ||
| knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ; | ||
| knora-base:creationDate "2018-08-07T13:27:18.518+00:00"^^xsd:dateTime ; | ||
| knora-base:lastModificationDate "2019-11-29T10:00:01.673298+00:00"^^xsd:dateTime ; | ||
| rdfs:label "a thing" ; | ||
| knora-base:isDeleted false . | ||
|
|
||
| <http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mc> | ||
| a knora-base:Resource ; | ||
| knora-base:attachedToUser <http://rdfh.ch/users/9XBCrDV3SRa7kS1WwynB4Q> ; | ||
| knora-base:attachedToProject <http://rdfh.ch/projects/0001> ; | ||
| knora-base:hasPermissions "V knora-admin:UnknownUser|M knora-admin:ProjectMember" ; | ||
| knora-base:creationDate "2018-08-07T13:27:18.518123Z"^^xsd:dateTime ; | ||
| knora-base:lastModificationDate "2019-11-29T10:00:01.673298+02:00"^^xsd:dateTime ; | ||
| knora-base:deleteDate "2020-04-07T14:59:28.960124+00:00"^^xsd:dateTime ; | ||
| rdfs:label "a thing" ; | ||
| knora-base:isDeleted false . |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
52 changes: 52 additions & 0 deletions
52
...c/main/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR2081.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| /* | ||
| * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.knora.webapi.store.triplestore.upgrade.plugins | ||
|
|
||
| import com.typesafe.scalalogging.Logger | ||
| import org.knora.webapi.feature.FeatureFactoryConfig | ||
| import org.knora.webapi.messages.OntologyConstants | ||
| import org.knora.webapi.messages.util.rdf._ | ||
| import org.knora.webapi.store.triplestore.upgrade.UpgradePlugin | ||
| import java.time.Instant | ||
|
|
||
| /** | ||
| * Transforms a repository for Knora PR 2081. | ||
| * Fixes wrong date serialisations (all `xsd:dateTime` in the database should end on `Z` rather than specifying a time zone). | ||
| */ | ||
| class UpgradePluginPR2081(featureFactoryConfig: FeatureFactoryConfig, log: Logger) extends UpgradePlugin { | ||
| private val nodeFactory: RdfNodeFactory = RdfFeatureFactory.getRdfNodeFactory(featureFactoryConfig) | ||
|
|
||
| override def transform(model: RdfModel): Unit = { | ||
| val statementsToRemove: collection.mutable.Set[Statement] = collection.mutable.Set.empty | ||
| val statementsToAdd: collection.mutable.Set[Statement] = collection.mutable.Set.empty | ||
|
|
||
| val newObjectValue: String => DatatypeLiteral = (in: String) => | ||
| nodeFactory.makeDatatypeLiteral(Instant.parse(in).toString, OntologyConstants.Xsd.DateTime) | ||
| val shouldTransform: DatatypeLiteral => Boolean = (literal: DatatypeLiteral) => | ||
| (literal.datatype == OntologyConstants.Xsd.DateTime && | ||
| literal.value != newObjectValue(literal.value).value) | ||
|
|
||
| for (statement: Statement <- model) { | ||
| statement.obj match { | ||
| case literal: DatatypeLiteral if shouldTransform(literal) => | ||
| val newValue = newObjectValue(literal.value) | ||
| log.debug(s"Transformed ${literal.value} => ${newValue.value}") | ||
| statementsToRemove += statement | ||
| statementsToAdd += nodeFactory.makeStatement( | ||
| subj = statement.subj, | ||
| pred = statement.pred, | ||
| obj = newValue, | ||
| context = statement.context | ||
| ) | ||
| case other => () | ||
| } | ||
| } | ||
|
|
||
| model.removeStatements(statementsToRemove.toSet) | ||
| model.addStatements(statementsToAdd.toSet) | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
76 changes: 76 additions & 0 deletions
76
...st/scala/org/knora/webapi/store/triplestore/upgrade/plugins/UpgradePluginPR2081Spec.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| /* | ||
| * Copyright © 2021 - 2022 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.knora.webapi.store.triplestore.upgrade.plugins | ||
|
|
||
| import com.typesafe.scalalogging.LazyLogging | ||
| import dsp.errors.AssertionException | ||
| import org.knora.webapi.messages.OntologyConstants | ||
| import org.knora.webapi.messages.util.rdf._ | ||
|
|
||
| class UpgradePluginPR2081Spec extends UpgradePluginSpec with LazyLogging { | ||
| private val nodeFactory: RdfNodeFactory = RdfFeatureFactory.getRdfNodeFactory(defaultFeatureFactoryConfig) | ||
|
|
||
| private def getDateValue(model: RdfModel, subj: IriNode, pred: IriNode): String = { | ||
| val statement = model.find(subj = Some(subj), pred = Some(pred), obj = None).toSet.head | ||
| statement.obj match { | ||
| case literal: DatatypeLiteral if literal.datatype == OntologyConstants.Xsd.DateTime => | ||
| literal.value | ||
| case other => throw AssertionException(s"Unexpected object for $pred: $other") | ||
| } | ||
| } | ||
|
|
||
| "Upgrade plugin PR2081" should { | ||
|
|
||
| "fix invalid date serializations" in { | ||
| val resource1 = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9ma") | ||
| val resource2 = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mb") | ||
| val resource3 = nodeFactory.makeIriNode("http://rdfh.ch/0001/55UrkgTKR2SEQgnsLWI9mc") | ||
| val creationDate = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.CreationDate) | ||
| val lastModificationDate = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.LastModificationDate) | ||
| val deletionDate = nodeFactory.makeIriNode(OntologyConstants.KnoraBase.DeleteDate) | ||
|
|
||
| // Parse the input file. | ||
| val model: RdfModel = trigFileToModel("../test_data/upgrade/pr2081.trig") | ||
|
|
||
| // Store previous values | ||
| val resource1CreationDate = getDateValue(model, resource1, creationDate) | ||
| val resource2CreationDate = getDateValue(model, resource2, creationDate) | ||
| val resource2LastModificationDate = getDateValue(model, resource2, lastModificationDate) | ||
| val resource3CreationDate = getDateValue(model, resource3, creationDate) // only this one should stay the same | ||
| val resource3LastModificationDate = getDateValue(model, resource3, lastModificationDate) | ||
| val resource3DeletionDate = getDateValue(model, resource3, deletionDate) | ||
|
|
||
| // Use the plugin to transform the input. | ||
| val plugin = new UpgradePluginPR2081(defaultFeatureFactoryConfig, log) | ||
| plugin.transform(model) | ||
|
|
||
| // get the new values after transformation | ||
| val newResource1CreationDate = getDateValue(model, resource1, creationDate) | ||
| val newResource2CreationDate = getDateValue(model, resource2, creationDate) | ||
| val newResource2LastModificationDate = getDateValue(model, resource2, lastModificationDate) | ||
| val newResource3CreationDate = | ||
| getDateValue(model, resource3, creationDate) // only this one should have stayed the same | ||
| val newResource3LastModificationDate = getDateValue(model, resource3, lastModificationDate) | ||
| val newResource3DeletionDate = getDateValue(model, resource3, deletionDate) | ||
|
|
||
| // Check that the dates were fixed. | ||
| newResource1CreationDate should not equal (resource1CreationDate) | ||
| newResource1CreationDate should endWith("Z") | ||
|
|
||
| newResource2CreationDate should not equal (resource2CreationDate) | ||
| newResource2CreationDate should endWith("Z") | ||
| newResource2LastModificationDate should not equal (resource2CreationDate) | ||
| newResource2LastModificationDate should endWith("Z") | ||
|
|
||
| newResource3CreationDate should equal(resource3CreationDate) // is equal! | ||
| newResource3CreationDate should endWith("Z") | ||
| newResource3LastModificationDate should not equal (resource3CreationDate) | ||
| newResource3LastModificationDate should endWith("Z") | ||
| newResource3DeletionDate should not equal (resource3CreationDate) | ||
| newResource3DeletionDate should endWith("Z") | ||
| } | ||
| } | ||
| } |