diff --git a/.vscode/settings.json b/.vscode/settings.json index de288e1..e9bc4e6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,12 @@ { - "python.formatting.provider": "black" + "python.formatting.provider": "black", + "cSpell.enableFiletypes": [ + "!css", + "!python", + "!yaml", + "!yml", + "!html" + ], + "isort.check": true, + "isort.importStrategy": "fromEnvironment" } \ No newline at end of file diff --git a/apis_core/apis_entities/tests.py b/apis_core/apis_entities/tests.py index d59077e..866bd13 100644 --- a/apis_core/apis_entities/tests.py +++ b/apis_core/apis_entities/tests.py @@ -5,6 +5,8 @@ from apis_core.apis_entities.forms import get_entities_form from apis_core.apis_entities.models import Person +from apis_core.apis_metainfo.models import Uri +from normdata.forms import NormDataImportForm client = Client() USER = {"username": "testuser", "password": "somepassword"} @@ -128,3 +130,45 @@ def test_010_delete_views(self): self.assertContains(response, "Löschen von") self.assertContains(response, item.id) item.delete() + + def test_010_import_nordmdata_view(self): + client.login(**USER) + payload = { + "normdata_url": "http://lobid.org/gnd/118566512", + "entity_type": "person", + } + url = reverse( + "normdata:import_from_normdata", + ) + response = client.post(url, payload, follow=True) + self.assertEqual(response.status_code, 200) + self.assertTrue(Uri.objects.filter(uri__icontains="118566512")) + payload = { + "normdata_url": "https://www.geonames.org/2772400/linz.html", + "entity_type": "place", + } + response = client.post(url, payload, follow=True) + self.assertEqual(response.status_code, 200) + self.assertTrue(Uri.objects.filter(uri__icontains="2772400")) + + payload = { + "normdata_url": "https://www.wikidata.org/wiki/Q119350694", + "entity_type": "person", + } + response = client.post(url, payload, follow=True) + self.assertEqual(response.status_code, 200) + + payload = { + "normdata_url": "https://www.wikidata.org/wiki/Q119350694", + "entity_type": "person", + } + response = client.post(url, payload, follow=True) + self.assertEqual(response.status_code, 200) + + def test_011_import_normdata_form(self): + payload = { + "normdata_url": "http://lobid.org/gnd/118566512", + "entity_type": "person" + } + form = NormDataImportForm(data=payload) + self.assertTrue(form.is_valid()) diff --git a/apis_core/apis_tei/tests.py b/apis_core/apis_tei/tests.py index 343f610..f436e75 100644 --- a/apis_core/apis_tei/tests.py +++ b/apis_core/apis_tei/tests.py @@ -16,21 +16,24 @@ class TeiTestCase(TestCase): - def setUp(self): # Create two users User.objects.create_user(**USER) Person.objects.create(**BAHR) def test_01_tei_autocomplete(self): - url = reverse("apis:apis_tei:generic_entities_autocomplete", kwargs={"entity": "person"}) + url = reverse( + "apis:apis_tei:generic_entities_autocomplete", kwargs={"entity": "person"} + ) response = client.get(url) self.assertEqual(response.status_code, 200) response = client.get(f"{url}?q=Bahr") self.assertTrue("Hermann" in str(response.content)) def test_02_tei_completer_autocomplete(self): - url = reverse("apis:apis_tei:tei_completer_autocomplete", kwargs={"entity": "person"}) + url = reverse( + "apis:apis_tei:tei_completer_autocomplete", kwargs={"entity": "person"} + ) response = client.get(url) self.assertEqual(response.status_code, 200) response = client.get(f"{url}?q=Bahr") diff --git a/normdata/__init__.py b/normdata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/normdata/apps.py b/normdata/apps.py new file mode 100644 index 0000000..74d00ba --- /dev/null +++ b/normdata/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class NormdataConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "normdata" diff --git a/normdata/forms.py b/normdata/forms.py new file mode 100644 index 0000000..aadba31 --- /dev/null +++ b/normdata/forms.py @@ -0,0 +1,14 @@ +from django import forms + + +class NormDataImportForm(forms.Form): + normdata_url = forms.URLField( + label="Normdata URL", + help_text="Zum Beispiel: http://lobid.org/gnd/118566512 oder https://www.geonames.org/2772400/linz.html", + max_length=100, + ) + entity_type = forms.ChoiceField( + label="Entität", + help_text="Wähle die Art der Entität: Person, Ort, ...", + choices=(("person", "Person"), ("place", "Ort")), + ) diff --git a/normdata/migrations/__init__.py b/normdata/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/normdata/templates/normdata/create_from_gnd.html b/normdata/templates/normdata/create_from_gnd.html new file mode 100644 index 0000000..14770a1 --- /dev/null +++ b/normdata/templates/normdata/create_from_gnd.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} +{% load crispy_forms_tags %} +{% load static %} +{% block content %} +
+

Neue Entität aus GND/WikiData/GeoNames importieren

+
+ {% csrf_token %} + {{ form|crispy }} + +
+
+ +{% endblock content %} \ No newline at end of file diff --git a/normdata/urls.py b/normdata/urls.py new file mode 100644 index 0000000..821c244 --- /dev/null +++ b/normdata/urls.py @@ -0,0 +1,14 @@ +from django.urls import path + +from . import views + +app_name = "normdata" + + +urlpatterns = [ + path( + "import-from-normdata/", + views.NormDataImportFormView.as_view(), + name="import_from_normdata", + ) +] diff --git a/normdata/utils.py b/normdata/utils.py new file mode 100644 index 0000000..07ef6c2 --- /dev/null +++ b/normdata/utils.py @@ -0,0 +1,87 @@ +from acdh_id_reconciler import geonames_to_wikidata, gnd_to_wikidata +from acdh_wikidata_pyutils import WikiDataPerson, WikiDataPlace +from AcdhArcheAssets.uri_norm_rules import get_normalized_uri +from django.core.exceptions import ObjectDoesNotExist + +from apis_core.apis_entities.models import Person, Place +from apis_core.apis_metainfo.models import Uri +from dumper.utils import DOMAIN_MAPPING + + +def get_uri_domain(uri): + for x in DOMAIN_MAPPING: + if x[0] in uri: + return x[1] + + +def import_from_wikidata(wikidata_url, entity_type): + if entity_type == "person": + wd_entity = WikiDataPerson(wikidata_url) + apis_entity = wd_entity.get_apis_entity() + entity = Person.objects.create(**apis_entity) + Uri.objects.create( + uri=get_normalized_uri(wikidata_url), + domain="wikidata", + entity=entity, + ) + if wd_entity.gnd_uri: + Uri.objects.create( + uri=get_normalized_uri(wd_entity.gnd_uri), + domain="gnd", + entity=entity, + ) + else: + wd_entity = WikiDataPlace(wikidata_url) + apis_entity = wd_entity.get_apis_entity() + entity = Place.objects.create(**apis_entity) + Uri.objects.create( + uri=get_normalized_uri(wikidata_url), + domain="wikidata", + entity=entity, + ) + if wd_entity.gnd_uri: + Uri.objects.create( + uri=get_normalized_uri(wd_entity.gnd_uri), + domain="gnd", + entity=entity, + ) + if wd_entity.geonames_uri: + Uri.objects.create( + uri=get_normalized_uri(wd_entity.geonames_uri), + domain="geonames", + entity=entity, + ) + return entity + + +def import_from_normdata(raw_url, entity_type): + normalized_url = get_normalized_uri(raw_url) + try: + entity = Uri.objects.get(uri=normalized_url).entity + return entity + except ObjectDoesNotExist: + pass + domain = get_uri_domain(normalized_url) + if domain == "gnd": + try: + wikidata_url = gnd_to_wikidata(normalized_url)["wikidata"] + except (IndexError, KeyError): + wikidata_url = False + elif domain == "geonames": + try: + wikidata_url = geonames_to_wikidata(normalized_url)["wikidata"] + except (IndexError, KeyError): + wikidata_url = False + elif domain == "wikidata": + wikidata_url = normalized_url + else: + wikidata_url = False + if wikidata_url: + try: + entity = Uri.objects.get(uri=normalized_url).entity + return entity + except ObjectDoesNotExist: + entity = import_from_wikidata(wikidata_url, entity_type) + else: + entity = None + return entity diff --git a/normdata/views.py b/normdata/views.py new file mode 100644 index 0000000..2834b84 --- /dev/null +++ b/normdata/views.py @@ -0,0 +1,19 @@ +from django.urls import reverse +from django.views.generic.edit import FormView + +from .forms import NormDataImportForm +from .utils import import_from_normdata + + +class NormDataImportFormView(FormView): + template_name = "normdata/create_from_gnd.html" + form_class = NormDataImportForm + + def get_success_url(self): + return reverse("apis:apis_entities:person_list_view") + + def form_valid(self, form): + raw_url = form.data["normdata_url"] + entity_type = form.data["entity_type"] + import_from_normdata(raw_url, entity_type) + return super().form_valid(form) diff --git a/pmb/settings.py b/pmb/settings.py index c95b8c7..9c55cdd 100644 --- a/pmb/settings.py +++ b/pmb/settings.py @@ -45,6 +45,7 @@ "apis_core.apis_vocabularies", "apis_core.apis_labels", "apis_core.apis_tei", + "normdata", "dumper", "archemd", ] diff --git a/pmb/urls.py b/pmb/urls.py index 37ad0d3..8e3dbf1 100644 --- a/pmb/urls.py +++ b/pmb/urls.py @@ -5,6 +5,7 @@ urlpatterns = [ path("apis/", include("apis_core.urls", namespace="apis")), + path("normdata/", include("normdata.urls", namespace="normdata")), path("admin/", admin.site.urls), path("arche/", include("archemd.urls")), path("", include("dumper.urls", namespace="dumper")), diff --git a/requirements.txt b/requirements.txt index b0ec769..b3614e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ apis-override-select2js==0.1 acdh-django-browsing acdh-id-reconciler>=0.2,<1 acdh-tei-pyutils>=0.34,<1 +acdh-wikidata-pyutils>=0.2.1,<1 Django>4.1,<6 django-admin-csvexport django-autocomplete-light @@ -11,6 +12,7 @@ django-model-utils django-tables2 djangorestframework pandas +pylobid psycopg2 pyocclient==0.6 icecream diff --git a/templates/partials/navbar.html b/templates/partials/navbar.html index 7eb5d56..6a27ab1 100644 --- a/templates/partials/navbar.html +++ b/templates/partials/navbar.html @@ -70,6 +70,12 @@ + {% if user.is_authenticated %} +
  • + Neue Entität aus GND/WikiData +
    • + {% endif %} + {% if user.is_authenticated %}