Merge pull request #13 from arthur-schnitzler/main

fall back to GND or Geonames if no Wikidata for given Place URI exists

.env

@@ -1,4 +1,4 @@
-POSTGRES_DB=pmb_play
+POSTGRES_DB=pmb
 # NEW_PMB=True
 # POSTGRES_HOST=172.17.0.1
 DEBUG=True

.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+multi_line_output=3

apis_core/apis_entities/tests.py

@@ -2,11 +2,17 @@
 from django.contrib.auth.models import User
 from django.test import Client, TestCase
 from django.urls import reverse
+from icecream import ic
 
 from apis_core.apis_entities.forms import get_entities_form
 from apis_core.apis_entities.models import Person
 from apis_core.apis_metainfo.models import Uri
 from normdata.forms import NormDataImportForm
+from normdata.utils import (
+    get_or_create_person_from_wikidata,
+    get_or_create_place_from_wikidata,
+    import_from_normdata,
+)
 
 client = Client()
 USER = {"username": "testuser", "password": "somepassword"}
@@ -172,3 +178,54 @@ def test_012_import_normdata_form(self):
         }
         form = NormDataImportForm(data=payload)
         self.assertTrue(form.is_valid())
+
+    def test_013_import_normdata_no_wikidata(self):
+        client.login(**USER)
+        payload = {
+            "normdata_url": "https://www.geonames.org/2461492/graret-um-igufen.html",
+            "entity_type": "person",
+        }
+        url = reverse(
+            "normdata:import_from_normdata",
+        )
+        response = client.post(url, payload, follow=True)
+        self.assertEqual(response.status_code, 200)
+
+    def test_014_wikidata_place_exist(self):
+        entity = get_or_create_place_from_wikidata(
+            "http://www.wikidata.org/entity/Q1741"
+        )
+        ic(entity)
+        for x in entity.uri_set.all():
+            entity = get_or_create_place_from_wikidata(x.uri)
+            self.assertTrue(entity)
+
+    def test_015_wikidata_person_exist(self):
+        entity = import_from_normdata("http://lobid.org/gnd/133430553", "person")
+        for x in entity.uri_set.all():
+            entity = get_or_create_person_from_wikidata(x.uri)
+            self.assertTrue(entity)
+
+    def test_016_import_nonsense_geonames(self):
+        client.login(**USER)
+        payload = {
+            "normdata_url": "https://www.geonames.org/2461123321492/graret-um-igufen.html",
+            "entity_type": "place",
+        }
+        url = reverse(
+            "normdata:import_from_normdata",
+        )
+        response = client.post(url, payload, follow=True)
+        self.assertEqual(response.status_code, 404)
+
+    def test_017_import_gndplacewithoutwikidata(self):
+        client.login(**USER)
+        payload = {
+            "normdata_url": "https://d-nb.info/gnd/10053010-2",
+            "entity_type": "place",
+        }
+        url = reverse(
+            "normdata:import_from_normdata",
+        )
+        response = client.post(url, payload, follow=True)
+        self.assertEqual(response.status_code, 200)

normdata/tests.py

@@ -0,0 +1,29 @@
+from django.test import TestCase
+
+from normdata.utils import (
+    get_or_create_place_from_geonames,
+    get_or_create_place_from_gnd,
+)
+
+GEONAMES_URL = "https://www.geonames.org/2461464/graret-oum-sedra.html"
+GND_URL = "http://lobid.org/gnd/4547867-3"
+
+
+class NormdataTestCase(TestCase):
+    def test_001_get_or_create_place_from_geonames(self):
+        entity = get_or_create_place_from_geonames(GEONAMES_URL)
+        self.assertEqual(entity.name, "Graret Oum Sedra")
+        entity = get_or_create_place_from_geonames(GEONAMES_URL)
+        entity.delete()
+
+    def test_002_get_or_create_place_from_gnd(self):
+        entity = get_or_create_place_from_gnd(GND_URL)
+        self.assertEqual(entity.name, "Gramastetten")
+        entity = get_or_create_place_from_gnd(GND_URL)
+        entity.delete()
+
+    def test_002_get_or_create_place_from_gnd_no_coords(self):
+        entity = get_or_create_place_from_gnd("http://lobid.org/gnd/10053010-2")
+        self.assertEqual(entity.name, "Horco Molle")
+        entity = get_or_create_place_from_gnd("http://lobid.org/gnd/10053010-2")
+        entity.delete()

normdata/utils.py

@@ -1,9 +1,12 @@
+from acdh_geonames_utils.gn_client import gn_as_object
 from acdh_id_reconciler import geonames_to_wikidata, gnd_to_wikidata
 from acdh_wikidata_pyutils import WikiDataPerson, WikiDataPlace
 from AcdhArcheAssets.uri_norm_rules import get_normalized_uri
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
 from django.db.utils import IntegrityError
+from icecream import ic
+from pylobid.pylobid import PyLobidPlace
 
 from apis_core.apis_entities.models import Person, Place
 from apis_core.apis_metainfo.models import Uri
@@ -21,6 +24,53 @@ def get_uri_domain(uri):
             return x[1]
 
 
+def get_or_create_place_from_gnd(uri):
+    uri = get_normalized_uri(uri)
+    try:
+        entity = Uri.objects.get(uri=uri).entity
+        entity = Place.objects.get(id=entity.id)
+        return entity
+    except ObjectDoesNotExist:
+        fetched_item = PyLobidPlace(uri)
+        apis_entity = {"name": fetched_item.pref_name}
+        try:
+            lng, lat = fetched_item.coords
+        except ValueError:
+            lng = False
+        if lng:
+            apis_entity["lat"] = lat
+            apis_entity["lng"] = lng
+        entity = Place.objects.create(**apis_entity)
+        Uri.objects.create(
+            uri=uri,
+            domain="gnd",
+            entity=entity,
+        )
+        return entity
+
+
+def get_or_create_place_from_geonames(uri):
+    uri = get_normalized_uri(uri)
+    try:
+        entity = Uri.objects.get(uri=uri).entity
+        entity = Place.objects.get(id=entity.id)
+        return entity
+    except ObjectDoesNotExist:
+        fetched_item = gn_as_object(uri)
+        apis_entity = {
+            "name": fetched_item["name"],
+            "lat": fetched_item["latitude"],
+            "lng": fetched_item["longitude"],
+        }
+        entity = Place.objects.create(**apis_entity)
+        Uri.objects.create(
+            uri=uri,
+            domain="geonames",
+            entity=entity,
+        )
+        return entity
+
+
 def get_or_create_place_from_wikidata(uri):
     try:
         entity = Uri.objects.get(uri=uri).entity
@@ -116,7 +166,7 @@ def get_or_create_person_from_wikidata(uri):
 
 
 def import_from_normdata(raw_url, entity_type):
-    normalized_url = get_normalized_uri(raw_url)
+    normalized_url = get_normalized_uri(raw_url.strip())
     try:
         entity = Uri.objects.get(uri=normalized_url).entity
         return entity
@@ -127,12 +177,24 @@ def import_from_normdata(raw_url, entity_type):
         try:
             wikidata_url = gnd_to_wikidata(normalized_url)["wikidata"]
         except (IndexError, KeyError):
+            if entity_type == "place":
+                try:
+                    entity = get_or_create_place_from_gnd(normalized_url)
+                    return entity
+                except Exception as e:
+                    ic(e)
+                    wikidata_url = False
             wikidata_url = False
     elif domain == "geonames":
         try:
             wikidata_url = geonames_to_wikidata(normalized_url)["wikidata"]
         except (IndexError, KeyError):
-            wikidata_url = False
+            try:
+                entity = get_or_create_place_from_geonames(normalized_url)
+                return entity
+            except Exception as e:
+                ic(e)
+                wikidata_url = False
     elif domain == "wikidata":
         wikidata_url = normalized_url
     else:

normdata/views.py

@@ -1,4 +1,5 @@
-from django.urls import reverse
+from django.http import HttpResponseNotFound
+from django.shortcuts import redirect
 from django.views.generic.edit import FormView
 
 from .forms import NormDataImportForm
@@ -9,11 +10,15 @@ class NormDataImportFormView(FormView):
     template_name = "normdata/create_from_gnd.html"
     form_class = NormDataImportForm
 
-    def get_success_url(self):
-        return reverse("apis:apis_entities:person_list_view")
-
     def form_valid(self, form):
         raw_url = form.data["normdata_url"]
         entity_type = form.data["entity_type"]
-        import_from_normdata(raw_url, entity_type)
-        return super().form_valid(form)
+        temp_ent = import_from_normdata(raw_url, entity_type)
+        if temp_ent is not None:
+            entity = temp_ent.get_child_entity()
+            redirect_url = entity.get_edit_url()
+            return redirect(redirect_url)
+        else:
+            return HttpResponseNotFound(
+                f"<h1>Error</h1><p>Zu <strong>{raw_url}</strong> konnte keine Entität angelegt werden</p>"
+            )

requirements.txt

@@ -1,5 +1,6 @@
 apis-override-select2js==0.1
 acdh-django-browsing
+acdh_geonames_utils
 acdh-id-reconciler>=0.2,<1
 acdh-tei-pyutils>=0.34,<1
 acdh-wikidata-pyutils>=0.2.1,<1