Merge pull request #35 from arthur-schnitzler/33-import-place-from-gn…

…d-if-no-matching-wikidata-id-exists

if no wikidata for gnd-place, use gnd

apis_core/apis_entities/detail_views.py

@@ -9,10 +9,7 @@
 from apis_core.apis_labels.models import Label
 from apis_core.apis_metainfo.models import Uri
 from apis_core.apis_relations.models import AbstractRelation
-from apis_core.apis_relations.tables import (
-    LabelTableBase,
-    get_generic_relations_table
-)
+from apis_core.apis_relations.tables import LabelTableBase, get_generic_relations_table
 
 from .models import BASE_URI, TempEntityClass
 

apis_core/apis_entities/list_view_event.py

@@ -13,7 +13,7 @@
     EventWorkRelation,
     InstitutionEventRelation,
     PersonEventRelation,
-    PlaceEventRelation
+    PlaceEventRelation,
 )
 from apis_core.helper_functions.utils import get_child_classes
 from browsing.browsing_utils import GenericListView

apis_core/apis_entities/list_view_institution.py

@@ -13,7 +13,7 @@
     InstitutionPlaceRelation,
     InstitutionType,
     InstitutionWorkRelation,
-    PersonInstitutionRelation
+    PersonInstitutionRelation,
 )
 from apis_core.helper_functions.utils import get_child_classes
 from browsing.browsing_utils import GenericListView

apis_core/apis_entities/list_view_person.py

@@ -12,7 +12,7 @@
     PersonPersonRelation,
     PersonPlaceRelation,
     PersonWorkRelation,
-    ProfessionType
+    ProfessionType,
 )
 from apis_core.helper_functions.utils import get_child_classes
 from browsing.browsing_utils import GenericListView

apis_core/apis_entities/list_view_place.py

@@ -10,7 +10,7 @@
 from apis_core.apis_vocabularies.models import (
     PersonPlaceRelation,
     PlaceType,
-    PlaceWorkRelation
+    PlaceWorkRelation,
 )
 from apis_core.helper_functions.utils import get_child_classes
 from browsing.browsing_utils import GenericListView

apis_core/apis_entities/list_view_work.py

@@ -12,7 +12,7 @@
     PersonWorkRelation,
     PlaceWorkRelation,
     WorkType,
-    WorkWorkRelation
+    WorkWorkRelation,
 )
 from apis_core.helper_functions.utils import get_child_classes
 from browsing.browsing_utils import GenericListView

apis_core/apis_entities/models.py

@@ -16,7 +16,7 @@
     PlaceType,
     ProfessionType,
     Title,
-    WorkType
+    WorkType,
 )
 from apis_core.helper_functions import EntityRelationFieldGenerator
 

apis_core/apis_entities/tests.py

@@ -11,7 +11,7 @@
 from normdata.utils import (
     get_or_create_person_from_wikidata,
     get_or_create_place_from_wikidata,
-    import_from_normdata
+    import_from_normdata,
 )
 
 client = Client()
@@ -217,3 +217,15 @@ def test_016_import_nonsense_geonames(self):
         )
         response = client.post(url, payload, follow=True)
         self.assertEqual(response.status_code, 404)
+
+    def test_017_import_gndplacewithoutwikidata(self):
+        client.login(**USER)
+        payload = {
+            "normdata_url": "https://d-nb.info/gnd/10053010-2",
+            "entity_type": "place",
+        }
+        url = reverse(
+            "normdata:import_from_normdata",
+        )
+        response = client.post(url, payload, follow=True)
+        self.assertEqual(response.status_code, 200)

apis_core/apis_entities/views.py

@@ -18,10 +18,7 @@
 from apis_core.apis_labels.models import Label
 from apis_core.apis_metainfo.models import Uri
 from apis_core.apis_relations.models import AbstractRelation
-from apis_core.apis_relations.tables import (
-    LabelTableEdit,
-    get_generic_relations_table
-)
+from apis_core.apis_relations.tables import LabelTableEdit, get_generic_relations_table
 
 from .forms import MergeForm, get_entities_form
 

apis_core/apis_metainfo/api_views.py

@@ -6,7 +6,7 @@
     SourceSerializer,
     TempEntityClassSerializer,
     TextSerializer,
-    UriSerializer
+    UriSerializer,
 )
 
 

apis_core/apis_metainfo/models.py

@@ -11,11 +11,7 @@
 from model_utils.managers import InheritanceManager
 
 from apis_core.apis_labels.models import Label
-from apis_core.apis_vocabularies.models import (
-    CollectionType,
-    LabelType,
-    TextType
-)
+from apis_core.apis_vocabularies.models import CollectionType, LabelType, TextType
 from apis_core.helper_functions import DateParser
 
 

apis_core/apis_relations/autocomplete_light_registry.py

@@ -9,21 +9,10 @@
 import requests
 from django.db.models import Q
 
-from apis_core.apis_entities.models import (
-    Event,
-    Institution,
-    Person,
-    Place,
-    Work
-)
+from apis_core.apis_entities.models import Event, Institution, Person, Place, Work
 from apis_core.apis_metainfo.models import Uri
-from apis_core.apis_relations.models import (
-    PersonInstitution,
-    PersonPerson,
-    PersonPlace
-)
-from apis_core.default_settings.NER_settings import \
-    autocomp_settings as ac_settings
+from apis_core.apis_relations.models import PersonInstitution, PersonPerson, PersonPlace
+from apis_core.default_settings.NER_settings import autocomp_settings as ac_settings
 
 
 class StanbolAutocompleteBase(al.AutocompleteListTemplate):

apis_core/apis_relations/forms2.py

@@ -14,6 +14,7 @@
 
 from apis_core.apis_entities.fields import ListSelect2
 from apis_core.apis_entities.models import AbstractEntity
+
 # from dal.autocomplete import ListSelect2
 from apis_core.apis_metainfo.models import TempEntityClass, Uri
 from apis_core.apis_relations.models import AbstractRelation

apis_core/apis_relations/serializers.py

@@ -1,12 +1,7 @@
 from rest_framework import serializers
 
-from apis_core.apis_entities.serializers import (
-    PersonSerializer,
-    PlaceSerializer
-)
-from apis_core.apis_vocabularies.serializers import (
-    PersonPlaceRelationSerializer
-)
+from apis_core.apis_entities.serializers import PersonSerializer, PlaceSerializer
+from apis_core.apis_vocabularies.serializers import PersonPlaceRelationSerializer
 
 from .models import (
     EventEvent,
@@ -23,7 +18,7 @@
     PlaceEvent,
     PlacePlace,
     PlaceWork,
-    WorkWork
+    WorkWork,
 )
 
 

apis_core/apis_relations/tables.py

@@ -8,7 +8,7 @@
     generic_order_end_date_written,
     generic_order_start_date_written,
     generic_render_end_date_written,
-    generic_render_start_date_written
+    generic_render_start_date_written,
 )
 from apis_core.apis_relations.models import AbstractRelation
 

apis_core/apis_relations/views.py

@@ -12,7 +12,7 @@
     Institution,
     Person,
     Place,
-    Work
+    Work,
 )
 from apis_core.apis_labels.models import Label
 from apis_core.apis_metainfo.models import TempEntityClass, Uri
@@ -34,7 +34,7 @@
     PlaceEvent,
     PlacePlace,
     PlaceWork,
-    WorkWork
+    WorkWork,
 )
 from .tables import LabelTableEdit
 

apis_core/apis_tei/tei.py

@@ -6,13 +6,7 @@
 from django.conf import settings
 from django.utils.text import slugify
 
-from apis_core.apis_entities.models import (
-    Event,
-    Institution,
-    Person,
-    Place,
-    Work
-)
+from apis_core.apis_entities.models import Event, Institution, Person, Place, Work
 from apis_core.apis_metainfo.models import Text
 
 from .partials import TEI_NSMAP, tei_gen_header

apis_core/apis_vocabularies/api_views.py

@@ -25,7 +25,7 @@
     VocabNames,
     VocabsBaseClass,
     WorkType,
-    WorkWorkRelation
+    WorkWorkRelation,
 )
 from .serializers import (
     CollectionTypeSerializer,
@@ -52,7 +52,7 @@
     VocabNamesSerializer,
     VocabsBaseClassSerializer,
     WorkTypeSerializer,
-    WorkWorkRelationSerializer
+    WorkWorkRelationSerializer,
 )
 
 ###########################################################

apis_core/apis_vocabularies/serializers.py

@@ -27,7 +27,7 @@
     VocabNames,
     VocabsBaseClass,
     WorkType,
-    WorkWorkRelation
+    WorkWorkRelation,
 )
 
 ###########################################################

apis_core/apis_vocabularies/urls.py

@@ -1,8 +1,6 @@
 from django.urls import path
 
-from apis_core.apis_entities.autocomplete3 import (
-    GenericVocabulariesAutocomplete
-)
+from apis_core.apis_entities.autocomplete3 import GenericVocabulariesAutocomplete
 from apis_core.apis_vocabularies import views
 
 app_name = "apis_vocabularies"

apis_core/openrefine/views.py

@@ -10,7 +10,7 @@
     Person,
     Place,
     TempEntityClass,
-    Work
+    Work,
 )
 
 from .utils import get_properties, get_service_mainfest

dumper/views.py

@@ -3,13 +3,7 @@
 from django.shortcuts import render
 from django.views.generic.base import TemplateView
 
-from apis_core.apis_entities.models import (
-    Event,
-    Institution,
-    Person,
-    Place,
-    Work
-)
+from apis_core.apis_entities.models import Event, Institution, Person, Place, Work
 from apis_core.apis_metainfo.models import Uri
 
 from .forms import form_user_login

normdata/tests.py

@@ -1,8 +1,12 @@
 from django.test import TestCase
 
-from normdata.utils import get_or_create_place_from_geonames
+from normdata.utils import (
+    get_or_create_place_from_geonames,
+    get_or_create_place_from_gnd,
+)
 
 GEONAMES_URL = "https://www.geonames.org/2461464/graret-oum-sedra.html"
+GND_URL = "http://lobid.org/gnd/4547867-3"
 
 
 class NormdataTestCase(TestCase):
@@ -11,3 +15,15 @@ def test_001_get_or_create_place_from_geonames(self):
         self.assertEqual(entity.name, "Graret Oum Sedra")
         entity = get_or_create_place_from_geonames(GEONAMES_URL)
         entity.delete()
+
+    def test_002_get_or_create_place_from_gnd(self):
+        entity = get_or_create_place_from_gnd(GND_URL)
+        self.assertEqual(entity.name, "Gramastetten")
+        entity = get_or_create_place_from_gnd(GND_URL)
+        entity.delete()
+
+    def test_002_get_or_create_place_from_gnd_no_coords(self):
+        entity = get_or_create_place_from_gnd("http://lobid.org/gnd/10053010-2")
+        self.assertEqual(entity.name, "Horco Molle")
+        entity = get_or_create_place_from_gnd("http://lobid.org/gnd/10053010-2")
+        entity.delete()

normdata/utils.py

@@ -6,6 +6,7 @@
 from django.core.exceptions import ObjectDoesNotExist
 from django.db.utils import IntegrityError
 from icecream import ic
+from pylobid.pylobid import PyLobidPlace
 
 from apis_core.apis_entities.models import Person, Place
 from apis_core.apis_metainfo.models import Uri
@@ -23,18 +24,43 @@ def get_uri_domain(uri):
             return x[1]
 
 
+def get_or_create_place_from_gnd(uri):
+    uri = get_normalized_uri(uri)
+    try:
+        entity = Uri.objects.get(uri=uri).entity
+        entity = Place.objects.get(id=entity.id)
+        return entity
+    except ObjectDoesNotExist:
+        fetched_item = PyLobidPlace(uri)
+        apis_entity = {"name": fetched_item.pref_name}
+        try:
+            lng, lat = fetched_item.coords
+        except ValueError:
+            lng = False
+        if lng:
+            apis_entity["lat"] = lat
+            apis_entity["lng"] = lng
+        entity = Place.objects.create(**apis_entity)
+        Uri.objects.create(
+            uri=uri,
+            domain="gnd",
+            entity=entity,
+        )
+        return entity
+
+
 def get_or_create_place_from_geonames(uri):
     uri = get_normalized_uri(uri)
     try:
         entity = Uri.objects.get(uri=uri).entity
         entity = Place.objects.get(id=entity.id)
         return entity
     except ObjectDoesNotExist:
-        geonames_obj = gn_as_object(uri)
+        fetched_item = gn_as_object(uri)
         apis_entity = {
-            "name": geonames_obj["name"],
-            "lat": geonames_obj["latitude"],
-            "lng": geonames_obj["longitude"],
+            "name": fetched_item["name"],
+            "lat": fetched_item["latitude"],
+            "lng": fetched_item["longitude"],
         }
         entity = Place.objects.create(**apis_entity)
         Uri.objects.create(
@@ -151,6 +177,13 @@ def import_from_normdata(raw_url, entity_type):
         try:
             wikidata_url = gnd_to_wikidata(normalized_url)["wikidata"]
         except (IndexError, KeyError):
+            if entity_type == "place":
+                try:
+                    entity = get_or_create_place_from_gnd(normalized_url)
+                    return entity
+                except Exception as e:
+                    ic(e)
+                    wikidata_url = False
             wikidata_url = False
     elif domain == "geonames":
         try: