diff --git a/apis_core/apis_entities/detail_views.py b/apis_core/apis_entities/detail_views.py index 94d1844..36c6e99 100644 --- a/apis_core/apis_entities/detail_views.py +++ b/apis_core/apis_entities/detail_views.py @@ -9,10 +9,7 @@ from apis_core.apis_labels.models import Label from apis_core.apis_metainfo.models import Uri from apis_core.apis_relations.models import AbstractRelation -from apis_core.apis_relations.tables import ( - LabelTableBase, - get_generic_relations_table -) +from apis_core.apis_relations.tables import LabelTableBase, get_generic_relations_table from .models import BASE_URI, TempEntityClass diff --git a/apis_core/apis_entities/list_view_event.py b/apis_core/apis_entities/list_view_event.py index c538265..0d27211 100644 --- a/apis_core/apis_entities/list_view_event.py +++ b/apis_core/apis_entities/list_view_event.py @@ -13,7 +13,7 @@ EventWorkRelation, InstitutionEventRelation, PersonEventRelation, - PlaceEventRelation + PlaceEventRelation, ) from apis_core.helper_functions.utils import get_child_classes from browsing.browsing_utils import GenericListView diff --git a/apis_core/apis_entities/list_view_institution.py b/apis_core/apis_entities/list_view_institution.py index fd3bf9c..243121c 100644 --- a/apis_core/apis_entities/list_view_institution.py +++ b/apis_core/apis_entities/list_view_institution.py @@ -13,7 +13,7 @@ InstitutionPlaceRelation, InstitutionType, InstitutionWorkRelation, - PersonInstitutionRelation + PersonInstitutionRelation, ) from apis_core.helper_functions.utils import get_child_classes from browsing.browsing_utils import GenericListView diff --git a/apis_core/apis_entities/list_view_person.py b/apis_core/apis_entities/list_view_person.py index 8954c56..3763c34 100644 --- a/apis_core/apis_entities/list_view_person.py +++ b/apis_core/apis_entities/list_view_person.py @@ -12,7 +12,7 @@ PersonPersonRelation, PersonPlaceRelation, PersonWorkRelation, - ProfessionType + ProfessionType, ) from apis_core.helper_functions.utils import get_child_classes from browsing.browsing_utils import GenericListView diff --git a/apis_core/apis_entities/list_view_place.py b/apis_core/apis_entities/list_view_place.py index 2f6ab97..8903cd3 100644 --- a/apis_core/apis_entities/list_view_place.py +++ b/apis_core/apis_entities/list_view_place.py @@ -10,7 +10,7 @@ from apis_core.apis_vocabularies.models import ( PersonPlaceRelation, PlaceType, - PlaceWorkRelation + PlaceWorkRelation, ) from apis_core.helper_functions.utils import get_child_classes from browsing.browsing_utils import GenericListView diff --git a/apis_core/apis_entities/list_view_work.py b/apis_core/apis_entities/list_view_work.py index df99af8..4b8be47 100644 --- a/apis_core/apis_entities/list_view_work.py +++ b/apis_core/apis_entities/list_view_work.py @@ -12,7 +12,7 @@ PersonWorkRelation, PlaceWorkRelation, WorkType, - WorkWorkRelation + WorkWorkRelation, ) from apis_core.helper_functions.utils import get_child_classes from browsing.browsing_utils import GenericListView diff --git a/apis_core/apis_entities/models.py b/apis_core/apis_entities/models.py index e1b8907..8cb447f 100644 --- a/apis_core/apis_entities/models.py +++ b/apis_core/apis_entities/models.py @@ -16,7 +16,7 @@ PlaceType, ProfessionType, Title, - WorkType + WorkType, ) from apis_core.helper_functions import EntityRelationFieldGenerator diff --git a/apis_core/apis_entities/tests.py b/apis_core/apis_entities/tests.py index 6752f69..4f1b711 100644 --- a/apis_core/apis_entities/tests.py +++ b/apis_core/apis_entities/tests.py @@ -11,7 +11,7 @@ from normdata.utils import ( get_or_create_person_from_wikidata, get_or_create_place_from_wikidata, - import_from_normdata + import_from_normdata, ) client = Client() @@ -217,3 +217,15 @@ def test_016_import_nonsense_geonames(self): ) response = client.post(url, payload, follow=True) self.assertEqual(response.status_code, 404) + + def test_017_import_gndplacewithoutwikidata(self): + client.login(**USER) + payload = { + "normdata_url": "https://d-nb.info/gnd/10053010-2", + "entity_type": "place", + } + url = reverse( + "normdata:import_from_normdata", + ) + response = client.post(url, payload, follow=True) + self.assertEqual(response.status_code, 200) diff --git a/apis_core/apis_entities/views.py b/apis_core/apis_entities/views.py index d2ce465..aab2956 100644 --- a/apis_core/apis_entities/views.py +++ b/apis_core/apis_entities/views.py @@ -18,10 +18,7 @@ from apis_core.apis_labels.models import Label from apis_core.apis_metainfo.models import Uri from apis_core.apis_relations.models import AbstractRelation -from apis_core.apis_relations.tables import ( - LabelTableEdit, - get_generic_relations_table -) +from apis_core.apis_relations.tables import LabelTableEdit, get_generic_relations_table from .forms import MergeForm, get_entities_form diff --git a/apis_core/apis_metainfo/api_views.py b/apis_core/apis_metainfo/api_views.py index cc373f4..62dba32 100644 --- a/apis_core/apis_metainfo/api_views.py +++ b/apis_core/apis_metainfo/api_views.py @@ -6,7 +6,7 @@ SourceSerializer, TempEntityClassSerializer, TextSerializer, - UriSerializer + UriSerializer, ) diff --git a/apis_core/apis_metainfo/models.py b/apis_core/apis_metainfo/models.py index 771ffe2..247be98 100644 --- a/apis_core/apis_metainfo/models.py +++ b/apis_core/apis_metainfo/models.py @@ -11,11 +11,7 @@ from model_utils.managers import InheritanceManager from apis_core.apis_labels.models import Label -from apis_core.apis_vocabularies.models import ( - CollectionType, - LabelType, - TextType -) +from apis_core.apis_vocabularies.models import CollectionType, LabelType, TextType from apis_core.helper_functions import DateParser diff --git a/apis_core/apis_relations/autocomplete_light_registry.py b/apis_core/apis_relations/autocomplete_light_registry.py index b64d75d..1155ebb 100644 --- a/apis_core/apis_relations/autocomplete_light_registry.py +++ b/apis_core/apis_relations/autocomplete_light_registry.py @@ -9,21 +9,10 @@ import requests from django.db.models import Q -from apis_core.apis_entities.models import ( - Event, - Institution, - Person, - Place, - Work -) +from apis_core.apis_entities.models import Event, Institution, Person, Place, Work from apis_core.apis_metainfo.models import Uri -from apis_core.apis_relations.models import ( - PersonInstitution, - PersonPerson, - PersonPlace -) -from apis_core.default_settings.NER_settings import \ - autocomp_settings as ac_settings +from apis_core.apis_relations.models import PersonInstitution, PersonPerson, PersonPlace +from apis_core.default_settings.NER_settings import autocomp_settings as ac_settings class StanbolAutocompleteBase(al.AutocompleteListTemplate): diff --git a/apis_core/apis_relations/forms2.py b/apis_core/apis_relations/forms2.py index 09b0206..f7b1a5e 100644 --- a/apis_core/apis_relations/forms2.py +++ b/apis_core/apis_relations/forms2.py @@ -14,6 +14,7 @@ from apis_core.apis_entities.fields import ListSelect2 from apis_core.apis_entities.models import AbstractEntity + # from dal.autocomplete import ListSelect2 from apis_core.apis_metainfo.models import TempEntityClass, Uri from apis_core.apis_relations.models import AbstractRelation diff --git a/apis_core/apis_relations/serializers.py b/apis_core/apis_relations/serializers.py index 57db8a9..75d9695 100644 --- a/apis_core/apis_relations/serializers.py +++ b/apis_core/apis_relations/serializers.py @@ -1,12 +1,7 @@ from rest_framework import serializers -from apis_core.apis_entities.serializers import ( - PersonSerializer, - PlaceSerializer -) -from apis_core.apis_vocabularies.serializers import ( - PersonPlaceRelationSerializer -) +from apis_core.apis_entities.serializers import PersonSerializer, PlaceSerializer +from apis_core.apis_vocabularies.serializers import PersonPlaceRelationSerializer from .models import ( EventEvent, @@ -23,7 +18,7 @@ PlaceEvent, PlacePlace, PlaceWork, - WorkWork + WorkWork, ) diff --git a/apis_core/apis_relations/tables.py b/apis_core/apis_relations/tables.py index a4caa4c..4a1974b 100644 --- a/apis_core/apis_relations/tables.py +++ b/apis_core/apis_relations/tables.py @@ -8,7 +8,7 @@ generic_order_end_date_written, generic_order_start_date_written, generic_render_end_date_written, - generic_render_start_date_written + generic_render_start_date_written, ) from apis_core.apis_relations.models import AbstractRelation diff --git a/apis_core/apis_relations/views.py b/apis_core/apis_relations/views.py index 128d6b4..290b05a 100644 --- a/apis_core/apis_relations/views.py +++ b/apis_core/apis_relations/views.py @@ -12,7 +12,7 @@ Institution, Person, Place, - Work + Work, ) from apis_core.apis_labels.models import Label from apis_core.apis_metainfo.models import TempEntityClass, Uri @@ -34,7 +34,7 @@ PlaceEvent, PlacePlace, PlaceWork, - WorkWork + WorkWork, ) from .tables import LabelTableEdit diff --git a/apis_core/apis_tei/tei.py b/apis_core/apis_tei/tei.py index c11b24c..cfb67e4 100644 --- a/apis_core/apis_tei/tei.py +++ b/apis_core/apis_tei/tei.py @@ -6,13 +6,7 @@ from django.conf import settings from django.utils.text import slugify -from apis_core.apis_entities.models import ( - Event, - Institution, - Person, - Place, - Work -) +from apis_core.apis_entities.models import Event, Institution, Person, Place, Work from apis_core.apis_metainfo.models import Text from .partials import TEI_NSMAP, tei_gen_header diff --git a/apis_core/apis_vocabularies/api_views.py b/apis_core/apis_vocabularies/api_views.py index 479fa81..d8ec410 100644 --- a/apis_core/apis_vocabularies/api_views.py +++ b/apis_core/apis_vocabularies/api_views.py @@ -25,7 +25,7 @@ VocabNames, VocabsBaseClass, WorkType, - WorkWorkRelation + WorkWorkRelation, ) from .serializers import ( CollectionTypeSerializer, @@ -52,7 +52,7 @@ VocabNamesSerializer, VocabsBaseClassSerializer, WorkTypeSerializer, - WorkWorkRelationSerializer + WorkWorkRelationSerializer, ) ########################################################### diff --git a/apis_core/apis_vocabularies/serializers.py b/apis_core/apis_vocabularies/serializers.py index 203e71a..1ad6bf7 100644 --- a/apis_core/apis_vocabularies/serializers.py +++ b/apis_core/apis_vocabularies/serializers.py @@ -27,7 +27,7 @@ VocabNames, VocabsBaseClass, WorkType, - WorkWorkRelation + WorkWorkRelation, ) ########################################################### diff --git a/apis_core/apis_vocabularies/urls.py b/apis_core/apis_vocabularies/urls.py index 10ae8a3..e840527 100644 --- a/apis_core/apis_vocabularies/urls.py +++ b/apis_core/apis_vocabularies/urls.py @@ -1,8 +1,6 @@ from django.urls import path -from apis_core.apis_entities.autocomplete3 import ( - GenericVocabulariesAutocomplete -) +from apis_core.apis_entities.autocomplete3 import GenericVocabulariesAutocomplete from apis_core.apis_vocabularies import views app_name = "apis_vocabularies" diff --git a/apis_core/openrefine/views.py b/apis_core/openrefine/views.py index 18a14f2..2735483 100644 --- a/apis_core/openrefine/views.py +++ b/apis_core/openrefine/views.py @@ -10,7 +10,7 @@ Person, Place, TempEntityClass, - Work + Work, ) from .utils import get_properties, get_service_mainfest diff --git a/dumper/views.py b/dumper/views.py index 92cb086..1ce157a 100644 --- a/dumper/views.py +++ b/dumper/views.py @@ -3,13 +3,7 @@ from django.shortcuts import render from django.views.generic.base import TemplateView -from apis_core.apis_entities.models import ( - Event, - Institution, - Person, - Place, - Work -) +from apis_core.apis_entities.models import Event, Institution, Person, Place, Work from apis_core.apis_metainfo.models import Uri from .forms import form_user_login diff --git a/normdata/tests.py b/normdata/tests.py index 6a4353e..abc2aaf 100644 --- a/normdata/tests.py +++ b/normdata/tests.py @@ -1,8 +1,12 @@ from django.test import TestCase -from normdata.utils import get_or_create_place_from_geonames +from normdata.utils import ( + get_or_create_place_from_geonames, + get_or_create_place_from_gnd, +) GEONAMES_URL = "https://www.geonames.org/2461464/graret-oum-sedra.html" +GND_URL = "http://lobid.org/gnd/4547867-3" class NormdataTestCase(TestCase): @@ -11,3 +15,15 @@ def test_001_get_or_create_place_from_geonames(self): self.assertEqual(entity.name, "Graret Oum Sedra") entity = get_or_create_place_from_geonames(GEONAMES_URL) entity.delete() + + def test_002_get_or_create_place_from_gnd(self): + entity = get_or_create_place_from_gnd(GND_URL) + self.assertEqual(entity.name, "Gramastetten") + entity = get_or_create_place_from_gnd(GND_URL) + entity.delete() + + def test_002_get_or_create_place_from_gnd_no_coords(self): + entity = get_or_create_place_from_gnd("http://lobid.org/gnd/10053010-2") + self.assertEqual(entity.name, "Horco Molle") + entity = get_or_create_place_from_gnd("http://lobid.org/gnd/10053010-2") + entity.delete() diff --git a/normdata/utils.py b/normdata/utils.py index d0a2c55..bd96c96 100644 --- a/normdata/utils.py +++ b/normdata/utils.py @@ -6,6 +6,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.db.utils import IntegrityError from icecream import ic +from pylobid.pylobid import PyLobidPlace from apis_core.apis_entities.models import Person, Place from apis_core.apis_metainfo.models import Uri @@ -23,6 +24,31 @@ def get_uri_domain(uri): return x[1] +def get_or_create_place_from_gnd(uri): + uri = get_normalized_uri(uri) + try: + entity = Uri.objects.get(uri=uri).entity + entity = Place.objects.get(id=entity.id) + return entity + except ObjectDoesNotExist: + fetched_item = PyLobidPlace(uri) + apis_entity = {"name": fetched_item.pref_name} + try: + lng, lat = fetched_item.coords + except ValueError: + lng = False + if lng: + apis_entity["lat"] = lat + apis_entity["lng"] = lng + entity = Place.objects.create(**apis_entity) + Uri.objects.create( + uri=uri, + domain="gnd", + entity=entity, + ) + return entity + + def get_or_create_place_from_geonames(uri): uri = get_normalized_uri(uri) try: @@ -30,11 +56,11 @@ def get_or_create_place_from_geonames(uri): entity = Place.objects.get(id=entity.id) return entity except ObjectDoesNotExist: - geonames_obj = gn_as_object(uri) + fetched_item = gn_as_object(uri) apis_entity = { - "name": geonames_obj["name"], - "lat": geonames_obj["latitude"], - "lng": geonames_obj["longitude"], + "name": fetched_item["name"], + "lat": fetched_item["latitude"], + "lng": fetched_item["longitude"], } entity = Place.objects.create(**apis_entity) Uri.objects.create( @@ -151,6 +177,13 @@ def import_from_normdata(raw_url, entity_type): try: wikidata_url = gnd_to_wikidata(normalized_url)["wikidata"] except (IndexError, KeyError): + if entity_type == "place": + try: + entity = get_or_create_place_from_gnd(normalized_url) + return entity + except Exception as e: + ic(e) + wikidata_url = False wikidata_url = False elif domain == "geonames": try: