Merge pull request #42 from arthur-schnitzler/main

tests and fixes (wikidata import)

.coveragerc

@@ -1,5 +1,5 @@
 [run]
 branch = True
 omit = 
-    dumper/management/commands/labels_to_uris.py
-    dumper/management/commands/add_gn_feature_codes.py
+    ./dumper/management/commands/*
+    manage.py

.gitignore

@@ -174,4 +174,5 @@ media/listperson.xml
 media/pmb-log.csv
 media/listorg.xml
 media/listbibl.xml
-staticfiles/
+staticfiles/
+hansi.csv

apis_core/apis_entities/tests.py

@@ -338,3 +338,37 @@ def test_023_save_uris(self):
         entity = Person.objects.last()
         new_uri = Uri.objects.create(uri=wd_uri, entity=entity)
         self.assertEqual(new_uri.uri, get_normalized_uri(wd_uri))
+
+    def test_024_autocompletes(self):
+        models = ["person", "place", "event", "institution", "work"]
+        for x in models:
+            url = reverse(
+                "apis:apis_entities:generic_entities_autocomplete", kwargs={"entity": x}
+            )
+            r = client.get(f"{url}?q=a")
+            self.assertEqual(r.status_code, 200)
+
+    def test_024_vocabs_dl(self):
+        models = ["placetype", "worktype"]
+        client.login(**USER)
+        for x in models:
+            url = reverse("apis:apis_vocabularies:dl-vocabs", kwargs={"model_name": x})
+            r = client.get(f"{url}")
+            self.assertEqual(r.status_code, 200)
+
+    def test_025_vocabs_ac(self):
+        models = ["placetype", "worktype", "personplacerelation", "personworkrelation"]
+        client = Client()
+        for x in models:
+            for y in ["normal", "reverse"]:
+                if y == "reverse" and x.endswith("type"):
+                    continue
+                r = None
+                url = reverse(
+                    "apis:apis_vocabularies:generic_vocabularies_autocomplete",
+                    kwargs={"vocab": x, "direct": y},
+                )
+                r = client.get(f"{url}")
+                self.assertEqual(r.status_code, 200)
+                r = client.get(f"{url}?q=g")
+                self.assertEqual(r.status_code, 200)

apis_core/apis_tei/tests.py

@@ -16,8 +16,11 @@
 
 
 class TeiTestCase(TestCase):
+    fixtures = [
+        "db.json",
+    ]
+
     def setUp(self):
-        # Create two users
         User.objects.create_user(**USER)
         Person.objects.create(**BAHR)
 
@@ -38,3 +41,17 @@ def test_02_tei_completer_autocomplete(self):
         self.assertEqual(response.status_code, 200)
         response = client.get(f"{url}?q=Bahr")
         self.assertTrue("Hermann" in str(response.content))
+
+    def test_03_tei_views(self):
+        for x in MODELS:
+            item = x.objects.first()
+            try:
+                url = item.get_tei_url()
+            except AttributeError:
+                url = False
+            if url:
+                try:
+                    response = client.get(url)
+                except TypeError:
+                    continue
+                self.assertEqual(response.status_code, 200)

archemd/tests.py

@@ -1,11 +1,9 @@
 from django.apps import apps
-from django.contrib.auth.models import User
 from django.test import Client, TestCase
 from django.urls import reverse
 
 from apis_core.apis_entities.models import Person, Place
 from archemd.arche_md_utils import ArcheMd
-from normdata.utils import import_from_normdata
 
 client = Client()
 USER = {"username": "testuser", "password": "somepassword"}
@@ -23,11 +21,9 @@
 
 
 class ArcheMedTestCase(TestCase):
-    def setUp(self):
-        User.objects.create_user(**USER)
-        Person.objects.create(**BAHR)
-        for x in URIS:
-            import_from_normdata(x[0], x[1])
+    fixtures = [
+        "db.json",
+    ]
 
     def test_01_archemd(self):
         item = Person.objects.last()

dumper/tests.py

@@ -1,67 +1,43 @@
-import os
-
 from django.apps import apps
 from django.conf import settings
-from django.contrib.auth.models import User
-from django.core.management import call_command
 from django.test import Client, TestCase
 from django.urls import reverse
 
-from apis_core.apis_entities.models import Person
-from apis_core.apis_metainfo.models import Uri
+from dumper.utils import gsheet_to_df, process_beacon
 from normdata.utils import import_from_normdata
 
+
 client = Client()
 USER = {"username": "testuser", "password": "password", "is_active": True}
-BAHR = {"name": "Bahr", "first_name": "Hermann", "start_date_written": "1900"}
+BAHR = {"name": "Schnitzler", "first_name": "Hermann", "start_date_written": "1900"}
 
 ENTITY_TYPES = ["person", "place", "event", "work", "institution"]
 
 MODELS = list(apps.all_models["apis_entities"].values())
 MEDIA_ROOT = settings.MEDIA_ROOT
 
-URIS = [
-    ("https://www.geonames.org/2772400/linz.html", "place"),
-    ("https://www.wikidata.org/wiki/Q100965214", "person"),
-    ("https://d-nb.info/gnd/118609807", "person"),
-    ("https://d-nb.info/gnd/118757393", "person"),
-]
 
+TEST_CSV = "hansi.csv"
 
-class DumperTestCase(TestCase):
-    def setUp(self):
-        User.objects.create_user(**USER)
-        Person.objects.create(**BAHR)
-        for x in URIS:
-            import_from_normdata(x[0], x[1])
 
-    def test_01_dump_data(self):
-        call_command("dump_entities", "--limit")
-        with open(os.path.join(MEDIA_ROOT, "listperson.xml"), "r") as f:
-            data = f.read()
-        self.assertTrue("Schnitzler" in data)
+class DumperTestCase(TestCase):
+    fixtures = [
+        "db.json",
+    ]
 
-    def test_02_wikipedia_minter(self):
-        call_command("wikipedia_minter")
-        uris = Uri.objects.filter(domain="wikipedia")
-        self.assertTrue(uris.count())
+    def test_01_dl_gsheet(self):
+        sheet_id = "1QVb62GiWx9MdEGUNTKaFsZ10rfV-s-PJW3QmVKG2EUQ"
+        df = gsheet_to_df(sheet_id)
+        df.to_csv(TEST_CSV, index=False)
+        self.assertTrue("foo" in list(df.keys()))
 
     def test_03_beacon(self):
+        import_from_normdata("https://d-nb.info/gnd/118757393", "person")
         domain = "thun-korrespondenz"
-        call_command(
-            "process_beacon",
-            beacon="https://thun-korrespondenz.acdh.oeaw.ac.at/beacon.txt",
-            domain=domain,
+        processed = process_beacon(
+            "https://thun-korrespondenz.acdh.oeaw.ac.at/beacon.txt", domain=domain
         )
-        uris = Uri.objects.filter(domain=domain)
-        self.assertTrue(uris.count())
-
-    def test_04_wikidata_minter(self):
-        Uri.objects.filter(domain="wikidata").first().delete()
-        before_uris = Uri.objects.all().count()
-        call_command("wikidata_minter")
-        after_uris = Uri.objects.all().count()
-        self.assertTrue(before_uris < after_uris)
+        self.assertTrue(processed > 0)
 
     def test_05_startpage(self):
         url = reverse("dumper:home")

dumper/urls.py

@@ -1,4 +1,3 @@
-from django.conf import settings
 from django.urls import path
 
 from dumper import views
@@ -12,8 +11,3 @@
     path("login/", views.user_login, name="user_login"),
     path("logout/", views.user_logout, name="user_logout"),
 ]
-
-if settings.DEBUG:
-    from django.conf.urls.static import static
-
-    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

dumper/utils.py

@@ -36,7 +36,7 @@
 
 def upload_files_to_owncloud(
     file_list, user=settings.OWNCLOUD_USER, pw=settings.OWNCLOUD_PW, folder=PMB_ENTITIES
-):
+):  # pragma: no cover
     collection = folder
     oc = owncloud.Client("https://oeawcloud.oeaw.ac.at")
     oc.login(user, pw)
@@ -54,7 +54,7 @@ def upload_files_to_owncloud(
         oc.put_file(owncloud_name, x)
 
 
-def write_report(report, report_file=settings.PMB_LOG_FILE):
+def write_report(report, report_file=settings.PMB_LOG_FILE):  # pragma: no cover
     with open(report_file, "a") as f:
         f.write(f'{",".join(report)}\n')
         return "done"

pmb/settings.py

@@ -18,6 +18,7 @@
     DEBUG = False
 
 ALLOWED_HOSTS = ["*"]
+FIXTURE_DIRS = [os.path.join(BASE_DIR, "fixtures")]
 
 APIS_LIST_VIEWS_ALLOWED = True
 APIS_DETAIL_VIEWS_ALLOWED = True

requirements.txt

@@ -2,7 +2,7 @@ acdh-django-browsing
 acdh_geonames_utils
 acdh-id-reconciler>=0.2,<1
 acdh-tei-pyutils>=0.34,<1
-acdh-wikidata-pyutils>=0.2.1,<1
+acdh-wikidata-pyutils>=0.4,<1
 apis-override-select2js==0.1
 Django>4.1,<6
 django-admin-csvexport