Merge pull request #2 from acdh-oeaw/zotero-support

Management command to cache Zotero data tagged with APIS

apis_ontology/management/commands/update_zotero_data.py

@@ -0,0 +1,54 @@
+import logging
+import os
+
+import pandas as pd
+import requests
+from apis_ontology.models import ZoteroEntry
+from django.core.management.base import BaseCommand
+from tqdm.auto import tqdm
+
+USER = os.environ.get("APIS_BIBSONOMY_USER")
+KEY = os.environ.get("APIS_BIBSONOMY_PASSWORD")
+GROUP = "4394244"
+
+
+QUERY_URL = f"https://api.zotero.org/groups/{GROUP}/items"
+HEADERS = {"Authorization": f"Bearer {KEY}"}
+PARAMS = {"v": 3, "format": "json"}
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--fake", action="store_true", help="Whether to fake or really update"
+        )
+
+    def handle(self, *args, **kwargs):
+        res = requests.get(QUERY_URL, headers=HEADERS, params=PARAMS)
+        res.raise_for_status()
+        num_records = res.headers["Total-Results"]
+
+        all_items = []
+        for start in tqdm(range(0, int(num_records), 25)):
+            PARAMS["start"] = start
+            res = requests.get(QUERY_URL, headers=HEADERS, params=PARAMS)
+
+            res.raise_for_status()
+            items = res.json()
+            all_items.extend(items)
+
+        df = pd.json_normalize(all_items)
+
+        for i, row in tqdm(df.iterrows(), total=df.shape[0]):
+            if "apis" in [tag["tag"].lower() for tag in row["data.tags"]]:
+                zotero_entry, _ = ZoteroEntry.objects.get_or_create(
+                    zoteroId=row["data.key"]
+                )
+                zotero_entry.shortTitle = row["data.shortTitle"]
+                zotero_entry.fullCitation = row["data.title"]
+                zotero_entry.year = row["data.date"]
+                zotero_entry.save()
+
+        print(f"Cached {len(ZoteroEntry.objects.all())} zotero entries.")

apis_ontology/migrations/0031_zoteroentry.py

@@ -0,0 +1,47 @@
+# Generated by Django 4.1.13 on 2023-11-29 11:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("apis_ontology", "0030_merge_20231016_0118"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ZoteroEntry",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "zoteroId",
+                    models.CharField(max_length=255, verbose_name="Zotero ID"),
+                ),
+                (
+                    "shortTitle",
+                    models.TextField(blank=True, null=True, verbose_name="Short title"),
+                ),
+                (
+                    "fullCitation",
+                    models.TextField(
+                        blank=True, null=True, verbose_name="Full Citation"
+                    ),
+                ),
+                (
+                    "year",
+                    models.IntegerField(
+                        blank=True, null=True, verbose_name="Year of publication"
+                    ),
+                ),
+            ],
+        ),
+    ]

apis_ontology/migrations/0032_alter_zoteroentry_year.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.13 on 2023-11-30 12:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("apis_ontology", "0031_zoteroentry"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="zoteroentry",
+            name="year",
+            field=models.CharField(
+                blank=True,
+                max_length=255,
+                null=True,
+                verbose_name="Year of publication",
+            ),
+        ),
+    ]

apis_ontology/models.py

@@ -193,3 +193,12 @@ class Place(TempEntityClass):
     alternative_names = models.TextField(
         blank=True, null=True, verbose_name="Alternative names"
     )
+
+
+class ZoteroEntry(models.Model):
+    zoteroId = models.CharField(max_length=255, verbose_name="Zotero ID")
+    shortTitle = models.TextField(blank=True, null=True, verbose_name="Short title")
+    fullCitation = models.TextField(blank=True, null=True, verbose_name="Full Citation")
+    year = models.CharField(
+        max_length=255, blank=True, null=True, verbose_name="Year of publication"
+    )