diff --git a/apis_core/apis_history/__init__.py b/apis_core/apis_history/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apis_core/apis_history/api_views.py b/apis_core/apis_history/api_views.py new file mode 100644 index 000000000..47dcc1a03 --- /dev/null +++ b/apis_core/apis_history/api_views.py @@ -0,0 +1,31 @@ +from django.http import Http404 +from rest_framework import routers +from rest_framework.viewsets import ReadOnlyModelViewSet +from apis_core.apis_history.serializers import HistoryLogSerializer +from django.contrib.contenttypes.models import ContentType +from drf_spectacular.openapi import OpenApiParameter, OpenApiTypes +from drf_spectacular.utils import extend_schema +from rest_framework import viewsets, mixins +from apis_core.apis_history.utils import get_history + + +class GenericHistoryLogs(viewsets.GenericViewSet, mixins.ListModelMixin): + serializer_class = HistoryLogSerializer + + def get_queryset(self): + id = self.request.query_params.get("id", None) + model = self.request.query_params.get("entity_type", None) + if id is None: + raise Http404("No id provided") + if model is None: + raise Http404("No model provided") + return get_history(model, id) + + @extend_schema( + parameters=[ + OpenApiParameter("id", OpenApiTypes.INT, OpenApiParameter.QUERY), + OpenApiParameter("entity_type", OpenApiTypes.STR, OpenApiParameter.QUERY), + ], + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) diff --git a/apis_core/apis_history/apps.py b/apis_core/apis_history/apps.py new file mode 100644 index 000000000..883a57c8f --- /dev/null +++ b/apis_core/apis_history/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class HistoryConfig(AppConfig): + name = "apis_core.apis_history" diff --git a/apis_core/apis_history/models.py b/apis_core/apis_history/models.py new file mode 100644 index 000000000..395843e48 --- /dev/null +++ b/apis_core/apis_history/models.py @@ -0,0 +1,90 @@ +from collections.abc import Iterable +from typing import Any +import pytz +from simple_history.models import HistoricalRecords +from django.core.exceptions import AppRegistryNotReady +import inspect +import django +from django.db import models +from datetime import datetime +from simple_history import utils +from django.db.models import Q + +# from apis_core.apis_history.mixins import TempTripleHistoryMixin +# from apis_core.apis_relations.models import TempTriple + + +class APISHistoricalRecords(HistoricalRecords): + def get_m2m_fields_from_model(self, model): + # Change the original simple history function to also return m2m fields + m2m_fields = [] + try: + for field in inspect.getmembers(model): + if isinstance( + field[1], + django.db.models.fields.related_descriptors.ManyToManyDescriptor, + ): + m2m_fields.append(getattr(model, field[0]).field) + except AppRegistryNotReady: + pass + return m2m_fields + + def get_prev_record(self): + """ + Get the previous history record for the instance. `None` if first. + """ + history = utils.get_history_manager_from_history(self) + return ( + history.filter(history_date__lt=self.history_date) + .order_by("history_date") + .first() + ) + + +class APISHistoryTableBase(models.Model): + version_tag = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + abstract = True + + def get_triples_for_version(self): + from apis_core.apis_relations.models import TempTriple + + triples = TempTriple.history.as_of(self.history_date).filter( + Q(subj=self.instance) | Q(obj=self.instance) + ) + return triples + + +class VersionMixin(models.Model): + history = APISHistoricalRecords( + inherit=True, + bases=[ + APISHistoryTableBase, + ], + ) + __history_date = None + + @property + def _history_date(self): + return self.__history_date + + @_history_date.setter + def _history_date(self, value): + self.__history_date = value + + class Meta: + abstract = True + + def save(self, *args, **kwargs) -> None: + if self._history_date is None: + self._history_date = datetime.now() + return super().save(*args, **kwargs) + + def delete(self, *args, **kwargs) -> tuple[int, dict[str, int]]: + if self._history_date is None: + self._history_date = datetime.now() + return super().delete(*args, **kwargs) + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) diff --git a/apis_core/apis_history/serializers.py b/apis_core/apis_history/serializers.py new file mode 100644 index 000000000..682bcb929 --- /dev/null +++ b/apis_core/apis_history/serializers.py @@ -0,0 +1,57 @@ +from rest_framework import serializers +from rest_framework.fields import empty +from django.core.exceptions import FieldDoesNotExist + + +class ModelChangeSerializer(serializers.Serializer): + field = serializers.CharField() + old = serializers.SerializerMethodField(method_name="get_field_data_old") + new = serializers.SerializerMethodField(method_name="get_field_data_new") + + def get_field_data(self, obj): + try: + self._parent_object._meta.get_field(obj["field"]) + return obj["value"] + except FieldDoesNotExist: + for field in self._parent_object._history_m2m_fields: + if field.attname == obj["field"]: + sst = [ + str(obj2) + for obj2 in field.related_model.objects.filter( + pk__in=[obj3[obj["field"]] for obj3 in obj["value"]] + ) + ] + return " | ".join(sst) + return obj + + def get_field_data_new(self, obj): + repr = self.get_field_data({"field": obj.field, "value": obj.new}) + return repr + + def get_field_data_old(self, obj): + repr = self.get_field_data({"field": obj.field, "value": obj.old}) + return repr + + def __init__(self, instance=None, parent_object=None, **kwargs): + self._parent_object = parent_object + super().__init__(instance, **kwargs) + + +class HistoryLogSerializer(serializers.Serializer): + diff = serializers.SerializerMethodField() + timestamp = serializers.DateTimeField(source="history_date") + version_tag = serializers.CharField() + user = serializers.CharField(source="history_user") + + def get_diff(self, obj): + if obj.prev_record is None: + return [] + diff = obj.diff_against(obj.prev_record) + changed_fields = [] + changes = [] + for change in diff.changes: + if change.new == "" and change.old == None: + continue + changed_fields.append(change.field) + changes.append(ModelChangeSerializer(change, obj).data) + return {"changed_fields": changed_fields, "changes": changes} diff --git a/apis_core/apis_history/utils.py b/apis_core/apis_history/utils.py new file mode 100644 index 000000000..a8d161746 --- /dev/null +++ b/apis_core/apis_history/utils.py @@ -0,0 +1,26 @@ +from apis_core.utils.caching import get_entity_class_of_name +from django.db.models.query import QuerySet + + +def get_history(model_class: str, id: int) -> QuerySet: + """Get history of a model instance. + + Args: + model_class (str): Model class name. + id (int): Model instance id. + + Returns: + QuerySet: Queryset of history of a model instance. + """ + cls = get_entity_class_of_name(model_class) + exclude = [] + qs = cls.history.filter(id=id) + for q in qs: + id1 = q.history_id + ts = q.history_date + exclude.extend( + cls.history.filter(history_id__lt=id1, history_date=ts).values_list( + "history_id", flat=True + ) + ) + return qs.exclude(history_id__in=exclude) diff --git a/apis_core/apis_relations/migrations/0006_historicaltriple_historicaltemptriple.py b/apis_core/apis_relations/migrations/0006_historicaltriple_historicaltemptriple.py new file mode 100644 index 000000000..2d493967f --- /dev/null +++ b/apis_core/apis_relations/migrations/0006_historicaltriple_historicaltemptriple.py @@ -0,0 +1,213 @@ +# Generated by Django 4.2.10 on 2024-03-01 10:17 + +import apis_core.apis_relations.models +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import simple_history.models + + +class Migration(migrations.Migration): + dependencies = [ + ("apis_metainfo", "0009_remove_collection_collection_type"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("apis_relations", "0005_alter_property_obj_class_alter_property_subj_class"), + ] + + operations = [ + migrations.CreateModel( + name="HistoricalTriple", + fields=[ + ( + "id", + models.IntegerField( + auto_created=True, blank=True, db_index=True, verbose_name="ID" + ), + ), + ( + "version_tag", + models.CharField(blank=True, max_length=255, null=True), + ), + ("history_id", models.AutoField(primary_key=True, serialize=False)), + ("history_date", models.DateTimeField(db_index=True)), + ("history_change_reason", models.CharField(max_length=100, null=True)), + ( + "history_type", + models.CharField( + choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")], + max_length=1, + ), + ), + ( + "history_user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "obj", + apis_core.apis_relations.models.InheritanceForeignKey( + blank=True, + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="+", + to="apis_metainfo.rootobject", + verbose_name="Object", + ), + ), + ( + "prop", + models.ForeignKey( + blank=True, + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="+", + to="apis_relations.property", + verbose_name="Property", + ), + ), + ( + "subj", + apis_core.apis_relations.models.InheritanceForeignKey( + blank=True, + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="+", + to="apis_metainfo.rootobject", + verbose_name="Subject", + ), + ), + ], + options={ + "verbose_name": "historical triple", + "verbose_name_plural": "historical triples", + "ordering": ("-history_date", "-history_id"), + "get_latest_by": ("history_date", "history_id"), + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + migrations.CreateModel( + name="HistoricalTempTriple", + fields=[ + ( + "triple_ptr", + models.ForeignKey( + auto_created=True, + blank=True, + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + parent_link=True, + related_name="+", + to="apis_relations.triple", + ), + ), + ( + "id", + models.IntegerField( + auto_created=True, blank=True, db_index=True, verbose_name="ID" + ), + ), + ( + "version_tag", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "review", + models.BooleanField( + default=False, + help_text="Should be set to True, if the data record holds up quality standards.", + ), + ), + ("start_date", models.DateField(blank=True, null=True)), + ("start_start_date", models.DateField(blank=True, null=True)), + ("start_end_date", models.DateField(blank=True, null=True)), + ("end_date", models.DateField(blank=True, null=True)), + ("end_start_date", models.DateField(blank=True, null=True)), + ("end_end_date", models.DateField(blank=True, null=True)), + ( + "start_date_written", + models.CharField( + blank=True, max_length=255, null=True, verbose_name="Start" + ), + ), + ( + "end_date_written", + models.CharField( + blank=True, max_length=255, null=True, verbose_name="End" + ), + ), + ("status", models.CharField(max_length=100)), + ("references", models.TextField(blank=True, null=True)), + ("notes", models.TextField(blank=True, null=True)), + ("history_id", models.AutoField(primary_key=True, serialize=False)), + ("history_date", models.DateTimeField(db_index=True)), + ("history_change_reason", models.CharField(max_length=100, null=True)), + ( + "history_type", + models.CharField( + choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")], + max_length=1, + ), + ), + ( + "history_user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "obj", + apis_core.apis_relations.models.InheritanceForeignKey( + blank=True, + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="+", + to="apis_metainfo.rootobject", + verbose_name="Object", + ), + ), + ( + "prop", + models.ForeignKey( + blank=True, + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="+", + to="apis_relations.property", + verbose_name="Property", + ), + ), + ( + "subj", + apis_core.apis_relations.models.InheritanceForeignKey( + blank=True, + db_constraint=False, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="+", + to="apis_metainfo.rootobject", + verbose_name="Subject", + ), + ), + ], + options={ + "verbose_name": "historical temp triple", + "verbose_name_plural": "historical temp triples", + "ordering": ("-history_date", "-history_id"), + "get_latest_by": ("history_date", "history_id"), + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + ] diff --git a/apis_core/tests/test_simple_history.py b/apis_core/tests/test_simple_history.py new file mode 100644 index 000000000..d1e1b8196 --- /dev/null +++ b/apis_core/tests/test_simple_history.py @@ -0,0 +1,84 @@ +import time +from apis_core.apis_relations.models import Property +from django.test import TestCase +from django.contrib.contenttypes.models import ContentType +from apis_ontology.models import Person, Place, Profession +from apis_core.apis_relations.models import TempTriple +from datetime import datetime, timedelta + + +class SimpleHistoryTestCase(TestCase): + """Test of the simple_history package for the ÖBL usecase. + This expects the ÖBL ontology to be installed and migrated: https://github.com/acdh-oeaw/apis-instance-oebl-pnp + """ + + def setUp(self): + prop = Property.objects.create(name="geboren in") + prop.subj_class.add(ContentType.objects.get(model="person")) + prop.obj_class.add(ContentType.objects.get(model="place")) + Person.objects.create(first_name="John", name="Doe") + place = Place.objects.create( + name="Steyr", _history_date=datetime.now() - timedelta(hours=0, minutes=50) + ) + + def test_history(self): + """Tests the simple version of attributes changed on a models instance.""" + pers = Person.objects.get(first_name="John") + pers.first_name = "Jane" + pers.save() + self.assertEqual(pers.first_name, "Jane") + pers_history = pers.history.all() + assert len(pers_history) == 2 + self.assertEqual("Jane", pers.history.most_recent().first_name) + self.assertEqual("John", pers.history.earliest().first_name) + + def test_history_through_triples(self): + """Tests the newly introduced function for retrieving triples for a specific version of a model instance.""" + pers = Person.objects.get(first_name="John") + place = Place.objects.first() + TempTriple.objects.create(subj=pers, prop=Property.objects.first(), obj=place) + place.name = "Ennsleite" + place.save() + assert len(Place.history.earliest().get_triples_for_version()) == 0 + assert len(Place.history.latest().get_triples_for_version()) == 1 + triples = Place.history.latest().get_triples_for_version() + print("fin") + + def test_history_delete_entry(self): + """Tests the deletion of an entry.""" + pers = Person.objects.get(first_name="John") + pers.delete() + assert len(Person.history.all()) == 2 + + def test_history_merge(self): + """Tests the merge function of the Place model. This is still expected to fail.""" + pl1 = Place.objects.first() + pl2 = Place.objects.create( + name="Test", _history_date=datetime.now() - timedelta(hours=0, minutes=10) + ) + pl1.merge_with(pl2) + print("save()") + + def test_m2m_save(self): + """Test if m2m profession is saved correctly.""" + pers = Person.objects.create( + first_name="John", + _history_date=datetime.now() - timedelta(hours=0, minutes=10), + ) + self.assertEqual(pers.first_name, "John") + pers_history = pers.history.all() + assert len(pers_history) == 1 + prof = Profession.objects.create(name="Test") + pers.profession.add(prof) + pers.first_name = "Jane" + pers.save() + assert len(pers.profession.all()) == 1 + assert len(pers.history.latest().profession.all()) == 1 + assert len(pers.history.earliest().profession.all()) == 0 + + def test_history_date(self): + """Test that history is set correctly when not manually set.""" + pers = Person.objects.all().first() + pers.first_name = "Jane" + pers.save() + assert pers.history.earliest().history_date < pers.history.latest().history_date diff --git a/apis_core/urls.py b/apis_core/urls.py index 67b4fe1d8..2b743a492 100644 --- a/apis_core/urls.py +++ b/apis_core/urls.py @@ -5,6 +5,8 @@ from django.views.static import serve from django.views.generic import TemplateView from rest_framework import routers +from django.conf import settings + from apis_core.api_routers import views @@ -15,6 +17,7 @@ from apis_core.utils import caching from apis_core.apis_metainfo.viewsets import UriToObjectViewSet from apis_core.core.views import Dumpdata +from apis_core.apis_history.api_views import GenericHistoryLogs from apis_core.apis_entities.api_views import GetEntityGeneric from drf_spectacular.views import ( @@ -42,6 +45,8 @@ # inject the manually created UriToObjectViewSet into the api router router.register(r"metainfo/uritoobject", UriToObjectViewSet, basename="uritoobject") +if "apis_core.apis_history" in settings.INSTALLED_APPS: + router.register(r"history/entity/edit_log", GenericHistoryLogs, "historylog") urlpatterns = [