From 2f2f73d22c52fcd5f94e5139e7fa2cfbd1165e9a Mon Sep 17 00:00:00 2001 From: Birger Schacht Date: Wed, 29 May 2024 12:11:59 +0200 Subject: [PATCH] refactor: replace caching --- .../management/commands/serialize_to_json.py | 116 ------------------ apis_core/apis_entities/models.py | 29 ++--- .../templatetags/apis_templatetags.py | 17 --- 3 files changed, 8 insertions(+), 154 deletions(-) delete mode 100644 apis_core/apis_entities/management/commands/serialize_to_json.py diff --git a/apis_core/apis_entities/management/commands/serialize_to_json.py b/apis_core/apis_entities/management/commands/serialize_to_json.py deleted file mode 100644 index 854d29943..000000000 --- a/apis_core/apis_entities/management/commands/serialize_to_json.py +++ /dev/null @@ -1,116 +0,0 @@ -import json -import os -import pickle - -from django.core.management.base import BaseCommand -from django.core.serializers.json import DjangoJSONEncoder - -from apis_core.apis_entities.serializers_generic import EntitySerializer -from apis_core.utils import caching - - -class Command(BaseCommand): - help = "Command to serialize APIS data to json and store the data." - - def add_arguments(self, parser): - parser.add_argument( - "--entity", - action="store", - dest="entity", - default="Person", - help="Specify the named entity to serialize.", - ) - - parser.add_argument( - "--output", - action="store", - dest="output", - default=False, - help="Path of file to store JSON in.", - ) - - parser.add_argument( - "--only-published", - action="store_true", - dest="only-published", - default=False, - help="Set if you want to include published relations only. (Boolean, Default: False).", - ) - - parser.add_argument( - "--filter", - action="store", - dest="filter", - default="{}", - help="Specify a dictionary of filter arguments for the Entity queryset.", - ) - - parser.add_argument( - "--use-cache", - action="store_true", - dest="use-cache", - default=False, - help="Set if you want to use the cached files instead of serializing them egain (Boolean, Default: False).", - ) - - parser.add_argument( - "--add-texts", - action="store_true", - dest="add-texts", - default=False, - help="Set if you want to add the texts attached to the entities (Boolean, Default: False).", - ) - - def handle(self, *args, **options): - ent = caching.get_ontology_class_of_name(options["entity"]) - res = [] - objcts = ent.objects.filter(**json.loads(options["filter"])) - if objcts.filter(uri__uri__icontains=" ").count() > 0: - self.stdout.write(self.style.ERROR("URIs found that contain whitespaces")) - return - if objcts.count() > 1000 and not options["use-cache"]: - self.stdout.write(self.style.NOTICE("More than 1000 objects, caching")) - cnt = 0 - while (cnt * 1000) < objcts.count(): - r = [] - for e in objcts[1000 * cnt : (1000 * cnt + 1000)]: - r.append( - EntitySerializer( - e, - only_published=options["only-published"], - add_texts=options["add-texts"], - ).data - ) - with open(f"serializer_cache/{cnt}.pkl", "wb") as out: - pickle.dump(r, out) - self.stdout.write( - self.style.NOTICE( - f"Pickle written to: serializer_cache/{cnt}.pkl" - ) - ) - cnt += 1 - res = "/home/sennierer/projects/apis-webpage-base/serializer_cache" - elif not options["use-cache"]: - for e in objcts: - res.append( - EntitySerializer( - e, - only_published=options["only-published"], - add_texts=options["add-texts"], - ).data - ) - elif options["use-cache"]: - self.stdout.write(self.style.NOTICE("using cache for serializing")) - res = "/home/sennierer/projects/apis-webpage-base/serializer_cache" - self.stdout.write(self.style.SUCCESS(f"serialized {len(res)} objects")) - self.stdout.write(self.style.NOTICE("Starting to create the json file")) - with open(options["output"], "w+") as outp: - if isinstance(res, str): - directory = os.fsencode(res) - data_lst = [] - for fn in os.listdir(directory): - with open(os.path.join(directory, fn), "rb") as inf: - data_lst.extend(pickle.load(inf)) - json.dump(data_lst, outp, cls=DjangoJSONEncoder) - elif isinstance(res, list): - json.dump(res, out, pcls=DjangoJSONEncoder) diff --git a/apis_core/apis_entities/models.py b/apis_core/apis_entities/models.py index 7703843fa..f785403a7 100644 --- a/apis_core/apis_entities/models.py +++ b/apis_core/apis_entities/models.py @@ -7,8 +7,7 @@ from django.urls import reverse from django.db.models.query import QuerySet -from apis_core.utils import caching -from apis_core.apis_metainfo.models import RootObject +from apis_core.apis_metainfo.models import RootObject, Uri from apis_core.apis_relations.models import TempTriple from apis_core.apis_entities import signals @@ -199,22 +198,10 @@ def get_serialization(self): @receiver(post_save, dispatch_uid="create_default_uri") -def create_default_uri(sender, instance, raw, **kwargs): - # with django reversion, browsing deleted entries in the admin interface - # leads to firing the `post_save` signal - # (https://github.com/etianen/django-reversion/issues/936) - a workaround - # is to check for the raw argument - if not raw: - from apis_core.apis_metainfo.models import Uri - - if kwargs["created"] and sender in caching.get_all_ontology_classes(): - if BASE_URI.endswith("/"): - base1 = BASE_URI[:-1] - else: - base1 = BASE_URI - uri_c = "{}{}".format( - base1, - reverse("GetEntityGenericRoot", kwargs={"pk": instance.pk}), - ) - uri2 = Uri(uri=uri_c, domain="apis default", root_object=instance) - uri2.save() +def create_default_uri(sender, instance, created, raw, using, update_fields, **kwargs): + if getattr(settings, "CREATE_DEFAULT_URI", True): + if isinstance(instance, AbstractEntity) and created: + base = BASE_URI.strip("/") + route = reverse("GetEntityGenericRoot", kwargs={"pk": instance.pk}) + uri = f"{base}{route}" + Uri.objects.create(uri=uri, domain="apis default", root_object=instance) diff --git a/apis_core/apis_entities/templatetags/apis_templatetags.py b/apis_core/apis_entities/templatetags/apis_templatetags.py index fea4ea9bb..9567e1a0e 100644 --- a/apis_core/apis_entities/templatetags/apis_templatetags.py +++ b/apis_core/apis_entities/templatetags/apis_templatetags.py @@ -1,6 +1,4 @@ -from operator import itemgetter from django import template -from apis_core.utils import caching from apis_core.utils.helpers import triple_sidebar from django.contrib.contenttypes.models import ContentType from apis_core.apis_entities.models import AbstractEntity @@ -16,21 +14,6 @@ def url_replace(request, field, value): return dict_.urlencode() -@register.simple_tag -def entities_list_links(): - """ - Retrieve all models which inherit from AbstractEntity class - and return their class name and verbose name. - """ - entities_classes = caching.get_all_entity_classes() or [] - entities_links = [ - (e.__name__.lower(), e._meta.verbose_name.title()) for e in entities_classes - ] - entities_links.sort(key=itemgetter(1)) - - return entities_links - - def is_entity(content_type: ContentType): model_class = content_type.model_class() return model_class is not None and issubclass(model_class, AbstractEntity)