From 7a166386a95def7e868bcb65886ad0e9d1f64817 Mon Sep 17 00:00:00 2001 From: Arne Gudermann Date: Mon, 21 Oct 2024 18:02:45 +0200 Subject: [PATCH 1/9] Add downloadUrl for derives --- src/viur/core/config.py | 13 +++++++++ src/viur/core/render/json/default.py | 43 ++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/viur/core/config.py b/src/viur/core/config.py index 2b6062e12..e38a44741 100644 --- a/src/viur/core/config.py +++ b/src/viur/core/config.py @@ -798,6 +798,19 @@ class Conf(ConfigType): file_thumbnailer_url: t.Optional[str] = None # TODO: """docstring""" + file_generate_download_url_for_derives: bool | list[str] = False + """ + If true for all derives a download Url is created + If this is a list all entries used as a search string: + Example: + class Test(Skeleton): + image_bar = FileBone(derive=conf["derives"]) + image_foo = FileBone(derive=conf["derives"]) + logo = FileBone(derive=conf["derives"]) + + conf.file_generate_download_url_for_derives = ["test.image*"] + """ + main_app: "Module" = None """Reference to our pre-build Application-Instance""" diff --git a/src/viur/core/render/json/default.py b/src/viur/core/render/json/default.py index bbd8e5b0e..67e5629ab 100644 --- a/src/viur/core/render/json/default.py +++ b/src/viur/core/render/json/default.py @@ -1,7 +1,9 @@ +import fnmatch import json +import logging import typing as t from enum import Enum - +from viur.core.modules.file import File from viur.core import bones, db, current from viur.core.render.abstract import AbstractRenderer from viur.core.skeleton import SkeletonInstance @@ -88,7 +90,11 @@ def renderSingleBoneValue(self, value: t.Any, if isinstance(bone, bones.RelationalBone): if isinstance(value, dict): return { - "dest": self.renderSkelValues(value["dest"], injectDownloadURL=isinstance(bone, bones.FileBone)), + "dest": self.renderSkelValues( + value["dest"], + bone_path=f"{skel.kindName}.{bone.name}", + injectDownloadURL=isinstance(bone, bones.FileBone) + ), "rel": (self.renderSkelValues(value["rel"], injectDownloadURL=isinstance(bone, bones.FileBone)) if value["rel"] else None), } @@ -122,7 +128,10 @@ def renderBoneValue(self, bone: bones.BaseBone, skel: SkeletonInstance, key: str res = self.renderSingleBoneValue(boneVal, bone, skel, key) return res - def renderSkelValues(self, skel: SkeletonInstance, injectDownloadURL: bool = False) -> t.Optional[dict]: + def renderSkelValues(self, + skel: SkeletonInstance, + bone_path: t.Optional[str] = None, + injectDownloadURL: bool = False) -> t.Optional[dict]: """ Prepares values of one :class:`viur.core.skeleton.Skeleton` or a list of skeletons for output. @@ -140,15 +149,39 @@ def renderSkelValues(self, skel: SkeletonInstance, injectDownloadURL: bool = Fal if ( injectDownloadURL - and (file := getattr(conf.main_app, "file", None)) and "dlkey" in skel and "name" in skel ): - res["downloadUrl"] = file.create_download_url( + res["downloadUrl"] = File.create_download_url( skel["dlkey"], skel["name"], expires=conf.render_json_download_url_expiration ) + + + # generate the downloadUrl for derives + search_paths= [] + if search_path := current.request.get().request.headers.get("X-VIUR-DERIVED-DOWNLOAD-URL"): + search_paths.append(search_path) + if conf.file_generate_download_url_for_derives: + + if isinstance(conf.file_generate_download_url_for_derives,list): + search_paths.extend(conf.file_generate_download_url_for_derives) + else: + search_path.append("*") + + for search_path in search_paths: + if fnmatch.fnmatch(bone_path, search_path): + break + else: + return res + + for derive_name in res.get("derived",{}).get("files",{}): + res["derived"]["files"][derive_name]["downloadUrl"]=File.create_download_url( + skel["dlkey"], + derive_name, + derived=True + ) return res def renderEntry(self, skel: SkeletonInstance, actionName, params=None): From 0c5f8674f828c8395a66886c6294326efccdb184 Mon Sep 17 00:00:00 2001 From: Arne Gudermann Date: Mon, 21 Oct 2024 18:06:26 +0200 Subject: [PATCH 2/9] PEP8 --- src/viur/core/render/json/default.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/viur/core/render/json/default.py b/src/viur/core/render/json/default.py index 67e5629ab..e4557eed3 100644 --- a/src/viur/core/render/json/default.py +++ b/src/viur/core/render/json/default.py @@ -158,14 +158,13 @@ def renderSkelValues(self, expires=conf.render_json_download_url_expiration ) - # generate the downloadUrl for derives - search_paths= [] + search_paths = [] if search_path := current.request.get().request.headers.get("X-VIUR-DERIVED-DOWNLOAD-URL"): search_paths.append(search_path) if conf.file_generate_download_url_for_derives: - if isinstance(conf.file_generate_download_url_for_derives,list): + if isinstance(conf.file_generate_download_url_for_derives, list): search_paths.extend(conf.file_generate_download_url_for_derives) else: search_path.append("*") @@ -176,8 +175,8 @@ def renderSkelValues(self, else: return res - for derive_name in res.get("derived",{}).get("files",{}): - res["derived"]["files"][derive_name]["downloadUrl"]=File.create_download_url( + for derive_name in res.get("derived", {}).get("files", {}): + res["derived"]["files"][derive_name]["downloadUrl"] = File.create_download_url( skel["dlkey"], derive_name, derived=True From a54d917de1d7567353ee0bc1935210a1bf2c8026 Mon Sep 17 00:00:00 2001 From: agudermann <47318461+ArneGudermann@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:31:23 +0200 Subject: [PATCH 3/9] Update src/viur/core/config.py Co-authored-by: Jan Max Meyer --- src/viur/core/config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/viur/core/config.py b/src/viur/core/config.py index e38a44741..b5a4a50b8 100644 --- a/src/viur/core/config.py +++ b/src/viur/core/config.py @@ -800,15 +800,20 @@ class Conf(ConfigType): file_generate_download_url_for_derives: bool | list[str] = False """ - If true for all derives a download Url is created - If this is a list all entries used as a search string: + If True, for all derives, a download URL is created as well. + + When an iterable of str is provided, for all entries that match a pattern, + a download URL will be created. + Example: + ```py class Test(Skeleton): image_bar = FileBone(derive=conf["derives"]) image_foo = FileBone(derive=conf["derives"]) logo = FileBone(derive=conf["derives"]) conf.file_generate_download_url_for_derives = ["test.image*"] + ``` """ main_app: "Module" = None From 7add5c00a1d18085fdc9f580e93600fc1f614818 Mon Sep 17 00:00:00 2001 From: agudermann <47318461+ArneGudermann@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:31:31 +0200 Subject: [PATCH 4/9] Update src/viur/core/config.py Co-authored-by: Jan Max Meyer --- src/viur/core/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/viur/core/config.py b/src/viur/core/config.py index b5a4a50b8..917c42b58 100644 --- a/src/viur/core/config.py +++ b/src/viur/core/config.py @@ -798,7 +798,7 @@ class Conf(ConfigType): file_thumbnailer_url: t.Optional[str] = None # TODO: """docstring""" - file_generate_download_url_for_derives: bool | list[str] = False + file_generate_download_url_for_derives: bool | t.Iterable[str] = False """ If True, for all derives, a download URL is created as well. From 15b83169fe05402324af7cf1f11c9f9067703c6b Mon Sep 17 00:00:00 2001 From: agudermann <47318461+ArneGudermann@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:32:11 +0200 Subject: [PATCH 5/9] Update src/viur/core/render/json/default.py Co-authored-by: Jan Max Meyer --- src/viur/core/render/json/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/viur/core/render/json/default.py b/src/viur/core/render/json/default.py index e4557eed3..cdc598b61 100644 --- a/src/viur/core/render/json/default.py +++ b/src/viur/core/render/json/default.py @@ -167,7 +167,7 @@ def renderSkelValues(self, if isinstance(conf.file_generate_download_url_for_derives, list): search_paths.extend(conf.file_generate_download_url_for_derives) else: - search_path.append("*") + search_paths.append("*") for search_path in search_paths: if fnmatch.fnmatch(bone_path, search_path): From 686b3a791c24d4914b84481d3d773d050e5e53c5 Mon Sep 17 00:00:00 2001 From: agudermann <47318461+ArneGudermann@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:32:28 +0200 Subject: [PATCH 6/9] Update src/viur/core/render/json/default.py Co-authored-by: Jan Max Meyer --- src/viur/core/render/json/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/viur/core/render/json/default.py b/src/viur/core/render/json/default.py index cdc598b61..817b5f3bc 100644 --- a/src/viur/core/render/json/default.py +++ b/src/viur/core/render/json/default.py @@ -164,7 +164,7 @@ def renderSkelValues(self, search_paths.append(search_path) if conf.file_generate_download_url_for_derives: - if isinstance(conf.file_generate_download_url_for_derives, list): + if isinstance(conf.file_generate_download_url_for_derives, t.Iterable): search_paths.extend(conf.file_generate_download_url_for_derives) else: search_paths.append("*") From 3423783a8d77cf81bf972b2793c32381ab6c150f Mon Sep 17 00:00:00 2001 From: Arne Gudermann Date: Tue, 22 Oct 2024 08:50:32 +0200 Subject: [PATCH 7/9] Remove unused code and pep8 fromat --- src/viur/core/config.py | 2 +- src/viur/core/render/json/default.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/viur/core/config.py b/src/viur/core/config.py index 917c42b58..059192365 100644 --- a/src/viur/core/config.py +++ b/src/viur/core/config.py @@ -801,7 +801,7 @@ class Conf(ConfigType): file_generate_download_url_for_derives: bool | t.Iterable[str] = False """ If True, for all derives, a download URL is created as well. - + When an iterable of str is provided, for all entries that match a pattern, a download URL will be created. diff --git a/src/viur/core/render/json/default.py b/src/viur/core/render/json/default.py index 817b5f3bc..8e88d128c 100644 --- a/src/viur/core/render/json/default.py +++ b/src/viur/core/render/json/default.py @@ -1,6 +1,5 @@ import fnmatch import json -import logging import typing as t from enum import Enum from viur.core.modules.file import File From bee04a24ea9cbd0336888510cf123cad923af21e Mon Sep 17 00:00:00 2001 From: Arne Gudermann Date: Thu, 24 Oct 2024 09:15:28 +0200 Subject: [PATCH 8/9] Add download url for json and html --- src/viur/core/bones/base.py | 2 -- src/viur/core/bones/file.py | 2 ++ src/viur/core/modules/file.py | 48 ++++++++++++++++++++++------------- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/viur/core/bones/base.py b/src/viur/core/bones/base.py index 502329fa8..17a42b292 100644 --- a/src/viur/core/bones/base.py +++ b/src/viur/core/bones/base.py @@ -985,14 +985,12 @@ def unserialize_compute(self, skel: "SkeletonInstance", name: str) -> bool: case ComputeMethod.Lifetime: now = utils.utcNow() from viur.core.skeleton import RefSkel # noqa: E402 # import works only here because circular imports - if issubclass(skel.skeletonCls, RefSkel): # we have a ref skel we must load the complete Entity db_obj = db.Get(skel["key"]) last_update = db_obj.get(f"_viur_compute_{name}_") else: last_update = skel.dbEntity.get(f"_viur_compute_{name}_") skel.accessedValues[f"_viur_compute_{name}_"] = last_update or now - if not last_update or last_update + self.compute.interval.lifetime <= now: # if so, recompute and refresh updated value skel.accessedValues[name] = value = self._compute(skel, name) diff --git a/src/viur/core/bones/file.py b/src/viur/core/bones/file.py index a4a0fdf70..c7a542d01 100644 --- a/src/viur/core/bones/file.py +++ b/src/viur/core/bones/file.py @@ -146,6 +146,8 @@ def __init__( "height", "derived", "public", + "download_url_json", + "download_url_html" ), public: bool = False, **kwargs diff --git a/src/viur/core/modules/file.py b/src/viur/core/modules/file.py index 063310253..bde0aa498 100644 --- a/src/viur/core/modules/file.py +++ b/src/viur/core/modules/file.py @@ -22,7 +22,8 @@ from google.cloud import storage from google.oauth2.service_account import Credentials as ServiceAccountCredentials from viur.core import conf, current, db, errors, utils -from viur.core.bones import BaseBone, BooleanBone, KeyBone, NumericBone, StringBone +from viur.core.bones import BaseBone, BooleanBone, KeyBone, NumericBone, StringBone, UriBone +from viur.core.bones import Compute, ComputeMethod, ComputeInterval from viur.core.decorators import * from viur.core.i18n import LanguageWrapper from viur.core.prototypes.tree import SkelType, Tree, TreeSkel @@ -281,20 +282,6 @@ def make_request(): return reslist -class DownloadUrlBone(BaseBone): - """ - This bone is used to inject a freshly signed download url into a FileSkel. - """ - - def unserialize(self, skel, name): - if "dlkey" in skel.dbEntity and "name" in skel.dbEntity: - skel.accessedValues[name] = File.create_download_url( - skel["dlkey"], skel["name"], expires=conf.render_json_download_url_expiration - ) - return True - - return False - class FileLeafSkel(TreeSkel): """ @@ -350,10 +337,37 @@ class FileLeafSkel(TreeSkel): searchable=True, ) - downloadUrl = DownloadUrlBone( - descr="Download-URL", + download_url_json = UriBone( + descr="Download-URL for HTML", readOnly=True, visible=False, + compute=Compute( + fn=lambda skel: File.create_download_url( + dlkey=skel["dlkey"], + filename=skel["name"], + expires=conf.render_json_download_url_expiration + ), + interval=ComputeInterval( + method=ComputeMethod.Lifetime, + lifetime=conf.render_json_download_url_expiration or datetime.timedelta(hours=1) + ) + ) + ) + download_url_html = UriBone( + descr="Download-URL for HTML", + readOnly=True, + visible=False, + compute=Compute( + fn=lambda skel: File.create_download_url( + dlkey=skel["dlkey"], + filename=skel["name"], + expires=conf.render_html_download_url_expiration + ), + interval=ComputeInterval( + method=ComputeMethod.Lifetime, + lifetime=conf.render_html_download_url_expiration or datetime.timedelta(hours=1) + ) + ) ) derived = BaseBone( From 05d2c14ca0113b91cd0af86d335bd3ac82953c64 Mon Sep 17 00:00:00 2001 From: agudermann <47318461+ArneGudermann@users.noreply.github.com> Date: Thu, 24 Oct 2024 11:53:54 +0200 Subject: [PATCH 9/9] Apply suggestions from code review Co-authored-by: Sven Eberth --- src/viur/core/render/json/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/viur/core/render/json/default.py b/src/viur/core/render/json/default.py index 8e88d128c..5d69461d9 100644 --- a/src/viur/core/render/json/default.py +++ b/src/viur/core/render/json/default.py @@ -160,7 +160,7 @@ def renderSkelValues(self, # generate the downloadUrl for derives search_paths = [] if search_path := current.request.get().request.headers.get("X-VIUR-DERIVED-DOWNLOAD-URL"): - search_paths.append(search_path) + search_paths.extend(search_path.split(",")) if conf.file_generate_download_url_for_derives: if isinstance(conf.file_generate_download_url_for_derives, t.Iterable):