From 1f54a9f56822ec1b61dfae77c47f7404923d35b9 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 9 Jan 2024 14:42:50 +0100 Subject: [PATCH 01/25] Update copyright year --- src/sciwyrm/__init__.py | 2 +- src/sciwyrm/assets/__init__.py | 2 +- src/sciwyrm/main.py | 2 +- src/sciwyrm/notebook/v1.py | 2 +- src/sciwyrm/typing.py | 2 +- tests/__init__.py | 2 +- tests/conftest.py | 2 +- tests/notebook/__init__.py | 2 +- tests/notebook/notebook_v1_test.py | 2 +- tests/seed.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/sciwyrm/__init__.py b/src/sciwyrm/__init__.py index 2ca7a3f..32ef800 100644 --- a/src/sciwyrm/__init__.py +++ b/src/sciwyrm/__init__.py @@ -1,3 +1,3 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 SciCat Project (https://github.com/SciCatProject/sciwyrm) +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Stand-alone service to serve jupyter notebooks to download SciCat datasets.""" diff --git a/src/sciwyrm/assets/__init__.py b/src/sciwyrm/assets/__init__.py index d01c0b2..4488646 100644 --- a/src/sciwyrm/assets/__init__.py +++ b/src/sciwyrm/assets/__init__.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 SciCat Project (https://github.com/SciCatProject/sciwyrm) +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Asset loaders for SciWyrm.""" import importlib.resources diff --git a/src/sciwyrm/main.py b/src/sciwyrm/main.py index a31be01..535a994 100644 --- a/src/sciwyrm/main.py +++ b/src/sciwyrm/main.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 SciCat Project (https://github.com/SciCatProject/sciwyrm) +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """The SciWym application.""" from __future__ import annotations diff --git a/src/sciwyrm/notebook/v1.py b/src/sciwyrm/notebook/v1.py index f92a2aa..f4f2725 100644 --- a/src/sciwyrm/notebook/v1.py +++ b/src/sciwyrm/notebook/v1.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 SciCat Project (https://github.com/SciCatProject/sciwyrm) +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Version 1 notebooks.""" from copy import deepcopy diff --git a/src/sciwyrm/typing.py b/src/sciwyrm/typing.py index d92f6ea..5569358 100644 --- a/src/sciwyrm/typing.py +++ b/src/sciwyrm/typing.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 SciCat Project (https://github.com/SciCatProject/sciwyrm) +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Utilities for type annotations.""" from typing import Any diff --git a/tests/__init__.py b/tests/__init__.py index 4941969..33edd30 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +1,2 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 SciCat Project (https://github.com/SciCatProject/sciwyrm) +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) diff --git a/tests/conftest.py b/tests/conftest.py index 18f776c..ef1e52f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 SciCat Project (https://github.com/SciCatProject/sciwyrm) +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) import os diff --git a/tests/notebook/__init__.py b/tests/notebook/__init__.py index 4941969..33edd30 100644 --- a/tests/notebook/__init__.py +++ b/tests/notebook/__init__.py @@ -1,2 +1,2 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 SciCat Project (https://github.com/SciCatProject/sciwyrm) +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) diff --git a/tests/notebook/notebook_v1_test.py b/tests/notebook/notebook_v1_test.py index a02943e..a376768 100644 --- a/tests/notebook/notebook_v1_test.py +++ b/tests/notebook/notebook_v1_test.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 SciCat Project (https://github.com/SciCatProject/sciwyrm) +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) from io import StringIO from typing import Any diff --git a/tests/seed.py b/tests/seed.py index c18a30b..0c65bb8 100644 --- a/tests/seed.py +++ b/tests/seed.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 SciCat Project (https://github.com/SciCatProject/sciwyrm) +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) import tempfile from contextlib import contextmanager from pathlib import Path From 8084a331dec316cb80574b972e0e7ca08e57ef17 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 9 Jan 2024 14:43:29 +0100 Subject: [PATCH 02/25] Move notebook template into subpackage --- src/sciwyrm/assets/__init__.py | 14 ++------ src/sciwyrm/assets/templates/__init__.py | 3 ++ .../assets/templates/notebook/__init__.py | 26 ++++++++++++++ .../notebook/generic_v1.ipynb} | 36 ++++++------------- src/sciwyrm/notebook/__init__.py | 2 +- src/sciwyrm/notebook/v1.py | 6 +--- 6 files changed, 44 insertions(+), 43 deletions(-) create mode 100644 src/sciwyrm/assets/templates/__init__.py create mode 100644 src/sciwyrm/assets/templates/notebook/__init__.py rename src/sciwyrm/assets/{notebook_template_v1.ipynb => templates/notebook/generic_v1.ipynb} (84%) diff --git a/src/sciwyrm/assets/__init__.py b/src/sciwyrm/assets/__init__.py index 4488646..06de048 100644 --- a/src/sciwyrm/assets/__init__.py +++ b/src/sciwyrm/assets/__init__.py @@ -2,16 +2,6 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Asset loaders for SciWyrm.""" -import importlib.resources -import json +from .templates.notebook import notebook_template -from ..typing import Notebook - - -def _read_text(filename: str) -> str: - return importlib.resources.files("sciwyrm.assets").joinpath(filename).read_text() - - -def notebook_template_v1() -> Notebook: - """Return notebook template version 1.""" - return json.loads(_read_text("notebook_template_v1.ipynb")) +__all__ = ["notebook_template"] diff --git a/src/sciwyrm/assets/templates/__init__.py b/src/sciwyrm/assets/templates/__init__.py new file mode 100644 index 0000000..3056702 --- /dev/null +++ b/src/sciwyrm/assets/templates/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) +"""Templates.""" diff --git a/src/sciwyrm/assets/templates/notebook/__init__.py b/src/sciwyrm/assets/templates/notebook/__init__.py new file mode 100644 index 0000000..021a7f2 --- /dev/null +++ b/src/sciwyrm/assets/templates/notebook/__init__.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) +"""Notebook templates for SciWyrm.""" + +import importlib.resources +import json +from copy import deepcopy +from functools import lru_cache + +from ....typing import Notebook + + +@lru_cache() +def _load(*, name: str, version: str) -> Notebook: + filename = f"{name}_v{version}.ipynb" + source = ( + importlib.resources.files("sciwyrm.assets.templates.notebook") + .joinpath(filename) + .read_text() + ) + return json.loads(source) + + +def notebook_template(*, name: str, version: str) -> Notebook: + """Return a notebook template.""" + return deepcopy(_load(name=name, version=version)) diff --git a/src/sciwyrm/assets/notebook_template_v1.ipynb b/src/sciwyrm/assets/templates/notebook/generic_v1.ipynb similarity index 84% rename from src/sciwyrm/assets/notebook_template_v1.ipynb rename to src/sciwyrm/assets/templates/notebook/generic_v1.ipynb index 0cb70ca..f7f160f 100644 --- a/src/sciwyrm/assets/notebook_template_v1.ipynb +++ b/src/sciwyrm/assets/templates/notebook/generic_v1.ipynb @@ -4,9 +4,7 @@ "cell_type": "code", "execution_count": null, "id": "923ba88e07905c7b", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "import scitacean\n", @@ -17,9 +15,7 @@ "cell_type": "code", "execution_count": null, "id": "91d1974c665d73f3", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "scicat_url = $SCICAT_URL\n", @@ -31,9 +27,7 @@ "cell_type": "code", "execution_count": null, "id": "31c3a003da7913e9", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "scicat_token = $SCICAT_TOKEN" @@ -42,22 +36,18 @@ { "cell_type": "code", "execution_count": null, + "id": "initial_id", + "metadata": {}, "outputs": [], "source": [ "input_dataset_pids = [$INPUT_DATASET_IDS]" - ], - "metadata": { - "collapsed": false - }, - "id": "initial_id" + ] }, { "cell_type": "code", "execution_count": null, "id": "8f1a34476bf70da7", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "file_download_folder = \"./download\"" @@ -67,9 +57,7 @@ "cell_type": "code", "execution_count": null, "id": "1d0f975884ac9e5b", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "scicat_client = scitacean.Client.from_token(\n", @@ -85,6 +73,8 @@ { "cell_type": "code", "execution_count": null, + "id": "dc6f6c8b285dfb16", + "metadata": {}, "outputs": [], "source": [ "datasets = [\n", @@ -94,11 +84,7 @@ " )\n", " for pid in input_dataset_pids\n", "]" - ], - "metadata": { - "collapsed": false - }, - "id": "dc6f6c8b285dfb16" + ] } ], "metadata": { diff --git a/src/sciwyrm/notebook/__init__.py b/src/sciwyrm/notebook/__init__.py index 676e623..9022f23 100644 --- a/src/sciwyrm/notebook/__init__.py +++ b/src/sciwyrm/notebook/__init__.py @@ -1,3 +1,3 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 SciCat Project (https://github.com/SciCatProject/sciwyrm) +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Notebook handling.""" diff --git a/src/sciwyrm/notebook/v1.py b/src/sciwyrm/notebook/v1.py index f4f2725..5c618d5 100644 --- a/src/sciwyrm/notebook/v1.py +++ b/src/sciwyrm/notebook/v1.py @@ -2,15 +2,11 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Version 1 notebooks.""" -from copy import deepcopy - from pydantic import BaseModel from .. import assets from ..typing import Notebook -_notebook_template_v1 = assets.notebook_template_v1() - class NotebookSpecV1(BaseModel): """Specifies which notebook to return and how to format it.""" @@ -42,7 +38,7 @@ def _pids_cell_source(pids: list[str]) -> list[str]: def format_notebook(spec: NotebookSpecV1) -> Notebook: """Return a formatted version 1 notebook.""" - nb = deepcopy(_notebook_template_v1) + nb = assets.notebook_template(name="generic", version="1") cells = nb["cells"] cells[1]["source"] = _scicat_url_cell_source( spec.scicat_url, spec.file_server_host, spec.file_server_port From 11d44d7cbefed5757fb710efabf9ae84fb2e454d Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 9 Jan 2024 14:45:42 +0100 Subject: [PATCH 03/25] Require notebook name and version --- src/sciwyrm/notebook/v1.py | 6 +++++- tests/notebook/notebook_v1_test.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/sciwyrm/notebook/v1.py b/src/sciwyrm/notebook/v1.py index 5c618d5..859e9fc 100644 --- a/src/sciwyrm/notebook/v1.py +++ b/src/sciwyrm/notebook/v1.py @@ -11,6 +11,8 @@ class NotebookSpecV1(BaseModel): """Specifies which notebook to return and how to format it.""" + notebook_name: str + notebook_version: str dataset_pids: list[str] file_server_host: str file_server_port: int @@ -38,7 +40,9 @@ def _pids_cell_source(pids: list[str]) -> list[str]: def format_notebook(spec: NotebookSpecV1) -> Notebook: """Return a formatted version 1 notebook.""" - nb = assets.notebook_template(name="generic", version="1") + nb = assets.notebook_template( + name=spec.notebook_name, version=spec.notebook_version + ) cells = nb["cells"] cells[1]["source"] = _scicat_url_cell_source( spec.scicat_url, spec.file_server_host, spec.file_server_port diff --git a/tests/notebook/notebook_v1_test.py b/tests/notebook/notebook_v1_test.py index a376768..3af8574 100644 --- a/tests/notebook/notebook_v1_test.py +++ b/tests/notebook/notebook_v1_test.py @@ -38,6 +38,8 @@ def test_notebook_contains_expected_url(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ + "notebook_name": "generic", + "notebook_version": "1", "scicat_url": url, "file_server_host": "login", "file_server_port": 22, @@ -53,6 +55,8 @@ def test_notebook_contains_expected_pids(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ + "notebook_name": "generic", + "notebook_version": "1", "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", "file_server_port": 22, @@ -71,6 +75,8 @@ def test_notebook_contains_only_expected_pids(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ + "notebook_name": "generic", + "notebook_version": "1", "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", "file_server_port": 22, @@ -82,6 +88,8 @@ def test_notebook_contains_only_expected_pids(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ + "notebook_name": "generic", + "notebook_version": "1", "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", "file_server_port": 22, @@ -100,6 +108,8 @@ def test_notebook_contains_expected_file_serve_host(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ + "notebook_name": "generic", + "notebook_version": "1", "scicat_url": "https://test-url.sci.cat", "file_server_host": file_server_host, "file_server_port": 22, @@ -115,6 +125,8 @@ def test_notebook_contains_expected_file_serve_port(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ + "notebook_name": "generic", + "notebook_version": "1", "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", "file_server_port": file_server_port, @@ -140,6 +152,8 @@ def test_notebook_run( response = sciwyrm_client.post( "/notebook/v1", json={ + "notebook_name": "generic", + "notebook_version": "1", "scicat_url": scicat_access.url, "file_server_host": sftp_access.host, "file_server_port": sftp_access.port, From 73587b7bc402bc2494799a2dcd7fb07c3025bb77 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 9 Jan 2024 16:59:28 +0100 Subject: [PATCH 04/25] Update dependencies --- pyproject.toml | 3 ++- requirements/base.in | 2 +- requirements/base.txt | 18 ++++++++---------- requirements/ci.txt | 2 +- requirements/dev.in | 1 + requirements/dev.txt | 21 ++++++++++++--------- requirements/docs.txt | 10 +++++----- requirements/static.txt | 6 +++--- requirements/test.in | 2 +- requirements/test.txt | 32 ++++++++++++++++---------------- 10 files changed, 50 insertions(+), 47 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9c1dd70..7550b36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ ] requires-python = ">=3.11" dependencies = [ - "fastapi", + "fastapi >= 0.108", + "pydantic", ] dynamic = ["version"] diff --git a/requirements/base.in b/requirements/base.in index 909e922..65deb2d 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,2 +1,2 @@ -fastapi +fastapi >= 0.108 pydantic diff --git a/requirements/base.txt b/requirements/base.txt index bc32116..d71671f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:df7ce1b491a85bde4c66124bd77383e903fb5722 +# SHA1:34d56b28a2716a86d91ccc9880d62ad149ec9196 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -7,25 +7,23 @@ # annotated-types==0.6.0 # via pydantic -anyio==3.7.1 - # via - # fastapi - # starlette -fastapi==0.104.1 +anyio==4.2.0 + # via starlette +fastapi==0.108.0 # via -r requirements/base.in idna==3.6 # via anyio -pydantic==2.5.2 +pydantic==2.5.3 # via # -r requirements/base.in # fastapi -pydantic-core==2.14.5 +pydantic-core==2.14.6 # via pydantic sniffio==1.3.0 # via anyio -starlette==0.27.0 +starlette==0.32.0.post1 # via fastapi -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via # fastapi # pydantic diff --git a/requirements/ci.txt b/requirements/ci.txt index fd1e86c..cfb293f 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -11,7 +11,7 @@ chardet==5.2.0 # via tox colorama==0.4.6 # via tox -distlib==0.3.7 +distlib==0.3.8 # via virtualenv filelock==3.13.1 # via diff --git a/requirements/dev.in b/requirements/dev.in index c721b92..0e0ba78 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -9,5 +9,6 @@ jupyterlab mypy pip-compile-multi pre-commit +uvicorn types-docutils diff --git a/requirements/dev.txt b/requirements/dev.txt index c9f16f3..cd6301e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,4 @@ -# SHA1:f36bab5c5bea14fadf704e7f0ba17b3061953cbf +# SHA1:1835a31fe992c4fa55da153dab45d5d296f65175 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -19,13 +19,14 @@ arrow==1.3.0 # via isoduration async-lru==2.0.4 # via jupyterlab -black==23.11.0 +black==23.12.1 # via -r requirements/dev.in click==8.1.7 # via # black # pip-compile-multi # pip-tools + # uvicorn fqdn==1.5.1 # via jsonschema isoduration==20.11.0 @@ -43,19 +44,19 @@ jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.1 # via jupyterlab -jupyter-server==2.11.2 +jupyter-server==2.12.2 # via # jupyter-lsp # jupyterlab # jupyterlab-server # notebook-shim -jupyter-server-terminals==0.4.4 +jupyter-server-terminals==0.5.1 # via jupyter-server -jupyterlab==4.0.9 +jupyterlab==4.0.10 # via -r requirements/dev.in jupyterlab-server==2.25.2 # via jupyterlab -mypy==1.7.1 +mypy==1.8.0 # via -r requirements/dev.in mypy-extensions==1.0.0 # via @@ -65,7 +66,7 @@ notebook-shim==0.2.3 # via jupyterlab overrides==7.4.0 # via jupyter-server -pathspec==0.11.2 +pathspec==0.12.1 # via black pip-compile-multi==2.6.3 # via -r requirements/dev.in @@ -91,12 +92,14 @@ terminado==0.18.0 # jupyter-server-terminals toposort==1.10 # via pip-compile-multi -types-docutils==0.20.0.3 +types-docutils==0.20.0.20240106 # via -r requirements/dev.in -types-python-dateutil==2.8.19.14 +types-python-dateutil==2.8.19.20240106 # via arrow uri-template==1.3.0 # via jsonschema +uvicorn==0.25.0 + # via -r requirements/dev.in webcolors==1.13 # via jsonschema websocket-client==1.7.0 diff --git a/requirements/docs.txt b/requirements/docs.txt index 241c815..24f60fe 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -9,15 +9,15 @@ -r test.txt accessible-pygments==0.0.4 # via pydata-sphinx-theme -alabaster==0.7.13 +alabaster==0.7.15 # via sphinx autodoc-pydantic==2.0.1 # via -r requirements/docs.in -babel==2.13.1 +babel==2.14.0 # via # pydata-sphinx-theme # sphinx -comm==0.2.0 +comm==0.2.1 # via ipykernel debugpy==1.8.0 # via ipykernel @@ -29,7 +29,7 @@ docutils==0.20.1 # sphinx imagesize==1.4.1 # via sphinx -ipykernel==6.27.1 +ipykernel==6.28.0 # via -r requirements/docs.in markdown-it-py==3.0.0 # via @@ -45,7 +45,7 @@ nbsphinx==0.9.3 # via -r requirements/docs.in nest-asyncio==1.5.8 # via ipykernel -psutil==5.9.6 +psutil==5.9.7 # via ipykernel pydantic-settings==2.1.0 # via autodoc-pydantic diff --git a/requirements/static.txt b/requirements/static.txt index 7375e27..e9f8d3d 100644 --- a/requirements/static.txt +++ b/requirements/static.txt @@ -7,17 +7,17 @@ # cfgv==3.4.0 # via pre-commit -distlib==0.3.7 +distlib==0.3.8 # via virtualenv filelock==3.13.1 # via virtualenv -identify==2.5.32 +identify==2.5.33 # via pre-commit nodeenv==1.8.0 # via pre-commit platformdirs==4.1.0 # via virtualenv -pre-commit==3.5.0 +pre-commit==3.6.0 # via -r requirements/static.in pyyaml==6.0.1 # via pre-commit diff --git a/requirements/test.in b/requirements/test.in index 928853a..b5308ae 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -1,5 +1,5 @@ -r base.in -httpx +httpx # for async testing ipython nbconvert pytest diff --git a/requirements/test.txt b/requirements/test.txt index 9799772..e141ac5 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,12 +8,12 @@ -r base.txt asttokens==2.4.1 # via stack-data -attrs==23.1.0 +attrs==23.2.0 # via # hypothesis # jsonschema # referencing -bcrypt==4.1.1 +bcrypt==4.1.2 # via paramiko beautifulsoup4==4.12.2 # via nbconvert @@ -48,7 +48,7 @@ executing==2.0.1 # via stack-data fabric==3.2.2 # via scitacean -fastjsonschema==2.19.0 +fastjsonschema==2.19.1 # via nbformat filelock==3.13.1 # via scitacean @@ -56,15 +56,15 @@ h11==0.14.0 # via httpcore httpcore==1.0.2 # via httpx -httpx==0.25.2 +httpx==0.26.0 # via -r requirements/test.in -hypothesis==6.91.0 +hypothesis==6.92.6 # via scitacean iniconfig==2.0.0 # via pytest invoke==2.2.0 # via fabric -ipython==8.18.1 +ipython==8.20.0 # via -r requirements/test.in jedi==0.19.1 # via ipython @@ -72,11 +72,11 @@ jinja2==3.1.2 # via nbconvert jsonschema==4.20.0 # via nbformat -jsonschema-specifications==2023.11.2 +jsonschema-specifications==2023.12.1 # via jsonschema jupyter-client==8.6.0 # via nbclient -jupyter-core==5.5.0 +jupyter-core==5.7.1 # via # jupyter-client # nbclient @@ -94,7 +94,7 @@ mistune==3.0.2 # via nbconvert nbclient==0.9.0 # via nbconvert -nbconvert==7.12.0 +nbconvert==7.14.0 # via -r requirements/test.in nbformat==5.9.2 # via @@ -106,7 +106,7 @@ packaging==23.2 # pytest pandocfilters==1.5.0 # via nbconvert -paramiko==3.3.1 +paramiko==3.4.0 # via # fabric # scitacean @@ -118,7 +118,7 @@ platformdirs==4.1.0 # via jupyter-core pluggy==1.3.0 # via pytest -prompt-toolkit==3.0.41 +prompt-toolkit==3.0.43 # via ipython ptyprocess==0.7.0 # via pexpect @@ -132,7 +132,7 @@ pygments==2.17.2 # nbconvert pynacl==1.5.0 # via paramiko -pytest==7.4.3 +pytest==7.4.4 # via # -r requirements/test.in # pytest-randomly @@ -146,13 +146,13 @@ pyyaml==6.0.1 # via scitacean pyzmq==25.1.2 # via jupyter-client -referencing==0.31.1 +referencing==0.32.1 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via scitacean -rpds-py==0.13.2 +rpds-py==0.16.2 # via # jsonschema # referencing @@ -173,7 +173,7 @@ tinycss2==1.2.1 # via nbconvert tornado==6.4 # via jupyter-client -traitlets==5.14.0 +traitlets==5.14.1 # via # ipython # jupyter-client @@ -184,7 +184,7 @@ traitlets==5.14.0 # nbformat urllib3==2.1.0 # via requests -wcwidth==0.2.12 +wcwidth==0.2.13 # via prompt-toolkit webencodings==0.5.1 # via From 55f10629137d60eb9a4292124d0bb0bd2b041825 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 10 Jan 2024 09:45:36 +0100 Subject: [PATCH 05/25] Use jinja2 for templating --- pyproject.toml | 1 + requirements/base.in | 1 + requirements/base.txt | 6 +- requirements/dev.txt | 2 +- requirements/docs.txt | 2 +- requirements/test.txt | 6 -- src/sciwyrm/assets/__init__.py | 52 ++++++++++++++++- src/sciwyrm/assets/templates/__init__.py | 3 - .../assets/templates/notebook/__init__.py | 26 --------- .../templates/notebook/generic_v1.ipynb | 12 ++-- src/sciwyrm/filters.py | 17 ++++++ src/sciwyrm/main.py | 10 ++-- src/sciwyrm/notebook/v1.py | 56 ++++++++----------- src/sciwyrm/typing.py | 8 --- tests/notebook/notebook_v1_test.py | 28 +++++----- 15 files changed, 125 insertions(+), 105 deletions(-) delete mode 100644 src/sciwyrm/assets/templates/__init__.py delete mode 100644 src/sciwyrm/assets/templates/notebook/__init__.py create mode 100644 src/sciwyrm/filters.py delete mode 100644 src/sciwyrm/typing.py diff --git a/pyproject.toml b/pyproject.toml index 7550b36..5c647e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ requires-python = ">=3.11" dependencies = [ "fastapi >= 0.108", + "jinja2", "pydantic", ] dynamic = ["version"] diff --git a/requirements/base.in b/requirements/base.in index 65deb2d..08744a5 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,2 +1,3 @@ fastapi >= 0.108 +jinja2 pydantic diff --git a/requirements/base.txt b/requirements/base.txt index d71671f..162a3d2 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:34d56b28a2716a86d91ccc9880d62ad149ec9196 +# SHA1:5bdee4462eb709fd035300da208c144e0abd18e2 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -13,6 +13,10 @@ fastapi==0.108.0 # via -r requirements/base.in idna==3.6 # via anyio +jinja2==3.1.2 + # via -r requirements/base.in +markupsafe==2.1.3 + # via jinja2 pydantic==2.5.3 # via # -r requirements/base.in diff --git a/requirements/dev.txt b/requirements/dev.txt index cd6301e..afab37c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -44,7 +44,7 @@ jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.1 # via jupyterlab -jupyter-server==2.12.2 +jupyter-server==2.12.3 # via # jupyter-lsp # jupyterlab diff --git a/requirements/docs.txt b/requirements/docs.txt index 24f60fe..91439ae 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -9,7 +9,7 @@ -r test.txt accessible-pygments==0.0.4 # via pydata-sphinx-theme -alabaster==0.7.15 +alabaster==0.7.16 # via sphinx autodoc-pydantic==2.0.1 # via -r requirements/docs.in diff --git a/requirements/test.txt b/requirements/test.txt index e141ac5..d7b2d69 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -68,8 +68,6 @@ ipython==8.20.0 # via -r requirements/test.in jedi==0.19.1 # via ipython -jinja2==3.1.2 - # via nbconvert jsonschema==4.20.0 # via nbformat jsonschema-specifications==2023.12.1 @@ -84,10 +82,6 @@ jupyter-core==5.7.1 # nbformat jupyterlab-pygments==0.3.0 # via nbconvert -markupsafe==2.1.3 - # via - # jinja2 - # nbconvert matplotlib-inline==0.1.6 # via ipython mistune==3.0.2 diff --git a/src/sciwyrm/assets/__init__.py b/src/sciwyrm/assets/__init__.py index 06de048..320d951 100644 --- a/src/sciwyrm/assets/__init__.py +++ b/src/sciwyrm/assets/__init__.py @@ -2,6 +2,54 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Asset loaders for SciWyrm.""" -from .templates.notebook import notebook_template +from pathlib import Path +from typing import Any -__all__ = ["notebook_template"] +from fastapi import Request +from fastapi.templating import Jinja2Templates +from jinja2 import Environment, FileSystemLoader + + +def _make_template_handler() -> Jinja2Templates: + from .. import filters + + templates = Jinja2Templates( + env=Environment( + loader=FileSystemLoader(Path(__file__).resolve().parent / "templates"), + autoescape=True, + ) + ) + templates.env.filters["quote"] = filters.quote + templates.env.filters["je"] = filters.json_escape + return templates + + +_templates = _make_template_handler() + + +def render_notebook_template( + *, name: str, version: str, request: Request, context: dict[str, Any] +) -> Any: + """Render a given notebook template into a response-compatible object. + + Parameters + ---------- + name: + Name of the template. + version: + Version of the template. + request: + FastAPI request that requested the notebook. + context: + Values to insert into the template. + + Returns + ------- + : + An objects that can be returned from an endpoint as a JSONResponse. + """ + return _templates.TemplateResponse( + name=f"notebook/{name}_v{version}.ipynb", + request=request, + context={**context}, + ) diff --git a/src/sciwyrm/assets/templates/__init__.py b/src/sciwyrm/assets/templates/__init__.py deleted file mode 100644 index 3056702..0000000 --- a/src/sciwyrm/assets/templates/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) -"""Templates.""" diff --git a/src/sciwyrm/assets/templates/notebook/__init__.py b/src/sciwyrm/assets/templates/notebook/__init__.py deleted file mode 100644 index 021a7f2..0000000 --- a/src/sciwyrm/assets/templates/notebook/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) -"""Notebook templates for SciWyrm.""" - -import importlib.resources -import json -from copy import deepcopy -from functools import lru_cache - -from ....typing import Notebook - - -@lru_cache() -def _load(*, name: str, version: str) -> Notebook: - filename = f"{name}_v{version}.ipynb" - source = ( - importlib.resources.files("sciwyrm.assets.templates.notebook") - .joinpath(filename) - .read_text() - ) - return json.loads(source) - - -def notebook_template(*, name: str, version: str) -> Notebook: - """Return a notebook template.""" - return deepcopy(_load(name=name, version=version)) diff --git a/src/sciwyrm/assets/templates/notebook/generic_v1.ipynb b/src/sciwyrm/assets/templates/notebook/generic_v1.ipynb index f7f160f..7b777a4 100644 --- a/src/sciwyrm/assets/templates/notebook/generic_v1.ipynb +++ b/src/sciwyrm/assets/templates/notebook/generic_v1.ipynb @@ -18,9 +18,9 @@ "metadata": {}, "outputs": [], "source": [ - "scicat_url = $SCICAT_URL\n", - "file_server_host = $FILE_SERVER_HOST\n", - "file_serve_port = $FILE_SERVER_PORT" + "scicat_url = {{ SCICAT_URL | quote | je | safe }}\n", + "file_server_host = {{ FILE_SERVER_HOST | quote | je | safe }}\n", + "file_server_port = {{ FILE_SERVER_PORT | int }}" ] }, { @@ -30,7 +30,7 @@ "metadata": {}, "outputs": [], "source": [ - "scicat_token = $SCICAT_TOKEN" + "scicat_token = {{ SCICAT_TOKEN | quote | je | safe }}" ] }, { @@ -40,7 +40,9 @@ "metadata": {}, "outputs": [], "source": [ - "input_dataset_pids = [$INPUT_DATASET_IDS]" + "input_dataset_pids = [{% for pid in DATASET_PIDS %}\n", + " {{ pid | quote | je | safe }},{% endfor %}\n", + "]" ] }, { diff --git a/src/sciwyrm/filters.py b/src/sciwyrm/filters.py new file mode 100644 index 0000000..a5508af --- /dev/null +++ b/src/sciwyrm/filters.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) +"""Template filters.""" + + +def quote(value: str) -> str: + """Surround a string in appropriate quotes.""" + if '"' in value: + if "'" in value: + return f'"""{value}"""' + return f"'{value}'" + return f'"{value}"' + + +def json_escape(value: str) -> str: + """Escape a string to be used in JSON.""" + return value.replace("\\", "\\\\").replace('"', '\\"') diff --git a/src/sciwyrm/main.py b/src/sciwyrm/main.py index 535a994..33c4edf 100644 --- a/src/sciwyrm/main.py +++ b/src/sciwyrm/main.py @@ -4,15 +4,15 @@ from __future__ import annotations -from fastapi import FastAPI +from fastapi import FastAPI, Request, Response +from fastapi.responses import JSONResponse from .notebook import v1 -from .typing import Notebook app = FastAPI() -@app.post("/notebook/v1") -def notebook_v1(spec: v1.NotebookSpecV1) -> Notebook: +@app.post("/notebook/v1", response_class=JSONResponse) +async def notebook_v1(request: Request, spec: v1.NotebookSpecV1) -> Response: """Format and return a notebook.""" - return v1.format_notebook(spec) + return v1.format_notebook(request, spec) diff --git a/src/sciwyrm/notebook/v1.py b/src/sciwyrm/notebook/v1.py index 859e9fc..743504c 100644 --- a/src/sciwyrm/notebook/v1.py +++ b/src/sciwyrm/notebook/v1.py @@ -2,51 +2,41 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Version 1 notebooks.""" +from typing import Any + +from fastapi import Request, Response from pydantic import BaseModel -from .. import assets -from ..typing import Notebook +from ..assets import render_notebook_template class NotebookSpecV1(BaseModel): - """Specifies which notebook to return and how to format it.""" + """Specifies which notebook to return and how to format it. + + This is for version 1 of the endpoint. + The template version is independent of that. + """ - notebook_name: str - notebook_version: str + template_name: str + template_version: str dataset_pids: list[str] file_server_host: str file_server_port: int scicat_url: str - scicat_token: str | None = None - - -def _scicat_url_cell_source( - scicat_url: str, file_server_host: str, file_server_port: int -) -> list[str]: - return [ - f'scicat_url = "{scicat_url}"\n', - f'file_server_host = "{file_server_host}"\n', - f'file_server_port = "{file_server_port}"', - ] + scicat_token: str = "INSERT-YOUR-SCICAT-TOKEN-HERE" -def _scicat_token_cell_source(scicat_token: str | None) -> list[str]: - return [f'scicat_token = "{scicat_token or "INSERT-YOUR-SCICAT-TOKEN-HERE"}"'] +def _build_context(spec: NotebookSpecV1) -> dict[str, Any]: + return { + key.upper(): value for key, value in spec.model_dump(exclude_none=True).items() + } -def _pids_cell_source(pids: list[str]) -> list[str]: - return ["input_dataset_pids = [\n", *(f' "{pid}",\n' for pid in pids), "]"] - - -def format_notebook(spec: NotebookSpecV1) -> Notebook: - """Return a formatted version 1 notebook.""" - nb = assets.notebook_template( - name=spec.notebook_name, version=spec.notebook_version - ) - cells = nb["cells"] - cells[1]["source"] = _scicat_url_cell_source( - spec.scicat_url, spec.file_server_host, spec.file_server_port +def format_notebook(request: Request, spec: NotebookSpecV1) -> Response: + """Return a formatted notebook.""" + return render_notebook_template( + name=spec.template_name, + version=spec.template_version, + request=request, + context=_build_context(spec), ) - cells[2]["source"] = _scicat_token_cell_source(spec.scicat_token) - cells[3]["source"] = _pids_cell_source(spec.dataset_pids) - return nb diff --git a/src/sciwyrm/typing.py b/src/sciwyrm/typing.py deleted file mode 100644 index 5569358..0000000 --- a/src/sciwyrm/typing.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) -"""Utilities for type annotations.""" - -from typing import Any - -Json = dict[str, "Json"] | list["Json"] | str | int | float | bool | None -Notebook = dict[str, Any] diff --git a/tests/notebook/notebook_v1_test.py b/tests/notebook/notebook_v1_test.py index 3af8574..963fdbe 100644 --- a/tests/notebook/notebook_v1_test.py +++ b/tests/notebook/notebook_v1_test.py @@ -38,8 +38,8 @@ def test_notebook_contains_expected_url(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ - "notebook_name": "generic", - "notebook_version": "1", + "template_name": "generic", + "template_version": "1", "scicat_url": url, "file_server_host": "login", "file_server_port": 22, @@ -55,8 +55,8 @@ def test_notebook_contains_expected_pids(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ - "notebook_name": "generic", - "notebook_version": "1", + "template_name": "generic", + "template_version": "1", "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", "file_server_port": 22, @@ -75,8 +75,8 @@ def test_notebook_contains_only_expected_pids(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ - "notebook_name": "generic", - "notebook_version": "1", + "template_name": "generic", + "template_version": "1", "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", "file_server_port": 22, @@ -88,8 +88,8 @@ def test_notebook_contains_only_expected_pids(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ - "notebook_name": "generic", - "notebook_version": "1", + "template_name": "generic", + "template_version": "1", "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", "file_server_port": 22, @@ -108,8 +108,8 @@ def test_notebook_contains_expected_file_serve_host(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ - "notebook_name": "generic", - "notebook_version": "1", + "template_name": "generic", + "template_version": "1", "scicat_url": "https://test-url.sci.cat", "file_server_host": file_server_host, "file_server_port": 22, @@ -125,8 +125,8 @@ def test_notebook_contains_expected_file_serve_port(sciwyrm_client): response = sciwyrm_client.post( "/notebook/v1", json={ - "notebook_name": "generic", - "notebook_version": "1", + "template_name": "generic", + "template_version": "1", "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", "file_server_port": file_server_port, @@ -152,8 +152,8 @@ def test_notebook_run( response = sciwyrm_client.post( "/notebook/v1", json={ - "notebook_name": "generic", - "notebook_version": "1", + "template_name": "generic", + "template_version": "1", "scicat_url": scicat_access.url, "file_server_host": sftp_access.host, "file_server_port": sftp_access.port, From 32d07204748d29afaacdffe2c5579889d3e007e7 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 10 Jan 2024 15:08:30 +0100 Subject: [PATCH 06/25] Add logging module --- src/sciwyrm/logging.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/sciwyrm/logging.py diff --git a/src/sciwyrm/logging.py b/src/sciwyrm/logging.py new file mode 100644 index 0000000..ed9ee4c --- /dev/null +++ b/src/sciwyrm/logging.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) +"""Logging tools and setup.""" + +import logging + + +def get_logger() -> logging.Logger: + """Return the Sciwyrm logger.""" + return logging.getLogger("sciwyrm") + + +def _configure() -> None: + logger = get_logger() + logger.setLevel(logging.INFO) + handler = logging.StreamHandler() + handler.setLevel(logging.INFO) + handler.setFormatter( + logging.Formatter( + "%(asctime)s %(name)s [%(levelname)s] %(message)s", + datefmt="%Y-%m-%dT%H:%M:%S%z", + ) + ) + logger.addHandler(handler) + + +_configure() From 559a0b9ad51d4d4599e40f4593de1d9e8167afd3 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 10 Jan 2024 15:13:07 +0100 Subject: [PATCH 07/25] Add pydantic-settings dependency --- pyproject.toml | 1 + requirements/base.in | 1 + requirements/base.txt | 7 ++++++- requirements/docs.txt | 4 ---- requirements/test.txt | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5c647e6..c4e73ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "fastapi >= 0.108", "jinja2", "pydantic", + "pydantic-settings", ] dynamic = ["version"] diff --git a/requirements/base.in b/requirements/base.in index 08744a5..623b750 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,3 +1,4 @@ fastapi >= 0.108 jinja2 pydantic +pydantic-settings diff --git a/requirements/base.txt b/requirements/base.txt index 162a3d2..9b5ed52 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:5bdee4462eb709fd035300da208c144e0abd18e2 +# SHA1:3cd8a9bfb6fac9081e7d66680df6a85f7ab6a5d8 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -21,8 +21,13 @@ pydantic==2.5.3 # via # -r requirements/base.in # fastapi + # pydantic-settings pydantic-core==2.14.6 # via pydantic +pydantic-settings==2.1.0 + # via -r requirements/base.in +python-dotenv==1.0.0 + # via pydantic-settings sniffio==1.3.0 # via anyio starlette==0.32.0.post1 diff --git a/requirements/docs.txt b/requirements/docs.txt index 91439ae..de0d67c 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -47,12 +47,8 @@ nest-asyncio==1.5.8 # via ipykernel psutil==5.9.7 # via ipykernel -pydantic-settings==2.1.0 - # via autodoc-pydantic pydata-sphinx-theme==0.14.0 # via -r requirements/docs.in -python-dotenv==1.0.0 - # via pydantic-settings snowballstemmer==2.2.0 # via sphinx sphinx==7.2.6 diff --git a/requirements/test.txt b/requirements/test.txt index d7b2d69..ae32bb0 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -58,7 +58,7 @@ httpcore==1.0.2 # via httpx httpx==0.26.0 # via -r requirements/test.in -hypothesis==6.92.6 +hypothesis==6.92.7 # via scitacean iniconfig==2.0.0 # via pytest From d101a14399f06daf583068feba8f84408743be99 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 10 Jan 2024 15:26:51 +0100 Subject: [PATCH 08/25] Use config file --- src/sciwyrm/assets/__init__.py | 50 +++++------------ src/sciwyrm/config.py | 89 ++++++++++++++++++++++++++++++ src/sciwyrm/main.py | 18 ++++-- src/sciwyrm/notebook/v1.py | 16 +----- tests/conftest.py | 29 ++++++++++ tests/notebook/notebook_v1_test.py | 8 --- 6 files changed, 149 insertions(+), 61 deletions(-) create mode 100644 src/sciwyrm/config.py diff --git a/src/sciwyrm/assets/__init__.py b/src/sciwyrm/assets/__init__.py index 320d951..9e9bd14 100644 --- a/src/sciwyrm/assets/__init__.py +++ b/src/sciwyrm/assets/__init__.py @@ -2,54 +2,34 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Asset loaders for SciWyrm.""" +from functools import lru_cache from pathlib import Path -from typing import Any +from typing import Annotated -from fastapi import Request +from fastapi import Depends from fastapi.templating import Jinja2Templates from jinja2 import Environment, FileSystemLoader +from ..config import AppConfig, app_config -def _make_template_handler() -> Jinja2Templates: + +def get_templates(config: Annotated[AppConfig, Depends(app_config)]) -> Jinja2Templates: + """Return a handler for loading and rendering templates.""" + return _make_template_handler(config.template_dir) + + +@lru_cache(maxsize=1) +def _make_template_handler(template_dir: Path) -> Jinja2Templates: from .. import filters + from ..logging import get_logger + get_logger().info("Loading templates from %s", template_dir) templates = Jinja2Templates( env=Environment( - loader=FileSystemLoader(Path(__file__).resolve().parent / "templates"), + loader=FileSystemLoader(template_dir), autoescape=True, ) ) templates.env.filters["quote"] = filters.quote templates.env.filters["je"] = filters.json_escape return templates - - -_templates = _make_template_handler() - - -def render_notebook_template( - *, name: str, version: str, request: Request, context: dict[str, Any] -) -> Any: - """Render a given notebook template into a response-compatible object. - - Parameters - ---------- - name: - Name of the template. - version: - Version of the template. - request: - FastAPI request that requested the notebook. - context: - Values to insert into the template. - - Returns - ------- - : - An objects that can be returned from an endpoint as a JSONResponse. - """ - return _templates.TemplateResponse( - name=f"notebook/{name}_v{version}.ipynb", - request=request, - context={**context}, - ) diff --git a/src/sciwyrm/config.py b/src/sciwyrm/config.py new file mode 100644 index 0000000..75c7fc2 --- /dev/null +++ b/src/sciwyrm/config.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) +"""Application and template configuration.""" + +import json +from functools import lru_cache +from pathlib import Path +from typing import Any, Generator + +from pydantic.fields import FieldInfo +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, +) + + +class _JSONFileSettingsSource(PydanticBaseSettingsSource): + def __init__( + self, *, filename: Path | str, settings_cls: type[BaseSettings] + ) -> None: + super().__init__(settings_cls) + try: + self._file_content = json.loads(Path(filename).read_text(encoding="utf-8")) + except FileNotFoundError: + self._file_content = {} + + def get_field_value( + self, field: FieldInfo, field_name: str + ) -> tuple[Any, str, bool]: + field_value = self._file_content.get(field_name) + return field_value, field_name, field_name == "nums" + + def prepare_field_value( + self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool + ) -> Any: + # Json.loads already decoded complex fields. + return value + + def _get_fields(self) -> Generator[tuple[str, Any], None, None]: + for field_name, field in self.settings_cls.model_fields.items(): + field_value, field_key, value_is_complex = self.get_field_value( + field, field_name + ) + field_value = self.prepare_field_value( + field_name, field, field_value, value_is_complex + ) + yield field_key, field_value + + def __call__(self) -> dict[str, Any]: + return dict(self._get_fields()) + + +class AppConfig(BaseSettings): + """Sciwyrm application config.""" + + template_dir: Path + + model_config = SettingsConfigDict(env_prefix="sciwyrm_") + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Select settings sources.""" + _ = dotenv_settings + return ( + init_settings, + env_settings, + _JSONFileSettingsSource( + filename="./settings.json", settings_cls=settings_cls + ), + file_secret_settings, + ) + + +@lru_cache(maxsize=1) +def app_config() -> AppConfig: + """Return the application config.""" + from .logging import get_logger + + config = AppConfig() + get_logger().info("Loaded app config: %r", config) + return config diff --git a/src/sciwyrm/main.py b/src/sciwyrm/main.py index 33c4edf..64ec532 100644 --- a/src/sciwyrm/main.py +++ b/src/sciwyrm/main.py @@ -2,17 +2,27 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """The SciWym application.""" -from __future__ import annotations +from typing import Annotated -from fastapi import FastAPI, Request, Response +from fastapi import Depends, FastAPI, Request, Response from fastapi.responses import JSONResponse +from fastapi.templating import Jinja2Templates +from .assets import get_templates from .notebook import v1 app = FastAPI() @app.post("/notebook/v1", response_class=JSONResponse) -async def notebook_v1(request: Request, spec: v1.NotebookSpecV1) -> Response: +async def format_notebook( + request: Request, + spec: v1.NotebookSpecV1, + templates: Annotated[Jinja2Templates, Depends(get_templates)], +) -> Response: """Format and return a notebook.""" - return v1.format_notebook(request, spec) + return templates.TemplateResponse( + name=f"notebook/{spec.template_name}_v{spec.template_version}.ipynb", + request=request, + context=v1.render_context(spec), + ) diff --git a/src/sciwyrm/notebook/v1.py b/src/sciwyrm/notebook/v1.py index 743504c..0a965b2 100644 --- a/src/sciwyrm/notebook/v1.py +++ b/src/sciwyrm/notebook/v1.py @@ -4,11 +4,8 @@ from typing import Any -from fastapi import Request, Response from pydantic import BaseModel -from ..assets import render_notebook_template - class NotebookSpecV1(BaseModel): """Specifies which notebook to return and how to format it. @@ -26,17 +23,8 @@ class NotebookSpecV1(BaseModel): scicat_token: str = "INSERT-YOUR-SCICAT-TOKEN-HERE" -def _build_context(spec: NotebookSpecV1) -> dict[str, Any]: +def render_context(spec: NotebookSpecV1) -> dict[str, Any]: + """Return a dict that can be used to render a notebook template.""" return { key.upper(): value for key, value in spec.model_dump(exclude_none=True).items() } - - -def format_notebook(request: Request, spec: NotebookSpecV1) -> Response: - """Return a formatted notebook.""" - return render_notebook_template( - name=spec.template_name, - version=spec.template_version, - request=request, - context=_build_context(spec), - ) diff --git a/tests/conftest.py b/tests/conftest.py index ef1e52f..5070b55 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,13 +2,17 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) import os +from pathlib import Path import pytest import scitacean +from fastapi.testclient import TestClient from scitacean.testing.backend import add_pytest_option as add_backend_option from scitacean.testing.sftp import add_pytest_option as add_sftp_option from scitacean.transfer.sftp import SFTPFileTransfer +from sciwyrm.config import AppConfig, app_config + from .seed import seed_scicat # Silence warning from Jupyter @@ -50,3 +54,28 @@ def scicat_client( real_client, _local_scicat, require_scicat_backend, require_sftp_fileserver ) -> scitacean.Client: return real_client + + +def _app_config_override(): + return AppConfig( + template_dir=Path(__file__).resolve().parent.parent + / "src" + / "sciwyrm" + / "assets" + / "templates" + ) + + +@pytest.fixture(scope="session") +def app(): + from sciwyrm.main import app + + old_overrides = dict(app.dependency_overrides) + app.dependency_overrides[app_config] = _app_config_override + yield app + app.dependency_overrides = old_overrides + + +@pytest.fixture +def sciwyrm_client(app): + return TestClient(app) diff --git a/tests/notebook/notebook_v1_test.py b/tests/notebook/notebook_v1_test.py index 963fdbe..b9ca948 100644 --- a/tests/notebook/notebook_v1_test.py +++ b/tests/notebook/notebook_v1_test.py @@ -6,19 +6,11 @@ import nbformat import pytest -from fastapi.testclient import TestClient from nbconvert import PythonExporter -from sciwyrm.main import app - from ..seed import SEED -@pytest.fixture -def sciwyrm_client(): - return TestClient(app) - - @pytest.fixture def scicat_token(scicat_client): return scicat_client.scicat._token.get_str() From 480a99ec2096b71f09ed14ba8537ca8bba94cf24 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 10 Jan 2024 15:35:46 +0100 Subject: [PATCH 09/25] Move templates out of source tree --- src/sciwyrm/main.py | 4 +-- .../{assets/__init__.py => templates.py} | 26 ++++++++++++++++--- .../notebook/generic_v1.ipynb | 0 tests/conftest.py | 8 +----- 4 files changed, 25 insertions(+), 13 deletions(-) rename src/sciwyrm/{assets/__init__.py => templates.py} (64%) rename {src/sciwyrm/assets/templates => templates}/notebook/generic_v1.ipynb (100%) diff --git a/src/sciwyrm/main.py b/src/sciwyrm/main.py index 64ec532..68a2d85 100644 --- a/src/sciwyrm/main.py +++ b/src/sciwyrm/main.py @@ -8,8 +8,8 @@ from fastapi.responses import JSONResponse from fastapi.templating import Jinja2Templates -from .assets import get_templates from .notebook import v1 +from .templates import get_templates, notebook_template_path app = FastAPI() @@ -22,7 +22,7 @@ async def format_notebook( ) -> Response: """Format and return a notebook.""" return templates.TemplateResponse( - name=f"notebook/{spec.template_name}_v{spec.template_version}.ipynb", + name=notebook_template_path(spec.template_name, spec.template_version), request=request, context=v1.render_context(spec), ) diff --git a/src/sciwyrm/assets/__init__.py b/src/sciwyrm/templates.py similarity index 64% rename from src/sciwyrm/assets/__init__.py rename to src/sciwyrm/templates.py index 9e9bd14..0e3c930 100644 --- a/src/sciwyrm/assets/__init__.py +++ b/src/sciwyrm/templates.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) -"""Asset loaders for SciWyrm.""" +"""Template loading.""" from functools import lru_cache from pathlib import Path @@ -10,7 +10,7 @@ from fastapi.templating import Jinja2Templates from jinja2 import Environment, FileSystemLoader -from ..config import AppConfig, app_config +from .config import AppConfig, app_config def get_templates(config: Annotated[AppConfig, Depends(app_config)]) -> Jinja2Templates: @@ -18,10 +18,28 @@ def get_templates(config: Annotated[AppConfig, Depends(app_config)]) -> Jinja2Te return _make_template_handler(config.template_dir) +def notebook_template_path(name: str, version: str) -> str: + """Return the relative path to a given notebook template. + + Parameters + ---------- + name: + Name of the template. + version: + Version of the template. + + Returns + ------- + : + The path to the template relative to the base template directory. + """ + return f"notebook/{name}_v{version}.ipynb" + + @lru_cache(maxsize=1) def _make_template_handler(template_dir: Path) -> Jinja2Templates: - from .. import filters - from ..logging import get_logger + from . import filters + from .logging import get_logger get_logger().info("Loading templates from %s", template_dir) templates = Jinja2Templates( diff --git a/src/sciwyrm/assets/templates/notebook/generic_v1.ipynb b/templates/notebook/generic_v1.ipynb similarity index 100% rename from src/sciwyrm/assets/templates/notebook/generic_v1.ipynb rename to templates/notebook/generic_v1.ipynb diff --git a/tests/conftest.py b/tests/conftest.py index 5070b55..12f21a2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,13 +57,7 @@ def scicat_client( def _app_config_override(): - return AppConfig( - template_dir=Path(__file__).resolve().parent.parent - / "src" - / "sciwyrm" - / "assets" - / "templates" - ) + return AppConfig(template_dir=Path(__file__).resolve().parent.parent / "templates") @pytest.fixture(scope="session") From 56b819811e103c2efd4629fbfdb07cc78ef4e533 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 10 Jan 2024 15:38:55 +0100 Subject: [PATCH 10/25] Remove v1 from API and package --- src/sciwyrm/main.py | 8 ++--- src/sciwyrm/notebook/__init__.py | 27 +++++++++++++++++ src/sciwyrm/notebook/v1.py | 30 ------------------- .../{notebook_v1_test.py => notebook_test.py} | 14 ++++----- 4 files changed, 38 insertions(+), 41 deletions(-) delete mode 100644 src/sciwyrm/notebook/v1.py rename tests/notebook/{notebook_v1_test.py => notebook_test.py} (96%) diff --git a/src/sciwyrm/main.py b/src/sciwyrm/main.py index 68a2d85..51b679a 100644 --- a/src/sciwyrm/main.py +++ b/src/sciwyrm/main.py @@ -8,21 +8,21 @@ from fastapi.responses import JSONResponse from fastapi.templating import Jinja2Templates -from .notebook import v1 +from . import notebook from .templates import get_templates, notebook_template_path app = FastAPI() -@app.post("/notebook/v1", response_class=JSONResponse) +@app.post("/notebook", response_class=JSONResponse) async def format_notebook( request: Request, - spec: v1.NotebookSpecV1, + spec: notebook.NotebookSpecV1, templates: Annotated[Jinja2Templates, Depends(get_templates)], ) -> Response: """Format and return a notebook.""" return templates.TemplateResponse( name=notebook_template_path(spec.template_name, spec.template_version), request=request, - context=v1.render_context(spec), + context=notebook.render_context(spec), ) diff --git a/src/sciwyrm/notebook/__init__.py b/src/sciwyrm/notebook/__init__.py index 9022f23..e67b1f3 100644 --- a/src/sciwyrm/notebook/__init__.py +++ b/src/sciwyrm/notebook/__init__.py @@ -1,3 +1,30 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Notebook handling.""" + +from typing import Any + +from pydantic import BaseModel + + +class NotebookSpecV1(BaseModel): + """Specifies which notebook to return and how to format it. + + This is for version 1 of the endpoint. + The template version is independent of that. + """ + + template_name: str + template_version: str + dataset_pids: list[str] + file_server_host: str + file_server_port: int + scicat_url: str + scicat_token: str = "INSERT-YOUR-SCICAT-TOKEN-HERE" + + +def render_context(spec: NotebookSpecV1) -> dict[str, Any]: + """Return a dict that can be used to render a notebook template.""" + return { + key.upper(): value for key, value in spec.model_dump(exclude_none=True).items() + } diff --git a/src/sciwyrm/notebook/v1.py b/src/sciwyrm/notebook/v1.py deleted file mode 100644 index 0a965b2..0000000 --- a/src/sciwyrm/notebook/v1.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) -"""Version 1 notebooks.""" - -from typing import Any - -from pydantic import BaseModel - - -class NotebookSpecV1(BaseModel): - """Specifies which notebook to return and how to format it. - - This is for version 1 of the endpoint. - The template version is independent of that. - """ - - template_name: str - template_version: str - dataset_pids: list[str] - file_server_host: str - file_server_port: int - scicat_url: str - scicat_token: str = "INSERT-YOUR-SCICAT-TOKEN-HERE" - - -def render_context(spec: NotebookSpecV1) -> dict[str, Any]: - """Return a dict that can be used to render a notebook template.""" - return { - key.upper(): value for key, value in spec.model_dump(exclude_none=True).items() - } diff --git a/tests/notebook/notebook_v1_test.py b/tests/notebook/notebook_test.py similarity index 96% rename from tests/notebook/notebook_v1_test.py rename to tests/notebook/notebook_test.py index b9ca948..8b0b4be 100644 --- a/tests/notebook/notebook_v1_test.py +++ b/tests/notebook/notebook_test.py @@ -28,7 +28,7 @@ def exec_notebook(nb_code: str) -> dict[str, Any]: def test_notebook_contains_expected_url(sciwyrm_client): url = "https://test-url.sci.cat" response = sciwyrm_client.post( - "/notebook/v1", + "/notebook", json={ "template_name": "generic", "template_version": "1", @@ -45,7 +45,7 @@ def test_notebook_contains_expected_url(sciwyrm_client): def test_notebook_contains_expected_pids(sciwyrm_client): pids = ["7192983", "7ca7/31a.2as"] response = sciwyrm_client.post( - "/notebook/v1", + "/notebook", json={ "template_name": "generic", "template_version": "1", @@ -65,7 +65,7 @@ def test_notebook_contains_only_expected_pids(sciwyrm_client): pids1 = ["9391"] response = sciwyrm_client.post( - "/notebook/v1", + "/notebook", json={ "template_name": "generic", "template_version": "1", @@ -78,7 +78,7 @@ def test_notebook_contains_only_expected_pids(sciwyrm_client): assert response.status_code == 200 response = sciwyrm_client.post( - "/notebook/v1", + "/notebook", json={ "template_name": "generic", "template_version": "1", @@ -98,7 +98,7 @@ def test_notebook_contains_only_expected_pids(sciwyrm_client): def test_notebook_contains_expected_file_serve_host(sciwyrm_client): file_server_host = "test.host.cat" response = sciwyrm_client.post( - "/notebook/v1", + "/notebook", json={ "template_name": "generic", "template_version": "1", @@ -115,7 +115,7 @@ def test_notebook_contains_expected_file_serve_host(sciwyrm_client): def test_notebook_contains_expected_file_serve_port(sciwyrm_client): file_server_port = 2200 response = sciwyrm_client.post( - "/notebook/v1", + "/notebook", json={ "template_name": "generic", "template_version": "1", @@ -142,7 +142,7 @@ def test_notebook_run( require_sftp_fileserver, ): response = sciwyrm_client.post( - "/notebook/v1", + "/notebook", json={ "template_name": "generic", "template_version": "1", From 391e5ac5a0747603cba876a9201e5e95f71a7844 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 10 Jan 2024 15:57:31 +0100 Subject: [PATCH 11/25] Add endpoint to list templates --- src/sciwyrm/main.py | 11 ++++++++++- src/sciwyrm/templates.py | 14 ++++++++++++++ tests/notebook/notebook_test.py | 6 ++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/sciwyrm/main.py b/src/sciwyrm/main.py index 51b679a..070835d 100644 --- a/src/sciwyrm/main.py +++ b/src/sciwyrm/main.py @@ -9,11 +9,20 @@ from fastapi.templating import Jinja2Templates from . import notebook -from .templates import get_templates, notebook_template_path +from .config import AppConfig, app_config +from .templates import get_templates, list_notebook_templates, notebook_template_path app = FastAPI() +@app.get("/notebook/templates") +async def list_templates( + config: Annotated[AppConfig, Depends(app_config)] +) -> JSONResponse: + """Return a list of available notebook templates.""" + return JSONResponse(list_notebook_templates(config)) + + @app.post("/notebook", response_class=JSONResponse) async def format_notebook( request: Request, diff --git a/src/sciwyrm/templates.py b/src/sciwyrm/templates.py index 0e3c930..812103f 100644 --- a/src/sciwyrm/templates.py +++ b/src/sciwyrm/templates.py @@ -36,6 +36,20 @@ def notebook_template_path(name: str, version: str) -> str: return f"notebook/{name}_v{version}.ipynb" +def list_notebook_templates(config: AppConfig) -> list[dict[str, str]]: + """List available notebook templates.""" + return [ + _split_notebook_template_name(path.stem) + for path in config.template_dir.joinpath("notebook").iterdir() + if path.suffix == ".ipynb" + ] + + +def _split_notebook_template_name(full_name: str) -> dict[str, str]: + name, version = full_name.split("_v") + return {"name": name, "version": version} + + @lru_cache(maxsize=1) def _make_template_handler(template_dir: Path) -> Jinja2Templates: from . import filters diff --git a/tests/notebook/notebook_test.py b/tests/notebook/notebook_test.py index 8b0b4be..3961751 100644 --- a/tests/notebook/notebook_test.py +++ b/tests/notebook/notebook_test.py @@ -25,6 +25,12 @@ def exec_notebook(nb_code: str) -> dict[str, Any]: return namespace +def test_list_notebook_templates(sciwyrm_client): + response = sciwyrm_client.get("/notebook/templates") + assert response.status_code == 200 + assert response.json() == [{"name": "generic", "version": "1"}] + + def test_notebook_contains_expected_url(sciwyrm_client): url = "https://test-url.sci.cat" response = sciwyrm_client.post( From 8678d0a50340def8b2cdc30bd04dfca60084252e Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 10 Jan 2024 16:02:06 +0100 Subject: [PATCH 12/25] Test that all placeholders are replaced --- tests/notebook/notebook_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/notebook/notebook_test.py b/tests/notebook/notebook_test.py index 3961751..39e2686 100644 --- a/tests/notebook/notebook_test.py +++ b/tests/notebook/notebook_test.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) +import re from io import StringIO from typing import Any @@ -135,6 +136,23 @@ def test_notebook_contains_expected_file_serve_port(sciwyrm_client): assert str(file_server_port) in response.text +def test_notebook_contains_no_placeholders(sciwyrm_client): + response = sciwyrm_client.post( + "/notebook", + json={ + "template_name": "generic", + "template_version": "1", + "scicat_url": "https://test-url.sci.cat", + "file_server_host": "login", + "file_server_port": 22, + "dataset_pids": ["abcd/123.522"], + }, + ) + assert response.status_code == 200 + assert not re.search(r"{{[^}]*}}", response.text) + assert not re.search(r"{%[^}]*%}", response.text) + + # This requires a way to either pass a custom connect function to the # SFTPFileTransfer in the notebook or a proper auth through SciCat. # The former is very tricky; so waiting for the latter for now. From 7cb99c4934414b952885ae1c0d54bd7cae9b4aa5 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 10 Jan 2024 16:26:13 +0100 Subject: [PATCH 13/25] Nest notebook parameters --- src/sciwyrm/main.py | 2 +- src/sciwyrm/notebook/__init__.py | 20 ++------ tests/notebook/notebook_test.py | 82 +++++++++++++++++++------------- 3 files changed, 55 insertions(+), 49 deletions(-) diff --git a/src/sciwyrm/main.py b/src/sciwyrm/main.py index 070835d..24930e9 100644 --- a/src/sciwyrm/main.py +++ b/src/sciwyrm/main.py @@ -26,7 +26,7 @@ async def list_templates( @app.post("/notebook", response_class=JSONResponse) async def format_notebook( request: Request, - spec: notebook.NotebookSpecV1, + spec: notebook.NotebookSpec, templates: Annotated[Jinja2Templates, Depends(get_templates)], ) -> Response: """Format and return a notebook.""" diff --git a/src/sciwyrm/notebook/__init__.py b/src/sciwyrm/notebook/__init__.py index e67b1f3..564115c 100644 --- a/src/sciwyrm/notebook/__init__.py +++ b/src/sciwyrm/notebook/__init__.py @@ -7,24 +7,14 @@ from pydantic import BaseModel -class NotebookSpecV1(BaseModel): - """Specifies which notebook to return and how to format it. - - This is for version 1 of the endpoint. - The template version is independent of that. - """ +class NotebookSpec(BaseModel): + """Specifies which notebook to return and how to format it.""" template_name: str template_version: str - dataset_pids: list[str] - file_server_host: str - file_server_port: int - scicat_url: str - scicat_token: str = "INSERT-YOUR-SCICAT-TOKEN-HERE" + parameters: dict[str, Any] -def render_context(spec: NotebookSpecV1) -> dict[str, Any]: +def render_context(spec: NotebookSpec) -> dict[str, Any]: """Return a dict that can be used to render a notebook template.""" - return { - key.upper(): value for key, value in spec.model_dump(exclude_none=True).items() - } + return {key.upper(): value for key, value in spec.parameters.items()} diff --git a/tests/notebook/notebook_test.py b/tests/notebook/notebook_test.py index 39e2686..3f289f5 100644 --- a/tests/notebook/notebook_test.py +++ b/tests/notebook/notebook_test.py @@ -39,10 +39,12 @@ def test_notebook_contains_expected_url(sciwyrm_client): json={ "template_name": "generic", "template_version": "1", - "scicat_url": url, - "file_server_host": "login", - "file_server_port": 22, - "dataset_pids": [], + "parameters": { + "scicat_url": url, + "file_server_host": "login", + "file_server_port": 22, + "dataset_pids": [], + }, }, ) assert response.status_code == 200 @@ -56,10 +58,12 @@ def test_notebook_contains_expected_pids(sciwyrm_client): json={ "template_name": "generic", "template_version": "1", - "scicat_url": "https://test-url.sci.cat", - "file_server_host": "login", - "file_server_port": 22, - "dataset_pids": pids, + "parameters": { + "scicat_url": "https://test-url.sci.cat", + "file_server_host": "login", + "file_server_port": 22, + "dataset_pids": pids, + }, }, ) assert response.status_code == 200 @@ -76,10 +80,12 @@ def test_notebook_contains_only_expected_pids(sciwyrm_client): json={ "template_name": "generic", "template_version": "1", - "scicat_url": "https://test-url.sci.cat", - "file_server_host": "login", - "file_server_port": 22, - "dataset_pids": pids0, + "parameters": { + "scicat_url": "https://test-url.sci.cat", + "file_server_host": "login", + "file_server_port": 22, + "dataset_pids": pids0, + }, }, ) assert response.status_code == 200 @@ -89,10 +95,12 @@ def test_notebook_contains_only_expected_pids(sciwyrm_client): json={ "template_name": "generic", "template_version": "1", - "scicat_url": "https://test-url.sci.cat", - "file_server_host": "login", - "file_server_port": 22, - "dataset_pids": pids1, + "parameters": { + "scicat_url": "https://test-url.sci.cat", + "file_server_host": "login", + "file_server_port": 22, + "dataset_pids": pids1, + }, }, ) assert response.status_code == 200 @@ -109,10 +117,12 @@ def test_notebook_contains_expected_file_serve_host(sciwyrm_client): json={ "template_name": "generic", "template_version": "1", - "scicat_url": "https://test-url.sci.cat", - "file_server_host": file_server_host, - "file_server_port": 22, - "dataset_pids": [], + "parameters": { + "scicat_url": "https://test-url.sci.cat", + "file_server_host": file_server_host, + "file_server_port": 22, + "dataset_pids": [], + }, }, ) assert response.status_code == 200 @@ -126,10 +136,12 @@ def test_notebook_contains_expected_file_serve_port(sciwyrm_client): json={ "template_name": "generic", "template_version": "1", - "scicat_url": "https://test-url.sci.cat", - "file_server_host": "login", - "file_server_port": file_server_port, - "dataset_pids": [], + "parameters": { + "scicat_url": "https://test-url.sci.cat", + "file_server_host": "login", + "file_server_port": file_server_port, + "dataset_pids": [], + }, }, ) assert response.status_code == 200 @@ -142,10 +154,12 @@ def test_notebook_contains_no_placeholders(sciwyrm_client): json={ "template_name": "generic", "template_version": "1", - "scicat_url": "https://test-url.sci.cat", - "file_server_host": "login", - "file_server_port": 22, - "dataset_pids": ["abcd/123.522"], + "parameters": { + "scicat_url": "https://test-url.sci.cat", + "file_server_host": "login", + "file_server_port": 22, + "dataset_pids": ["abcd/123.522"], + }, }, ) assert response.status_code == 200 @@ -170,11 +184,13 @@ def test_notebook_run( json={ "template_name": "generic", "template_version": "1", - "scicat_url": scicat_access.url, - "file_server_host": sftp_access.host, - "file_server_port": sftp_access.port, - "scicat_token": scicat_token, - "dataset_pids": list(map(str, SEED)), + "parameters": { + "scicat_url": scicat_access.url, + "file_server_host": sftp_access.host, + "file_server_port": sftp_access.port, + "scicat_token": scicat_token, + "dataset_pids": list(map(str, SEED)), + }, }, ) assert response.status_code == 200 From fe11286c15dd0101649215006e7ee8e9686f9b48 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 10 Jan 2024 16:46:50 +0100 Subject: [PATCH 14/25] Add jsonschema dependency --- pyproject.toml | 1 + requirements/base.in | 1 + requirements/base.txt | 18 +++++++++++++++++- requirements/dev.txt | 1 + requirements/test.txt | 17 ----------------- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c4e73ba..e6f82dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ requires-python = ">=3.11" dependencies = [ "fastapi >= 0.108", "jinja2", + "jsonschema", "pydantic", "pydantic-settings", ] diff --git a/requirements/base.in b/requirements/base.in index 623b750..2badffa 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,4 +1,5 @@ fastapi >= 0.108 jinja2 +jsonschema pydantic pydantic-settings diff --git a/requirements/base.txt b/requirements/base.txt index 9b5ed52..d63ee6b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:3cd8a9bfb6fac9081e7d66680df6a85f7ab6a5d8 +# SHA1:914f531ccf8fcb2d3b73647c301af71df7db7d6d # # This file is autogenerated by pip-compile-multi # To update, run: @@ -9,12 +9,20 @@ annotated-types==0.6.0 # via pydantic anyio==4.2.0 # via starlette +attrs==23.2.0 + # via + # jsonschema + # referencing fastapi==0.108.0 # via -r requirements/base.in idna==3.6 # via anyio jinja2==3.1.2 # via -r requirements/base.in +jsonschema==4.20.0 + # via -r requirements/base.in +jsonschema-specifications==2023.12.1 + # via jsonschema markupsafe==2.1.3 # via jinja2 pydantic==2.5.3 @@ -28,6 +36,14 @@ pydantic-settings==2.1.0 # via -r requirements/base.in python-dotenv==1.0.0 # via pydantic-settings +referencing==0.32.1 + # via + # jsonschema + # jsonschema-specifications +rpds-py==0.16.2 + # via + # jsonschema + # referencing sniffio==1.3.0 # via anyio starlette==0.32.0.post1 diff --git a/requirements/dev.txt b/requirements/dev.txt index afab37c..a3de101 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -37,6 +37,7 @@ jsonpointer==2.4 # via jsonschema jsonschema[format-nongpl]==4.20.0 # via + # -r requirements/base.in # jupyter-events # jupyterlab-server # nbformat diff --git a/requirements/test.txt b/requirements/test.txt index ae32bb0..c8bbc8b 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,11 +8,6 @@ -r base.txt asttokens==2.4.1 # via stack-data -attrs==23.2.0 - # via - # hypothesis - # jsonschema - # referencing bcrypt==4.1.2 # via paramiko beautifulsoup4==4.12.2 @@ -68,10 +63,6 @@ ipython==8.20.0 # via -r requirements/test.in jedi==0.19.1 # via ipython -jsonschema==4.20.0 - # via nbformat -jsonschema-specifications==2023.12.1 - # via jsonschema jupyter-client==8.6.0 # via nbclient jupyter-core==5.7.1 @@ -140,16 +131,8 @@ pyyaml==6.0.1 # via scitacean pyzmq==25.1.2 # via jupyter-client -referencing==0.32.1 - # via - # jsonschema - # jsonschema-specifications requests==2.31.0 # via scitacean -rpds-py==0.16.2 - # via - # jsonschema - # referencing scitacean[sftp,ssh,test]==23.10.0 # via -r requirements/test.in six==1.16.0 From 5e1229a793536e94ec7c5f180f7fdcc522d92202 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 10 Jan 2024 16:47:53 +0100 Subject: [PATCH 15/25] Validate parameter schema --- src/sciwyrm/filters.py | 1 + src/sciwyrm/notebook/__init__.py | 25 +++++++++++++- src/sciwyrm/templates.py | 11 +++++- templates/notebook/generic_v1.json | 41 +++++++++++++++++++++++ tests/notebook/notebook_test.py | 54 ++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 templates/notebook/generic_v1.json diff --git a/src/sciwyrm/filters.py b/src/sciwyrm/filters.py index a5508af..ba05bc6 100644 --- a/src/sciwyrm/filters.py +++ b/src/sciwyrm/filters.py @@ -5,6 +5,7 @@ def quote(value: str) -> str: """Surround a string in appropriate quotes.""" + value = str(value) if '"' in value: if "'" in value: return f'"""{value}"""' diff --git a/src/sciwyrm/notebook/__init__.py b/src/sciwyrm/notebook/__init__.py index 564115c..6a6e19e 100644 --- a/src/sciwyrm/notebook/__init__.py +++ b/src/sciwyrm/notebook/__init__.py @@ -4,7 +4,11 @@ from typing import Any -from pydantic import BaseModel +import jsonschema +from pydantic import BaseModel, ValidationInfo, field_validator + +from ..config import app_config +from ..templates import get_template_config class NotebookSpec(BaseModel): @@ -14,6 +18,25 @@ class NotebookSpec(BaseModel): template_version: str parameters: dict[str, Any] + @field_validator("parameters") + @classmethod + def validate_parameters( + cls, parameters: Any, info: ValidationInfo + ) -> dict[str, Any]: + """Validate parameters against the template schema.""" + if not isinstance(parameters, dict): + raise AssertionError("Parameters must be a dict.") + + schema = get_template_config( + info.data["template_name"], info.data["template_version"], app_config() + ) + try: + jsonschema.validate(parameters, schema) + except jsonschema.ValidationError as err: + # TODO better message + raise AssertionError(str(err)) from None + return parameters + def render_context(spec: NotebookSpec) -> dict[str, Any]: """Return a dict that can be used to render a notebook template.""" diff --git a/src/sciwyrm/templates.py b/src/sciwyrm/templates.py index 812103f..410c48e 100644 --- a/src/sciwyrm/templates.py +++ b/src/sciwyrm/templates.py @@ -2,9 +2,10 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Template loading.""" +import json from functools import lru_cache from pathlib import Path -from typing import Annotated +from typing import Annotated, Any from fastapi import Depends from fastapi.templating import Jinja2Templates @@ -18,6 +19,14 @@ def get_templates(config: Annotated[AppConfig, Depends(app_config)]) -> Jinja2Te return _make_template_handler(config.template_dir) +def get_template_config(name: str, version: str, config: AppConfig) -> dict[str, Any]: + """Return a template configuration.""" + with config.template_dir.joinpath( + "notebook", f"{name}_v{version}.json" + ).open() as f: + return json.load(f) + + def notebook_template_path(name: str, version: str) -> str: """Return the relative path to a given notebook template. diff --git a/templates/notebook/generic_v1.json b/templates/notebook/generic_v1.json new file mode 100644 index 0000000..316a014 --- /dev/null +++ b/templates/notebook/generic_v1.json @@ -0,0 +1,41 @@ +{ + "description": "Specifies which notebook to return and how to format it.\n\nThis is for version 1 of the endpoint.\nThe template version is independent of that.", + "properties": { + "dataset_pids": { + "items": { + "type": "string" + }, + "title": "Dataset Pids", + "type": "array" + }, + "file_server_host": { + "title": "File Server Host", + "type": "string" + }, + "file_server_port": { + "title": "File Server Port", + "type": "integer" + }, + "scicat_url": { + "format": "uri", + "maxLength": 2083, + "minLength": 1, + "title": "Scicat Url", + "type": "string" + }, + "scicat_token": { + "default": "INSERT-YOUR-SCICAT-TOKEN-HERE", + "title": "Scicat Token", + "type": "string" + } + }, + "required": [ + "dataset_pids", + "file_server_host", + "file_server_port", + "scicat_url" + ], + "additionalProperties": false, + "title": "Generic v1", + "type": "object" +} diff --git a/tests/notebook/notebook_test.py b/tests/notebook/notebook_test.py index 3f289f5..6c693f2 100644 --- a/tests/notebook/notebook_test.py +++ b/tests/notebook/notebook_test.py @@ -167,6 +167,60 @@ def test_notebook_contains_no_placeholders(sciwyrm_client): assert not re.search(r"{%[^}]*%}", response.text) +def test_notebook_bad_parameter(sciwyrm_client): + response = sciwyrm_client.post( + "/notebook", + json={ + "template_name": "generic", + "template_version": "1", + "parameters": { + "scicat_url": "https://test-url.sci.cat", + "file_server_host": "login", + "file_server_port": 22, + "dataset_pids": "abcd/123.522", + }, + }, + ) + assert response.status_code == 422 + assert "dataset_pids" in response.text + + +def test_notebook_missing_parameter(sciwyrm_client): + response = sciwyrm_client.post( + "/notebook", + json={ + "template_name": "generic", + "template_version": "1", + "parameters": { + "scicat_url": "https://test-url.sci.cat", + "file_server_port": 22, + "dataset_pids": ["abcd/123.522"], + }, + }, + ) + assert response.status_code == 422 + assert "file_server_host" in response.text + + +def test_notebook_extra_parameter(sciwyrm_client): + response = sciwyrm_client.post( + "/notebook", + json={ + "template_name": "generic", + "template_version": "1", + "parameters": { + "scicat_url": "https://test-url.sci.cat", + "file_server_host": "login", + "file_server_port": 22, + "dataset_pids": ["abcd/123.522"], + "extra": "not allowed", + }, + }, + ) + assert response.status_code == 422 + assert "extra" in response.text + + # This requires a way to either pass a custom connect function to the # SFTPFileTransfer in the notebook or a proper auth through SciCat. # The former is very tricky; so waiting for the latter for now. From c5539e2398d556cd68e6a44523ec0a2c5646194b Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 11 Jan 2024 09:14:24 +0100 Subject: [PATCH 16/25] Turn notebook package into a module --- src/sciwyrm/{notebook/__init__.py => notebook.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/sciwyrm/{notebook/__init__.py => notebook.py} (94%) diff --git a/src/sciwyrm/notebook/__init__.py b/src/sciwyrm/notebook.py similarity index 94% rename from src/sciwyrm/notebook/__init__.py rename to src/sciwyrm/notebook.py index 6a6e19e..a3affc0 100644 --- a/src/sciwyrm/notebook/__init__.py +++ b/src/sciwyrm/notebook.py @@ -7,8 +7,8 @@ import jsonschema from pydantic import BaseModel, ValidationInfo, field_validator -from ..config import app_config -from ..templates import get_template_config +from .config import app_config +from .templates import get_template_config class NotebookSpec(BaseModel): From 2c048ebcc0af4f55fd258c57d0acf96b9e4e56ef Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 11 Jan 2024 09:37:08 +0100 Subject: [PATCH 17/25] Better param validation error --- src/sciwyrm/notebook.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/sciwyrm/notebook.py b/src/sciwyrm/notebook.py index a3affc0..a6e2219 100644 --- a/src/sciwyrm/notebook.py +++ b/src/sciwyrm/notebook.py @@ -6,6 +6,7 @@ import jsonschema from pydantic import BaseModel, ValidationInfo, field_validator +from pydantic_core import PydanticCustomError from .config import app_config from .templates import get_template_config @@ -25,7 +26,7 @@ def validate_parameters( ) -> dict[str, Any]: """Validate parameters against the template schema.""" if not isinstance(parameters, dict): - raise AssertionError("Parameters must be a dict.") + raise AssertionError("'parameters' must be a dict.") schema = get_template_config( info.data["template_name"], info.data["template_version"], app_config() @@ -33,8 +34,21 @@ def validate_parameters( try: jsonschema.validate(parameters, schema) except jsonschema.ValidationError as err: - # TODO better message - raise AssertionError(str(err)) from None + raise PydanticCustomError( + "Validation Error", + "{message}", + { + "message": err.message, + "template_name": info.data["template_name"], + "template_version": info.data["template_version"], + "instance": err.instance, + "jsonpath": err.json_path, + "schema": err.schema, + "schema_path": err.schema_path, + "validator": err.validator, + "validator_value": err.validator_value, + }, + ) from None return parameters From 62302d291d32cac9a9965500b8bb1c7b57d9373e Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 11 Jan 2024 11:43:02 +0100 Subject: [PATCH 18/25] Include metadata in notebook --- src/sciwyrm/filters.py | 2 +- src/sciwyrm/main.py | 6 ++- src/sciwyrm/notebook.py | 26 +++++++-- src/sciwyrm/templates.py | 29 ++++++++-- templates/notebook/generic_v1.ipynb | 16 ++++++ templates/notebook/generic_v1.json | 83 ++++++++++++++++------------- tests/notebook/notebook_test.py | 41 ++++++++++++++ 7 files changed, 158 insertions(+), 45 deletions(-) diff --git a/src/sciwyrm/filters.py b/src/sciwyrm/filters.py index ba05bc6..becbcd1 100644 --- a/src/sciwyrm/filters.py +++ b/src/sciwyrm/filters.py @@ -15,4 +15,4 @@ def quote(value: str) -> str: def json_escape(value: str) -> str: """Escape a string to be used in JSON.""" - return value.replace("\\", "\\\\").replace('"', '\\"') + return str(value).replace("\\", "\\\\").replace('"', '\\"') diff --git a/src/sciwyrm/main.py b/src/sciwyrm/main.py index 24930e9..bf7b9a4 100644 --- a/src/sciwyrm/main.py +++ b/src/sciwyrm/main.py @@ -2,6 +2,7 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """The SciWym application.""" +import json from typing import Annotated from fastapi import Depends, FastAPI, Request, Response @@ -30,8 +31,11 @@ async def format_notebook( templates: Annotated[Jinja2Templates, Depends(get_templates)], ) -> Response: """Format and return a notebook.""" - return templates.TemplateResponse( + formatted = templates.TemplateResponse( name=notebook_template_path(spec.template_name, spec.template_version), request=request, context=notebook.render_context(spec), ) + nb = json.loads(formatted.body) + nb["metadata"]["sciwyrm"] = notebook.notebook_metadata(spec) + return JSONResponse(nb) diff --git a/src/sciwyrm/notebook.py b/src/sciwyrm/notebook.py index a6e2219..9b69490 100644 --- a/src/sciwyrm/notebook.py +++ b/src/sciwyrm/notebook.py @@ -2,6 +2,7 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Notebook handling.""" +from datetime import datetime, timezone from typing import Any import jsonschema @@ -9,7 +10,7 @@ from pydantic_core import PydanticCustomError from .config import app_config -from .templates import get_template_config +from .templates import get_template_config, notebook_template_hash class NotebookSpec(BaseModel): @@ -30,7 +31,7 @@ def validate_parameters( schema = get_template_config( info.data["template_name"], info.data["template_version"], app_config() - ) + )["parameter_schema"] try: jsonschema.validate(parameters, schema) except jsonschema.ValidationError as err: @@ -54,4 +55,23 @@ def validate_parameters( def render_context(spec: NotebookSpec) -> dict[str, Any]: """Return a dict that can be used to render a notebook template.""" - return {key.upper(): value for key, value in spec.parameters.items()} + context = spec.parameters | notebook_metadata(spec) + return {key.upper(): value for key, value in context.items()} + + +def notebook_metadata(spec: NotebookSpec) -> dict[str, Any]: + """Return metadata for a requested notebook. + + Here, metadata is any data that was not explicitly requested by the user. + """ + app_conf = app_config() + config = get_template_config(spec.template_name, spec.template_version, app_conf) + return { + "template_name": spec.template_name, + "template_version": spec.template_version, + "template_authors": config["authors"], + "template_rendered": datetime.now(tz=timezone.utc).isoformat(), + "template_hash": notebook_template_hash( + spec.template_name, spec.template_version, app_conf + ), + } diff --git a/src/sciwyrm/templates.py b/src/sciwyrm/templates.py index 410c48e..a81783e 100644 --- a/src/sciwyrm/templates.py +++ b/src/sciwyrm/templates.py @@ -2,6 +2,7 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Template loading.""" +import hashlib import json from functools import lru_cache from pathlib import Path @@ -19,11 +20,19 @@ def get_templates(config: Annotated[AppConfig, Depends(app_config)]) -> Jinja2Te return _make_template_handler(config.template_dir) -def get_template_config(name: str, version: str, config: AppConfig) -> dict[str, Any]: +def get_template_config( + name: str, version: str, config: AppConfig, category: str = "notebook" +) -> dict[str, Any]: """Return a template configuration.""" - with config.template_dir.joinpath( - "notebook", f"{name}_v{version}.json" - ).open() as f: + return _load_template_config( + config.template_dir.joinpath(category, f"{name}_v{version}.json") + ) + + +@lru_cache() +def _load_template_config(path: Path) -> dict[str, Any]: + # AppConfig cannot be hashed and used with lru_cache. + with path.open() as f: return json.load(f) @@ -54,6 +63,18 @@ def list_notebook_templates(config: AppConfig) -> list[dict[str, str]]: ] +def notebook_template_hash(name: str, version: str, config: AppConfig) -> str: + """Return a hash for a notebook template.""" + return ( + "blake2b:" + + hashlib.blake2b( + config.template_dir.joinpath( + notebook_template_path(name, version) + ).read_bytes() + ).hexdigest() + ) + + def _split_notebook_template_name(full_name: str) -> dict[str, str]: name, version = full_name.split("_v") return {"name": name, "version": version} diff --git a/templates/notebook/generic_v1.ipynb b/templates/notebook/generic_v1.ipynb index 7b777a4..f69665b 100644 --- a/templates/notebook/generic_v1.ipynb +++ b/templates/notebook/generic_v1.ipynb @@ -1,5 +1,21 @@ { "cells": [ + { + "cell_type": "markdown", + "id": "803f7b636cfa4b4c", + "metadata": {}, + "source": [ + "# {{ TEMPLATE_NAME | capitalize }}" + ] + }, + { + "cell_type": "markdown", + "id": "9e7442fc9ec68975", + "metadata": {}, + "source": [ + "Notebook generated from template '{{ TEMPLATE_NAME }}' version {{ TEMPLATE_VERSION }}." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/templates/notebook/generic_v1.json b/templates/notebook/generic_v1.json index 316a014..c6d22e5 100644 --- a/templates/notebook/generic_v1.json +++ b/templates/notebook/generic_v1.json @@ -1,41 +1,52 @@ { - "description": "Specifies which notebook to return and how to format it.\n\nThis is for version 1 of the endpoint.\nThe template version is independent of that.", - "properties": { - "dataset_pids": { - "items": { + "description": "Generic template for downloading datasets and files.", + "authors": [ + { + "name": "Jan-Lukas Wynen", + "email": "jan-lukas.wynen@ess.eu" + } + ], + "parameter_schema": { + "description": "Specifies which notebook to return and how to format it.", + "properties": { + "dataset_pids": { + "items": { + "type": "string" + }, + "title": "Dataset Pids", + "type": "array" + }, + "file_server_host": { + "title": "File Server Host", "type": "string" }, - "title": "Dataset Pids", - "type": "array" - }, - "file_server_host": { - "title": "File Server Host", - "type": "string" - }, - "file_server_port": { - "title": "File Server Port", - "type": "integer" - }, - "scicat_url": { - "format": "uri", - "maxLength": 2083, - "minLength": 1, - "title": "Scicat Url", - "type": "string" + "file_server_port": { + "title": "File Server Port", + "type": "integer" + }, + "scicat_url": { + "format": "uri", + "maxLength": 2083, + "minLength": 1, + "title": "Scicat Url", + "type": "string" + }, + "scicat_token": { + "default": "INSERT-YOUR-SCICAT-TOKEN-HERE", + "format": "password", + "title": "Scicat Token", + "type": "string", + "writeOnly": true + } }, - "scicat_token": { - "default": "INSERT-YOUR-SCICAT-TOKEN-HERE", - "title": "Scicat Token", - "type": "string" - } - }, - "required": [ - "dataset_pids", - "file_server_host", - "file_server_port", - "scicat_url" - ], - "additionalProperties": false, - "title": "Generic v1", - "type": "object" + "required": [ + "dataset_pids", + "file_server_host", + "file_server_port", + "scicat_url" + ], + "additionalProperties": false, + "title": "Generic v1", + "type": "object" + } } diff --git a/tests/notebook/notebook_test.py b/tests/notebook/notebook_test.py index 6c693f2..e872409 100644 --- a/tests/notebook/notebook_test.py +++ b/tests/notebook/notebook_test.py @@ -32,6 +32,24 @@ def test_list_notebook_templates(sciwyrm_client): assert response.json() == [{"name": "generic", "version": "1"}] +def test_notebook_is_valid_json(sciwyrm_client): + response = sciwyrm_client.post( + "/notebook", + json={ + "template_name": "generic", + "template_version": "1", + "parameters": { + "scicat_url": "https://test-url.sci.cat", + "file_server_host": "login", + "file_server_port": 22, + "dataset_pids": ["abcd/123.522"], + }, + }, + ) + assert response.status_code == 200 + assert response.json() is not None + + def test_notebook_contains_expected_url(sciwyrm_client): url = "https://test-url.sci.cat" response = sciwyrm_client.post( @@ -167,6 +185,29 @@ def test_notebook_contains_no_placeholders(sciwyrm_client): assert not re.search(r"{%[^}]*%}", response.text) +def test_notebook_contains_metadata(sciwyrm_client): + response = sciwyrm_client.post( + "/notebook", + json={ + "template_name": "generic", + "template_version": "1", + "parameters": { + "scicat_url": "https://test-url.sci.cat", + "file_server_host": "login", + "file_server_port": 22, + "dataset_pids": ["abcd/123.522"], + }, + }, + ) + assert response.status_code == 200 + metadata = response.json()["metadata"]["sciwyrm"] + assert metadata["template_name"] == "generic" + assert metadata["template_version"] == "1" + assert metadata["template_authors"] == [ + {"name": "Jan-Lukas Wynen", "email": "jan-lukas.wynen@ess.eu"} + ] + + def test_notebook_bad_parameter(sciwyrm_client): response = sciwyrm_client.post( "/notebook", From 21ec0d09c55ecdc90e78cffc03672e3c3937429e Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 11 Jan 2024 11:50:11 +0100 Subject: [PATCH 19/25] Update python version in CI --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97ff043..ddb25b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: submodules: true - uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: 3.11 - run: python -m pip install --upgrade pip - run: python -m pip install -r requirements/ci.txt - run: tox -e static diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a33d4b3..69b7277 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: 3.11 - run: python -m pip install --upgrade pip - run: python -m pip install -r requirements/wheels.txt From 8de0f3f0c7b8456dc9b2619aeb63dbc9d50760eb Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 11 Jan 2024 13:49:32 +0100 Subject: [PATCH 20/25] Depend on email-validator --- pyproject.toml | 1 + requirements/base.in | 1 + requirements/base.txt | 12 +++++++++--- requirements/dev.txt | 2 +- requirements/test.txt | 8 ++------ 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e6f82dc..3d587ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ ] requires-python = ">=3.11" dependencies = [ + "email-validator", "fastapi >= 0.108", "jinja2", "jsonschema", diff --git a/requirements/base.in b/requirements/base.in index 2badffa..d360e1b 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,3 +1,4 @@ +email-validator fastapi >= 0.108 jinja2 jsonschema diff --git a/requirements/base.txt b/requirements/base.txt index d63ee6b..8d8e9ee 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:914f531ccf8fcb2d3b73647c301af71df7db7d6d +# SHA1:bb330365f0386228cefde857153626da1b4f23cd # # This file is autogenerated by pip-compile-multi # To update, run: @@ -13,11 +13,17 @@ attrs==23.2.0 # via # jsonschema # referencing +dnspython==2.4.2 + # via email-validator +email-validator==2.1.0.post1 + # via -r requirements/base.in fastapi==0.108.0 # via -r requirements/base.in idna==3.6 - # via anyio -jinja2==3.1.2 + # via + # anyio + # email-validator +jinja2==3.1.3 # via -r requirements/base.in jsonschema==4.20.0 # via -r requirements/base.in diff --git a/requirements/dev.txt b/requirements/dev.txt index a3de101..9c46199 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -45,7 +45,7 @@ jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.1 # via jupyterlab -jupyter-server==2.12.3 +jupyter-server==2.12.4 # via # jupyter-lsp # jupyterlab diff --git a/requirements/test.txt b/requirements/test.txt index c8bbc8b..0b87fc5 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -35,10 +35,6 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.14 # via fabric -dnspython==2.4.2 - # via email-validator -email-validator==2.1.0.post1 - # via scitacean executing==2.0.1 # via stack-data fabric==3.2.2 @@ -53,7 +49,7 @@ httpcore==1.0.2 # via httpx httpx==0.26.0 # via -r requirements/test.in -hypothesis==6.92.7 +hypothesis==6.92.8 # via scitacean iniconfig==2.0.0 # via pytest @@ -79,7 +75,7 @@ mistune==3.0.2 # via nbconvert nbclient==0.9.0 # via nbconvert -nbconvert==7.14.0 +nbconvert==7.14.1 # via -r requirements/test.in nbformat==5.9.2 # via From df798425c0d7e36570483632c61b3ef6f88adc79 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 11 Jan 2024 13:53:19 +0100 Subject: [PATCH 21/25] Inject template config Before, the notebook functions would get the default app config and would not allow for overriding it in tests. --- src/sciwyrm/config.py | 6 ++- src/sciwyrm/main.py | 29 +++++++++++++- src/sciwyrm/notebook.py | 57 +++++++++++++++------------ src/sciwyrm/templates.py | 85 ++++++++++++++++++++++++---------------- 4 files changed, 114 insertions(+), 63 deletions(-) diff --git a/src/sciwyrm/config.py b/src/sciwyrm/config.py index 75c7fc2..2e2c952 100644 --- a/src/sciwyrm/config.py +++ b/src/sciwyrm/config.py @@ -81,7 +81,11 @@ def settings_customise_sources( @lru_cache(maxsize=1) def app_config() -> AppConfig: - """Return the application config.""" + """Return the application config. + + This function should only be called by the app. + Otherwise, tests cannot override the configuration. + """ from .logging import get_logger config = AppConfig() diff --git a/src/sciwyrm/main.py b/src/sciwyrm/main.py index bf7b9a4..4b5a894 100644 --- a/src/sciwyrm/main.py +++ b/src/sciwyrm/main.py @@ -6,12 +6,19 @@ from typing import Annotated from fastapi import Depends, FastAPI, Request, Response +from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from fastapi.templating import Jinja2Templates +from pydantic import ValidationError from . import notebook from .config import AppConfig, app_config -from .templates import get_templates, list_notebook_templates, notebook_template_path +from .templates import ( + get_notebook_template_config, + get_templates, + list_notebook_templates, + notebook_template_path, +) app = FastAPI() @@ -24,11 +31,29 @@ async def list_templates( return JSONResponse(list_notebook_templates(config)) +def _inject_app_config( + spec: notebook.NotebookSpec, config: Annotated[AppConfig, Depends(app_config)] +) -> notebook.NotebookSpecWithConfig: + try: + return spec.with_config( + get_notebook_template_config( + spec.template_name, spec.template_version, config + ) + ) + except ValidationError as exc: + # FastAPI cannot handle a ValidationError at this point. + # So make it look like this came from the initial spec. + errors = exc.errors() + for error in errors: + error["input"].pop("config") + raise RequestValidationError(errors) from None + + @app.post("/notebook", response_class=JSONResponse) async def format_notebook( request: Request, - spec: notebook.NotebookSpec, templates: Annotated[Jinja2Templates, Depends(get_templates)], + spec: notebook.NotebookSpecWithConfig = Depends(_inject_app_config), # noqa: B008 ) -> Response: """Format and return a notebook.""" formatted = templates.TemplateResponse( diff --git a/src/sciwyrm/notebook.py b/src/sciwyrm/notebook.py index 9b69490..0f030d2 100644 --- a/src/sciwyrm/notebook.py +++ b/src/sciwyrm/notebook.py @@ -2,15 +2,16 @@ # Copyright (c) 2024 SciCat Project (https://github.com/SciCatProject/sciwyrm) """Notebook handling.""" +from __future__ import annotations + from datetime import datetime, timezone from typing import Any import jsonschema -from pydantic import BaseModel, ValidationInfo, field_validator +from pydantic import BaseModel, model_validator from pydantic_core import PydanticCustomError -from .config import app_config -from .templates import get_template_config, notebook_template_hash +from .templates import NotebookTemplateConfig class NotebookSpec(BaseModel): @@ -20,28 +21,36 @@ class NotebookSpec(BaseModel): template_version: str parameters: dict[str, Any] - @field_validator("parameters") - @classmethod - def validate_parameters( - cls, parameters: Any, info: ValidationInfo - ) -> dict[str, Any]: - """Validate parameters against the template schema.""" - if not isinstance(parameters, dict): - raise AssertionError("'parameters' must be a dict.") + def with_config(self, config: NotebookTemplateConfig) -> NotebookSpecWithConfig: + """Return a new spec with template config.""" + return NotebookSpecWithConfig( + **{**self.model_dump(), "config": config}, + ) + + +class NotebookSpecWithConfig(NotebookSpec): + """Internal spec with added template config. - schema = get_template_config( - info.data["template_name"], info.data["template_version"], app_config() - )["parameter_schema"] + This should not be exposed to the API because notebook template configs + are provided by the server. + """ + + config: NotebookTemplateConfig + + @model_validator(mode="after") + def validate_parameters(self) -> NotebookSpecWithConfig: + """Validate parameters against the template schema.""" + schema = self.config.parameter_schema try: - jsonschema.validate(parameters, schema) + jsonschema.validate(self.parameters, schema) except jsonschema.ValidationError as err: raise PydanticCustomError( "Validation Error", "{message}", { "message": err.message, - "template_name": info.data["template_name"], - "template_version": info.data["template_version"], + "template_name": self.template_name, + "template_version": self.template_version, "instance": err.instance, "jsonpath": err.json_path, "schema": err.schema, @@ -50,28 +59,24 @@ def validate_parameters( "validator_value": err.validator_value, }, ) from None - return parameters + return self -def render_context(spec: NotebookSpec) -> dict[str, Any]: +def render_context(spec: NotebookSpecWithConfig) -> dict[str, Any]: """Return a dict that can be used to render a notebook template.""" context = spec.parameters | notebook_metadata(spec) return {key.upper(): value for key, value in context.items()} -def notebook_metadata(spec: NotebookSpec) -> dict[str, Any]: +def notebook_metadata(spec: NotebookSpecWithConfig) -> dict[str, Any]: """Return metadata for a requested notebook. Here, metadata is any data that was not explicitly requested by the user. """ - app_conf = app_config() - config = get_template_config(spec.template_name, spec.template_version, app_conf) return { "template_name": spec.template_name, "template_version": spec.template_version, - "template_authors": config["authors"], + "template_authors": [author.model_dump() for author in spec.config.authors], "template_rendered": datetime.now(tz=timezone.utc).isoformat(), - "template_hash": notebook_template_hash( - spec.template_name, spec.template_version, app_conf - ), + "template_hash": spec.config.template_hash, } diff --git a/src/sciwyrm/templates.py b/src/sciwyrm/templates.py index a81783e..233b21f 100644 --- a/src/sciwyrm/templates.py +++ b/src/sciwyrm/templates.py @@ -11,29 +11,70 @@ from fastapi import Depends from fastapi.templating import Jinja2Templates from jinja2 import Environment, FileSystemLoader +from pydantic import BaseModel, EmailStr from .config import AppConfig, app_config +class Author(BaseModel): + """Author of the template.""" + + name: str + email: EmailStr | None = None + + +class NotebookTemplateConfig(BaseModel): + """Template configuration.""" + + name: str + version: str + description: str + authors: list[Author] + parameter_schema: dict[str, Any] + template_hash: str + + def get_templates(config: Annotated[AppConfig, Depends(app_config)]) -> Jinja2Templates: """Return a handler for loading and rendering templates.""" return _make_template_handler(config.template_dir) -def get_template_config( - name: str, version: str, config: AppConfig, category: str = "notebook" -) -> dict[str, Any]: - """Return a template configuration.""" - return _load_template_config( - config.template_dir.joinpath(category, f"{name}_v{version}.json") +@lru_cache(maxsize=1) +def _make_template_handler(template_dir: Path) -> Jinja2Templates: + from . import filters + from .logging import get_logger + + get_logger().info("Loading templates from %s", template_dir) + templates = Jinja2Templates( + env=Environment( + loader=FileSystemLoader(template_dir), + autoescape=True, + ) ) + templates.env.filters["quote"] = filters.quote + templates.env.filters["je"] = filters.json_escape + return templates + + +def get_notebook_template_config( + name: str, version: str, config: AppConfig +) -> NotebookTemplateConfig: + """Return a template configuration.""" + return _load_notebook_template_config(name, version, config.template_dir) @lru_cache() -def _load_template_config(path: Path) -> dict[str, Any]: +def _load_notebook_template_config( + name: str, version: str, template_dir: Path +) -> NotebookTemplateConfig: # AppConfig cannot be hashed and used with lru_cache. + path = template_dir.joinpath("notebook", f"{name}_v{version}.json") with path.open() as f: - return json.load(f) + fields = json.load(f) + fields["name"] = name + fields["version"] = version + fields["template_hash"] = _notebook_template_hash(path) + return NotebookTemplateConfig(**fields) def notebook_template_path(name: str, version: str) -> str: @@ -63,35 +104,11 @@ def list_notebook_templates(config: AppConfig) -> list[dict[str, str]]: ] -def notebook_template_hash(name: str, version: str, config: AppConfig) -> str: +def _notebook_template_hash(path: Path) -> str: """Return a hash for a notebook template.""" - return ( - "blake2b:" - + hashlib.blake2b( - config.template_dir.joinpath( - notebook_template_path(name, version) - ).read_bytes() - ).hexdigest() - ) + return "blake2b:" + hashlib.blake2b(path.read_bytes()).hexdigest() def _split_notebook_template_name(full_name: str) -> dict[str, str]: name, version = full_name.split("_v") return {"name": name, "version": version} - - -@lru_cache(maxsize=1) -def _make_template_handler(template_dir: Path) -> Jinja2Templates: - from . import filters - from .logging import get_logger - - get_logger().info("Loading templates from %s", template_dir) - templates = Jinja2Templates( - env=Environment( - loader=FileSystemLoader(template_dir), - autoescape=True, - ) - ) - templates.env.filters["quote"] = filters.quote - templates.env.filters["je"] = filters.json_escape - return templates From 436243c795e900e63bd231fe66802973864a6974 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 23 Jan 2024 16:35:43 +0100 Subject: [PATCH 22/25] Add deffault settings --- settings.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 settings.json diff --git a/settings.json b/settings.json new file mode 100644 index 0000000..a646e5a --- /dev/null +++ b/settings.json @@ -0,0 +1 @@ +{"template_dir":"./templates"} From cd74ca3f678cbe9d03ee6e084f48296356f91e31 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Mon, 29 Jan 2024 14:46:32 +0100 Subject: [PATCH 23/25] Use uuid to identify templates --- src/sciwyrm/main.py | 8 +--- src/sciwyrm/notebook.py | 14 +++--- src/sciwyrm/templates.py | 32 +++++-------- ...32f6992-0355-4759-b780-ececd4957c23.ipynb} | 4 +- ...b32f6992-0355-4759-b780-ececd4957c23.json} | 3 ++ tests/notebook/notebook_test.py | 46 ++++++++----------- 6 files changed, 44 insertions(+), 63 deletions(-) rename templates/notebook/{generic_v1.ipynb => b32f6992-0355-4759-b780-ececd4957c23.ipynb} (93%) rename templates/notebook/{generic_v1.json => b32f6992-0355-4759-b780-ececd4957c23.json} (93%) diff --git a/src/sciwyrm/main.py b/src/sciwyrm/main.py index 4b5a894..0460bf8 100644 --- a/src/sciwyrm/main.py +++ b/src/sciwyrm/main.py @@ -35,11 +35,7 @@ def _inject_app_config( spec: notebook.NotebookSpec, config: Annotated[AppConfig, Depends(app_config)] ) -> notebook.NotebookSpecWithConfig: try: - return spec.with_config( - get_notebook_template_config( - spec.template_name, spec.template_version, config - ) - ) + return spec.with_config(get_notebook_template_config(spec.template_id, config)) except ValidationError as exc: # FastAPI cannot handle a ValidationError at this point. # So make it look like this came from the initial spec. @@ -57,7 +53,7 @@ async def format_notebook( ) -> Response: """Format and return a notebook.""" formatted = templates.TemplateResponse( - name=notebook_template_path(spec.template_name, spec.template_version), + name=notebook_template_path(spec.template_id), request=request, context=notebook.render_context(spec), ) diff --git a/src/sciwyrm/notebook.py b/src/sciwyrm/notebook.py index 0f030d2..06f84c5 100644 --- a/src/sciwyrm/notebook.py +++ b/src/sciwyrm/notebook.py @@ -17,8 +17,7 @@ class NotebookSpec(BaseModel): """Specifies which notebook to return and how to format it.""" - template_name: str - template_version: str + template_id: str parameters: dict[str, Any] def with_config(self, config: NotebookTemplateConfig) -> NotebookSpecWithConfig: @@ -49,8 +48,7 @@ def validate_parameters(self) -> NotebookSpecWithConfig: "{message}", { "message": err.message, - "template_name": self.template_name, - "template_version": self.template_version, + "template_id": self.template_id, "instance": err.instance, "jsonpath": err.json_path, "schema": err.schema, @@ -74,9 +72,11 @@ def notebook_metadata(spec: NotebookSpecWithConfig) -> dict[str, Any]: Here, metadata is any data that was not explicitly requested by the user. """ return { - "template_name": spec.template_name, - "template_version": spec.template_version, + "template_id": spec.template_id, + "template_submission_name": spec.config.submission_name, + "template_display_name": spec.config.display_name, + "template_version": spec.config.version, "template_authors": [author.model_dump() for author in spec.config.authors], - "template_rendered": datetime.now(tz=timezone.utc).isoformat(), + "template_rendered_at": datetime.now(tz=timezone.utc).isoformat(), "template_hash": spec.config.template_hash, } diff --git a/src/sciwyrm/templates.py b/src/sciwyrm/templates.py index 233b21f..b16c6ff 100644 --- a/src/sciwyrm/templates.py +++ b/src/sciwyrm/templates.py @@ -26,7 +26,8 @@ class Author(BaseModel): class NotebookTemplateConfig(BaseModel): """Template configuration.""" - name: str + submission_name: str + display_name: str version: str description: str authors: list[Author] @@ -57,48 +58,44 @@ def _make_template_handler(template_dir: Path) -> Jinja2Templates: def get_notebook_template_config( - name: str, version: str, config: AppConfig + template_id: str, config: AppConfig ) -> NotebookTemplateConfig: """Return a template configuration.""" - return _load_notebook_template_config(name, version, config.template_dir) + return _load_notebook_template_config(template_id, config.template_dir) @lru_cache() def _load_notebook_template_config( - name: str, version: str, template_dir: Path + template_id: str, template_dir: Path ) -> NotebookTemplateConfig: # AppConfig cannot be hashed and used with lru_cache. - path = template_dir.joinpath("notebook", f"{name}_v{version}.json") + path = template_dir.joinpath("notebook", f"{template_id}.json") with path.open() as f: fields = json.load(f) - fields["name"] = name - fields["version"] = version fields["template_hash"] = _notebook_template_hash(path) return NotebookTemplateConfig(**fields) -def notebook_template_path(name: str, version: str) -> str: +def notebook_template_path(template_id: str) -> str: """Return the relative path to a given notebook template. Parameters ---------- - name: - Name of the template. - version: - Version of the template. + template_id: + Unique ID of the template. Returns ------- : The path to the template relative to the base template directory. """ - return f"notebook/{name}_v{version}.ipynb" + return f"notebook/{template_id}.ipynb" -def list_notebook_templates(config: AppConfig) -> list[dict[str, str]]: +def list_notebook_templates(config: AppConfig) -> list[str]: """List available notebook templates.""" return [ - _split_notebook_template_name(path.stem) + path.stem for path in config.template_dir.joinpath("notebook").iterdir() if path.suffix == ".ipynb" ] @@ -107,8 +104,3 @@ def list_notebook_templates(config: AppConfig) -> list[dict[str, str]]: def _notebook_template_hash(path: Path) -> str: """Return a hash for a notebook template.""" return "blake2b:" + hashlib.blake2b(path.read_bytes()).hexdigest() - - -def _split_notebook_template_name(full_name: str) -> dict[str, str]: - name, version = full_name.split("_v") - return {"name": name, "version": version} diff --git a/templates/notebook/generic_v1.ipynb b/templates/notebook/b32f6992-0355-4759-b780-ececd4957c23.ipynb similarity index 93% rename from templates/notebook/generic_v1.ipynb rename to templates/notebook/b32f6992-0355-4759-b780-ececd4957c23.ipynb index f69665b..4aadfb1 100644 --- a/templates/notebook/generic_v1.ipynb +++ b/templates/notebook/b32f6992-0355-4759-b780-ececd4957c23.ipynb @@ -5,7 +5,7 @@ "id": "803f7b636cfa4b4c", "metadata": {}, "source": [ - "# {{ TEMPLATE_NAME | capitalize }}" + "# {{ TEMPLATE_DISPLAY_NAME | capitalize }}" ] }, { @@ -13,7 +13,7 @@ "id": "9e7442fc9ec68975", "metadata": {}, "source": [ - "Notebook generated from template '{{ TEMPLATE_NAME }}' version {{ TEMPLATE_VERSION }}." + "Notebook generated from template '{{ TEMPLATE_DISPLAY_NAME }}' version {{ TEMPLATE_VERSION }} (id: {{ TEMPLATE_ID }})." ] }, { diff --git a/templates/notebook/generic_v1.json b/templates/notebook/b32f6992-0355-4759-b780-ececd4957c23.json similarity index 93% rename from templates/notebook/generic_v1.json rename to templates/notebook/b32f6992-0355-4759-b780-ececd4957c23.json index c6d22e5..d645151 100644 --- a/templates/notebook/generic_v1.json +++ b/templates/notebook/b32f6992-0355-4759-b780-ececd4957c23.json @@ -1,4 +1,7 @@ { + "submission_name": "generic", + "display_name": "Generic", + "version": "1", "description": "Generic template for downloading datasets and files.", "authors": [ { diff --git a/tests/notebook/notebook_test.py b/tests/notebook/notebook_test.py index e872409..f23194f 100644 --- a/tests/notebook/notebook_test.py +++ b/tests/notebook/notebook_test.py @@ -11,6 +11,8 @@ from ..seed import SEED +TEMPLATE_IDS = {"generic": "b32f6992-0355-4759-b780-ececd4957c23"} + @pytest.fixture def scicat_token(scicat_client): @@ -29,15 +31,14 @@ def exec_notebook(nb_code: str) -> dict[str, Any]: def test_list_notebook_templates(sciwyrm_client): response = sciwyrm_client.get("/notebook/templates") assert response.status_code == 200 - assert response.json() == [{"name": "generic", "version": "1"}] + assert response.json() == [TEMPLATE_IDS["generic"]] def test_notebook_is_valid_json(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", @@ -55,8 +56,7 @@ def test_notebook_contains_expected_url(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": url, "file_server_host": "login", @@ -74,8 +74,7 @@ def test_notebook_contains_expected_pids(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", @@ -96,8 +95,7 @@ def test_notebook_contains_only_expected_pids(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", @@ -111,8 +109,7 @@ def test_notebook_contains_only_expected_pids(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", @@ -133,8 +130,7 @@ def test_notebook_contains_expected_file_serve_host(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": "https://test-url.sci.cat", "file_server_host": file_server_host, @@ -152,8 +148,7 @@ def test_notebook_contains_expected_file_serve_port(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", @@ -170,8 +165,7 @@ def test_notebook_contains_no_placeholders(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", @@ -189,8 +183,7 @@ def test_notebook_contains_metadata(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", @@ -201,7 +194,8 @@ def test_notebook_contains_metadata(sciwyrm_client): ) assert response.status_code == 200 metadata = response.json()["metadata"]["sciwyrm"] - assert metadata["template_name"] == "generic" + assert metadata["template_id"] == TEMPLATE_IDS["generic"] + assert metadata["template_submission_name"] == "generic" assert metadata["template_version"] == "1" assert metadata["template_authors"] == [ {"name": "Jan-Lukas Wynen", "email": "jan-lukas.wynen@ess.eu"} @@ -212,8 +206,7 @@ def test_notebook_bad_parameter(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", @@ -230,8 +223,7 @@ def test_notebook_missing_parameter(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": "https://test-url.sci.cat", "file_server_port": 22, @@ -247,8 +239,7 @@ def test_notebook_extra_parameter(sciwyrm_client): response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": "https://test-url.sci.cat", "file_server_host": "login", @@ -277,8 +268,7 @@ def test_notebook_run( response = sciwyrm_client.post( "/notebook", json={ - "template_name": "generic", - "template_version": "1", + "template_id": TEMPLATE_IDS["generic"], "parameters": { "scicat_url": scicat_access.url, "file_server_host": sftp_access.host, From 632d2b88262a894c2de23f485f621cb09a7e9ae6 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Mon, 29 Jan 2024 15:11:34 +0100 Subject: [PATCH 24/25] Better API docs --- src/sciwyrm/main.py | 9 +++---- src/sciwyrm/notebook.py | 45 ++++++++++++++++++++++++++++++--- tests/notebook/notebook_test.py | 9 ++++++- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/sciwyrm/main.py b/src/sciwyrm/main.py index 0460bf8..457119b 100644 --- a/src/sciwyrm/main.py +++ b/src/sciwyrm/main.py @@ -16,19 +16,18 @@ from .templates import ( get_notebook_template_config, get_templates, - list_notebook_templates, notebook_template_path, ) app = FastAPI() -@app.get("/notebook/templates") +@app.get("/notebook/templates", response_description="Available templates") async def list_templates( config: Annotated[AppConfig, Depends(app_config)] -) -> JSONResponse: +) -> list[notebook.TemplateSummary]: """Return a list of available notebook templates.""" - return JSONResponse(list_notebook_templates(config)) + return notebook.available_templates(config) def _inject_app_config( @@ -45,7 +44,7 @@ def _inject_app_config( raise RequestValidationError(errors) from None -@app.post("/notebook", response_class=JSONResponse) +@app.post("/notebook", response_model=dict, response_description="Rendered notebook") async def format_notebook( request: Request, templates: Annotated[Jinja2Templates, Depends(get_templates)], diff --git a/src/sciwyrm/notebook.py b/src/sciwyrm/notebook.py index 06f84c5..01aacfe 100644 --- a/src/sciwyrm/notebook.py +++ b/src/sciwyrm/notebook.py @@ -8,17 +8,25 @@ from typing import Any import jsonschema -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, Field, model_validator from pydantic_core import PydanticCustomError -from .templates import NotebookTemplateConfig +from .config import AppConfig +from .templates import ( + NotebookTemplateConfig, + get_notebook_template_config, + list_notebook_templates, +) class NotebookSpec(BaseModel): """Specifies which notebook to return and how to format it.""" - template_id: str - parameters: dict[str, Any] + template_id: str = Field(description="ID of the template to render.") + parameters: dict[str, Any] = Field( + description="Parameters for the template. " + "The schema depends on the concrete template." + ) def with_config(self, config: NotebookTemplateConfig) -> NotebookSpecWithConfig: """Return a new spec with template config.""" @@ -60,6 +68,35 @@ def validate_parameters(self) -> NotebookSpecWithConfig: return self +class TemplateSummary(BaseModel): + """Short overview of a notebook template.""" + + template_id: str = Field(description="ID of the template.") + submission_name: str = Field(description="Template name given during submission.") + display_name: str = Field(description="Template name meant for display to users.") + version: str = Field(description="Template version.") + + @classmethod + def from_config( + cls, template_id: str, config: NotebookTemplateConfig + ) -> TemplateSummary: + """Construct from a template config.""" + return cls( + template_id=template_id, + submission_name=config.submission_name, + display_name=config.display_name, + version=config.version, + ) + + +def available_templates(config: AppConfig) -> list[TemplateSummary]: + """Summarise available templates.""" + return [ + TemplateSummary.from_config(tid, get_notebook_template_config(tid, config)) + for tid in list_notebook_templates(config) + ] + + def render_context(spec: NotebookSpecWithConfig) -> dict[str, Any]: """Return a dict that can be used to render a notebook template.""" context = spec.parameters | notebook_metadata(spec) diff --git a/tests/notebook/notebook_test.py b/tests/notebook/notebook_test.py index f23194f..e5ee138 100644 --- a/tests/notebook/notebook_test.py +++ b/tests/notebook/notebook_test.py @@ -31,7 +31,14 @@ def exec_notebook(nb_code: str) -> dict[str, Any]: def test_list_notebook_templates(sciwyrm_client): response = sciwyrm_client.get("/notebook/templates") assert response.status_code == 200 - assert response.json() == [TEMPLATE_IDS["generic"]] + assert response.json() == [ + { + "template_id": TEMPLATE_IDS["generic"], + "submission_name": "generic", + "display_name": "Generic", + "version": "1", + } + ] def test_notebook_is_valid_json(sciwyrm_client): From fba6fa5dd2f997da9b02929aa117d0d6d603fd4c Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Mon, 29 Jan 2024 15:27:42 +0100 Subject: [PATCH 25/25] More info in generic template --- ...b32f6992-0355-4759-b780-ececd4957c23.ipynb | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/templates/notebook/b32f6992-0355-4759-b780-ececd4957c23.ipynb b/templates/notebook/b32f6992-0355-4759-b780-ececd4957c23.ipynb index 4aadfb1..e050651 100644 --- a/templates/notebook/b32f6992-0355-4759-b780-ececd4957c23.ipynb +++ b/templates/notebook/b32f6992-0355-4759-b780-ececd4957c23.ipynb @@ -5,7 +5,8 @@ "id": "803f7b636cfa4b4c", "metadata": {}, "source": [ - "# {{ TEMPLATE_DISPLAY_NAME | capitalize }}" + "# {{ TEMPLATE_DISPLAY_NAME | capitalize }}\n", + "Notebook generated from template '{{ TEMPLATE_DISPLAY_NAME }}' version {{ TEMPLATE_VERSION }} (id: `{{ TEMPLATE_ID }}`, time: )." ] }, { @@ -13,7 +14,7 @@ "id": "9e7442fc9ec68975", "metadata": {}, "source": [ - "Notebook generated from template '{{ TEMPLATE_DISPLAY_NAME }}' version {{ TEMPLATE_VERSION }} (id: {{ TEMPLATE_ID }})." + "This notebook downloads selected datasets from SciCat and all files for those datasets." ] }, { @@ -27,6 +28,14 @@ "from scitacean.transfer.sftp import SFTPFileTransfer" ] }, + { + "cell_type": "markdown", + "id": "d0336f3a7a2d37eb", + "metadata": {}, + "source": [ + "Scicat configuration:" + ] + }, { "cell_type": "code", "execution_count": null, @@ -39,6 +48,14 @@ "file_server_port = {{ FILE_SERVER_PORT | int }}" ] }, + { + "cell_type": "markdown", + "id": "9bdf7ecbf3cd7df", + "metadata": {}, + "source": [ + "Login token for SciCat. If it expires, replace it with a new one obtained from the SciCat web interface." + ] + }, { "cell_type": "code", "execution_count": null, @@ -49,6 +66,14 @@ "scicat_token = {{ SCICAT_TOKEN | quote | je | safe }}" ] }, + { + "cell_type": "markdown", + "id": "590b037151d5a71d", + "metadata": {}, + "source": [ + "Select datasets to downloads:" + ] + }, { "cell_type": "code", "execution_count": null, @@ -61,6 +86,14 @@ "]" ] }, + { + "cell_type": "markdown", + "id": "a3358b891c9c476d", + "metadata": {}, + "source": [ + "Download files to this folder. You may change it freely." + ] + }, { "cell_type": "code", "execution_count": null,