From 805c7203952261318fa9fa949231cf7349db089d Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 3 Nov 2024 16:41:43 +0100 Subject: [PATCH] Testing: Add more utilities to `pueblo.testing.notebook` - list_notebooks - generate_notebook_tests - run_notebook --- CHANGES.md | 1 + pueblo/testing/notebook.py | 57 ++++++++++++++++++++++++++++++++++++-- tests/test_code.py | 41 +++++++++++++++++++++++---- 3 files changed, 90 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 79e7fe3..d11e9dc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,7 @@ ## Unreleased - nlp: Updated dependencies langchain, langchain-text-splitters, unstructured - CI: Verify compatibility with Python 3.13 +- Testing: Add `pueblo.testing.notebook.{list_notebooks,generate_notebook_tests,run_notebook}` ## 2024-03-07 v0.0.9 - Testing: Add `pueblo.testing.notebook.{list_path,generate_tests}` diff --git a/pueblo/testing/notebook.py b/pueblo/testing/notebook.py index 9c8e321..518911a 100644 --- a/pueblo/testing/notebook.py +++ b/pueblo/testing/notebook.py @@ -59,7 +59,12 @@ def list_path(path: Path, suffix: str = ".ipynb"): yield item -def generate_tests(metafunc, paths: t.Union[t.List[Path], None] = None, path: t.Union[Path, None] = None): +def generate_tests( + metafunc, + paths: t.Union[t.List[Path], None] = None, + path: t.Union[Path, None] = None, + fixture_name: str = "notebook", +): """ Generate test cases for Jupyter Notebooks. To be used from `pytest_generate_tests`. @@ -70,6 +75,52 @@ def generate_tests(metafunc, paths: t.Union[t.List[Path], None] = None, path: t. paths = list(paths) else: raise ValueError("Path is missing") - if "notebook" in metafunc.fixturenames: + if fixture_name in metafunc.fixturenames: names = [nb_path.name for nb_path in paths] - metafunc.parametrize("notebook", paths, ids=names) + metafunc.parametrize(fixture_name, paths, ids=names) + + +def list_notebooks(path: Path) -> t.List[Path]: + """ + Enumerate all Jupyter Notebook files found in given directory. + """ + return list(path.rglob("*.ipynb")) + + +def generate_notebook_tests(metafunc, notebook_paths: t.List[Path], fixture_name: str = "notebook"): + """ + Generate test cases for Jupyter Notebooks. + To be used from `pytest_generate_tests`. + """ + if fixture_name in metafunc.fixturenames: + names = [nb_path.name for nb_path in notebook_paths] + metafunc.parametrize(fixture_name, notebook_paths, ids=names) + + +def run_notebook(notebook, enable_skipping=True, timeout=60, **kwargs): + """ + Execute Jupyter Notebook, one test case per .ipynb file, with optional skipping. + + Skip executing a notebook by using this code within a cell:: + + pytest.exit("Something failed but let's skip! [skip-notebook]") + + For example, this is used by `pueblo.util.environ.getenvpass()`, to + skip executing the notebook when an authentication token is not supplied. + """ + + from nbclient.exceptions import CellExecutionError + from testbook import testbook + + with testbook(notebook, timeout=timeout, **kwargs) as tb: + try: + tb.execute() + + # Skip notebook if `pytest.exit()` is invoked, + # including the `[skip-notebook]` label. + except CellExecutionError as ex: + if enable_skipping: + msg = str(ex) + if "[skip-notebook]" in msg: + raise pytest.skip(msg) from ex + raise diff --git a/tests/test_code.py b/tests/test_code.py index 46760db..d13591e 100644 --- a/tests/test_code.py +++ b/tests/test_code.py @@ -1,8 +1,5 @@ from pathlib import Path -from pueblo.testing.notebook import generate_tests, monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip -from pueblo.testing.snippet import pytest_module_function, pytest_notebook - HERE = Path(__file__).parent TESTDATA_FOLDER = HERE / "testdata" / "folder" TESTDATA_SNIPPET = HERE / "testdata" / "snippet" @@ -12,6 +9,8 @@ def test_monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip(): """ Verify loading a monkeypatch supporting Jupyter Notebook testing. """ + from pueblo.testing.notebook import monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip + monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip() @@ -19,6 +18,8 @@ def test_pytest_module_function(request, capsys): """ Verify running an arbitrary Python function from an arbitrary Python file. """ + from pueblo.testing.snippet import pytest_module_function + outcome = pytest_module_function(request=request, filepath=TESTDATA_FOLDER / "dummy.py") assert isinstance(outcome[0], Path) assert outcome[0].name == "dummy.py" @@ -35,6 +36,8 @@ def test_pytest_notebook(request): """ from _pytest._py.path import LocalPath + from pueblo.testing.snippet import pytest_notebook + outcomes = pytest_notebook(request=request, filepath=TESTDATA_FOLDER / "dummy.ipynb") assert isinstance(outcomes[0][0], LocalPath) assert outcomes[0][0].basename == "dummy.ipynb" @@ -52,7 +55,7 @@ def test_list_python_files(): assert outcome == ["dummy.py"] -def test_list_notebooks(): +def test_folder_list_notebooks(): """ Verify utility function for enumerating all Jupyter Notebook files in given directory. """ @@ -62,6 +65,16 @@ def test_list_notebooks(): assert outcome == ["dummy.ipynb"] +def test_notebook_list_notebooks(): + """ + Verify recursive Jupyter Notebook enumerator utility. + """ + from pueblo.testing.notebook import list_notebooks + + outcome = list_notebooks(TESTDATA_FOLDER) + assert outcome[0].name == "dummy.ipynb" + + def test_notebook_injection(): """ Execute a Jupyter Notebook with custom code injected into a cell. @@ -101,10 +114,17 @@ def pytest_generate_tests(metafunc): """ Generate test cases for Jupyter Notebooks, one test case per .ipynb file. """ - generate_tests(metafunc, path=TESTDATA_FOLDER) + from pueblo.testing.notebook import generate_notebook_tests, generate_tests, list_notebooks + + # That's for testing. "foobar" and "bazqux" features are never used. + generate_tests(metafunc, path=TESTDATA_FOLDER, fixture_name="foobar") + generate_notebook_tests(metafunc, notebook_paths=list_notebooks(TESTDATA_FOLDER), fixture_name="bazqux") + # That's for real. + generate_notebook_tests(metafunc, notebook_paths=list_notebooks(TESTDATA_FOLDER)) -def test_notebook(notebook): + +def test_notebook_run_direct(notebook): """ Execute Jupyter Notebook, one test case per .ipynb file. """ @@ -112,3 +132,12 @@ def test_notebook(notebook): with testbook(notebook) as tb: tb.execute() + + +def test_notebook_run_api(notebook): + """ + Execute Jupyter Notebook using API. + """ + from pueblo.testing.notebook import run_notebook + + run_notebook(notebook)