Testing: Add more utilities to pueblo.testing.notebook

- list_notebooks
- generate_notebook_tests
- run_notebook

CHANGES.md

@@ -3,6 +3,7 @@
 ## Unreleased
 - nlp: Updated dependencies langchain, langchain-text-splitters, unstructured
 - CI: Verify compatibility with Python 3.13
+- Testing: Add `pueblo.testing.notebook.{list_notebooks,generate_notebook_tests,run_notebook}`
 
 ## 2024-03-07 v0.0.9
 - Testing: Add `pueblo.testing.notebook.{list_path,generate_tests}`

pueblo/testing/notebook.py

@@ -59,7 +59,12 @@ def list_path(path: Path, suffix: str = ".ipynb"):
             yield item
 
 
-def generate_tests(metafunc, paths: t.Union[t.List[Path], None] = None, path: t.Union[Path, None] = None):
+def generate_tests(
+    metafunc,
+    paths: t.Union[t.List[Path], None] = None,
+    path: t.Union[Path, None] = None,
+    fixture_name: str = "notebook",
+):
     """
     Generate test cases for Jupyter Notebooks.
     To be used from `pytest_generate_tests`.
@@ -70,6 +75,52 @@ def generate_tests(metafunc, paths: t.Union[t.List[Path], None] = None, path: t.
         paths = list(paths)
     else:
         raise ValueError("Path is missing")
-    if "notebook" in metafunc.fixturenames:
+    if fixture_name in metafunc.fixturenames:
         names = [nb_path.name for nb_path in paths]
-        metafunc.parametrize("notebook", paths, ids=names)
+        metafunc.parametrize(fixture_name, paths, ids=names)
+
+
+def list_notebooks(path: Path) -> t.List[Path]:
+    """
+    Enumerate all Jupyter Notebook files found in given directory.
+    """
+    return list(path.rglob("*.ipynb"))
+
+
+def generate_notebook_tests(metafunc, notebook_paths: t.List[Path], fixture_name: str = "notebook"):
+    """
+    Generate test cases for Jupyter Notebooks.
+    To be used from `pytest_generate_tests`.
+    """
+    if fixture_name in metafunc.fixturenames:
+        names = [nb_path.name for nb_path in notebook_paths]
+        metafunc.parametrize(fixture_name, notebook_paths, ids=names)
+
+
+def run_notebook(notebook, enable_skipping=True, timeout=60, **kwargs):
+    """
+    Execute Jupyter Notebook, one test case per .ipynb file, with optional skipping.
+
+    Skip executing a notebook by using this code within a cell::
+
+        pytest.exit("Something failed but let's skip! [skip-notebook]")
+
+    For example, this is used by `pueblo.util.environ.getenvpass()`, to
+    skip executing the notebook when an authentication token is not supplied.
+    """
+
+    from nbclient.exceptions import CellExecutionError
+    from testbook import testbook
+
+    with testbook(notebook, timeout=timeout, **kwargs) as tb:
+        try:
+            tb.execute()
+
+        # Skip notebook if `pytest.exit()` is invoked,
+        # including the `[skip-notebook]` label.
+        except CellExecutionError as ex:
+            if enable_skipping:
+                msg = str(ex)
+                if "[skip-notebook]" in msg:
+                    raise pytest.skip(msg) from ex
+            raise

tests/test_code.py

@@ -1,8 +1,5 @@
 from pathlib import Path
 
-from pueblo.testing.notebook import generate_tests, monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip
-from pueblo.testing.snippet import pytest_module_function, pytest_notebook
-
 HERE = Path(__file__).parent
 TESTDATA_FOLDER = HERE / "testdata" / "folder"
 TESTDATA_SNIPPET = HERE / "testdata" / "snippet"
@@ -12,13 +9,17 @@ def test_monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip():
     """
     Verify loading a monkeypatch supporting Jupyter Notebook testing.
     """
+    from pueblo.testing.notebook import monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip
+
     monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip()
 
 
 def test_pytest_module_function(request, capsys):
     """
     Verify running an arbitrary Python function from an arbitrary Python file.
     """
+    from pueblo.testing.snippet import pytest_module_function
+
     outcome = pytest_module_function(request=request, filepath=TESTDATA_FOLDER / "dummy.py")
     assert isinstance(outcome[0], Path)
     assert outcome[0].name == "dummy.py"
@@ -35,6 +36,8 @@ def test_pytest_notebook(request):
     """
     from _pytest._py.path import LocalPath
 
+    from pueblo.testing.snippet import pytest_notebook
+
     outcomes = pytest_notebook(request=request, filepath=TESTDATA_FOLDER / "dummy.ipynb")
     assert isinstance(outcomes[0][0], LocalPath)
     assert outcomes[0][0].basename == "dummy.ipynb"
@@ -52,7 +55,7 @@ def test_list_python_files():
     assert outcome == ["dummy.py"]
 
 
-def test_list_notebooks():
+def test_folder_list_notebooks():
     """
     Verify utility function for enumerating all Jupyter Notebook files in given directory.
     """
@@ -62,6 +65,16 @@ def test_list_notebooks():
     assert outcome == ["dummy.ipynb"]
 
 
+def test_notebook_list_notebooks():
+    """
+    Verify recursive Jupyter Notebook enumerator utility.
+    """
+    from pueblo.testing.notebook import list_notebooks
+
+    outcome = list_notebooks(TESTDATA_FOLDER)
+    assert outcome[0].name == "dummy.ipynb"
+
+
 def test_notebook_injection():
     """
     Execute a Jupyter Notebook with custom code injected into a cell.
@@ -101,14 +114,30 @@ def pytest_generate_tests(metafunc):
     """
     Generate test cases for Jupyter Notebooks, one test case per .ipynb file.
     """
-    generate_tests(metafunc, path=TESTDATA_FOLDER)
+    from pueblo.testing.notebook import generate_notebook_tests, generate_tests, list_notebooks
+
+    # That's for testing. "foobar" and "bazqux" features are never used.
+    generate_tests(metafunc, path=TESTDATA_FOLDER, fixture_name="foobar")
+    generate_notebook_tests(metafunc, notebook_paths=list_notebooks(TESTDATA_FOLDER), fixture_name="bazqux")
 
+    # That's for real.
+    generate_notebook_tests(metafunc, notebook_paths=list_notebooks(TESTDATA_FOLDER))
 
-def test_notebook(notebook):
+
+def test_notebook_run_direct(notebook):
     """
     Execute Jupyter Notebook, one test case per .ipynb file.
     """
     from testbook import testbook
 
     with testbook(notebook) as tb:
         tb.execute()
+
+
+def test_notebook_run_api(notebook):
+    """
+    Execute Jupyter Notebook using API.
+    """
+    from pueblo.testing.notebook import run_notebook
+
+    run_notebook(notebook)