From 0402531763a788ca107d44085b0f790589c714ef Mon Sep 17 00:00:00 2001
From: Andreas Motl <andreas.motl@panodata.org>
Date: Tue, 7 Nov 2023 11:14:01 +0100
Subject: [PATCH] Add a few testing helper utilities to `pueblo.testing`

---
 CHANGES.md                 |  1 +
 pueblo/testing/folder.py   | 35 +++++++++++++++++++++
 pueblo/testing/nlp.py      | 13 ++++++++
 pueblo/testing/notebook.py |  2 +-
 pueblo/testing/snippet.py  | 48 +++++++++++++++++++++++++++++
 pyproject.toml             | 27 +++++++++++++++--
 tests/test_nlp.py          | 13 ++++++++
 tests/test_testing.py      | 40 ++++++++++++++++++++++++
 tests/testing/dummy.ipynb  | 62 ++++++++++++++++++++++++++++++++++++++
 tests/testing/dummy.py     |  3 ++
 10 files changed, 240 insertions(+), 4 deletions(-)
 create mode 100644 pueblo/testing/folder.py
 create mode 100644 pueblo/testing/nlp.py
 create mode 100644 pueblo/testing/snippet.py
 create mode 100644 tests/testing/dummy.ipynb
 create mode 100644 tests/testing/dummy.py

diff --git a/CHANGES.md b/CHANGES.md
index 48cee72..0802c81 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,6 +1,7 @@
 # Changes for pueblo
 
 ## Unreleased
+- Add a few testing helper utilities to `pueblo.testing`
  
 ## 2023-11-06 v0.0.3
 - ngr: Fix `contextlib.chdir` only available on Python 3.11 and newer
diff --git a/pueblo/testing/folder.py b/pueblo/testing/folder.py
new file mode 100644
index 0000000..6e09a70
--- /dev/null
+++ b/pueblo/testing/folder.py
@@ -0,0 +1,35 @@
+from pathlib import Path
+
+
+def list_files(path: Path, pattern: str):
+    """
+    Enumerate all files in given directory.
+    """
+    files = path.glob(pattern)
+    return [item.relative_to(path) for item in files]
+
+
+def list_notebooks(path: Path, pattern: str = "*.ipynb"):
+    """
+    Enumerate all Jupyter Notebook files found in given directory.
+    """
+    return list_files(path, pattern)
+
+
+def list_python_files(path: Path, pattern: str = "*.py"):
+    """
+    Enumerate all regular Python files found in given directory.
+    """
+    pyfiles = []
+    for item in list_files(path, pattern):
+        if item.name in ["conftest.py"] or item.name.startswith("test"):
+            continue
+        pyfiles.append(item)
+    return pyfiles
+
+
+def str_list(things):
+    """
+    Converge list to list of strings.
+    """
+    return list(map(str, things))
diff --git a/pueblo/testing/nlp.py b/pueblo/testing/nlp.py
new file mode 100644
index 0000000..d50a1bd
--- /dev/null
+++ b/pueblo/testing/nlp.py
@@ -0,0 +1,13 @@
+import pytest
+
+
+@pytest.fixture(scope="session", autouse=True)
+def nltk_init():
+    """
+    Initialize nltk upfront, so that it does not run stray output into Jupyter Notebooks.
+    """
+    download_items = ["averaged_perceptron_tagger", "punkt"]
+    import nltk
+
+    for item in download_items:
+        nltk.download(item)
diff --git a/pueblo/testing/notebook.py b/pueblo/testing/notebook.py
index 5f75002..e72c73a 100644
--- a/pueblo/testing/notebook.py
+++ b/pueblo/testing/notebook.py
@@ -46,4 +46,4 @@ async def async_execute_cell(
             else:  # noqa: RET506
                 raise
 
-    NotebookClient.async_execute_cell = async_execute_cell  # type: ignore[method-assign]
+    NotebookClient.async_execute_cell = async_execute_cell  # type: ignore[method-assign,unused-ignore]
diff --git a/pueblo/testing/snippet.py b/pueblo/testing/snippet.py
new file mode 100644
index 0000000..1d589f0
--- /dev/null
+++ b/pueblo/testing/snippet.py
@@ -0,0 +1,48 @@
+import importlib
+import typing as t
+from pathlib import Path
+
+import pytest
+
+
+def pytest_module_function(request: pytest.FixtureRequest, filepath: t.Union[str, Path], entrypoint: str = "main"):
+    """
+    From individual Python file, collect and wrap the `main` function into a test case.
+    """
+    from _pytest.monkeypatch import MonkeyPatch
+    from _pytest.python import Function
+
+    path = Path(filepath)
+
+    # Temporarily add parent directory to module search path.
+    with MonkeyPatch.context() as m:
+        m.syspath_prepend(path.parent)
+
+        # Import file as Python module.
+        mod = importlib.import_module(path.stem)
+        fun = getattr(mod, entrypoint)
+
+        # Wrap the entrypoint function into a pytest test case, and run it.
+        test = Function.from_parent(request.node, name=entrypoint, callobj=fun)
+        test.runtest()
+        return test.reportinfo()
+
+
+def pytest_notebook(request: pytest.FixtureRequest, filepath: t.Union[str, Path]):
+    """
+    From individual Jupyter Notebook file, collect cells as pytest
+    test cases, and run them.
+
+    Not using `NBRegressionFixture`, because it would manually need to be configured.
+    """
+    from _pytest._py.path import LocalPath
+    from pytest_notebook.plugin import pytest_collect_file
+
+    tests = pytest_collect_file(LocalPath(filepath), request.node)
+    if not tests:
+        raise ValueError(f"No tests collected from notebook: {filepath}")
+    infos = []
+    for test in tests.collect():
+        test.runtest()
+        infos.append(test.reportinfo())
+    return infos
diff --git a/pyproject.toml b/pyproject.toml
index 96b5c0e..f040132 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -69,7 +69,7 @@ dependencies = [
 
 [project.optional-dependencies]
 all = [
-  "pueblo[cli,ngr,nlp,web]",
+  "pueblo[cli,ngr,nlp,testing,web]",
 ]
 cli = [
   "click<9",
@@ -78,7 +78,7 @@ cli = [
   "python-dotenv<2",
 ]
 develop = [
-  "black<24",
+  "black[jupyter]<24",
   "mypy==1.6.1",
   "poethepoet<1",
   "pyproject-fmt<1.5,>=1.3",
@@ -102,6 +102,14 @@ test = [
   "pytest-cov<5",
   "pytest-mock<4",
 ]
+testing = [
+  "coverage~=7.3",
+  "ipykernel",
+  "pytest<8",
+  "pytest-cov<5",
+  "pytest-env<2",
+  "pytest-notebook<0.9",
+]
 web = [
   "requests-cache<2",
 ]
@@ -131,7 +139,7 @@ skip_gitignore = false
 minversion = "2.0"
 addopts = """
   -rfEX -p pytester --strict-markers --verbosity=3
-  --cov --cov-report=term-missing --cov-report=xml
+  --cov=pueblo --cov-report=term-missing --cov-report=xml
   --capture=no
   --ignore=tests/ngr
   """
@@ -142,6 +150,19 @@ xfail_strict = true
 markers = [
   "ngr",
 ]
+env = [
+    "PYDEVD_DISABLE_FILE_VALIDATION=1",
+]
+
+
+# pytest-notebook settings
+nb_test_files = "true"
+nb_coverage = "false"
+nb_diff_ignore = [
+    "/metadata/language_info",
+    "/cells/*/execution_count",
+    "/cells/*/outputs/*/execution_count",
+]
 
 [tool.coverage.run]
 branch = false
diff --git a/tests/test_nlp.py b/tests/test_nlp.py
index 4414689..753e5a3 100644
--- a/tests/test_nlp.py
+++ b/tests/test_nlp.py
@@ -1,3 +1,7 @@
+# TODO: Publish as real package.
+from pueblo.testing.nlp import nltk_init  # noqa: F401
+
+
 def test_cached_web_resource():
     from pueblo.nlp.resource import CachedWebResource
 
@@ -8,3 +12,12 @@ def test_cached_web_resource():
     from langchain.schema import Document
 
     assert isinstance(docs[0], Document)
+
+
+def test_nltk_init(nltk_init):  # noqa: F811
+    """
+    Just _use_ the fixture to check if it works well.
+
+    TODO: Anything else that could be verified here?
+    """
+    pass
diff --git a/tests/test_testing.py b/tests/test_testing.py
index 45f6c0d..c2f194f 100644
--- a/tests/test_testing.py
+++ b/tests/test_testing.py
@@ -1,5 +1,45 @@
+from pathlib import Path
+
 from pueblo.testing.notebook import monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip
+from pueblo.testing.snippet import pytest_module_function, pytest_notebook
+
+HERE = Path(__file__).parent
 
 
 def test_monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip():
     monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip()
+
+
+def test_pytest_module_function(request, capsys):
+    outcome = pytest_module_function(request=request, filepath=HERE / "testing" / "dummy.py")
+    assert isinstance(outcome[0], Path)
+    assert outcome[0].name == "dummy.py"
+    assert outcome[1] == 0
+    assert outcome[2] == "test_pytest_module_function.main"
+
+    out, err = capsys.readouterr()
+    assert out == "Hallo, Räuber Hotzenplotz.\n"
+
+
+def test_pytest_notebook(request):
+    from _pytest._py.path import LocalPath
+
+    outcomes = pytest_notebook(request=request, filepath=HERE / "testing" / "dummy.ipynb")
+    assert isinstance(outcomes[0][0], LocalPath)
+    assert outcomes[0][0].basename == "dummy.ipynb"
+    assert outcomes[0][1] == 0
+    assert outcomes[0][2] == "notebook: nbregression(dummy)"
+
+
+def test_list_python_files():
+    from pueblo.testing.folder import list_python_files, str_list
+
+    outcome = str_list(list_python_files(HERE / "testing"))
+    assert outcome == ["dummy.py"]
+
+
+def test_list_notebooks():
+    from pueblo.testing.folder import list_notebooks, str_list
+
+    outcome = str_list(list_notebooks(HERE / "testing"))
+    assert outcome == ["dummy.ipynb"]
diff --git a/tests/testing/dummy.ipynb b/tests/testing/dummy.ipynb
new file mode 100644
index 0000000..c2f4e68
--- /dev/null
+++ b/tests/testing/dummy.ipynb
@@ -0,0 +1,62 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "source": [
+    "# A little notebook."
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Hallo, Räuber Hotzenplotz.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Hallo, Räuber Hotzenplotz.\")"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [],
+   "metadata": {
+    "collapsed": false
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/tests/testing/dummy.py b/tests/testing/dummy.py
new file mode 100644
index 0000000..f6705b5
--- /dev/null
+++ b/tests/testing/dummy.py
@@ -0,0 +1,3 @@
+def main():
+    print("Hallo, Räuber Hotzenplotz.")  # noqa: T201
+    return 42