diff --git a/CHANGES.md b/CHANGES.md index 48cee72..0802c81 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # Changes for pueblo ## Unreleased +- Add a few testing helper utilities to `pueblo.testing` ## 2023-11-06 v0.0.3 - ngr: Fix `contextlib.chdir` only available on Python 3.11 and newer diff --git a/pueblo/testing/folder.py b/pueblo/testing/folder.py new file mode 100644 index 0000000..6e09a70 --- /dev/null +++ b/pueblo/testing/folder.py @@ -0,0 +1,35 @@ +from pathlib import Path + + +def list_files(path: Path, pattern: str): + """ + Enumerate all files in given directory. + """ + files = path.glob(pattern) + return [item.relative_to(path) for item in files] + + +def list_notebooks(path: Path, pattern: str = "*.ipynb"): + """ + Enumerate all Jupyter Notebook files found in given directory. + """ + return list_files(path, pattern) + + +def list_python_files(path: Path, pattern: str = "*.py"): + """ + Enumerate all regular Python files found in given directory. + """ + pyfiles = [] + for item in list_files(path, pattern): + if item.name in ["conftest.py"] or item.name.startswith("test"): + continue + pyfiles.append(item) + return pyfiles + + +def str_list(things): + """ + Converge list to list of strings. + """ + return list(map(str, things)) diff --git a/pueblo/testing/nlp.py b/pueblo/testing/nlp.py new file mode 100644 index 0000000..d50a1bd --- /dev/null +++ b/pueblo/testing/nlp.py @@ -0,0 +1,13 @@ +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def nltk_init(): + """ + Initialize nltk upfront, so that it does not run stray output into Jupyter Notebooks. + """ + download_items = ["averaged_perceptron_tagger", "punkt"] + import nltk + + for item in download_items: + nltk.download(item) diff --git a/pueblo/testing/notebook.py b/pueblo/testing/notebook.py index 5f75002..e72c73a 100644 --- a/pueblo/testing/notebook.py +++ b/pueblo/testing/notebook.py @@ -46,4 +46,4 @@ async def async_execute_cell( else: # noqa: RET506 raise - NotebookClient.async_execute_cell = async_execute_cell # type: ignore[method-assign] + NotebookClient.async_execute_cell = async_execute_cell # type: ignore[method-assign,unused-ignore] diff --git a/pueblo/testing/snippet.py b/pueblo/testing/snippet.py new file mode 100644 index 0000000..1d589f0 --- /dev/null +++ b/pueblo/testing/snippet.py @@ -0,0 +1,48 @@ +import importlib +import typing as t +from pathlib import Path + +import pytest + + +def pytest_module_function(request: pytest.FixtureRequest, filepath: t.Union[str, Path], entrypoint: str = "main"): + """ + From individual Python file, collect and wrap the `main` function into a test case. + """ + from _pytest.monkeypatch import MonkeyPatch + from _pytest.python import Function + + path = Path(filepath) + + # Temporarily add parent directory to module search path. + with MonkeyPatch.context() as m: + m.syspath_prepend(path.parent) + + # Import file as Python module. + mod = importlib.import_module(path.stem) + fun = getattr(mod, entrypoint) + + # Wrap the entrypoint function into a pytest test case, and run it. + test = Function.from_parent(request.node, name=entrypoint, callobj=fun) + test.runtest() + return test.reportinfo() + + +def pytest_notebook(request: pytest.FixtureRequest, filepath: t.Union[str, Path]): + """ + From individual Jupyter Notebook file, collect cells as pytest + test cases, and run them. + + Not using `NBRegressionFixture`, because it would manually need to be configured. + """ + from _pytest._py.path import LocalPath + from pytest_notebook.plugin import pytest_collect_file + + tests = pytest_collect_file(LocalPath(filepath), request.node) + if not tests: + raise ValueError(f"No tests collected from notebook: {filepath}") + infos = [] + for test in tests.collect(): + test.runtest() + infos.append(test.reportinfo()) + return infos diff --git a/pyproject.toml b/pyproject.toml index 96b5c0e..f040132 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ dependencies = [ [project.optional-dependencies] all = [ - "pueblo[cli,ngr,nlp,web]", + "pueblo[cli,ngr,nlp,testing,web]", ] cli = [ "click<9", @@ -78,7 +78,7 @@ cli = [ "python-dotenv<2", ] develop = [ - "black<24", + "black[jupyter]<24", "mypy==1.6.1", "poethepoet<1", "pyproject-fmt<1.5,>=1.3", @@ -102,6 +102,14 @@ test = [ "pytest-cov<5", "pytest-mock<4", ] +testing = [ + "coverage~=7.3", + "ipykernel", + "pytest<8", + "pytest-cov<5", + "pytest-env<2", + "pytest-notebook<0.9", +] web = [ "requests-cache<2", ] @@ -131,7 +139,7 @@ skip_gitignore = false minversion = "2.0" addopts = """ -rfEX -p pytester --strict-markers --verbosity=3 - --cov --cov-report=term-missing --cov-report=xml + --cov=pueblo --cov-report=term-missing --cov-report=xml --capture=no --ignore=tests/ngr """ @@ -142,6 +150,19 @@ xfail_strict = true markers = [ "ngr", ] +env = [ + "PYDEVD_DISABLE_FILE_VALIDATION=1", +] + + +# pytest-notebook settings +nb_test_files = "true" +nb_coverage = "false" +nb_diff_ignore = [ + "/metadata/language_info", + "/cells/*/execution_count", + "/cells/*/outputs/*/execution_count", +] [tool.coverage.run] branch = false diff --git a/tests/test_nlp.py b/tests/test_nlp.py index 4414689..753e5a3 100644 --- a/tests/test_nlp.py +++ b/tests/test_nlp.py @@ -1,3 +1,7 @@ +# TODO: Publish as real package. +from pueblo.testing.nlp import nltk_init # noqa: F401 + + def test_cached_web_resource(): from pueblo.nlp.resource import CachedWebResource @@ -8,3 +12,12 @@ def test_cached_web_resource(): from langchain.schema import Document assert isinstance(docs[0], Document) + + +def test_nltk_init(nltk_init): # noqa: F811 + """ + Just _use_ the fixture to check if it works well. + + TODO: Anything else that could be verified here? + """ + pass diff --git a/tests/test_testing.py b/tests/test_testing.py index 45f6c0d..c2f194f 100644 --- a/tests/test_testing.py +++ b/tests/test_testing.py @@ -1,5 +1,45 @@ +from pathlib import Path + from pueblo.testing.notebook import monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip +from pueblo.testing.snippet import pytest_module_function, pytest_notebook + +HERE = Path(__file__).parent def test_monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip(): monkeypatch_pytest_notebook_treat_cell_exit_as_notebook_skip() + + +def test_pytest_module_function(request, capsys): + outcome = pytest_module_function(request=request, filepath=HERE / "testing" / "dummy.py") + assert isinstance(outcome[0], Path) + assert outcome[0].name == "dummy.py" + assert outcome[1] == 0 + assert outcome[2] == "test_pytest_module_function.main" + + out, err = capsys.readouterr() + assert out == "Hallo, Räuber Hotzenplotz.\n" + + +def test_pytest_notebook(request): + from _pytest._py.path import LocalPath + + outcomes = pytest_notebook(request=request, filepath=HERE / "testing" / "dummy.ipynb") + assert isinstance(outcomes[0][0], LocalPath) + assert outcomes[0][0].basename == "dummy.ipynb" + assert outcomes[0][1] == 0 + assert outcomes[0][2] == "notebook: nbregression(dummy)" + + +def test_list_python_files(): + from pueblo.testing.folder import list_python_files, str_list + + outcome = str_list(list_python_files(HERE / "testing")) + assert outcome == ["dummy.py"] + + +def test_list_notebooks(): + from pueblo.testing.folder import list_notebooks, str_list + + outcome = str_list(list_notebooks(HERE / "testing")) + assert outcome == ["dummy.ipynb"] diff --git a/tests/testing/dummy.ipynb b/tests/testing/dummy.ipynb new file mode 100644 index 0000000..c2f4e68 --- /dev/null +++ b/tests/testing/dummy.ipynb @@ -0,0 +1,62 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# A little notebook." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 1, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hallo, Räuber Hotzenplotz.\n" + ] + } + ], + "source": [ + "print(\"Hallo, Räuber Hotzenplotz.\")" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tests/testing/dummy.py b/tests/testing/dummy.py new file mode 100644 index 0000000..f6705b5 --- /dev/null +++ b/tests/testing/dummy.py @@ -0,0 +1,3 @@ +def main(): + print("Hallo, Räuber Hotzenplotz.") # noqa: T201 + return 42