From caf2e157f017c4de1dca864fd5af0f4fe237d5e7 Mon Sep 17 00:00:00 2001 From: Pedro Brochado Date: Fri, 11 Oct 2024 06:51:25 -0300 Subject: [PATCH 1/2] Implement doc tree builder with toml, yaml and custom format This is to help create fixtures, so I can write meaningfull tests without too much pain and then refactor this code without having to manually look at the pages. --- pyproject.toml | 6 ++ src/pulp_docs/test_tools/__init__.py | 0 src/pulp_docs/test_tools/doctree_writer.py | 72 +++++++++++++++ src/pulp_docs/utils/__init__.py | 0 tests/test_doctree_writer.py | 101 +++++++++++++++++++++ 5 files changed, 179 insertions(+) create mode 100644 src/pulp_docs/test_tools/__init__.py create mode 100644 src/pulp_docs/test_tools/doctree_writer.py create mode 100644 src/pulp_docs/utils/__init__.py create mode 100644 tests/test_doctree_writer.py diff --git a/pyproject.toml b/pyproject.toml index 51846cd..7877975 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,3 +30,9 @@ pulp_docs = ["data/**"] [tool.setuptools.packages.find] where = ["src"] +[tool.pytest.ini_options] +pythonpath = "src" +addopts = [ + "--import-mode=importlib", +] + diff --git a/src/pulp_docs/test_tools/__init__.py b/src/pulp_docs/test_tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pulp_docs/test_tools/doctree_writer.py b/src/pulp_docs/test_tools/doctree_writer.py new file mode 100644 index 0000000..68e747b --- /dev/null +++ b/src/pulp_docs/test_tools/doctree_writer.py @@ -0,0 +1,72 @@ +from pathlib import Path + +import tomllib +import yaml +import re + + +def parse_doctree_file(doctree_file: Path, target: Path, project_name: str = "foobar"): + """Create a whole documentation tree base on @doctree_file on @target. + + The goal is to facilitate creating fixtures for testing complex build cases, such + as pulp structure. + + The declarative doctree file specifies a list of (path,content) tuples, with an semantic + header separation.. + + The overall structure is: + + ```pseudo-format + { + projet-name-1: [{path: content}, ..., {path: content}], + ... + projet-name-N: [{path: content}, ..., {path: content}], + } + ``` + + See `test_doctree_writer` for samples. + + Params: + doctree_file: The file with a supported extenstion format. E.g: `.toml` `.yml` and `.doctree` + target: The directory where the project should be writter to. + """ + + def custom_parser(file: Path): + _data = file.read_text() + section_match = r"\n*\[\[\s*[\w-]+\s*\]\]\n" + item_match = r"----+\n" + section_split = [ + section for section in re.split(section_match, _data) if section + ] + item_split = [ + item + for section in section_split + for item in re.split(item_match, section) + if section and item + ] + item_partition = [t.partition("\n\n") for t in item_split if t] + + def sanitize_path(s): + return s.partition("\n")[0].strip(" ") + + items = [{"path": sanitize_path(s[0]), "data": s[2]} for s in item_partition] + return {"foobar": items} + + # Open and parse doctree file + if doctree_file.suffix in (".yml", ".yaml"): + data = yaml.load(doctree_file.read_text(), Loader=yaml.SafeLoader) + elif doctree_file.suffix in (".toml",): + data = tomllib.loads(doctree_file.read_text()) + elif doctree_file.suffix in (".doctree",): + data = custom_parser(doctree_file) + # breakpoint() + else: + raise NotImplementedError(f"File type not supported: {doctree_file.name}") + + # Create all directories + for prj_name, contents in data.items(): + for item in contents: + basedir, _, filename = item["path"].strip("/").rpartition("/") + basedir = target / basedir + basedir.mkdir(parents=True, exist_ok=True) + Path(target / basedir / filename).write_text(item["data"]) diff --git a/src/pulp_docs/utils/__init__.py b/src/pulp_docs/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_doctree_writer.py b/tests/test_doctree_writer.py new file mode 100644 index 0000000..8aa8c74 --- /dev/null +++ b/tests/test_doctree_writer.py @@ -0,0 +1,101 @@ +from pathlib import Path + +import pytest +import textwrap + +from pulp_docs.test_tools.doctree_writer import parse_doctree_file + +file_sample = """\ +# check-title + +check-content + +--- +not/a/path (separator must have 4+ ---) + +dont split.""" + +yaml_sample = f"""\ +project1: + - path: docs/index.md + data: | +{textwrap.indent(file_sample, " " * 6)} + - path: docs/guides/foo.md + data: | +{textwrap.indent(file_sample, " " * 6)} +project2: + - path: docs/guides/bar.md + data: | +{textwrap.indent(file_sample, " " * 6)} +""" + +toml_sample = f"""\ +[[project1]] +path = 'docs/index.md' +data = ''' +{file_sample} +''' + +[[project1]] +path = 'docs/guides/foo.md' +data = ''' +{file_sample} +''' + +[[project2]] +path = 'docs/guides/bar.md' +data = ''' +{file_sample} +''' +""" + +# .doctree extenstion +custom_sample = f"""\ +[[ project1 ]] +------------------------ +docs/index.md + +{file_sample} +----------------- +docs/guides/foo.md +---------------#ignore + +{file_sample} + +[[ project2 ]] +----- +docs/guides/bar.md + +{file_sample} +""" + + +@pytest.mark.parametrize( + "file_ext,content", + [ + pytest.param("toml", toml_sample, id="toml"), + pytest.param("yaml", yaml_sample, id="yaml"), + pytest.param("yml", yaml_sample, id="yml"), + pytest.param("doctree", custom_sample, id="doctree"), + ], +) +def test_doctree_write(file_ext, content, tmp_path): + sample_file = tmp_path / f"declarative_fixture.{file_ext}" + sample_file.write_text(content) + parse_doctree_file(sample_file, tmp_path) + + pages = ("docs/index.md", "docs/guides/foo.md", "docs/guides/bar.md") + for page_path in pages: + assert Path(tmp_path / page_path).exists() + + contents = [] + for page_path in pages: + content = Path(tmp_path / page_path).read_text() + contents.append(content) + assert "# check-title" in content + assert "check-content" in content + assert "[[ project1 ]]" not in content + assert "[[ project2 ]]" not in content + + print() + print(f"To check manually cd to:\n{tmp_path}") From b8ed48612244a87675c1cfae6f830e4ef60cf989 Mon Sep 17 00:00:00 2001 From: Pedro Brochado Date: Fri, 11 Oct 2024 17:56:20 -0300 Subject: [PATCH 2/2] Add html-snapshot testing with basic fixture --- src/pulp_docs/main.py | 5 ++- src/pulp_docs/test_tools/snapshot.py | 46 +++++++++++++++++++++++++++ tests/doctrees/pulpcore_only.toml | 47 ++++++++++++++++++++++++++++ tests/test_fixture_snapshot.py | 10 ++++++ 4 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 src/pulp_docs/test_tools/snapshot.py create mode 100644 tests/doctrees/pulpcore_only.toml create mode 100644 tests/test_fixture_snapshot.py diff --git a/src/pulp_docs/main.py b/src/pulp_docs/main.py index 57e16c8..6413708 100644 --- a/src/pulp_docs/main.py +++ b/src/pulp_docs/main.py @@ -85,7 +85,10 @@ def serve(self, config: Config, dry_run: bool = False): return subprocess.run(cmd, env=env) - def build(self, config: Config, dry_run: bool = False): + def build( + self, config: Config, dry_run: bool = False, target: t.Optional[Path] = None + ): + # TODO: implement target # Process option to pass to command cmd = ["mkdocs", "build"] diff --git a/src/pulp_docs/test_tools/snapshot.py b/src/pulp_docs/test_tools/snapshot.py new file mode 100644 index 0000000..8ee52e2 --- /dev/null +++ b/src/pulp_docs/test_tools/snapshot.py @@ -0,0 +1,46 @@ +from pathlib import Path +from pulp_docs.main import PulpDocs, Config + + +def snapshot_fixture(fixture_dir: Path, repolist: Path, target: Path) -> Path: + """Builds snapshot of the fixture-docs using @fixture_dir and @repolist at @target. + + The snapshot should be taken after someone carefully inspect the core elements of + the site looks as expected, like: + * Navigation display: nav items that should and shouldnt be there. + * Special pages behave as expected, like RestAPI, Changes and index pages. + * Regular pages exists (or dont exist) where expected inside plugins and sections. + + The snapshot is not intended to provide a 1:1 comparision, but more of a structural + comparision, so at least we catch obivous structural regressions. + + Params: + fixture_dir: A dir which should contain `{repository_name}/{repository_tree}` + repolist: A yaml file containing the aggregation config. + target: Dir where to write the build. + + Returns: + The Path of the new snapshot. The dirname is commit hash at the moment of the + which snapshot. + """ + # Guards to avoid surprises + if not fixture_dir.is_dir(): + raise ValueError(f"'fixture_dir' should be a dir: {fixture_dir}") + if not list(fixture_dir.iterdir()): + raise ValueError(f"'fixture_dir' should NOT be empty.: {fixture_dir}") + + if not target.is_dir(): + raise ValueError(f"'fixture_dir' should be a dir: {target}") + if list(fixture_dir.iterdir()): + raise ValueError(f"'target' must be empty.: {target}") + + if repolist.suffix not in (".yml", "yaml"): + raise ValueError(f"'repolist' must be a YAML file: {repolist.name}") + + # TODO: test this. + config = Config() + config.repolist = repolist.absolute() + pulp_docs = PulpDocs(config) + pulp_docs.build(target=target) + + return Path() diff --git a/tests/doctrees/pulpcore_only.toml b/tests/doctrees/pulpcore_only.toml new file mode 100644 index 0000000..80d2dfa --- /dev/null +++ b/tests/doctrees/pulpcore_only.toml @@ -0,0 +1,47 @@ +################ +### PULPCORE ### +################ +# Minimal setup with pulpcore only. + + [[ pulpcore ]] + +path = 'pulpcore/CHANGES.md' +data = ''' +# Changelog + +## x.y.z (YYYY-MM-DD) + +#### Features +- feature-changes-check + +#### Bugfixes +- bugfixes-changes-check +''' + + [[ pulpcore ]] + +path = 'pulpcore/index.md' +data = ''' +# Welcome to Pulpcore + +pulpcore-index-check +''' + + [[ pulpcore ]] + +path = 'pulpcore/guides/guides1.md' +data = ''' +# Guide1 + +guide1-check +''' + + [[ pulpcore ]] + +path = 'pulpcore/guides/guides2.md' + +data = ''' +# Guide2 + +guide2-check +''' diff --git a/tests/test_fixture_snapshot.py b/tests/test_fixture_snapshot.py new file mode 100644 index 0000000..22b65a4 --- /dev/null +++ b/tests/test_fixture_snapshot.py @@ -0,0 +1,10 @@ +from pulp_docs.test_tools.snapshot import snapshot_fixture +from pathlib import Path + + +def test_snapshot_fixture(tmp_path): + """Test that using different fixture_dir or repolist the snapshot is different.""" + fixture_dir = Path() + target = Path() + repolist = Path() + dirname = snapshot_fixture(fixture_dir, repolist, target)