From e4bf0a4632f33131e4311844733963fd5f644a6d Mon Sep 17 00:00:00 2001 From: Pedro Brochado Date: Fri, 2 Feb 2024 17:42:18 -0300 Subject: [PATCH] Refactor, improve organization and fix edit_this_page --- src/pulp_docs/constants.py | 39 ++ .../data/docs/{tags.md => gitignore.md} | 0 src/pulp_docs/data/mkdocs.yml | 8 +- src/pulp_docs/data/repolist.yml | 4 + src/pulp_docs/mkdocs_macros.py | 24 +- src/pulp_docs/navigation.py | 230 ++--------- src/pulp_docs/repository.py | 23 +- src/pulp_docs/utils/aggregation.py | 180 +++++++++ src/pulp_docs/utils/fake_repo_generator.py | 370 ------------------ src/pulp_docs/utils/general.py | 18 + .../docs => staging_docs}/assets/hero-img.png | Bin .../docs => staging_docs}/assets/logo.png | Bin .../data/docs => staging_docs}/css/extra.css | 0 .../css/mkdocstrings.css | 0 staging_docs/dev/tutorials/quickstart.md | 3 + .../docs/.gitkeep => staging_docs/tags.md | 0 tests/fixtures/pulp-docs/staging_docs/tags.md | 0 17 files changed, 297 insertions(+), 602 deletions(-) create mode 100644 src/pulp_docs/constants.py rename src/pulp_docs/data/docs/{tags.md => gitignore.md} (100%) create mode 100644 src/pulp_docs/utils/aggregation.py delete mode 100644 src/pulp_docs/utils/fake_repo_generator.py create mode 100644 src/pulp_docs/utils/general.py rename {src/pulp_docs/data/docs => staging_docs}/assets/hero-img.png (100%) rename {src/pulp_docs/data/docs => staging_docs}/assets/logo.png (100%) rename {src/pulp_docs/data/docs => staging_docs}/css/extra.css (100%) rename {src/pulp_docs/data/docs => staging_docs}/css/mkdocstrings.css (100%) create mode 100644 staging_docs/dev/tutorials/quickstart.md rename src/pulp_docs/docs/.gitkeep => staging_docs/tags.md (100%) create mode 100644 tests/fixtures/pulp-docs/staging_docs/tags.md diff --git a/src/pulp_docs/constants.py b/src/pulp_docs/constants.py new file mode 100644 index 0000000..6bd24c0 --- /dev/null +++ b/src/pulp_docs/constants.py @@ -0,0 +1,39 @@ +ADMIN_NAME = "admin" +USER_NAME = "user" + +DISPLAY_NAMES = { + "guides": "How-to Guides", + "learn": "Learn More", + "tutorials": "Tutorials", + "reference": "Reference", +} +GUIDES = DISPLAY_NAMES["guides"] +LEARN = DISPLAY_NAMES["learn"] +TUTORIALS = DISPLAY_NAMES["tutorials"] + + +class Names: + """Display Names""" + + # content types + GUIDES = "How-to Guides" + LEARN = "Learn More" + TUTORIALS = "Tutorials" + REFERENCE = "Reference" + + # repo-types + OTHER = "Extra" + CORE = "Pulpcore" + CONTENT = "Plugins" + + # personas + USER = "User" + ADMIN = "Admin" + DEV = "Dev" + + # other + PULPCORE_TUTORIAL = "Getting Started" + + @staticmethod + def get(name: str): + return getattr(Names, name.upper()) diff --git a/src/pulp_docs/data/docs/tags.md b/src/pulp_docs/data/docs/gitignore.md similarity index 100% rename from src/pulp_docs/data/docs/tags.md rename to src/pulp_docs/data/docs/gitignore.md diff --git a/src/pulp_docs/data/mkdocs.yml b/src/pulp_docs/data/mkdocs.yml index be08a59..08f8f6e 100644 --- a/src/pulp_docs/data/mkdocs.yml +++ b/src/pulp_docs/data/mkdocs.yml @@ -6,7 +6,7 @@ repo_name: pulp/pulpcore docs_dir: docs theme: name: material - logo: pulp-docs/assets/logo.png + logo: pulp-docs/docs/assets/logo.png features: - content.code.annotate - navigation.tabs @@ -41,7 +41,7 @@ plugins: - macros: module_name: '../mkdocs_macros' - tags: - tags_file: pulp-docs/tags.md + tags_file: pulp-docs/docs/tags.md - mkdocstrings: handlers: python: @@ -66,8 +66,8 @@ extra: link: https://discourse.pulpproject.org/ extra_css: - - pulp-docs/css/extra.css - - pulp-docs/css/mkdocstrings.css + - pulp-docs/docs/css/extra.css + - pulp-docs/docs/css/mkdocstrings.css markdown_extensions: # Python Markdown diff --git a/src/pulp_docs/data/repolist.yml b/src/pulp_docs/data/repolist.yml index f26d06a..50a7c74 100644 --- a/src/pulp_docs/data/repolist.yml +++ b/src/pulp_docs/data/repolist.yml @@ -22,3 +22,7 @@ repos: owner: pulp title: Pulp OCI Images branch: latest + - name: pulp-docs + owner: pedro-psb + title: Docs Tool + branch: main diff --git a/src/pulp_docs/mkdocs_macros.py b/src/pulp_docs/mkdocs_macros.py index 5cec9d4..00a4d01 100644 --- a/src/pulp_docs/mkdocs_macros.py +++ b/src/pulp_docs/mkdocs_macros.py @@ -18,11 +18,10 @@ import logging import shutil import tempfile -import typing as t +import time from pathlib import Path import rich -from importlib_resources import as_file, files from pulp_docs.cli import Config from pulp_docs.navigation import get_navigation @@ -91,6 +90,9 @@ def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config): shutil.rmtree(repo_docs, ignore_errors=True) for repo in repos.all: + start = time.perf_counter() + # if repo.name == "pulp-docs": + # breakpoint() # 1. Download repo (copy locally or fetch from GH) this_src_dir = repo_sources / repo.name repo.download(dest_dir=this_src_dir, clear_cache=config.clear_cache) @@ -125,11 +127,11 @@ def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config): md_body = f"[{repo.rest_api_link}]({repo.rest_api_link})" rest_api_page.write_text(f"{md_title}\n\n{md_body}") + end = time.perf_counter() + duration = end - start + log.info(f"{repo.name} completed in {duration:.2} sec") + # Copy template-files (from this plugin) to tmpdir - log.info("[pulp-docs] Moving pulp-docs /docs to final destination") - data_file_docs = files("pulp_docs").joinpath("data/docs") - with as_file(data_file_docs) as _docs: - shutil.copytree(_docs, repo_docs / "pulp-docs") shutil.copy( repo_sources / repos.core_repo.name / SRC_DOCS_DIRNAME / "index.md", repo_docs / "index.md", @@ -158,7 +160,7 @@ def print_user_repo(repos: Repos, config: Config): record = { "name": repo.name, "download_source": repo.status.download_source, - "refs": repo.branch, + "refs": repo.branch_in_use, } if repo.status.use_local_checkout is True: local_checkouts.append(record) @@ -168,9 +170,9 @@ def print_user_repo(repos: Repos, config: Config): downloaded_repos.append(record) # TODO: improve this refspec comparision heuristics - if repo.status.original_refs and (repo.status.original_refs not in repo.branch): + if repo.branch not in repo.branch_in_use: warn_msgs.append( - f"[pulp-docs] Original {repo.name!r} ref is {repo.status.original_refs!r}, but local one is '{repo.branch}'." + f"[pulp-docs] Original {repo.name!r} ref is {repo.branch!r}, but local one is '{repo.branch_in_use}'." ) if len(local_checkouts) == 0: @@ -249,9 +251,7 @@ def on_pre_page_macros(env): path = f"{SRC_DOCS_DIRNAME}/index.md" repo_obj = repos.get(repo) - repo_branch = "main" - if repo_obj: - repo_branch = repo_obj.status.original_refs + repo_branch = getattr(repo_obj, "branch", "main") edit_url = f"https://github.com/pulp/{repo}/edit/{repo_branch}/{path}" env.page.edit_url = edit_url diff --git a/src/pulp_docs/navigation.py b/src/pulp_docs/navigation.py index 92c3530..05c8b33 100644 --- a/src/pulp_docs/navigation.py +++ b/src/pulp_docs/navigation.py @@ -26,24 +26,11 @@ """ from __future__ import annotations -import os -import typing as t from pathlib import Path +from pulp_docs.constants import Names from pulp_docs.repository import Repos - -ADMIN_NAME = "admin" -USER_NAME = "user" - -DISPLAY_NAMES = { - "guides": "How-to Guides", - "learn": "Learn More", - "tutorials": "Tutorials", - "reference": "Reference", -} -GUIDES = DISPLAY_NAMES["guides"] -LEARN = DISPLAY_NAMES["learn"] -TUTORIALS = DISPLAY_NAMES["tutorials"] +from pulp_docs.utils.aggregation import AgregationUtils def get_navigation(tmpdir: Path, repos: Repos): @@ -64,11 +51,11 @@ def grouped_by_persona(tmpdir: Path, repos: Repos): Organizes content (roughly) with the pattern. {persona}/ Overview/ - {repos}/ - {content-type} + {repos-type}/ + {repos}/ + {content-type} """ f = AgregationUtils(tmpdir, repos) - # Aggregation and Grouping help_section = [ {"Overview": f.section_file("help/index.md")}, { @@ -77,16 +64,17 @@ def grouped_by_persona(tmpdir: Path, repos: Repos): ) }, ] - # pulpcore_tutorial usage_section = [ {"Overview": f.section_file("usage/index.md")}, { - "Getting Started": [ + "Pulpcore": [ f.section( - "Intro Tutorial", f.get_children, "pulpcore/docs/user/tutorials" + Names.PULPCORE_TUTORIAL, + f.get_children, + "pulpcore/docs/user/tutorials", ), - f.section(LEARN, f.get_children, "pulpcore/docs/user/learn"), - f.section(GUIDES, f.get_children, "pulpcore/docs/user/guides"), + f.section(Names.LEARN, f.get_children, "pulpcore/docs/user/learn"), + f.section(Names.GUIDES, f.get_children, "pulpcore/docs/user/guides"), ] }, { @@ -99,12 +87,14 @@ def grouped_by_persona(tmpdir: Path, repos: Repos): admin_section = [ {"Overview": f.section_file("admin/index.md")}, { - "Getting Started": [ + "Pulpcore": [ f.section( - "Intro Tutorial", f.get_children, "pulpcore/docs/admin/tutorials" + Names.PULPCORE_TUTORIAL, + f.get_children, + "pulpcore/docs/admin/tutorials", ), - f.section(LEARN, f.get_children, "pulpcore/docs/admin/learn"), - f.section(GUIDES, f.get_children, "pulpcore/docs/admin/guides"), + f.section(Names.LEARN, f.get_children, "pulpcore/docs/admin/learn"), + f.section(Names.GUIDES, f.get_children, "pulpcore/docs/admin/guides"), ] }, f.section( @@ -125,10 +115,14 @@ def grouped_by_persona(tmpdir: Path, repos: Repos): development_section = [ {"Overview": f.section_file("development/index.md")}, { - "Getting Started": [ - f.section("Quickstarts", f.get_children, "pulpcore/docs/dev/tutorials"), - f.section(LEARN, f.get_children, "pulpcore/docs/dev/learn"), - f.section(GUIDES, f.get_children, "pulpcore/docs/dev/guides"), + "Pulpcore": [ + f.section( + Names.PULPCORE_TUTORIAL, + f.get_children, + "pulpcore/docs/dev/tutorials", + ), + f.section(Names.LEARN, f.get_children, "pulpcore/docs/dev/learn"), + f.section(Names.GUIDES, f.get_children, "pulpcore/docs/dev/guides"), ] }, { @@ -155,177 +149,3 @@ def grouped_by_persona(tmpdir: Path, repos: Repos): {"Help": help_section}, ] return navigation - - -class AgregationUtils: - def __init__(self, tmpdir: Path, repos: Repos): - self.tmpdir = tmpdir - self.repos = repos - - def section(self, name: str, fn: t.Callable, *args, **kwargs) -> dict: - """ - Create section with @name by calling the @fn aggregation function. - - A section look like {"Name": [file1, file2, ...]} - - Params: - hide_empty_section: Hide section if @fn return is empty. Default=True - """ - hide_empty_section = kwargs.pop("hide_empty_section", False) - section = fn(*args, **kwargs) - if hide_empty_section is False: - return {name: section} - return {name: section} if section else {"": ""} - - def get_children(self, path: t.Union[str, Path]) -> list[str]: - """ - Get all markdown files contained in @path recursively. - - Uses the dirname.title() as the subsection name if there are subdirs. - """ - basepath = self.tmpdir / path if isinstance(path, str) else path - if not basepath.exists(): - return [] - - def _get_tree(_path): - """Recursive scandir""" - with os.scandir(_path) as it: - children = [] - for entry in sorted(it, key=lambda x: x.name): - if entry.is_file() and entry.name.endswith(".md"): - filename = str(Path(entry.path).relative_to(self.tmpdir)) - children.append(filename) - elif entry.is_dir(): - sub_section = {entry.name.title(): _get_tree(entry)} - children.append(sub_section) - return children - - result = _get_tree(basepath) - return result - - def repo_grouping( - self, - template_str: str, - repo_types: t.Optional[list[str]] = None, - content_types: t.Optional[list[str]] = None, - ): - """ - Get all markdown files that matches @template_str basepath and group by repos. - - Arguments: - template_str: The template with fields to expand. Accepts combination of '{repo}' and '{content}' - repos: The set of repos to use. Accepts list with combination of "core", "content" and "other" - content_types: The set of content-types to use. Accepts combination of "guides", "learn" and "tutorial" - - Example: - ```python - >>> repo_grouping("{repo}/docs/dev/guides") - { - "repoA": ["docs/dev/guides/file1.md", "docs/dev/guides/file2.md"], - "repoB": ["docs/dev/guides/file3.md", "docs/dev/guides/file4.md"], - } - >>> repo_grouping("{repo}/docs/dev/{content}", content=["guide", "learn"]) - { - "repoA": [ - {"How-to": ["docs/dev/guides/file1.md", "docs/dev/guides/file2.md"]}, - {"Learn": ["docs/dev/learn/file1.md", "docs/dev/learn/file2.md"]}, - ], - "repoB": [...] - - } - ``` - """ - _nav = {} - _expand_content_types = "{content}" in template_str - - # Selected a set of repos - selected_repos = [] - selected_content = content_types or ["tutorials", "guides", "learn"] - if not repo_types: # default case - selected_repos = self.repos.all - else: - for repo_name in repo_types: - selected_repos.extend(getattr(self.repos, f"{repo_name}_repos")) - - # Dont expand content-types - if not _expand_content_types: - for repo in selected_repos: - lookup_path = self.tmpdir / template_str.format( - repo=repo.name, admin=ADMIN_NAME, user=USER_NAME - ) - _repo_content = self.get_children(lookup_path) - if _repo_content: - _nav[repo.title] = _repo_content - # Expand content-types - else: - for repo in selected_repos: - repo_nav = {} - for content_type in selected_content: - lookup_path = self.tmpdir / template_str.format( - repo=repo.name, - admin=ADMIN_NAME, - user=USER_NAME, - content=content_type, - ) - _repo_content = self.get_children(lookup_path) - - # special treatment to quickstart tutorial - quickstart_file = self._pop_quickstart_from(_repo_content) - if content_type.lower() == "tutorials" and quickstart_file: - repo_nav["Quickstart"] = quickstart_file # type: ignore - - # doesnt render content-type section if no files inside - if _repo_content: - content_type_name = DISPLAY_NAMES[content_type] - repo_nav[content_type_name] = _repo_content # type: ignore - if repo_nav: - _nav[repo.title] = repo_nav - return _nav - - def _pop_quickstart_from(self, pathlist: list[str]) -> t.Optional[str]: - """Get quickstart.md file from filelist with case and variations tolerance""" - for path in pathlist: - filename = path.split("/")[-1] - if filename.lower() in ("quickstart.md", "quick-start.md"): - pathlist.remove(path) - return path - return None - - def repo_reference_grouping(self): - """ - Create reference section by aggregating some specific files. - - Group according to the pattern: - {repo}/ - Readme.md - Rest Api.md - Code Api/ - Module A.md - Module B.md - Changelog.md - - """ - template_str = "{repo}/docs/reference" - _nav = {} - for repo in self.repos.all: - lookup_path = self.tmpdir / template_str.format(repo=repo.name) - _repo_content = self.get_children(lookup_path) - reference_section = [ - {"REST API": f"{repo.name}/docs/rest_api.md"}, - {"Readme": f"{repo.name}/README.md"}, - {"Code API": _repo_content}, - {"Changelog": f"{repo.name}/CHANGELOG.md"}, - ] - _nav[repo.title] = reference_section - return _nav - - def section_file(self, section_and_filename: str): - """Get a markdown file from the website section folder.""" - basepath = "pulpcore/docs/sections" - return f"{basepath}/{section_and_filename}" - - def section_children(self, section_name: str): - """Get children markdown files from the website section folder.""" - basepath = "pulpcore/docs/sections" - section = self.get_children(f"{basepath}/{section_name}") - return section diff --git a/src/pulp_docs/repository.py b/src/pulp_docs/repository.py index 6574818..651ba73 100644 --- a/src/pulp_docs/repository.py +++ b/src/pulp_docs/repository.py @@ -19,6 +19,8 @@ import httpx import yaml +from pulp_docs.utils.general import get_git_ignored_files + log = logging.getLogger("mkdocs") FIXTURE_WORKDIR = Path("tests/fixtures").absolute() @@ -57,6 +59,7 @@ class Repo: name: str owner: str = "pulp" branch: str = "main" + branch_in_use: str = "main" local_basepath: t.Optional[Path] = None status: RepoStatus = field(default_factory=lambda: RepoStatus()) type: t.Optional[str] = None @@ -109,21 +112,17 @@ def download(self, dest_dir: Path, clear_cache: bool = False) -> str: elif not cached_repo.exists(): log_header = "Downloading from remote" download_from = download_from_gh_main( - DOWNLOAD_CACHE_DIR / self.name, self.owner, self.name, self.branch + DOWNLOAD_CACHE_DIR / self.name, + self.owner, + self.name, + self.branch_in_use, ) src_copy_path = DOWNLOAD_CACHE_DIR / self.name # copy from source/cache to pulp-docs workdir log.info(f"{log_header}: source={download_from}, copied_from={src_copy_path}") - gitignore_files = [] - repo_gitignore = Path(src_copy_path / ".gitignore") - if repo_gitignore.exists(): - gitignore_files = [ - f - for f in repo_gitignore.read_text().splitlines() - if f and not f.startswith("#") - ] + gitignore_files = get_git_ignored_files(Path(src_copy_path)) shutil.copytree( src_copy_path, dest_dir, @@ -206,8 +205,7 @@ def update_local_checkouts(self): # looks like 'refs/head/main' checkout_refs = Path(checkout_dir / ".git" / "HEAD").read_text() checkout_refs = checkout_refs[len("ref: ") :].replace("\n", "") - repo.status.original_refs = repo.branch - repo.branch = checkout_refs + repo.branch_in_use = checkout_refs def get(self, repo_name: str) -> t.Optional[Repo]: repo = [r for r in self.all if r.name == repo_name] or [None] @@ -270,5 +268,8 @@ def test_fixtures(cls): type="content", ), Repo("Maven", "new_repo3", local_basepath=FIXTURE_WORKDIR, type="content"), + Repo( + "Docs Tool", "pulp-docs", local_basepath=FIXTURE_WORKDIR, type="other" + ), ] return Repos(core_repo=DEFAULT_CORE, content_repos=DEFAULT_CONTENT_REPOS) diff --git a/src/pulp_docs/utils/aggregation.py b/src/pulp_docs/utils/aggregation.py new file mode 100644 index 0000000..4122342 --- /dev/null +++ b/src/pulp_docs/utils/aggregation.py @@ -0,0 +1,180 @@ +import os +import typing as t +from pathlib import Path + +from pulp_docs.constants import Names +from pulp_docs.repository import Repos + + +class AgregationUtils: + def __init__(self, tmpdir: Path, repos: Repos): + self.tmpdir = tmpdir + self.repos = repos + + def section(self, name: str, fn: t.Callable, *args, **kwargs) -> dict: + """ + Create section with @name by calling the @fn aggregation function. + + A section look like {"Name": [file1, file2, ...]} + + Params: + hide_empty_section: Hide section if @fn return is empty. Default=True + """ + hide_empty_section = kwargs.pop("hide_empty_section", False) + section = fn(*args, **kwargs) + if hide_empty_section is False: + return {name: section} + return {name: section} if section else {"": ""} + + def get_children(self, path: t.Union[str, Path]) -> t.List[str]: + """ + Get all markdown files contained in @path recursively. + + Uses the dirname.title() as the subsection name if there are subdirs. + """ + basepath = self.tmpdir / path if isinstance(path, str) else path + if not basepath.exists(): + return [] + + def _get_tree(_path): + """Recursive scandir""" + with os.scandir(_path) as it: + children = [] + for entry in sorted(it, key=lambda x: x.name): + if entry.is_file() and entry.name.endswith(".md"): + filename = str(Path(entry.path).relative_to(self.tmpdir)) + children.append(filename) + elif entry.is_dir(): + sub_section = {entry.name.title(): _get_tree(entry)} + children.append(sub_section) + return children + + result = _get_tree(basepath) + return result + + def repo_grouping( + self, + template_str: str, + repo_types: t.Optional[t.List[str]] = None, + content_types: t.Optional[t.List[str]] = None, + ): + """ + Get all markdown files that matches @template_str basepath and group by repos. + + Arguments: + template_str: The template with fields to expand. Accepts combination of '{repo}' and '{content}' + repos: The set of repos to use. Accepts list with combination of "core", "content" and "other" + content_types: The set of content-types to use. Accepts combination of "guides", "learn" and "tutorial" + + Example: + ```python + >>> repo_grouping("{repo}/docs/dev/guides") + { + "repoA": ["docs/dev/guides/file1.md", "docs/dev/guides/file2.md"], + "repoB": ["docs/dev/guides/file3.md", "docs/dev/guides/file4.md"], + } + >>> repo_grouping("{repo}/docs/dev/{content}", content=["guide", "learn"]) + { + "repoA": [ + {"How-to": ["docs/dev/guides/file1.md", "docs/dev/guides/file2.md"]}, + {"Learn": ["docs/dev/learn/file1.md", "docs/dev/learn/file2.md"]}, + ], + "repoB": [...] + + } + ``` + """ + _nav = {} + _expand_content_types = "{content}" in template_str + + # Selected a set of repos + selected_repos = [] + selected_content = content_types or ["tutorials", "guides", "learn"] + if not repo_types: # default case + selected_repos = self.repos.all + else: + for repo_name in repo_types: + selected_repos.extend(getattr(self.repos, f"{repo_name}_repos")) + + # Dont expand content-types + if not _expand_content_types: + for repo in selected_repos: + lookup_path = self.tmpdir / template_str.format( + repo=repo.name, admin=Names.ADMIN, user=Names.USER + ) + _repo_content = self.get_children(lookup_path) + if _repo_content: + _nav[repo.title] = _repo_content + # Expand content-types + else: + for repo in selected_repos: + repo_nav = {} + for content_type in selected_content: + lookup_path = self.tmpdir / template_str.format( + repo=repo.name, + admin=Names.ADMIN, + user=Names.USER, + content=content_type, + ) + _repo_content = self.get_children(lookup_path) + + # special treatment to quickstart tutorial + quickstart_file = self._pop_quickstart_from(_repo_content) + if content_type.lower() == "tutorials" and quickstart_file: + repo_nav["Quickstart"] = quickstart_file # type: ignore + + # doesnt render content-type section if no files inside + if _repo_content: + content_type_name = Names.get(content_type) + repo_nav[content_type_name] = _repo_content # type: ignore + if repo_nav: + _nav[repo.title] = repo_nav + return _nav or ["#"] + + def _pop_quickstart_from(self, pathlist: t.List[str]) -> t.Optional[str]: + """Get quickstart.md file from filelist with case and variations tolerance""" + for path in pathlist: + filename = path.split("/")[-1] + if filename.lower() in ("quickstart.md", "quick-start.md"): + pathlist.remove(path) + return path + return None + + def repo_reference_grouping(self): + """ + Create reference section by aggregating some specific files. + + Group according to the pattern: + {repo}/ + Readme.md + Rest Api.md + Code Api/ + Module A.md + Module B.md + Changelog.md + + """ + template_str = "{repo}/docs/reference" + _nav = {} + for repo in self.repos.all: + lookup_path = self.tmpdir / template_str.format(repo=repo.name) + _repo_content = self.get_children(lookup_path) + reference_section = [ + {"REST API": f"{repo.name}/docs/rest_api.md"}, + {"Readme": f"{repo.name}/README.md"}, + {"Code API": _repo_content}, + {"Changelog": f"{repo.name}/CHANGELOG.md"}, + ] + _nav[repo.title] = reference_section + return _nav + + def section_file(self, section_and_filename: str): + """Get a markdown file from the website section folder.""" + basepath = "pulpcore/docs/sections" + return f"{basepath}/{section_and_filename}" + + def section_children(self, section_name: str): + """Get children markdown files from the website section folder.""" + basepath = "pulpcore/docs/sections" + section = self.get_children(f"{basepath}/{section_name}") + return section diff --git a/src/pulp_docs/utils/fake_repo_generator.py b/src/pulp_docs/utils/fake_repo_generator.py deleted file mode 100644 index 85f556a..0000000 --- a/src/pulp_docs/utils/fake_repo_generator.py +++ /dev/null @@ -1,370 +0,0 @@ -from __future__ import annotations - -import argparse -import os -import shutil -import sys -import typing as t -from dataclasses import dataclass -from pathlib import Path -from random import randint - -from faker import Faker -from jinja2 import Environment, PackageLoader - - -@dataclass -class Repository: - """A fake repository""" - - basedir: str - name: str - owner: str - - def generate_documentation_files(self): - """Generates markdown documenation files according to the guidelines.""" - - -@dataclass -class MultirepoFactory: - """Responsible for creating the multi-repo setup.""" - - repositories: t.List[Repository] - - def generate_multirepo_setup(self): - """Generate multirepo directories with initial git commits.""" - - def add_changes_and_bump_versions(self): - """Randomly add changes to repository files and bump version (minor,major or patch).""" - - -#: This are the allowed content types -ContentType = t.Union[ - t.Literal["learn"], - t.Literal["guide"], - t.Literal["reference"], - t.Literal["tutorial"], -] - - -@dataclass -class Document: - """A markdown document which belongs to some content-type.""" - - name: str - template_name: str - template_vars: dict - relative_path: str = "" - basepath: t.Optional[Path] = None - jinja = Environment(loader=PackageLoader("mockrepos")) - - def create(self, write: bool = False): - """Create document based on content-type""" - template = self.jinja.get_template(self.template_name) - data = template.render(**self.template_vars) - - # write - if write and self.basepath: - filename = "{}.md".format(self.name) - if self.relative_path: - dir_path = self.basepath / self.relative_path - else: - dir_path = self.basepath - dir_path.mkdir(parents=True, exist_ok=True) - Path(dir_path / filename).write_text(data) - return data - - -@dataclass -class ContentFactory: - """Creates a Document or Documents list with content-type templates.""" - - basepath: t.Optional[Path] = None - template_name = "content_templates" - templates = ("guide_page", "tutorial_page", "tutorial_main", "learn_page") - content = ("learn", "guide", "tutorial") - faker = Faker() - - def create_guide( - self, - title: t.Optional[str] = None, - repository: t.Optional[str] = None, - persona: t.Optional[str] = None, - ) -> Document: - """Creates a guide""" - verbs = ["do", "create", "generate", "bind", "delete", "update"] - - def gen_paragraph(): - return " ".join((self.faker.paragraph()) for _ in range(3)) - - def gen_imperative(): - return "{} {}".format( - self.faker.word(ext_word_list=verbs), - self.faker.sentence(nb_words=3).lower(), - ) - - # optional kwargs - title = title or "How to {}".format(gen_imperative()[:-1]) - repository = repository or "default" - persona = persona or self.faker.word(ext_word_list=["admin", "dev", "norm"]) - - # guide generation - n_steps = randint(2, 4) - steps = [gen_imperative() for _ in range(n_steps)] - - template_vars = { - "tags": ["guide", persona], - "created": "0.1.0", - "updated": [], - "title": title, - "overview": gen_paragraph(), - "steps": steps, - "sections": [ - ("{} - {}".format(i + 1, step), gen_paragraph()) - for i, step in enumerate(steps) - ], - } - relative_path = "{}/guides".format(persona) - return Document( - title, - "guide_page.md.j2", - template_vars, - relative_path=relative_path, - basepath=self.basepath, - ) - - def create_learn( - self, - title: t.Optional[str] = None, - repository: t.Optional[str] = None, - persona: t.Optional[str] = None, - ) -> Document: - """Creates a learn""" - subject = ["database", "model", "content", "artifact", "repository"] - subject_complement = ["configuration", "deployment", "ecossystem", "setup"] - adjective = [ - "resilient", - "complex", - "flexible", - "autonomous", - "modular", - "scalable", - ] - introductory = ["On", "About", "In perspective:", "Understanding"] - - def gen_paragraphs(): - return " ".join((self.faker.paragraph()) for _ in range(3)) - - def gen_title(): - return " ".join( - [ - self.faker.word(ext_word_list=introductory), - self.faker.word(ext_word_list=adjective), - self.faker.word(ext_word_list=subject).title(), - self.faker.word(ext_word_list=subject_complement), - ] - ) - - def gen_section(): - return ( - self.faker.sentence()[:-1], - "\n\n".join( - self.faker.texts( - nb_texts=randint(3, 5), max_nb_chars=randint(200, 400) - ) - ), - ) - - title = title or gen_title() - persona = persona or self.faker.word(ext_word_list=["admin", "dev", "norm"]) - repository = repository or "default" - - template_vars = { - "tags": ["learn", persona], - "created": "0.1.0", - "updated": [], - "title": title, - "overview": gen_paragraphs(), - "sections": [gen_section() for _ in range(randint(2, 4))], - } - relative_path = "{}/learn".format(persona) - return Document( - title, - "learn_page.md.j2", - template_vars, - relative_path=relative_path, - basepath=self.basepath, - ) - - def create_tutorial( - self, - title: t.Optional[str] = None, - repository: t.Optional[str] = None, - persona: t.Optional[str] = None, - ) -> t.List[Document]: - """Creates a tutorial""" - documents = [] - subject = [ - "the database", - "the model", - "the content", - "the artifact", - "the repository", - ] - verb = [ - "configuring", - "deploying", - "setting", - "organizing", - "doing", - "handling", - ] - adjective = [ - "infrastructure", - "internals", - "contexts", - "dependencie", - "relationships", - "signals", - ] - topics = ["Content Management", "Development", "System Administration"] - - def gen_paragraphs(n: t.Optional[int] = None): - n = n or 3 - return " ".join((self.faker.paragraph()) for _ in range(n)) - - def gen_title(): - return "Getting started with {}".format( - self.faker.word(ext_word_list=topics) - ) - - def gen_section_title(): - return " ".join( - [ - self.faker.word(ext_word_list=verb).title(), - self.faker.word(ext_word_list=subject), - self.faker.word(ext_word_list=adjective), - ] - ) - - def gen_section(): - return ( - self.faker.sentence()[:-1], - "\n\n".join( - self.faker.texts( - nb_texts=randint(3, 5), max_nb_chars=randint(200, 400) - ) - ), - ) - - persona = persona or self.faker.word(ext_word_list=["admin", "dev", "norm"]) - title = title or "Getting started as {}".format(persona) - repository = repository or "default" - tags = ["tutorial", persona] - - # generate main page - main_title = "00 - {}".format(title) - template_vars = { - "tags": tags, - "created": "0.1.0", - "updated": [], - "title": main_title, - "overview": gen_paragraphs(5), - "sections": [gen_section() for _ in range(randint(2, 4))], - } - relative_path = "{}/tutorials/{}".format(persona, title) - documents.append( - Document( - main_title, - "tutorial_main.md.j2", - template_vars, - relative_path=relative_path, - basepath=self.basepath, - ) - ) - - # generate each page of the tutorial - for i in range(randint(2, 4)): - section_title = "0{} - {}".format(i + 1, gen_section_title()) - template_vars = { - "tags": ["tutorial", persona], - "created": "0.1.0", - "updated": [], - "title": section_title, - "overview": gen_paragraphs(), - "sections": [gen_section() for _ in range(randint(2, 4))], - } - documents.append( - Document( - section_title, - "tutorial_main.md.j2", - template_vars, - relative_path=relative_path, - basepath=self.basepath, - ) - ) - - return documents - - -@dataclass -class DocumentationFactory: - """Generates Diataxis-based documentation for Pulp""" - - basepath: Path - - def create_documentation(self) -> t.List[Document]: - """Create Documents based for all repositories.""" - documents: t.List[Document] = [] - - # create content for one repo - content_factory = ContentFactory(self.basepath) - for persona in ("developer", "content-manager", "sys-admin"): - [ - content_factory.create_guide(persona=persona).create(write=True) - for _ in range(randint(1, 5)) - ] - [ - content_factory.create_learn(persona=persona).create(write=True) - for _ in range(randint(1, 5)) - ] - [ - tuto.create(write=True) - for tuto in content_factory.create_tutorial(persona=persona) - ] - - return documents - - -def main(): - # parse args - parser = argparse.ArgumentParser( - "docs_generator", - description="Genreate a doc structure populated with fake data", - ) - parser.add_argument( - "basepath", help="The basepath where the documentation folder will be created" - ) - args = parser.parse_args() - basepath = Path() / args.basepath - - # ensure won't unintended override a direcotory - if basepath.exists(): - overwrite = input( - f"{basepath} exists, do you wanna override? (N/y): " - ).lower() in ("y", "yes") - if overwrite: - print(f"Cleaned: {basepath.absolute()}.") - shutil.rmtree(basepath) - else: - print("Did not overwritte. Nothing to do.") - return 0 - - # create documentation - basepath.mkdir() - documentation_factory = DocumentationFactory(basepath / "docs") - documentation_factory.create_documentation() - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/pulp_docs/utils/general.py b/src/pulp_docs/utils/general.py new file mode 100644 index 0000000..805a504 --- /dev/null +++ b/src/pulp_docs/utils/general.py @@ -0,0 +1,18 @@ +import typing as t +from pathlib import Path + + +def get_git_ignored_files(repo_path: Path) -> t.List[str]: + """Get list of ignored files as defined in the repo .gitignore""" + repo_gitignore = Path(repo_path / ".gitignore") + gitignore_files = [] + if repo_gitignore.exists(): + gitignore_files.extend( + [ + f.strip("/") + for f in repo_gitignore.read_text().splitlines() + if f and not f.startswith("#") + ] + ) + gitignore_files.append("tests") + return gitignore_files diff --git a/src/pulp_docs/data/docs/assets/hero-img.png b/staging_docs/assets/hero-img.png similarity index 100% rename from src/pulp_docs/data/docs/assets/hero-img.png rename to staging_docs/assets/hero-img.png diff --git a/src/pulp_docs/data/docs/assets/logo.png b/staging_docs/assets/logo.png similarity index 100% rename from src/pulp_docs/data/docs/assets/logo.png rename to staging_docs/assets/logo.png diff --git a/src/pulp_docs/data/docs/css/extra.css b/staging_docs/css/extra.css similarity index 100% rename from src/pulp_docs/data/docs/css/extra.css rename to staging_docs/css/extra.css diff --git a/src/pulp_docs/data/docs/css/mkdocstrings.css b/staging_docs/css/mkdocstrings.css similarity index 100% rename from src/pulp_docs/data/docs/css/mkdocstrings.css rename to staging_docs/css/mkdocstrings.css diff --git a/staging_docs/dev/tutorials/quickstart.md b/staging_docs/dev/tutorials/quickstart.md new file mode 100644 index 0000000..412701a --- /dev/null +++ b/staging_docs/dev/tutorials/quickstart.md @@ -0,0 +1,3 @@ +# Quickstart + +Learn how to use `pulp-docs` diff --git a/src/pulp_docs/docs/.gitkeep b/staging_docs/tags.md similarity index 100% rename from src/pulp_docs/docs/.gitkeep rename to staging_docs/tags.md diff --git a/tests/fixtures/pulp-docs/staging_docs/tags.md b/tests/fixtures/pulp-docs/staging_docs/tags.md new file mode 100644 index 0000000..e69de29