From c7ceecbde7dc3add9a37db4507484c496daa002a Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Thu, 4 Jul 2024 13:35:11 +0300 Subject: [PATCH] Generate developer docs for Python API (#503) Co-authored-by: Salomon Popp --- docs/docs/developer/api.md | 16 +++++++ docs/mkdocs.yml | 17 +++++++ kpops/api/__init__.py | 96 ++++++++++++++++++++++++++++++++++++-- poetry.lock | 77 ++++++++++++++++++++++++++++-- pyproject.toml | 1 + tests/api/test_handlers.py | 6 +-- 6 files changed, 203 insertions(+), 10 deletions(-) create mode 100644 docs/docs/developer/api.md diff --git a/docs/docs/developer/api.md b/docs/docs/developer/api.md new file mode 100644 index 000000000..9bf22d809 --- /dev/null +++ b/docs/docs/developer/api.md @@ -0,0 +1,16 @@ +# Python API + + + +::: kpops.api + options: + filters: + - "!^_" + +::: kpops.pipeline.Pipeline + options: + filters: + - "!^_" + - "!^model_config$" + + diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index f4e3a8ba6..0cadd93c5 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -37,6 +37,22 @@ plugins: exclude: - resources/* - glightbox + - mkdocstrings: + handlers: + python: + paths: [..] + options: + docstring_style: sphinx + show_if_no_docstring: true + line_length: 60 + separate_signature: true + show_signature_annotations: true + docstring_section_style: spacy + show_root_heading: true + # The options below are not yet publicly available, + # but would be a nice addition when released. + #show_inheritance_diagram: true # https://mkdocstrings.github.io/python/usage/configuration/general/?h=show_inheri#show_inheritance_diagram + #modernize_annotations: true # https://mkdocstrings.github.io/python/usage/configuration/signatures/#modernize_annotations extra: version: @@ -120,6 +136,7 @@ nav: - GitHub Actions: user/references/ci-integration/github-actions.md - Developer Guide: - Getting Started: developer/getting-started.md + - API: developer/api.md - Contributing: developer/contributing.md - Code base: - Auto generation: developer/auto-generation.md diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index 826307308..3073d9452 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -37,13 +37,24 @@ def generate( environment: str | None = None, verbose: bool = False, ) -> Pipeline: + """Generate enriched pipeline representation. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :return: Generated `Pipeline` object. + """ kpops_config = KpopsConfig.create( config, dotenv, environment, verbose, ) - pipeline = create_pipeline(pipeline_path, kpops_config, environment) + pipeline = _create_pipeline(pipeline_path, kpops_config, environment) log.info(f"Picked up pipeline '{pipeline_path.parent.name}'") if steps: component_names = steps @@ -68,6 +79,17 @@ def manifest( environment: str | None = None, verbose: bool = False, ) -> list[Resource]: + """Generate pipeline, return final resource representations for each step. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :return: Resources. + """ pipeline = kpops.generate( pipeline_path=pipeline_path, dotenv=dotenv, @@ -95,6 +117,18 @@ def deploy( verbose: bool = True, parallel: bool = False, ): + """Deploy pipeline steps. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param dry_run: Whether to dry run the command or execute it. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :param parallel: Enable or disable parallel execution of pipeline steps. + """ pipeline = kpops.generate( pipeline_path=pipeline_path, dotenv=dotenv, @@ -131,6 +165,18 @@ def destroy( verbose: bool = True, parallel: bool = False, ): + """Destroy pipeline steps. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param dry_run: Whether to dry run the command or execute it. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :param parallel: Enable or disable parallel execution of pipeline steps. + """ pipeline = kpops.generate( pipeline_path=pipeline_path, dotenv=dotenv, @@ -169,6 +215,18 @@ def reset( verbose: bool = True, parallel: bool = False, ): + """Reset pipeline steps. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param dry_run: Whether to dry run the command or execute it. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :param parallel: Enable or disable parallel execution of pipeline steps. + """ pipeline = kpops.generate( pipeline_path=pipeline_path, dotenv=dotenv, @@ -206,6 +264,18 @@ def clean( verbose: bool = True, parallel: bool = False, ): + """Clean pipeline steps. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param dry_run: Whether to dry run the command or execute it. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :param parallel: Enable or disable parallel execution of pipeline steps. + """ pipeline = kpops.generate( pipeline_path=pipeline_path, dotenv=dotenv, @@ -236,6 +306,12 @@ def init( path: Path, config_include_opt: bool = False, ): + """Initiate a default empty project. + + :param path: Directory in which the project should be initiated. + :param conf_incl_opt: Whether to include non-required settings + in the generated config file. + """ if not path.exists(): path.mkdir(parents=False) elif next(path.iterdir(), False): @@ -244,22 +320,34 @@ def init( init_project(path, config_include_opt) -def create_pipeline( +def _create_pipeline( pipeline_path: Path, kpops_config: KpopsConfig, environment: str | None, ) -> Pipeline: + """Create pipeline. + + :param pipeline_path: Path to pipeline definition yaml file. + :param config: KPOps Config. + :param environment: The environment to generate and deploy the pipeline to. + :return: Created `Pipeline` object. + """ registry = Registry() if kpops_config.components_module: registry.find_components(kpops_config.components_module) registry.find_components("kpops.components") - handlers = setup_handlers(kpops_config) + handlers = _setup_handlers(kpops_config) parser = PipelineGenerator(kpops_config, registry, handlers) return parser.load_yaml(pipeline_path, environment) -def setup_handlers(config: KpopsConfig) -> ComponentHandlers: +def _setup_handlers(config: KpopsConfig) -> ComponentHandlers: + """Set up handlers for a component. + + :param config: KPOps config. + :return: Handlers for a component. + """ schema_handler = SchemaHandler.load_schema_handler(config) connector_handler = KafkaConnectHandler.from_kpops_config(config) proxy_wrapper = ProxyWrapper(config.kafka_rest) diff --git a/poetry.lock b/poetry.lock index e6f270466..da1889908 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiofiles" @@ -550,6 +550,20 @@ python-dateutil = ">=2.8.1" [package.extras] dev = ["flake8", "markdown", "twine", "wheel"] +[[package]] +name = "griffe" +version = "0.45.2" +description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." +optional = false +python-versions = ">=3.8" +files = [ + {file = "griffe-0.45.2-py3-none-any.whl", hash = "sha256:297ec8530d0c68e5b98ff86fb588ebc3aa3559bb5dc21f3caea8d9542a350133"}, + {file = "griffe-0.45.2.tar.gz", hash = "sha256:83ce7dcaafd8cb7f43cbf1a455155015a1eb624b1ffd93249e5e1c4a22b2fdb2"}, +] + +[package.dependencies] +colorama = ">=0.4" + [[package]] name = "h11" version = "0.14.0" @@ -840,6 +854,22 @@ watchdog = ">=2.0" i18n = ["babel (>=2.9.0)"] min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-import (==1.0)", "importlib-metadata (==4.3)", "jinja2 (==2.11.1)", "markdown (==3.2.1)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "packaging (==20.5)", "pathspec (==0.11.1)", "platformdirs (==2.2.0)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "typing-extensions (==3.10)", "watchdog (==2.0)"] +[[package]] +name = "mkdocs-autorefs" +version = "1.0.1" +description = "Automatically link across pages in MkDocs." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mkdocs_autorefs-1.0.1-py3-none-any.whl", hash = "sha256:aacdfae1ab197780fb7a2dac92ad8a3d8f7ca8049a9cbe56a4218cd52e8da570"}, + {file = "mkdocs_autorefs-1.0.1.tar.gz", hash = "sha256:f684edf847eced40b570b57846b15f0bf57fb93ac2c510450775dcf16accb971"}, +] + +[package.dependencies] +Markdown = ">=3.3" +markupsafe = ">=2.0.1" +mkdocs = ">=1.1" + [[package]] name = "mkdocs-exclude-search" version = "0.6.5" @@ -926,6 +956,48 @@ files = [ {file = "mkdocs_material_extensions-1.3.tar.gz", hash = "sha256:f0446091503acb110a7cab9349cbc90eeac51b58d1caa92a704a81ca1e24ddbd"}, ] +[[package]] +name = "mkdocstrings" +version = "0.25.1" +description = "Automatic documentation from sources, for MkDocs." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mkdocstrings-0.25.1-py3-none-any.whl", hash = "sha256:da01fcc2670ad61888e8fe5b60afe9fee5781017d67431996832d63e887c2e51"}, + {file = "mkdocstrings-0.25.1.tar.gz", hash = "sha256:c3a2515f31577f311a9ee58d089e4c51fc6046dbd9e9b4c3de4c3194667fe9bf"}, +] + +[package.dependencies] +click = ">=7.0" +Jinja2 = ">=2.11.1" +Markdown = ">=3.3" +MarkupSafe = ">=1.1" +mkdocs = ">=1.4" +mkdocs-autorefs = ">=0.3.1" +mkdocstrings-python = {version = ">=0.5.2", optional = true, markers = "extra == \"python\""} +platformdirs = ">=2.2.0" +pymdown-extensions = ">=6.3" + +[package.extras] +crystal = ["mkdocstrings-crystal (>=0.3.4)"] +python = ["mkdocstrings-python (>=0.5.2)"] +python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] + +[[package]] +name = "mkdocstrings-python" +version = "1.10.3" +description = "A Python handler for mkdocstrings." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mkdocstrings_python-1.10.3-py3-none-any.whl", hash = "sha256:11ff6d21d3818fb03af82c3ea6225b1534837e17f790aa5f09626524171f949b"}, + {file = "mkdocstrings_python-1.10.3.tar.gz", hash = "sha256:321cf9c732907ab2b1fedaafa28765eaa089d89320f35f7206d00ea266889d03"}, +] + +[package.dependencies] +griffe = ">=0.44" +mkdocstrings = ">=0.25" + [[package]] name = "multidict" version = "6.0.5" @@ -1682,7 +1754,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2289,4 +2360,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10, <3.13" -content-hash = "80ac595bb43560de5e123076830390054b2c9c15b865f3665e1e1b114a31faad" +content-hash = "b2a054d94283805037c411df17b8adc527dc9a63adb3892c37d383c10a485802" diff --git a/pyproject.toml b/pyproject.toml index 49ee0b459..16da43444 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ mkdocs-material = "^9.0.0" mkdocs-glightbox = "^0.3.1" mkdocs-exclude-search = "^0.6.5" mike = "^1.1.2" +mkdocstrings = { extras = ["python"], version = "^0.25.1" } [tool.poetry_bumpversion.file."kpops/__init__.py"] diff --git a/tests/api/test_handlers.py b/tests/api/test_handlers.py index 18d35131b..c3a92784d 100644 --- a/tests/api/test_handlers.py +++ b/tests/api/test_handlers.py @@ -1,6 +1,6 @@ from pytest_mock import MockerFixture -from kpops.api import setup_handlers +from kpops.api import _setup_handlers from kpops.component_handlers import ComponentHandlers from kpops.component_handlers.kafka_connect.kafka_connect_handler import ( KafkaConnectHandler, @@ -35,7 +35,7 @@ def test_set_up_handlers_with_no_schema_handler(mocker: MockerFixture): topic_handler=topic_handler, ) - actual_handlers = setup_handlers(config) + actual_handlers = _setup_handlers(config) connector_handler_mock.from_kpops_config.assert_called_once_with(config) @@ -72,7 +72,7 @@ def test_set_up_handlers_with_schema_handler(mocker: MockerFixture): topic_handler=topic_handler, ) - actual_handlers = setup_handlers(config) + actual_handlers = _setup_handlers(config) schema_handler_mock.load_schema_handler.assert_called_once_with(config)