From 67ddc66e692afbde5faf742ff9e1c7b914025123 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 5 Jun 2024 14:46:19 +0200 Subject: [PATCH 01/14] Try out mkdocstrings --- docs/docs/developer/api.md | 27 ++++++++++++ docs/mkdocs.yml | 2 + poetry.lock | 87 ++++++++++++++++++++++++++++++++------ pyproject.toml | 1 + 4 files changed, 104 insertions(+), 13 deletions(-) create mode 100644 docs/docs/developer/api.md diff --git a/docs/docs/developer/api.md b/docs/docs/developer/api.md new file mode 100644 index 000000000..623b6a574 --- /dev/null +++ b/docs/docs/developer/api.md @@ -0,0 +1,27 @@ +# API + + + +::: kpops.api + options: + members: + - generate + - manifest + - deploy + - destroy + - reset + - clean + - init + show_root_heading: true + show_source: true + + + + + +::: kpops.pipeline.Pipeline + options: + show_root_heading: true + show_source: true + + diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index f4e3a8ba6..0126f535f 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -37,6 +37,7 @@ plugins: exclude: - resources/* - glightbox + - mkdocstrings extra: version: @@ -120,6 +121,7 @@ nav: - GitHub Actions: user/references/ci-integration/github-actions.md - Developer Guide: - Getting Started: developer/getting-started.md + - API: developer/api.md - Contributing: developer/contributing.md - Code base: - Auto generation: developer/auto-generation.md diff --git a/poetry.lock b/poetry.lock index e6f270466..63b6c9e9d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiofiles" @@ -550,6 +550,20 @@ python-dateutil = ">=2.8.1" [package.extras] dev = ["flake8", "markdown", "twine", "wheel"] +[[package]] +name = "griffe" +version = "0.45.2" +description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." +optional = false +python-versions = ">=3.8" +files = [ + {file = "griffe-0.45.2-py3-none-any.whl", hash = "sha256:297ec8530d0c68e5b98ff86fb588ebc3aa3559bb5dc21f3caea8d9542a350133"}, + {file = "griffe-0.45.2.tar.gz", hash = "sha256:83ce7dcaafd8cb7f43cbf1a455155015a1eb624b1ffd93249e5e1c4a22b2fdb2"}, +] + +[package.dependencies] +colorama = ">=0.4" + [[package]] name = "h11" version = "0.14.0" @@ -840,6 +854,22 @@ watchdog = ">=2.0" i18n = ["babel (>=2.9.0)"] min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-import (==1.0)", "importlib-metadata (==4.3)", "jinja2 (==2.11.1)", "markdown (==3.2.1)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "packaging (==20.5)", "pathspec (==0.11.1)", "platformdirs (==2.2.0)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "typing-extensions (==3.10)", "watchdog (==2.0)"] +[[package]] +name = "mkdocs-autorefs" +version = "1.0.1" +description = "Automatically link across pages in MkDocs." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mkdocs_autorefs-1.0.1-py3-none-any.whl", hash = "sha256:aacdfae1ab197780fb7a2dac92ad8a3d8f7ca8049a9cbe56a4218cd52e8da570"}, + {file = "mkdocs_autorefs-1.0.1.tar.gz", hash = "sha256:f684edf847eced40b570b57846b15f0bf57fb93ac2c510450775dcf16accb971"}, +] + +[package.dependencies] +Markdown = ">=3.3" +markupsafe = ">=2.0.1" +mkdocs = ">=1.1" + [[package]] name = "mkdocs-exclude-search" version = "0.6.5" @@ -926,6 +956,48 @@ files = [ {file = "mkdocs_material_extensions-1.3.tar.gz", hash = "sha256:f0446091503acb110a7cab9349cbc90eeac51b58d1caa92a704a81ca1e24ddbd"}, ] +[[package]] +name = "mkdocstrings" +version = "0.25.1" +description = "Automatic documentation from sources, for MkDocs." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mkdocstrings-0.25.1-py3-none-any.whl", hash = "sha256:da01fcc2670ad61888e8fe5b60afe9fee5781017d67431996832d63e887c2e51"}, + {file = "mkdocstrings-0.25.1.tar.gz", hash = "sha256:c3a2515f31577f311a9ee58d089e4c51fc6046dbd9e9b4c3de4c3194667fe9bf"}, +] + +[package.dependencies] +click = ">=7.0" +Jinja2 = ">=2.11.1" +Markdown = ">=3.3" +MarkupSafe = ">=1.1" +mkdocs = ">=1.4" +mkdocs-autorefs = ">=0.3.1" +mkdocstrings-python = {version = ">=0.5.2", optional = true, markers = "extra == \"python\""} +platformdirs = ">=2.2.0" +pymdown-extensions = ">=6.3" + +[package.extras] +crystal = ["mkdocstrings-crystal (>=0.3.4)"] +python = ["mkdocstrings-python (>=0.5.2)"] +python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] + +[[package]] +name = "mkdocstrings-python" +version = "1.10.3" +description = "A Python handler for mkdocstrings." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mkdocstrings_python-1.10.3-py3-none-any.whl", hash = "sha256:11ff6d21d3818fb03af82c3ea6225b1534837e17f790aa5f09626524171f949b"}, + {file = "mkdocstrings_python-1.10.3.tar.gz", hash = "sha256:321cf9c732907ab2b1fedaafa28765eaa089d89320f35f7206d00ea266889d03"}, +] + +[package.dependencies] +griffe = ">=0.44" +mkdocstrings = ">=0.25" + [[package]] name = "multidict" version = "6.0.5" @@ -1669,7 +1741,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1677,16 +1748,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1703,7 +1766,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1711,7 +1773,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2289,4 +2350,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10, <3.13" -content-hash = "80ac595bb43560de5e123076830390054b2c9c15b865f3665e1e1b114a31faad" +content-hash = "b2a054d94283805037c411df17b8adc527dc9a63adb3892c37d383c10a485802" diff --git a/pyproject.toml b/pyproject.toml index 49ee0b459..16da43444 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ mkdocs-material = "^9.0.0" mkdocs-glightbox = "^0.3.1" mkdocs-exclude-search = "^0.6.5" mike = "^1.1.2" +mkdocstrings = { extras = ["python"], version = "^0.25.1" } [tool.poetry_bumpversion.file."kpops/__init__.py"] From b89cf9349581988f8849bf1b5c0e0ee138cbed1e Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Thu, 27 Jun 2024 15:50:15 +0300 Subject: [PATCH 02/14] Poetry lock --- poetry.lock | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/poetry.lock b/poetry.lock index 63b6c9e9d..da1889908 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1741,6 +1741,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1748,8 +1749,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1766,6 +1774,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1773,6 +1782,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, From 9a3aba9e81eb8ee30fa040cdd9f2724d2ab84f67 Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Thu, 27 Jun 2024 16:48:41 +0300 Subject: [PATCH 03/14] Configure api docs gen --- docs/mkdocs.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 0126f535f..e1e73f136 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -37,7 +37,22 @@ plugins: exclude: - resources/* - glightbox - - mkdocstrings + - mkdocstrings: + handlers: + python: + paths: [..] + options: + docstring_style: sphinx + show_if_no_docstring: true + line_length: 60 + separate_signature: true + show_signature_annotations: true + docstring_section_style: spacy + show_root_heading: true + # TODO(Ivan Yortdanov): Uncomment when option becomes available + show_inheritance_diagram: true + modernize_annotations: true + parameter_headings: true extra: version: From 67034bd153645eca7cb9d81a1e3e29065beeda82 Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Fri, 28 Jun 2024 11:23:46 +0300 Subject: [PATCH 04/14] Remove local settings --- docs/docs/developer/api.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docs/docs/developer/api.md b/docs/docs/developer/api.md index 623b6a574..348b666f0 100644 --- a/docs/docs/developer/api.md +++ b/docs/docs/developer/api.md @@ -3,25 +3,11 @@ ::: kpops.api - options: - members: - - generate - - manifest - - deploy - - destroy - - reset - - clean - - init - show_root_heading: true - show_source: true ::: kpops.pipeline.Pipeline - options: - show_root_heading: true - show_source: true From bded0d45ef8fcf6842ac87ead8d34b4d3f0e7c97 Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Fri, 28 Jun 2024 11:30:40 +0300 Subject: [PATCH 05/14] Change comment --- docs/mkdocs.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index e1e73f136..797f42051 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -49,10 +49,11 @@ plugins: show_signature_annotations: true docstring_section_style: spacy show_root_heading: true - # TODO(Ivan Yortdanov): Uncomment when option becomes available - show_inheritance_diagram: true - modernize_annotations: true - parameter_headings: true + # The options below are not yet publicly available, + # but would be a nice addition when released. + #show_inheritance_diagram: true + #modernize_annotations: true + #parameter_headings: true extra: version: From e0143bf71efbba6d6e9ec9ce02dbe1a2821ebabf Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Fri, 28 Jun 2024 14:48:38 +0300 Subject: [PATCH 06/14] Add links to future options --- docs/mkdocs.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 797f42051..0cadd93c5 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -51,9 +51,8 @@ plugins: show_root_heading: true # The options below are not yet publicly available, # but would be a nice addition when released. - #show_inheritance_diagram: true - #modernize_annotations: true - #parameter_headings: true + #show_inheritance_diagram: true # https://mkdocstrings.github.io/python/usage/configuration/general/?h=show_inheri#show_inheritance_diagram + #modernize_annotations: true # https://mkdocstrings.github.io/python/usage/configuration/signatures/#modernize_annotations extra: version: From 0672b8a4a7771053dbbf729bcf537625caa094e0 Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Fri, 28 Jun 2024 14:59:02 +0300 Subject: [PATCH 07/14] Omit `setup_handlers` and `create_pipeline` --- docs/docs/developer/api.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/docs/developer/api.md b/docs/docs/developer/api.md index 348b666f0..fe3af4814 100644 --- a/docs/docs/developer/api.md +++ b/docs/docs/developer/api.md @@ -3,11 +3,9 @@ ::: kpops.api - - - - - -::: kpops.pipeline.Pipeline + options: + filters: + - "!^setup_handlers$" + - "!^create_pipeline$" From b6830a1f1d1d6642c838f3021dbd4e3cad5c5435 Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Fri, 28 Jun 2024 16:57:36 +0300 Subject: [PATCH 08/14] Add docstrings in API --- kpops/api/__init__.py | 88 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index 826307308..79fe0ccd8 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -37,6 +37,17 @@ def generate( environment: str | None = None, verbose: bool = False, ) -> Pipeline: + """Generate enriched pipeline representation. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :return: Generated `Pipeline` object. + """ kpops_config = KpopsConfig.create( config, dotenv, @@ -68,6 +79,17 @@ def manifest( environment: str | None = None, verbose: bool = False, ) -> list[Resource]: + """Generate pipeline, return final resource representations for each step. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :return: Resources. + """ pipeline = kpops.generate( pipeline_path=pipeline_path, dotenv=dotenv, @@ -95,6 +117,18 @@ def deploy( verbose: bool = True, parallel: bool = False, ): + """Deploy pipeline steps. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param dry_run: Whether to dry run the command or execute it. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :param parallel: Enable or disable parallel execution of pipeline steps. + """ pipeline = kpops.generate( pipeline_path=pipeline_path, dotenv=dotenv, @@ -131,6 +165,18 @@ def destroy( verbose: bool = True, parallel: bool = False, ): + """Destroy pipeline steps. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param dry_run: Whether to dry run the command or execute it. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :param parallel: Enable or disable parallel execution of pipeline steps. + """ pipeline = kpops.generate( pipeline_path=pipeline_path, dotenv=dotenv, @@ -169,6 +215,18 @@ def reset( verbose: bool = True, parallel: bool = False, ): + """Reset pipeline steps. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param dry_run: Whether to dry run the command or execute it. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :param parallel: Enable or disable parallel execution of pipeline steps. + """ pipeline = kpops.generate( pipeline_path=pipeline_path, dotenv=dotenv, @@ -206,6 +264,18 @@ def clean( verbose: bool = True, parallel: bool = False, ): + """Clean pipeline steps. + + :param pipeline_path: Path to pipeline definition yaml file. + :param dotenv: Paths to dotenv files. + :param config: Path to the dir containing config.yaml files. + :param steps: Set of steps (components) to apply the command on. + :param filter_type: Whether `steps` should include/exclude the steps. + :param dry_run: Whether to dry run the command or execute it. + :param environment: The environment to generate and deploy the pipeline to. + :param verbose: Enable verbose printing. + :param parallel: Enable or disable parallel execution of pipeline steps. + """ pipeline = kpops.generate( pipeline_path=pipeline_path, dotenv=dotenv, @@ -236,6 +306,12 @@ def init( path: Path, config_include_opt: bool = False, ): + """Initiate a default empty project. + + :param path: Directory in which the project should be initiated. + :param conf_incl_opt: Whether to include non-required settings + in the generated config file. + """ if not path.exists(): path.mkdir(parents=False) elif next(path.iterdir(), False): @@ -249,6 +325,13 @@ def create_pipeline( kpops_config: KpopsConfig, environment: str | None, ) -> Pipeline: + """Create pipeline. + + :param pipeline_path: Path to pipeline definition yaml file. + :param config: KPOps Config. + :param environment: The environment to generate and deploy the pipeline to. + :return: Created `Pipeline` object. + """ registry = Registry() if kpops_config.components_module: registry.find_components(kpops_config.components_module) @@ -260,6 +343,11 @@ def create_pipeline( def setup_handlers(config: KpopsConfig) -> ComponentHandlers: + """Set up handlers for a component. + + :param config: KPOps config. + :return: Handlers for a component. + """ schema_handler = SchemaHandler.load_schema_handler(config) connector_handler = KafkaConnectHandler.from_kpops_config(config) proxy_wrapper = ProxyWrapper(config.kafka_rest) From 084c70d5c76415c76f88fbad8b5c8177a796d858 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 11:55:35 +0200 Subject: [PATCH 09/14] Exclude private functions prefixed with `_` --- docs/docs/developer/api.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/docs/developer/api.md b/docs/docs/developer/api.md index fe3af4814..d4b2434a0 100644 --- a/docs/docs/developer/api.md +++ b/docs/docs/developer/api.md @@ -5,7 +5,6 @@ ::: kpops.api options: filters: - - "!^setup_handlers$" - - "!^create_pipeline$" + - "!^_" From 7a6ae64595829126d3585cce517f17dbf85f2694 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 11:56:46 +0200 Subject: [PATCH 10/14] Rename functions --- kpops/api/__init__.py | 8 ++++---- tests/api/test_handlers.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index 79fe0ccd8..3073d9452 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -54,7 +54,7 @@ def generate( environment, verbose, ) - pipeline = create_pipeline(pipeline_path, kpops_config, environment) + pipeline = _create_pipeline(pipeline_path, kpops_config, environment) log.info(f"Picked up pipeline '{pipeline_path.parent.name}'") if steps: component_names = steps @@ -320,7 +320,7 @@ def init( init_project(path, config_include_opt) -def create_pipeline( +def _create_pipeline( pipeline_path: Path, kpops_config: KpopsConfig, environment: str | None, @@ -337,12 +337,12 @@ def create_pipeline( registry.find_components(kpops_config.components_module) registry.find_components("kpops.components") - handlers = setup_handlers(kpops_config) + handlers = _setup_handlers(kpops_config) parser = PipelineGenerator(kpops_config, registry, handlers) return parser.load_yaml(pipeline_path, environment) -def setup_handlers(config: KpopsConfig) -> ComponentHandlers: +def _setup_handlers(config: KpopsConfig) -> ComponentHandlers: """Set up handlers for a component. :param config: KPOps config. diff --git a/tests/api/test_handlers.py b/tests/api/test_handlers.py index 18d35131b..c3a92784d 100644 --- a/tests/api/test_handlers.py +++ b/tests/api/test_handlers.py @@ -1,6 +1,6 @@ from pytest_mock import MockerFixture -from kpops.api import setup_handlers +from kpops.api import _setup_handlers from kpops.component_handlers import ComponentHandlers from kpops.component_handlers.kafka_connect.kafka_connect_handler import ( KafkaConnectHandler, @@ -35,7 +35,7 @@ def test_set_up_handlers_with_no_schema_handler(mocker: MockerFixture): topic_handler=topic_handler, ) - actual_handlers = setup_handlers(config) + actual_handlers = _setup_handlers(config) connector_handler_mock.from_kpops_config.assert_called_once_with(config) @@ -72,7 +72,7 @@ def test_set_up_handlers_with_schema_handler(mocker: MockerFixture): topic_handler=topic_handler, ) - actual_handlers = setup_handlers(config) + actual_handlers = _setup_handlers(config) schema_handler_mock.load_schema_handler.assert_called_once_with(config) From a561b479dc24d8deef1f9ebfc0219525c6bcf879 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 11:58:12 +0200 Subject: [PATCH 11/14] Add back `Pipeline` --- docs/docs/developer/api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/docs/developer/api.md b/docs/docs/developer/api.md index d4b2434a0..f94cdd9c8 100644 --- a/docs/docs/developer/api.md +++ b/docs/docs/developer/api.md @@ -7,4 +7,6 @@ filters: - "!^_" +::: kpops.pipeline.Pipeline + From 844331c20fa8acddbc0c9585fc48aee672108549 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 13:07:14 +0200 Subject: [PATCH 12/14] Exclude Pipeline private methods --- docs/docs/developer/api.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/docs/developer/api.md b/docs/docs/developer/api.md index f94cdd9c8..829549189 100644 --- a/docs/docs/developer/api.md +++ b/docs/docs/developer/api.md @@ -8,5 +8,8 @@ - "!^_" ::: kpops.pipeline.Pipeline + options: + filters: + - "!^_" From 3b4271341234c5e05380227afbe0585eefa9c5a0 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 13:08:49 +0200 Subject: [PATCH 13/14] Exclude Pydantic model config --- docs/docs/developer/api.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/developer/api.md b/docs/docs/developer/api.md index 829549189..ab89cd5f5 100644 --- a/docs/docs/developer/api.md +++ b/docs/docs/developer/api.md @@ -11,5 +11,6 @@ options: filters: - "!^_" + - "!^model_config$" From 8cc664571833a7def549d291e2babeac22004332 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 13:10:24 +0200 Subject: [PATCH 14/14] Rename --- docs/docs/developer/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/developer/api.md b/docs/docs/developer/api.md index ab89cd5f5..9bf22d809 100644 --- a/docs/docs/developer/api.md +++ b/docs/docs/developer/api.md @@ -1,4 +1,4 @@ -# API +# Python API