From 867ee89ce4cfa12f2dabf4abf4c32e2ea8a14306 Mon Sep 17 00:00:00 2001 From: MichaelKora Date: Wed, 22 Nov 2023 14:53:03 +0100 Subject: [PATCH 1/8] Test doc --- .github/workflows/_generate_doc.yaml | 72 ++++++++++++++++++++++++++++ .github/workflows/shell.sh | 15 ++++++ 2 files changed, 87 insertions(+) create mode 100644 .github/workflows/_generate_doc.yaml create mode 100644 .github/workflows/shell.sh diff --git a/.github/workflows/_generate_doc.yaml b/.github/workflows/_generate_doc.yaml new file mode 100644 index 000000000..0413e2639 --- /dev/null +++ b/.github/workflows/_generate_doc.yaml @@ -0,0 +1,72 @@ +name: Generate Workflow doc + +on: + push: + # workflow_dispatch: + # inputs: + # filename: + # description: "Workflow ou action name" + # type: string + # required: false + # default: ".github/workflows/workflow.yaml" + # file-type: + # description: "Type of the file." + # type: choice + # required: true + # default: patch + # options: + # - action + # - workflow + # output-file: + # description: "File name of the documentation to be generated" + # type: string + # # required: true + # default: ".github/workflows/TESTDOC.md" + +jobs: + workflow-doc: + name: Generate workflow documentation + runs-on: ubuntu-22.04 + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # - name: Install CLI + # shell: bash + # run: go get -u github.com/tj-actions/auto-doc/v3 + + - name: Generates docs + shell: bash + run: | + # actions + # for actionsubdir in actions/ ; do + # actionfilename=$actionsubdir/action.yaml + # outputdocfile="docs/references/$actionsubdir" + + # echo "# Refenrences $actionsubdir" > $actionfilename + # echo "## Inputs" >> $actionfilename + # echo "## Outputs" >> $actionfilename + + # auto-doc -f actionfilename --colMaxWidth 10000 --colMaxWords 2000 -o $outputdocfile + # done + # docname="$(echo $workflowname | cut -d'.' -f1)" + + # workflows + for workflows in .github/workflows ; do + if [[ "$worklows" != "_*" && "$worklows" != "README"; then + echo "$worklows" + fi + done + + - name: Commit and push + uses: bakdata/ci-templates/actions/commit-and-push@1.39.1 + with: + add-untracked: "true" + pass-empty-commit: "true" + commit-message: "Committing all the awesome changes in my repository!" + github-email: "${{ secrets.GH_EMAIL }}" + github-username: "${{ secrets.GH_USERNAME }}" + github-token: "${{ secrets.GH_TOKEN }}" diff --git a/.github/workflows/shell.sh b/.github/workflows/shell.sh new file mode 100644 index 000000000..80f515ab8 --- /dev/null +++ b/.github/workflows/shell.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# for workflows in .github/workflows ; do +# # if [[ "$worklows" != "_*" && "$worklows" != "README" ]]; then +# echo "$worklows" +# # fi +# done + +# for actionsubdir in actions/ ; do +# echo $actionsubdir +# done + +for d in */ ; do + echo "$d" +done From 956865b852ce7a113fb8afbdb56c4e8e7d9124dd Mon Sep 17 00:00:00 2001 From: MichaelKora Date: Fri, 26 Jan 2024 13:31:49 +0100 Subject: [PATCH 2/8] First commit --- .github/dependencies.py | 101 +++++++++++++++++++ .github/generate-doc.py | 8 +- action-README.md | 0 docs/references/workflows/shell/Variables.md | 4 + workflow-README.md | 0 5 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 .github/dependencies.py create mode 100644 action-README.md create mode 100644 docs/references/workflows/shell/Variables.md create mode 100644 workflow-README.md diff --git a/.github/dependencies.py b/.github/dependencies.py new file mode 100644 index 000000000..c4158377e --- /dev/null +++ b/.github/dependencies.py @@ -0,0 +1,101 @@ +import glob +import os +import yaml + + +class Colors: + RED = '\033[91m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + BLUE = '\033[94m' + RESET = '\033[0m' + + +def print_colored(text, color): + print(f"{color}{text}{Colors.RESET}") + + +def extract_dependencies(data): + uses_values = [] + + if isinstance(data, dict): + for key, run in data.items(): + # if key == "jobs" and isinstance(value, list): + if key == "runs": + if isinstance(run, dict): + steps = run.get("steps", []) + if isinstance(steps, list): + for step in steps: + if isinstance(step, dict): + uses_value = step.get("uses") + if uses_value is not None: + uses_values.append(uses_value) + elif key == "jobs": + if isinstance(run, dict): + jobs = run.items() + for _, job in jobs: + steps = job.get("steps", []) + if isinstance(steps, list): + for step in steps: + if isinstance(step, dict): + uses_value = step.get("uses") + if uses_value is not None: + uses_values.append(uses_value) + + return uses_values + + +def generate_link(dependency): + link = "" + base = "https://github.com/" + if "bakdata" in dependency: + separator = "ci-templates/" + prefix, sufix = dependency.split(separator) + base += prefix + separator + sufix, tag = sufix.split("@") + tag = f"blob/{tag}/" + link = base + tag + sufix + else: + link = base + dependency.replace("@", "/tree/") + return link + + +def main(): + + # go through actions + action_readme_path = "action-README.md" + action_files = glob.glob("actions/**/action.yaml") + action_files.extend(glob.glob("actions/**/action.yml")) + for action_file in action_files: + action_name = os.path.basename(os.path.dirname(action_file)) + + with open(action_file, 'r') as file: + yaml_data = yaml.safe_load(file) + uses_values = extract_dependencies(yaml_data) + + with open(action_readme_path, 'a') as file: + file.write(f"\n# Dependencies for {action_name}\n") + for dep in uses_values: + link = generate_link(dep) + file.write(f"- [{dep}]({link})\n") + + # go through workflows + workflow_readme_path = "workflow-README.md" + workflow_dir = ".github/workflows" + for workflow in os.listdir(workflow_dir): + workflow_name = workflow.split(".")[0] + if not workflow.startswith("_") and workflow != "README.md": + workflow_path = os.path.join(workflow_dir, workflow) + with open(workflow_path, 'r') as workflow_file: + workflow_data = yaml.safe_load(workflow_file) + workflow_uses_values = extract_dependencies(workflow_data) + + with open(workflow_readme_path, 'a') as workflow_file: + workflow_file.write(f"\n# Dependencies for {workflow_name}\n") + for dep in workflow_uses_values: + link = generate_link(dep) + workflow_file.write(f"- [{dep}]({link})\n") + + +if __name__ == "__main__": + main() diff --git a/.github/generate-doc.py b/.github/generate-doc.py index 09aac60d9..0f542286b 100644 --- a/.github/generate-doc.py +++ b/.github/generate-doc.py @@ -12,6 +12,10 @@ class Colors: RESET = '\033[0m' +def print_colored(text, color): + print(f"{color}{text}{Colors.RESET}") + + def auto_doc_installed(): auto_doc_cmd = os.environ.get("DOC_CMD") if auto_doc_cmd is None or auto_doc_cmd == "": @@ -24,10 +28,6 @@ def auto_doc_installed(): return f"Error: {e.stderr}" -def print_colored(text, color): - print(f"{color}{text}{Colors.RESET}") - - class DocGenerationError(Exception): def __init__(self): self.message = "Error: The documentation is not up to date. Re running pre-commit may help." diff --git a/action-README.md b/action-README.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/references/workflows/shell/Variables.md b/docs/references/workflows/shell/Variables.md new file mode 100644 index 000000000..2c613d86a --- /dev/null +++ b/docs/references/workflows/shell/Variables.md @@ -0,0 +1,4 @@ +# Refenrences shell reusable Workflow +## Inputs +## Outputs +## Secrets diff --git a/workflow-README.md b/workflow-README.md new file mode 100644 index 000000000..e69de29bb From 1cbc2e86b94cabe7de98c9eae35ad6507dba4186 Mon Sep 17 00:00:00 2001 From: MichaelKora Date: Mon, 29 Jan 2024 16:16:43 +0100 Subject: [PATCH 3/8] Integrate dependencies in readme and clean up links in globa readmes for workflows and actions --- .github/workflows/_generate_doc.yaml | 72 ---------------------------- 1 file changed, 72 deletions(-) delete mode 100644 .github/workflows/_generate_doc.yaml diff --git a/.github/workflows/_generate_doc.yaml b/.github/workflows/_generate_doc.yaml deleted file mode 100644 index 0413e2639..000000000 --- a/.github/workflows/_generate_doc.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: Generate Workflow doc - -on: - push: - # workflow_dispatch: - # inputs: - # filename: - # description: "Workflow ou action name" - # type: string - # required: false - # default: ".github/workflows/workflow.yaml" - # file-type: - # description: "Type of the file." - # type: choice - # required: true - # default: patch - # options: - # - action - # - workflow - # output-file: - # description: "File name of the documentation to be generated" - # type: string - # # required: true - # default: ".github/workflows/TESTDOC.md" - -jobs: - workflow-doc: - name: Generate workflow documentation - runs-on: ubuntu-22.04 - - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - # - name: Install CLI - # shell: bash - # run: go get -u github.com/tj-actions/auto-doc/v3 - - - name: Generates docs - shell: bash - run: | - # actions - # for actionsubdir in actions/ ; do - # actionfilename=$actionsubdir/action.yaml - # outputdocfile="docs/references/$actionsubdir" - - # echo "# Refenrences $actionsubdir" > $actionfilename - # echo "## Inputs" >> $actionfilename - # echo "## Outputs" >> $actionfilename - - # auto-doc -f actionfilename --colMaxWidth 10000 --colMaxWords 2000 -o $outputdocfile - # done - # docname="$(echo $workflowname | cut -d'.' -f1)" - - # workflows - for workflows in .github/workflows ; do - if [[ "$worklows" != "_*" && "$worklows" != "README"; then - echo "$worklows" - fi - done - - - name: Commit and push - uses: bakdata/ci-templates/actions/commit-and-push@1.39.1 - with: - add-untracked: "true" - pass-empty-commit: "true" - commit-message: "Committing all the awesome changes in my repository!" - github-email: "${{ secrets.GH_EMAIL }}" - github-username: "${{ secrets.GH_USERNAME }}" - github-token: "${{ secrets.GH_TOKEN }}" From 30f075d753af8d247487807ab0df2562e79b1336 Mon Sep 17 00:00:00 2001 From: MichaelKora Date: Mon, 29 Jan 2024 16:43:33 +0100 Subject: [PATCH 4/8] Test ref fail 1 --- docs/actions/action-lint/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/actions/action-lint/README.md b/docs/actions/action-lint/README.md index 3cf35716f..9deb8b229 100644 --- a/docs/actions/action-lint/README.md +++ b/docs/actions/action-lint/README.md @@ -23,7 +23,7 @@ steps: | INPUT | TYPE | REQUIRED | DEFAULT | DESCRIPTION | | ------------------- | ------ | -------- | ----------- | ------------------------------------------ | | action-lint-version | string | false | `"v1.6.22"` | The action lint repository version to use. | -| ref | string | false | | The ref name to checkout the repository. | +| ref2 | string | false | | The ref name to checkout the repository. | From ec16c154dc33a4de4a0061e556259ef94e8a147b Mon Sep 17 00:00:00 2001 From: MichaelKora Date: Mon, 29 Jan 2024 17:19:57 +0100 Subject: [PATCH 5/8] Test dep fail 1 --- docs/actions/action-lint/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/actions/action-lint/README.md b/docs/actions/action-lint/README.md index 9deb8b229..b812a653c 100644 --- a/docs/actions/action-lint/README.md +++ b/docs/actions/action-lint/README.md @@ -12,7 +12,7 @@ steps: ## Dependencies -- [bakdata/ci-templates/actions/checkout@1.32.0](https://github.com/bakdata/ci-templates/blob/1.32.0/actions/checkout) +- [bakdata/ci-templates/actions/checkout@1.33.0](https://github.com/bakdata/ci-templates/blob/1.32.0/actions/checkout) ## References @@ -23,7 +23,7 @@ steps: | INPUT | TYPE | REQUIRED | DEFAULT | DESCRIPTION | | ------------------- | ------ | -------- | ----------- | ------------------------------------------ | | action-lint-version | string | false | `"v1.6.22"` | The action lint repository version to use. | -| ref2 | string | false | | The ref name to checkout the repository. | +| ref | string | false | | The ref name to checkout the repository. | From 366e169ce6f999746f72d539e00b0985bacb496f Mon Sep 17 00:00:00 2001 From: MichaelKora Date: Mon, 29 Jan 2024 17:22:12 +0100 Subject: [PATCH 6/8] Clean code --- docs/actions/action-lint/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/actions/action-lint/README.md b/docs/actions/action-lint/README.md index b812a653c..3cf35716f 100644 --- a/docs/actions/action-lint/README.md +++ b/docs/actions/action-lint/README.md @@ -12,7 +12,7 @@ steps: ## Dependencies -- [bakdata/ci-templates/actions/checkout@1.33.0](https://github.com/bakdata/ci-templates/blob/1.32.0/actions/checkout) +- [bakdata/ci-templates/actions/checkout@1.32.0](https://github.com/bakdata/ci-templates/blob/1.32.0/actions/checkout) ## References From 9c45d8e904355b024848218e75a7633d66f885cc Mon Sep 17 00:00:00 2001 From: MichaelKora Date: Mon, 29 Jan 2024 17:31:11 +0100 Subject: [PATCH 7/8] Clean code and file reading function --- .github/dependencies.py | 15 +++++++++------ .github/generate-doc.py | 15 ++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/dependencies.py b/.github/dependencies.py index a8343b54b..f2d2949e4 100644 --- a/.github/dependencies.py +++ b/.github/dependencies.py @@ -7,6 +7,12 @@ 'No external actions in use here.'] +def read_file(file_path: str): + with open(file_path, 'r') as file: + content = file.read() + return content + + def remove_formatting(content): # Remove whitespaces and newlines and hyphens try: @@ -17,8 +23,7 @@ def remove_formatting(content): def replace_string_in_markdown(file_path, old_string, new_string): - with open(file_path, 'r') as file: - content = file.read() + content = read_file(file_path) modified_content = content.replace(old_string, new_string) with open(file_path, 'w') as file: file.write(modified_content) @@ -111,8 +116,7 @@ def generate_links(used_ci): def update_dependencies(readme_path: str, dependencies: list): updated = False if dependencies: - with open(readme_path, 'r') as file1: - readme_content = file1.read() + readme_content = read_file(readme_path) if f"## {DEPENDENCIES_SUBSECTION_TITLE}" not in readme_content: try: @@ -122,8 +126,7 @@ def update_dependencies(readme_path: str, dependencies: list): file_readme.write("\n") except Exception as e: print(f"An error occurred: {e}") - with open(readme_path, 'r') as file2: - new_readme_content = file2.read() + new_readme_content = read_file(readme_path) readme_extraction_result = extract_subsection_content( new_readme_content, DEPENDENCIES_SUBSECTION_TITLE) param_join_result = ''.join(dependencies) diff --git a/.github/generate-doc.py b/.github/generate-doc.py index 101fbdad3..5d2f453ce 100644 --- a/.github/generate-doc.py +++ b/.github/generate-doc.py @@ -1,4 +1,4 @@ -from dependencies import contents_equal, extract_dependencies, extract_subsection_content, generate_links, replace_string_in_markdown, update_dependencies +from dependencies import contents_equal, extract_dependencies, extract_subsection_content, generate_links, read_file, replace_string_in_markdown, update_dependencies import glob import os import shutil @@ -38,10 +38,8 @@ def update_doc(readme_path, reference_path): for line in subsection_placeholder: file.write(line + "\n") - with open(readme_path, 'r') as file1: - readme_content = file1.read() - with open(reference_path, 'r') as file2: - reference_content = file2.read() + readme_content = read_file(readme_path) + reference_content = read_file(reference_path) # add subsection if it does not exist if f"## {TARGET_SUBSECTION_TITLE}" not in readme_content: @@ -51,15 +49,14 @@ def update_doc(readme_path, reference_path): file_readme.write(line + "\n") except Exception as e: print(f"An error occurred: {e}") - with open(readme_path, 'r') as file3: - new_readme_content = file3.read() + new_readme_content = read_file(readme_path) readme_extraction_result = extract_subsection_content( new_readme_content, TARGET_SUBSECTION_TITLE) readme_references_subsection = readme_extraction_result.split("\n## ")[0] - + reference_extraction_result = extract_subsection_content( reference_content, TARGET_SUBSECTION_TITLE) - + if not contents_equal(readme_references_subsection, reference_extraction_result): replace_string_in_markdown( readme_path, readme_references_subsection, reference_extraction_result) From 23866c810962e38003dae6d094c306024b5fe838 Mon Sep 17 00:00:00 2001 From: MichaelKora Date: Wed, 31 Jan 2024 10:59:22 +0100 Subject: [PATCH 8/8] Allow file to be created if not exist and update changelog readme --- .github/dependencies.py | 19 ++++++++ .github/generate-doc.py | 16 ++----- docs/actions/changelog-generate/README.md | 54 +++++++++++++++++++++++ 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/.github/dependencies.py b/.github/dependencies.py index f2d2949e4..92338059c 100644 --- a/.github/dependencies.py +++ b/.github/dependencies.py @@ -1,3 +1,4 @@ +import os import re import yaml @@ -7,6 +8,21 @@ 'No external actions in use here.'] +def create_non_existing_docu(file_path: str, placeholder: list): + # Create the dir in case documentation is not created yet + directory_path = os.path.dirname(file_path) + os.makedirs(directory_path, exist_ok=True) + + # Create Readme if it does not exist yet + ci_name = os.path.basename(directory_path) + file_exist = os.path.exists(file_path) + if not file_exist: + with open(file_path, 'w') as file: + file.write(f"# Documentation for {ci_name}\n") + for line in placeholder: + file.write(line + "\n") + + def read_file(file_path: str): with open(file_path, 'r') as file: content = file.read() @@ -115,6 +131,9 @@ def generate_links(used_ci): def update_dependencies(readme_path: str, dependencies: list): updated = False + + create_non_existing_docu(file_path=readme_path, + placeholder=DEPENDENCIES_PLACEHOLDER) if dependencies: readme_content = read_file(readme_path) diff --git a/.github/generate-doc.py b/.github/generate-doc.py index 5d2f453ce..97228355c 100644 --- a/.github/generate-doc.py +++ b/.github/generate-doc.py @@ -1,4 +1,4 @@ -from dependencies import contents_equal, extract_dependencies, extract_subsection_content, generate_links, read_file, replace_string_in_markdown, update_dependencies +from dependencies import create_non_existing_docu, contents_equal, extract_dependencies, extract_subsection_content, generate_links, read_file, replace_string_in_markdown, update_dependencies import glob import os import shutil @@ -25,18 +25,8 @@ def update_doc(readme_path, reference_path): '### Subsubsection', 'Placeholder'] updated = False - # Create the dir in case documentation is not created yet - directory_path = os.path.dirname(readme_path) - os.makedirs(directory_path, exist_ok=True) - - # Create Readme if it does not exist yet - ci_name = os.path.basename(directory_path) - file_exist = os.path.exists(readme_path) - if not file_exist: - with open(readme_path, 'w') as file: - file.write(f"# Documentation for {ci_name}") - for line in subsection_placeholder: - file.write(line + "\n") + create_non_existing_docu(file_path=readme_path, + placeholder=subsection_placeholder) readme_content = read_file(readme_path) reference_content = read_file(reference_path) diff --git a/docs/actions/changelog-generate/README.md b/docs/actions/changelog-generate/README.md index 587f2cdfc..c7e212691 100644 --- a/docs/actions/changelog-generate/README.md +++ b/docs/actions/changelog-generate/README.md @@ -20,6 +20,60 @@ The upper bound might be either existing or new. If the new tag does not yet exist, the action will nevertheless create the changelog so that it may be included in the release. +## Prerequisites + +Create a file called `changelog-config.json` that contains the changelog configurations. +The mentioned action's documentation goes into great detail about how to create and utilize config +files. A simple configuration may look like this: + +```json +{ + "categories": [{ "title": "### Merged pull requests:" }], + "ignore_labels": ["ignore"], + "sort": { "order": "ASC", "on_property": "mergedAt" }, + "template": "# [#{{TO_TAG}}](https://github.com/#{{OWNER}}/#{{REPO}}/releases/tag/#{{TO_TAG}}) - Release Date: #{{TO_TAG_DATE}}\n\n#{{CHANGELOG}}", + "pr_template": "- #{{TITLE}} [##{{NUMBER}}](#{{URL}}) ([@#{{AUTHOR}}](https://github.com/#{{AUTHOR}}))\n", + "empty_template": "- no changes!" +} +``` + +Make sure to update the link +`https://github.com///releases/tag/${{TO_TAG}}` +accordingly and make sure to include `- Release Date: ${{TO_TAG_DATE}}` +because the action looks for this pattern to make the date format easily readable. + +Additional configuration options can be explored +[here](https://github.com/mikepenz/release-changelog-builder-action#configuration-specification). + +## Usage + +By default, just a single commit for the ref/SHA that started the process is retrieved. +In the [checkout action](https://github.com/actions/checkout), enter `fetch-depth: 0` to retrieve +all history for all branches and tags. +Without it, the changelog action will be unable to track down previous tags. + +```yaml +steps: + - uses: bakdata/ci-templates/actions/checkout@1.32.0 + with: + persist-credentials: false + fetch-depth: 0 + - name: Create changelog + id: build_changelog + uses: bakdata/ci-templates/actions/changelog-generate@main + with: + github-token: ${{ secrets.GH_TOKEN }} + new-tag: "1.0.0" + changelog-file: "CHANGELOG.md" + fetch-reviewers: "true" + fetch-release-information: "true" + - name: Use output + run: | + echo "${{ steps.build_changelog.outputs.single-changelog }}" > tag_changelog.md + echo "${{ steps.build_changelog.outputs.merged-changelog }}" > global_changelog.md + shell: bash +``` + ## Dependencies - [mikepenz/release-changelog-builder-action@v4](https://github.com/mikepenz/release-changelog-builder-action/tree/v4)