From aab3234f3a0d23fee5d6298b2f54f8e69fef53b7 Mon Sep 17 00:00:00 2001 From: David Sillman Date: Sat, 25 Nov 2023 18:20:35 -0800 Subject: [PATCH] sets up code splitting using yaml imports (#4) * sets up code splitting using yaml imports * making `key` optional, fixing order of ops in yaml processing * doing some introspective metadocking on jinja helpers * some reorg, metadock-generation of snippets * fixing precommit * starting the process of adding jinja docs to readme using metadock * completing html.summary jinja helper metadock * fixing pre-commit install * trying double linebreaking code blocks * linebreaking code blocks in readme * reverting to md.converted code blocks * reverting to html.inlined code blocks in tables * fleshing out jinja helper metadocks * minor metadock correction in jinja helper section * trying converting the tables * committing test * committing pre_test * using html.pre for multiline code blocks * fixing readme multiline code blocks * escaping grave in metadock * adding docs for yaml imports to README * fixing import key typo in readme ex project --- .gitignore | 2 +- .metadock/content_schematics/cli.yml | 45 +++ .../documents/repo_readme.yml | 26 ++ .../vscode_jinja_md_metadock_snippets.yml | 9 + .../content_schematics/example_project.yml | 159 ++++++++ .../content_schematics/jinja_helpers.yml | 10 + .../jinja_helpers/global.yml | 122 ++++++ .../content_schematics/jinja_helpers/html.yml | 151 +++++++ .../content_schematics/jinja_helpers/md.yml | 162 ++++++++ .../content_schematics/python_interfaces.yml | 27 ++ .metadock/content_schematics/repo_readme.yml | 216 ---------- .metadock/generated_documents/README.html | 372 +++++++++++++++++- .metadock/generated_documents/README.md | 281 ++++++++++++- .../jinja-md.code-snippets | 215 ++++++++++ .metadock/metadock-build-docs.sh | 18 + .../repo_readme_template.md | 141 ++++++- .../vscode_jinja_md_metadock.code-snippets | 31 ++ .pre-commit-config.yaml | 22 +- .vscode/jinja-md.code-snippets | 215 ++++++++++ README.md | 281 ++++++++++++- metadock/engine.py | 23 +- metadock/env.py | 204 ++++++---- metadock/exceptions.py | 4 + metadock/target_formats.py | 3 +- metadock/yaml_utils.py | 67 +++- pre_test.md | 5 + test.md | 5 + tests/test_engine.py | 65 ++- tests/test_env.py | 23 ++ tests/test_yaml_utils.py | 61 +++ 30 files changed, 2582 insertions(+), 383 deletions(-) create mode 100644 .metadock/content_schematics/cli.yml create mode 100644 .metadock/content_schematics/documents/repo_readme.yml create mode 100644 .metadock/content_schematics/documents/vscode_jinja_md_metadock_snippets.yml create mode 100644 .metadock/content_schematics/example_project.yml create mode 100644 .metadock/content_schematics/jinja_helpers.yml create mode 100644 .metadock/content_schematics/jinja_helpers/global.yml create mode 100644 .metadock/content_schematics/jinja_helpers/html.yml create mode 100644 .metadock/content_schematics/jinja_helpers/md.yml create mode 100644 .metadock/content_schematics/python_interfaces.yml delete mode 100644 .metadock/content_schematics/repo_readme.yml create mode 100644 .metadock/generated_documents/jinja-md.code-snippets create mode 100644 .metadock/metadock-build-docs.sh create mode 100644 .metadock/templated_documents/vscode_jinja_md_metadock.code-snippets create mode 100644 .vscode/jinja-md.code-snippets create mode 100644 pre_test.md create mode 100644 test.md diff --git a/.gitignore b/.gitignore index b408cf4..3c91cf1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ poetry.lock -.vscode/ +.vscode/settings.json **/__pycache__/ .pytest_cache/ .hypothesis/ diff --git a/.metadock/content_schematics/cli.yml b/.metadock/content_schematics/cli.yml new file mode 100644 index 0000000..a298d64 --- /dev/null +++ b/.metadock/content_schematics/cli.yml @@ -0,0 +1,45 @@ +cli: + usage_string: | + usage: metadock [-h] [-p PROJECT_DIR] {init,validate,build,list,clean} ... + + Generates and formats Jinja documentation templates from yaml sources. + + positional arguments: + {init,validate,build,list,clean} + Metadock command + init Initialize a new Metadock project in a folder which does not currently have one. + validate Validate the structure of an existing Metadock project. + build Build a Metadock project, rendering some or all documents. + list List all recognized documents which can be generated from a given selection. + clean Cleans the generated_documents directory for the Metadock project. + + options: + -h, --help show this help message and exit + -p PROJECT_DIR, --project-dir PROJECT_DIR + Project directory containing a .metadock directory. + + commands: + init: + description: Used to initialize a fresh Metadock project in a folder which does not currently have one. + usage: metadock [-p PROJECT_DIR] init + python_interface: { import: python_interfaces.yml, key: python_interfaces.init } + + validate: + description: Used to validate the structure of an existing Metadock project. + usage: metadock [-p PROJECT_DIR] validate + python_interface: { import: python_interfaces.yml, key: python_interfaces.validate } + + build: + description: Used to build a Metadock project, rendering some or all documents. + usage: metadock [-p PROJECT_DIR] build [-s SCHEMATIC_GLOBS [SCHEMATIC_GLOBS ...]] [-t TEMPLATE_GLOBS [TEMPLATE_GLOBS ...]] + python_interface: { import: python_interfaces.yml, key: python_interfaces.build } + + list: + description: Used to list all recognized documents which can be generated from a given selection. + usage: metadock [-p PROJECT_DIR] list [-s SCHEMATIC_GLOBS [SCHEMATIC_GLOBS ...]] [-t TEMPLATE_GLOBS [TEMPLATE_GLOBS ...]] + python_interface: { import: python_interfaces.yml, key: python_interfaces.list } + + clean: + description: Used to clean the generated_documents directory for the Metadock project. + usage: metadock [-p PROJECT_DIR] clean + python_interface: { import: python_interfaces.yml, key: python_interfaces.clean } \ No newline at end of file diff --git a/.metadock/content_schematics/documents/repo_readme.yml b/.metadock/content_schematics/documents/repo_readme.yml new file mode 100644 index 0000000..d4fa749 --- /dev/null +++ b/.metadock/content_schematics/documents/repo_readme.yml @@ -0,0 +1,26 @@ +content_schematics: + - name: README + template: repo_readme_template.md + target_formats: [md+html, md] + context: + # Hyperlinks in readme + links: + Jinja2: "[Jinja2](https://github.com/pallets/jinja)" + marko: "[marko](https://github.com/frostming/marko)" + + <<: + # Import example project configuration + - import: example_project.yml + # Import CLI information + - import: cli.yml + # Import Jinja2 helpers + - import: jinja_helpers.yml + + # Target formats + target_formats: + md+html: + description: Generates the given template, parses it into a markdown document, and then generates HTML from it. + + # Project author(s) + authors: + - David Sillman diff --git a/.metadock/content_schematics/documents/vscode_jinja_md_metadock_snippets.yml b/.metadock/content_schematics/documents/vscode_jinja_md_metadock_snippets.yml new file mode 100644 index 0000000..efa3795 --- /dev/null +++ b/.metadock/content_schematics/documents/vscode_jinja_md_metadock_snippets.yml @@ -0,0 +1,9 @@ +content_schematics: + + - name: jinja-md + template: vscode_jinja_md_metadock.code-snippets + target_formats: [ code-snippets ] + context: + + # Import jinja helpers context + <<: { import: jinja_helpers.yml } \ No newline at end of file diff --git a/.metadock/content_schematics/example_project.yml b/.metadock/content_schematics/example_project.yml new file mode 100644 index 0000000..fa166fa --- /dev/null +++ b/.metadock/content_schematics/example_project.yml @@ -0,0 +1,159 @@ +example_project: + structure: | + MyProject/ + - ... + - + - ... + - .metadock/ + - templated_documents/ + - gitlab_mr_template.md + - content_schematics/ + - gitlab_mr__feature1.yml + - gitlab_mr__otherfeature.yml + - generated_documents/ + - gitlab_mr__feature1.md + - gitlab_mr__feature1.html + - gitlab_mr__otherfeature.md + - gitlab_mr__otherfeature.html + + templated_documents: + gitlab_mr_template.md: | + {%- set jira_project_name = jira.get('project_name') -%} + {%- set jira_project_id = jira.get('project_id') -%} + {%- set jira_ticket_num = jira.get('ticket_num') -%} + {%- set jira_ticket_id = jira_project_name ~ "-" ~ jira_ticket_num -%} + {%- set mr_summary = merge_request.get('summary') -%} + # [{{ jira_ticket_id }}] {{ mr_summary }} + + Welcome to my MR. Some of the changes are listed below: + + {% for change in merge_request.get('changes', []) -%} + {{ loop.index }}. {{ change }}{{ "\n" if not loop.last else "" }} + {%- endfor %} + + {% if merge_request.get('breaking_changes') -%} + In addition to the changes above, there are also a few breaking changes introduced in this MR: + + {% for breaking_change in merge_request.get('breaking_changes') -%} + - {{ breaking_change.get('summary') }} + - **Affected downstream stakeholders**: {{ breaking_change.get('affected_downstream', [{'id': 'None'}]) | map(attribute='id') | join(", ") }}. + - **Suggested remedy**: {{ breaking_change.get('suggested_remedy', 'None') }}{{ "\n" if not loop.last else "" }} + {%- endfor -%} + {%- endif %} + + For more information, please check out the Jira ticket associated with this MR, {{ jira_ticket_id }}. + + content_schematics: + gitlab_mr__feature1.yml: | + content_schematics: + + - name: gitlab_mr__feature1 + template: gitlab_mr_template.md + target_formats: [ md+html, md ] + + context: + + jira: + project_name: "IGDP" + project_id: "12001" + ticket_num: "13" + + merge_request: + summary: Adding software version as hard requirement for staging + changes: + - "Added software version to staging model." + - "Added unit tests for valid software version, invalid software version, missing software version." + breaking_changes: + - summary: "Dropping all records which are missing software version." + affected_downstream: + - id: Service + email: service@company.com + - id: Analytics + email: analytics-data@company.com + suggested_remedy: | + - Drop all records which are missing software version. + - Add software version as a hard requirement for staging. + + import_key_examples.yml: | + content_schematics: + + - name: alerting_project_proposal + template: airflow_project_proposal_template.md + target_formats: [ md+html, md ] + + context: + + jira: + + # "block" syntax for importing a root-level key "IGDP" + project: + import: jira/projects.yml + key: IGDP + + # "flow" syntax for importing a sub-key, "David_Sillman" inside "eng_identity" + code_owners: + - { import: jira/identities.yml, key: eng_identity.David_Sillman } + + # "flow" syntax for importing a sub-key using a merge key ("<<"), + <<: { import: team_contexts/data.yml, key: resources.alerting_channels } + + # "block" syntax for importing multiple subkeys from multiple files using a merge key, + <<: + - import: team_contexts/data_contacts.yml + key: contacts.email + - import: team_contexts/data_push_api.yml + key: push_api.contracts + + import_key_examples_2.yml: | + content_schematics: + + - name: confluence_docs_summary + template: confluence/data_docs/confluence_docs_summary_template.md + target_formats: [ md+html, md ] + context: + + # "flow" syntax for a single whole-file import, + all_contracts: { import: confluence/data_docs/contracts.yml } + + # "block" syntax for importing multiple whole files using a merge key, + <<: + - import: confluence/data_docs/projects.yml + - import: confluence/data_docs/sources.yml + + generated_documents: + gitlab_mr__feature1.md: | + # [IGDP-13] Adding software version as hard requirement for staging + + Welcome to my MR. Some of the changes are listed below: + + 1. Added software version to staging model. + 2. Added unit tests for valid software version, invalid software version, missing software version. + + In addition to the changes above, there are also a few breaking changes introduced in this MR: + + - Dropping all records which are missing software version. + - **Affected downstream stakeholders**: Service, Analytics. + - **Suggested remedy**: + - Drop all records which are missing software version. + - Add software version as a hard requirement for staging. + + For more information, please check out the Jira ticket associated with this MR, IGDP-13. + + gitlab_mr__feature1.html: | +

[IGDP-13] Adding software version as hard requirement for staging

+

Welcome to my MR. Some of the changes are listed below:

+
    +
  1. Added software version to staging model.
  2. +
  3. Added unit tests for valid software version, invalid software version, missing software version.
  4. +
+

In addition to the changes above, there are also a few breaking changes introduced in this MR:

+
    +
  • + Dropping all records which are missing software version.
      +
    • Affected downstream stakeholders: Service, Analytics.
    • +
    • Suggested remedy: Handle deletions manualy, using the software version column in the exposures to identify source records + which will be dropped, and drop them in the target environment after our change is deployed.
    • +
    +
  • +
+

For more information, please check out the Jira ticket associated with this MR, IGDP-13.

diff --git a/.metadock/content_schematics/jinja_helpers.yml b/.metadock/content_schematics/jinja_helpers.yml new file mode 100644 index 0000000..4a73b86 --- /dev/null +++ b/.metadock/content_schematics/jinja_helpers.yml @@ -0,0 +1,10 @@ +jinja_helpers: + + <<: + # Import global macros, filters context + - import: jinja_helpers/global.yml + # TODO: import markdown macros, filters context + - import: jinja_helpers/md.yml + # TODO: import html macros, filters context + - import: jinja_helpers/html.yml + \ No newline at end of file diff --git a/.metadock/content_schematics/jinja_helpers/global.yml b/.metadock/content_schematics/jinja_helpers/global.yml new file mode 100644 index 0000000..b92d753 --- /dev/null +++ b/.metadock/content_schematics/jinja_helpers/global.yml @@ -0,0 +1,122 @@ +global: + docstring: | + Jinja namespace for the global Metadock environment, including all global macros, filters, and namespaces. + + macros: + debug: + docstring: Prints a debug message to stdout, and returns an empty string. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("No changes!{{ debug('This is a debug message.') }}").render() + This is a debug message. + 'No changes!' + source_file: metadock/env.py + method_name: metadock.env.MetadockEnv.debug + signature: "(self, message: str) -> None" + intellisense: + snippet_key: Debug message + snippet_body: + - debug($1) + + filters: + chain: + docstring: | + Filter which flattens a sequence of iterables into a single iterable. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string('{{ {"first": 1, "second": 2}.items() | chain | join(" ") }}').render() + 'first 1 second 2' + source_file: metadock/env.py + method_name: metadock.env.MetadockEnv.chain_filter + signature: "(self, iterables: Sequence[Iterable[Any]]) -> Iterable[Any]" + intellisense: + snippet_key: Chain iterables + snippet_body: + - chain + + inline: + docstring: | + Filter which inlines a string by replacing all newlines with spaces, and all double spaces with single spaces. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ 'This is a multi-line string.\nThis is the second line.\nAnd the third.' | inline }}").render() + 'This is a multi-line string. This is the second line. And the third.' + source_file: metadock/env.py + method_name: metadock.env.MetadockEnv.inline_filter + signature: "(self, value: str) -> str" + intellisense: + snippet_key: Inline text + snippet_body: + - inline + + with_prefix: + docstring: | + Filter which prepends a prefix to a string, with an optional separator. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ 'This is a string.' | with_prefix('Prefix') }}").render() + 'PrefixThis is a string.' + >>> env.from_string("{{ 'This is a string.' | with_prefix('Prefix: ', sep = ' : ') }}").render() + 'Prefix : This is a string.' + source_file: metadock/env.py + method_name: metadock.env.MetadockEnv.with_prefix_filter + signature: "(self, value: str, prefix: str, sep: str = '') -> str" + intellisense: + snippet_key: With prefix + snippet_body: + - with_prefix($1) + + with_suffix: + docstring: | + Filter which appends a suffix to a string, with an optional separator. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ 'This is a string' | with_suffix('Suffix') }}").render() + 'This is a stringSuffix' + >>> env.from_string("{{ 'This is a string' | with_suffix('Suffix', sep = ' : ') }}").render() + 'This is a string : Suffix' + source_file: metadock/env.py + method_name: metadock.env.MetadockEnv.with_suffix_filter + signature: "(self, value: str, suffix: str, sep: str = '') -> str" + intellisense: + snippet_key: With suffix + snippet_body: + - with_suffix($1) + + wrap: + docstring: | + Filter which wraps an inner string with a given outer string. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> # Wrap with graves, like md.code(...) + >>> env.from_string("{{ 'This is a string.' | wrap('\`') }}").render() + '\`This is a string.\`' + source_file: metadock/env.py + method_name: metadock.env.MetadockEnv.wrap_filter + signature: "(self, value: str, wrap: str) -> str" + intellisense: + snippet_key: Wrap text + snippet_body: + - wrap($1) + + zip: + docstring: | + Filter which zips an input iterable with one or more iterables. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ ['a', 'b', 'c'] | zip([1, 2, 3]) | list }}").render() + "[('a', 1), ('b', 2), ('c', 3)]" + source_file: metadock/env.py + method_name: metadock.env.MetadockEnv.zip_filter + signature: "(self, input_iterable: Iterable[Any], *iterables: Iterable[Any]) -> Iterable[tuple[Any, ...]]" + intellisense: + snippet_key: Zip iterables + snippet_body: + - zip($1) diff --git a/.metadock/content_schematics/jinja_helpers/html.yml b/.metadock/content_schematics/jinja_helpers/html.yml new file mode 100644 index 0000000..b35e2aa --- /dev/null +++ b/.metadock/content_schematics/jinja_helpers/html.yml @@ -0,0 +1,151 @@ +html: + docstring: | + Jinja namespace which owns HTML-related functions and filters. + + macros: + bold: + docstring: | + Wraps a string in HTML bold tags (). + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ html.bold('This is bold text.') }}").render() + 'This is bold text.' + source_file: metadock/env.py + method_name: metadock.env.MetadockHtmlNamespace.bold + signature: "(self, content: str) -> str" + intellisense: + snippet_key: HTML bold + snippet_body: + - html.bold($1) + + code: + docstring: | + Wraps a string in HTML code tags (). + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ html.code('This is code text.') }}").render() + 'This is code text.' + source_file: metadock/env.py + method_name: metadock.env.MetadockHtmlNamespace.code + signature: "(self, content: str) -> str" + intellisense: + snippet_key: HTML code + snippet_body: + - html.code($1) + + details: + docstring: | + Wraps a string in line-broken HTML details tags (
). Multiple arguments get separated by two + line breaks. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ html.details('This is details text.') }}").render() + '
\nThis is details text.\n
' + source_file: metadock/env.py + method_name: metadock.env.MetadockHtmlNamespace.details + signature: "(self, *contents: str) -> str" + intellisense: + snippet_key: HTML details + snippet_body: + - html.details($1) + + italic: + docstring: | + Wraps a string in HTML italic tags (). + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ html.italic('This is italic text.') }}").render() + 'This is italic text.' + source_file: metadock/env.py + method_name: metadock.env.MetadockHtmlNamespace.italic + signature: "(self, content: str) -> str" + intellisense: + snippet_key: HTML italic + snippet_body: + - html.italic($1) + + pre: + docstring: | + Wraps a string in preformatted HTML pre tags (
), and indents the content by the
+        given amount.
+      example: |
+        >>> from metadock.env import MetadockEnv
+        >>> env = MetadockEnv().jinja_environment()
+        >>> env.from_string("{{ html.pre('This is code text.', indent = 4) }}").render()
+        '
    This is code text.
' + source_file: metadock/env.py + method_name: metadock.env.MetadockHtmlNamespace.pre + signature: "(self, content: str, indent: int = 0) -> str" + intellisense: + snippet_key: HTML pre + snippet_body: + - html.pre($1) + + summary: + docstring: | + Wraps a string in line-broken HTML summary tags (\n\n). + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ html.summary('This is summary text.') }}").render() + '\nThis is summary text.\n' + source_file: metadock/env.py + method_name: metadock.env.MetadockHtmlNamespace.summary + signature: "(self, content: str) -> str" + intellisense: + snippet_key: HTML summary + snippet_body: + - html.summary($1) + + underline: + docstring: | + Wraps a string in HTML underline tags (). + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ html.underline('This is underlined text.') }}").render() + 'This is underlined text.' + source_file: metadock/env.py + method_name: metadock.env.MetadockHtmlNamespace.underline + signature: "(self, content: str) -> str" + intellisense: + snippet_key: HTML underline + snippet_body: + - html.underline($1) + + filters: + escape: + docstring: | + Filter which escapes a string by replacing all HTML special characters with their HTML entity equivalents. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ '

This is a paragraph.

' | html.escape }}").render() + '<p>This is a paragraph.</p>' + source_file: metadock/env.py + method_name: metadock.env.MetadockHtmlNamespace.escape_filter + signature: "(self, content: str) -> str" + intellisense: + snippet_key: HTML escape + snippet_body: + - html.escape + + inline: + docstring: | + Filter which inlines a string by replacing all newlines with HTML line-breaks
singleton tags. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ 'This is a multi-line string.\nThis is the second line.\nAnd the third.' | html.inline }}").render() + 'This is a multi-line string.
This is the second line.
And the third.' + source_file: metadock/env.py + method_name: metadock.env.MetadockHtmlNamespace.inline_filter + signature: "(self, content: str) -> str" + intellisense: + snippet_key: HTML inline + snippet_body: + - html.inline diff --git a/.metadock/content_schematics/jinja_helpers/md.yml b/.metadock/content_schematics/jinja_helpers/md.yml new file mode 100644 index 0000000..2268a03 --- /dev/null +++ b/.metadock/content_schematics/jinja_helpers/md.yml @@ -0,0 +1,162 @@ +md: + docstring: | + Jinja Namespace for Markdown-related functions and filters. + + **Macros**: + + blockquote + code + codeblock + list + tablehead + tablerow + + **Filters**: + + convert + list + + macros: + blockquote: + docstring: | + Produces a Markdown blockquote from the given content by prepending each line with a gt symbol ("> "). + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ md.blockquote('This is a blockquote.') }}").render() + '> This is a blockquote.' + source_file: metadock/env.py + method_name: metadock.env.MetadockMdNamespace.blockquote + signature: "(self, content: str) -> str" + intellisense: + snippet_key: Markdown blockquote + snippet_body: + - md.blockquote($1) + + code: + docstring: | + Produces a Markdown inline code block from the given content by wrapping the string in graves ("\`"). + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ md.code('This is an inline code block.') }}").render() + '`This is an inline code block.`' + source_file: metadock/env.py + method_name: metadock.env.MetadockMdNamespace.code + signature: "(self, content: str) -> str" + intellisense: + snippet_key: Markdown inline code + snippet_body: + - md.code($1) + + codeblock: + docstring: | + Produces a Markdown codeblock from the given content by wrapping the string in triple-graves ("\`\`\`"), + and optionally specifies a language. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ md.codeblock('This is a codeblock.', language = 'sh') }}").render() + '```sh\nThis is a codeblock.\n```' + source_file: metadock/env.py + method_name: metadock.env.MetadockMdNamespace.codeblock + signature: "(self, content: str, language: str = '') -> str" + intellisense: + snippet_key: Markdown codeblock + snippet_body: + - md.codeblock($1) + + list: + docstring: | + Produces a Markdown list from the given content by prepending each line with a dash ("- "). If any of its + arguments are, themselves, formatted as Markdown lists, then they are simply indented as sublists. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string( + ... "{{ md.list('This is a list.', md.list('This is a sublist,', 'in two pieces.')) }}" + ... ).render() + '- This is a list.\n - This is a sublist,\n - in two pieces.' + source_file: metadock/env.py + method_name: metadock.env.MetadockMdNamespace.list + signature: "(self, *items: str) -> str" + intellisense: + snippet_key: Markdown list + snippet_body: + - md.list($1) + + tablehead: + docstring: | + Produces a Markdown table header from the given cells by joining each cell with pipes ("|") and wrapping the + result in pipes, plus adding a header divider row. Cell contents have their pipes escaped with a backslash + ("\\"). To bold the header cell contents, supply `bold = true`. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string( + ... "{{ md.tablehead('Column 1', 'Column 2', 'Column 3', bold = true) }}" + ... ).render() + '| Column 1 | Column 2 | Column 3 |\n| --- | --- | --- |' + source_file: metadock/env.py + method_name: metadock.env.MetadockMdNamespace.tablehead + signature: "(self, *header_cells: str, bold: bool = False) -> str" + intellisense: + snippet_key: Markdown table head + snippet_body: + - md.tablehead($1) + + tablerow: + docstring: | + Produces a Markdown table row from the given cells by joining each cell with pipes ("|") and wrapping the + result in pipes. Cell contents have their pipes escaped with a backslash ("\\"). + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string( + ... "{{ md.tablehead('Column 1', 'Column 2', 'Column 3') }}\n" + ... "{{ md.tablerow('Value 1', 'Value 2', 'Value 3') }}" + ... ).render() + '| Column 1 | Column 2 | Column 3 |\n| --- | --- | --- |\n| Value 1 | Value 2 | Value 3 |' + source_file: metadock/env.py + method_name: metadock.env.MetadockMdNamespace.tablerow + signature: "(self, *row_cells: str) -> str" + intellisense: + snippet_key: Markdown table row + snippet_body: + - md.tablerow($1) + + filters: + convert: + docstring: | + Filter which converts Markdown content to HTML, by invoking `marko.convert` (using github-flavored md). + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string("{{ '# This is a heading\n\n> And a block quote.' | md.convert }}").render() + '

This is a heading

\n
\n

And a block quote.

\n
\n' + source_file: metadock/env.py + method_name: metadock.env.MetadockMdNamespace.convert_filter + signature: "(self, md_content: str) -> str" + intellisense: + snippet_key: Markdown convert + snippet_body: + - md.convert($1) + + list: + docstring: | + Filter which unpacks an iterable of values into a Markdown list, or formats a single value as a Markdown list + element. + example: | + >>> from metadock.env import MetadockEnv + >>> env = MetadockEnv().jinja_environment() + >>> env.from_string( + ... "{{ ['This is a list.', 'This is a second element'] | md.list }}\n" + ... ).render() + '- This is a list.\n- This is a second element\n' + source_file: metadock/env.py + method_name: metadock.env.MetadockMdNamespace.list_filter + signature: "(self, values: str | Iterable[str]) -> str" + intellisense: + snippet_key: Markdown list + snippet_body: + - md.list diff --git a/.metadock/content_schematics/python_interfaces.yml b/.metadock/content_schematics/python_interfaces.yml new file mode 100644 index 0000000..e366122 --- /dev/null +++ b/.metadock/content_schematics/python_interfaces.yml @@ -0,0 +1,27 @@ +python_interfaces: + + init: + source_file: metadock/__init__.py + method_name: metadock.Metadock.init + signature: "(self, working_directory: Path | str = Path.cwd()) -> metadock.Metadock" + + validate: + source_file: metadock/__init__.py + method_name: metadock.Metadock.validate + signature: "(self) -> metadock.engine.MetadockProjectValidationResult" + + build: + source_file: metadock/__init__.py + method_name: metadock.Metadock.build + signature: | + "(self, schematic_globs: list[str] = [], template_globs: list[str] = []) -> metadock.engine.MetadockProjectBuildResult" + + list: + source_file: metadock/__init__.py + method_name: metadock.Metadock.list + signature: "(self, schematic_globs: list[str] = [], template_globs: list[str] = []) -> list[str]" + + clean: + source_file: metadock/__init__.py + method_name: metadock.Metadock.clean + signature: (self) -> None \ No newline at end of file diff --git a/.metadock/content_schematics/repo_readme.yml b/.metadock/content_schematics/repo_readme.yml deleted file mode 100644 index 2a8ab0d..0000000 --- a/.metadock/content_schematics/repo_readme.yml +++ /dev/null @@ -1,216 +0,0 @@ -.example_project_structure: &example_project_structure | - MyProject/ - - ... - - - - ... - - .metadock/ - - templated_documents/ - - gitlab_mr_template.md - - content_schematics/ - - gitlab_mr__feature1.yml - - gitlab_mr__otherfeature.yml - - generated_documents/ - - gitlab_mr__feature1.md - - gitlab_mr__feature1.html - - gitlab_mr__otherfeature.md - - gitlab_mr__otherfeature.html - -.example_project_templated_documents_gitlab_mr_template_md: &example_project_templated_documents_gitlab_mr_template_md | - {%- set jira_project_name = jira.get('project_name') -%} - {%- set jira_project_id = jira.get('project_id') -%} - {%- set jira_ticket_num = jira.get('ticket_num') -%} - {%- set jira_ticket_id = jira_project_name ~ "-" ~ jira_ticket_num -%} - {%- set mr_summary = merge_request.get('summary') -%} - # [{{ jira_ticket_id }}] {{ mr_summary }} - - Welcome to my MR. Some of the changes are listed below: - - {% for change in merge_request.get('changes', []) -%} - {{ loop.index }}. {{ change }}{{ "\n" if not loop.last else "" }} - {%- endfor %} - - {% if merge_request.get('breaking_changes') -%} - In addition to the changes above, there are also a few breaking changes introduced in this MR: - - {% for breaking_change in merge_request.get('breaking_changes') -%} - - {{ breaking_change.get('summary') }} - - **Affected downstream stakeholders**: {{ breaking_change.get('affected_downstream', [{'id': 'None'}]) | map(attribute='id') | join(", ") }}. - - **Suggested remedy**: {{ breaking_change.get('suggested_remedy', 'None') }}{{ "\n" if not loop.last else "" }} - {%- endfor -%} - {%- endif %} - - For more information, please check out the Jira ticket associated with this MR, {{ jira_ticket_id }}. - -.example_project_content_schematics_gitlab_mr__feature1_yml: &example_project_content_schematics_gitlab_mr__feature1_yml | - #... - # yaml anchor definitions - #... - - content_schematics: - - - name: gitlab_mr__feature1 - template: gitlab_mr_template.md - target_formats: [ md+html, md ] - - context: - - jira: - <<: *JiraProject-IGDP - ticket_num: "13" - - merge_request: - summary: Adding software version as hard requirement for staging - changes: - - "Added software version to staging model." - - "Added unit tests for valid software version, invalid software version, missing software version." - breaking_changes: - - summary: "Dropping all records which are missing software version." - affected_downstream: - - *Stakeholder-Service - - *Stakeholder-Analytics - suggested_remedy: | - - Drop all records which are missing software version. - - Add software version as a hard requirement for staging. - -.example_project_generated_documents_gitlab_mr__feature1_md: &example_project_generated_documents_gitlab_mr__feature1_md | - # [IGDP-13] Adding software version as hard requirement for staging - - Welcome to my MR. Some of the changes are listed below: - - 1. Added software version to staging model. - 2. Added unit tests for valid software version, invalid software version, missing software version. - - In addition to the changes above, there are also a few breaking changes introduced in this MR: - - - Dropping all records which are missing software version. - - **Affected downstream stakeholders**: Service, Analytics. - - **Suggested remedy**: - - Drop all records which are missing software version. - - Add software version as a hard requirement for staging. - - For more information, please check out the Jira ticket associated with this MR, IGDP-13. - -.example_project_generated_documents_gitlab_mr__feature1_html: &example_project_generated_documents_gitlab_mr__feature1_html | -

[IGDP-13] Adding software version as hard requirement for staging

-

Welcome to my MR. Some of the changes are listed below:

-
    -
  1. Added software version to staging model.
  2. -
  3. Added unit tests for valid software version, invalid software version, missing software version.
  4. -
-

In addition to the changes above, there are also a few breaking changes introduced in this MR:

-
    -
  • - Dropping all records which are missing software version.
      -
    • Affected downstream stakeholders: Service, Analytics.
    • -
    • Suggested remedy: Handle deletions manualy, using the software version column in the exposures to identify source records - which will be dropped, and drop them in the target environment after our change is deployed.
    • -
    -
  • -
-

For more information, please check out the Jira ticket associated with this MR, IGDP-13.

- -.cli_usage_string: &cli_usage_string | - usage: metadock [-h] [-p PROJECT_DIR] {init,validate,build,list,clean} ... - - Generates and formats Jinja documentation templates from yaml sources. - - positional arguments: - {init,validate,build,list,clean} - Metadock command - init Initialize a new Metadock project in a folder which does not currently have one. - validate Validate the structure of an existing Metadock project. - build Build a Metadock project, rendering some or all documents. - list List all recognized documents which can be generated from a given selection. - clean Cleans the generated_documents directory for the Metadock project. - - options: - -h, --help show this help message and exit - -p PROJECT_DIR, --project-dir PROJECT_DIR - Project directory containing a .metadock directory. - -.python_interface_init: &python_interface_init - source_file: metadock/__init__.py - method_name: metadock.Metadock.init - signature: (Path | str) -> metadock.Metadock - -.python_interface_validate: &python_interface_validate - source_file: metadock/__init__.py - method_name: metadock.Metadock.validate - signature: () -> metadock.engine.MetadockProjectValidationResult - -.python_interface_build: &python_interface_build - source_file: metadock/__init__.py - method_name: metadock.Metadock.build - signature: (list[str], list[str]) -> metadock.engine.MetadockProjectBuildResult - -.python_interface_list: &python_interface_list - source_file: metadock/__init__.py - method_name: metadock.Metadock.list - signature: (list[str], list[str]) -> metadock.engine.MetadockProjectListResult - -.python_interface_clean: &python_interface_clean - source_file: metadock/__init__.py - method_name: metadock.Metadock.clean - signature: () -> None - -content_schematics: - - - name: README - template: repo_readme_template.md - target_formats: [ md+html, md ] - context: - - # Hyperlinks in readme - links: - Jinja2: "[Jinja2](https://github.com/pallets/jinja)" - marko: "[marko](https://github.com/frostming/marko)" - - # Example project metadata - example_project: - structure: *example_project_structure - templated_documents: - gitlab_mr_template.md: *example_project_templated_documents_gitlab_mr_template_md - content_schematics: - gitlab_mr__feature1.yml: *example_project_content_schematics_gitlab_mr__feature1_yml - generated_documents: - gitlab_mr__feature1.md: *example_project_generated_documents_gitlab_mr__feature1_md - gitlab_mr__feature1.html: *example_project_generated_documents_gitlab_mr__feature1_html - - # CLI information - cli: - usage_string: *cli_usage_string - commands: - - init: - description: Used to initialize a fresh Metadock project in a folder which does not currently have one. - usage: metadock [-p PROJECT_DIR] init - python_interface: *python_interface_init - - validate: - description: Used to validate the structure of an existing Metadock project. - usage: metadock [-p PROJECT_DIR] validate - python_interface: *python_interface_validate - - build: - description: Used to build a Metadock project, rendering some or all documents. - usage: metadock [-p PROJECT_DIR] build [-s SCHEMATIC_GLOBS [SCHEMATIC_GLOBS ...]] [-t TEMPLATE_GLOBS [TEMPLATE_GLOBS ...]] - python_interface: *python_interface_build - - list: - description: Used to list all recognized documents which can be generated from a given selection. - usage: metadock [-p PROJECT_DIR] list [-s SCHEMATIC_GLOBS [SCHEMATIC_GLOBS ...]] [-t TEMPLATE_GLOBS [TEMPLATE_GLOBS ...]] - python_interface: *python_interface_list - - clean: - description: Used to clean the generated_documents directory for the Metadock project. - usage: metadock [-p PROJECT_DIR] clean - python_interface: *python_interface_clean - - # Target formats - target_formats: - md+html: - description: Generates the given template, parses it into a markdown document, and then generates HTML from it. - - # Project author(s) - authors: - - David Sillman \ No newline at end of file diff --git a/.metadock/generated_documents/README.html b/.metadock/generated_documents/README.html index 80c1957..3fb7f75 100644 --- a/.metadock/generated_documents/README.html +++ b/.metadock/generated_documents/README.html @@ -25,7 +25,8 @@

Quick Intro

The root of your project is expected to have a .metadock folder, which can be generated from the CLI using metadock init.

Basic CLI Usage

-

The metadock CLI, installed using pip install metadock, has 5 basic commands, spelled out in the help message:

+

The metadock CLI, installed using pip install metadock, has 5 basic commands, +spelled out in the help message:

usage: metadock [-h] [-p PROJECT_DIR] {init,validate,build,list,clean} ...
 
 Generates and formats Jinja documentation templates from yaml sources.
@@ -55,7 +56,7 @@ 

Basic CLI Usage

  • Python interface:
    • Name: metadock.Metadock.init
    • -
    • Signature: (Path | str) -> metadock.Metadock
    • +
    • Signature: (self, working_directory: Path | str = Path.cwd()) -> metadock.Metadock
  • @@ -70,7 +71,7 @@

    Basic CLI Usage

  • Python interface:
    • Name: metadock.Metadock.validate
    • -
    • Signature: () -> metadock.engine.MetadockProjectValidationResult
    • +
    • Signature: (self) -> metadock.engine.MetadockProjectValidationResult
  • @@ -85,7 +86,7 @@

    Basic CLI Usage

  • Python interface:
    • Name: metadock.Metadock.build
    • -
    • Signature: (list[str], list[str]) -> metadock.engine.MetadockProjectBuildResult
    • +
    • Signature: "(self, schematic_globs: list[str] = [], template_globs: list[str] = []) -> metadock.engine.MetadockProjectBuildResult"
  • @@ -100,7 +101,7 @@

    Basic CLI Usage

  • Python interface:
    • Name: metadock.Metadock.list
    • -
    • Signature: (list[str], list[str]) -> metadock.engine.MetadockProjectListResult
    • +
    • Signature: (self, schematic_globs: list[str] = [], template_globs: list[str] = []) -> list[str]
  • @@ -115,7 +116,7 @@

    Basic CLI Usage

  • Python interface:
    • Name: metadock.Metadock.clean
    • -
    • Signature: () -> None
    • +
    • Signature: (self) -> None
  • @@ -150,11 +151,7 @@

    Example Usage

    This is a very simple MR format which can easily be generalized to allow for quickly generating large sets of docs which meet the same format and style requirements. An example content schematic which could service this template could be in gitlab_mr__feature1.yml:

    -
    #...
    -# yaml anchor definitions
    -#...
    -
    -content_schematics:
    +
    content_schematics:
     
     - name: gitlab_mr__feature1
       template: gitlab_mr_template.md
    @@ -163,7 +160,8 @@ 

    Example Usage

    context: jira: - <<: *JiraProject-IGDP + project_name: "IGDP" + project_id: "12001" ticket_num: "13" merge_request: @@ -174,8 +172,10 @@

    Example Usage

    breaking_changes: - summary: "Dropping all records which are missing software version." affected_downstream: - - *Stakeholder-Service - - *Stakeholder-Analytics + - id: Service + email: service@company.com + - id: Analytics + email: analytics-data@company.com suggested_remedy: | - Drop all records which are missing software version. - Add software version as a hard requirement for staging. @@ -205,8 +205,8 @@

    [IGDP-13] Adding software version as hard requirement for staging

    For more information, please check out the Jira ticket associated with this MR, IGDP-13.

    -

    Because the target_formats we chose included md+html and md, we also get an HTML rendering of the document for free, -located at generated_documents/gitlab_mr__feature_1.html:

    +

    Because the target_formats we chose included md+html and md, we also get an HTML rendering of the document for +free, located at generated_documents/gitlab_mr__feature_1.html:

    <h1>[IGDP-13] Adding software version as hard requirement for staging</h1>
     <p>Welcome to my MR. Some of the changes are listed below:</p>
     <ol>
    @@ -237,10 +237,344 @@ 

    [IGDP-13] Adding software version as hard requirement for staging

  • Generates the given template, parses it into a markdown document, and then generates HTML from it.
  • -
  • Anything else, e.g. txt, sql or py
  • -
  • Generates the given template as plaintext, and adds the given string as a file extension, e.g. .txt, .sql or -.py.
  • +
  • +Anything else, e.g. txt, sql or py:
      +
    • Generates the given template as plaintext, and adds the given string as a file extension, e.g. +.txt, .sql or .py.
    • +
    +
  • +

    Code splitting with YAML imports

    +

    In order to keep your content schematics DRY, you can use YAML imports to split your content schematics into multiple +YAML files. For example, if you have a set of content schematics responsible for laying out a "knowledge base" of +services maintained by your team, you might have a YAML file for each service, e.g. +services/airflow/google_forms_scrubber.yml and services/pipelines/user_interaction_data_pipeline.yml which +separately model their respective service specifications.

    +

    A content schematic can import context from a specific YAML key in another YAML file by using the special import-key +object, e.g.:

    +
    content_schematics:
    +
    +- name: alerting_project_proposal
    +  template: airflow_project_proposal_template.md
    +  target_formats: [ md+html, md ]
    +
    +  context:
    +
    +    jira:
    +
    +      # "block" syntax for importing a root-level key "IGDP"
    +      project:
    +        import: jira/projects.yml
    +        key: IGDP
    +
    +      # "flow" syntax for importing a sub-key, "David_Sillman" inside "eng_identity"
    +      code_owners: 
    +        - { import: jira/identities.yml, key: eng_identity.David_Sillman }
    +
    +      # "flow" syntax for importing a sub-key using a merge key ("<<"),
    +      <<: { import: team_contexts/data.yml, key: resources.alerting_channels }
    +
    +      # "block" syntax for importing multiple subkeys from multiple files using a merge key,
    +      <<:
    +        - import: team_contexts/data_contacts.yml
    +          key: contacts.email
    +        - import: team_contexts/data_push_api.yml
    +          key: push_api.contracts
    +
    +

    Note that all paths for the import field are relative to the content_schematics folder for the project. +If you'd like to import the entire content of a file as context, you may omit the key field, e.g.:

    +
    content_schematics:
    +
    +- name: confluence_docs_summary
    +  template: confluence/data_docs/confluence_docs_summary_template.md
    +  target_formats: [ md+html, md ]
    +  context:
    +
    +    # "flow" syntax for a single whole-file import,
    +    all_contracts: { import: confluence/data_docs/contracts.yml }
    +
    +    # "block" syntax for importing multiple whole files using a merge key,
    +    <<:
    +      - import: confluence/data_docs/projects.yml
    +      - import: confluence/data_docs/sources.yml
    +
    +

    At the moment, no protection against cyclic dependencies are implemented (apart from a recursion depth exception which +will likely be thrown before memory is consumed). Users are responsible for ensuring that their imports do not create +cyclic dependencies.

    +

    Jinja Templating Helpers

    +

    In the Jinja templating context which is loaded for each templated document, there are a handful of helpful Jinja macros +and filters which can be used to make formatting content easier. The macros and filters are segregated into +3 namespaces, documented below:

    +

    Global namespace

    +

    Jinja namespace for the global Metadock environment, including all global macros, filters, and namespaces.

    +

    Jinja macros

    +

    The following macros are available in the global namespace:

    +
      +
    • debug
    • +
    +
    + +Jinja macro reference + + + + + + + + + + + + + + + +
    MacroSignatureDoc
    debug
    metadock.env.MetadockEnv.debug: (self, message: str) -> None
    Prints a debug message to stdout, and returns an empty string.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("No changes!{{ debug('This is a debug message.') }}").render()
    This is a debug message.
    'No changes!'
    +

    Jinja filters

    +

    The following filters are available in the global namespace:

    +
      +
    • chain
    • +
    • inline
    • +
    • with_prefix
    • +
    • with_suffix
    • +
    • wrap
    • +
    • zip
    • +
    +
    + +Jinja filter reference + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    FilterSignatureDoc
    chain
    metadock.env.MetadockEnv.chain_filter: (self, iterables: Sequence[Iterable[Any]]) -> Iterable[Any]
    Filter which flattens a sequence of iterables into a single iterable.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string('{{ {"first": 1, "second": 2}.items() | chain | join(" ") }}').render()
    'first 1 second 2'
    inline
    metadock.env.MetadockEnv.inline_filter: (self, value: str) -> str
    Filter which inlines a string by replacing all newlines with spaces, and all double spaces with single spaces.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a multi-line string.\nThis is the second line.\nAnd the third.' | inline }}").render()
    'This is a multi-line string. This is the second line. And the third.'
    with_prefix
    metadock.env.MetadockEnv.with_prefix_filter: (self, value: str, prefix: str, sep: str = '') -> str
    Filter which prepends a prefix to a string, with an optional separator.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a string.' | with_prefix('Prefix') }}").render()
    'PrefixThis is a string.'
    >>> env.from_string("{{ 'This is a string.' | with_prefix('Prefix: ', sep = ' : ') }}").render()
    'Prefix : This is a string.'
    with_suffix
    metadock.env.MetadockEnv.with_suffix_filter: (self, value: str, suffix: str, sep: str = '') -> str
    Filter which appends a suffix to a string, with an optional separator.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a string' | with_suffix('Suffix') }}").render()
    'This is a stringSuffix'
    >>> env.from_string("{{ 'This is a string' | with_suffix('Suffix', sep = ' : ') }}").render()
    'This is a string : Suffix'
    wrap
    metadock.env.MetadockEnv.wrap_filter: (self, value: str, wrap: str) -> str
    Filter which wraps an inner string with a given outer string.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> # Wrap with graves, like md.code(...)
    >>> env.from_string("{{ 'This is a string.' | wrap('`') }}").render()
    '`This is a string.`'
    zip
    metadock.env.MetadockEnv.zip_filter: (self, input_iterable: Iterable[Any], *iterables: Iterable[Any]) -> Iterable[tuple[Any, ...]]
    Filter which zips an input iterable with one or more iterables.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ ['a', 'b', 'c'] | zip([1, 2, 3]) | list }}").render()
    "[('a', 1), ('b', 2), ('c', 3)]"
    +

    md namespace

    +

    Jinja Namespace for Markdown-related functions and filters.

    +

    Macros:

    +
    blockquote
    +code
    +codeblock
    +list
    +tablehead
    +tablerow
    +
    +

    Filters:

    +
    convert
    +list
    +
    +

    Jinja macros

    +

    The following macros are available in the md namespace:

    +
      +
    • md.blockquote
    • +
    • md.code
    • +
    • md.codeblock
    • +
    • md.list
    • +
    • md.tablehead
    • +
    • md.tablerow
    • +
    +
    + +Jinja macro reference + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    MacroSignatureDoc
    md.blockquote
    metadock.env.MetadockMdNamespace.blockquote: (self, content: str) -> str
    Produces a Markdown blockquote from the given content by prepending each line with a gt symbol ("> ").

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ md.blockquote('This is a blockquote.') }}").render()
    '> This is a blockquote.'
    md.code
    metadock.env.MetadockMdNamespace.code: (self, content: str) -> str
    Produces a Markdown inline code block from the given content by wrapping the string in graves ("`").

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ md.code('This is an inline code block.') }}").render()
    '`This is an inline code block.`'
    md.codeblock
    metadock.env.MetadockMdNamespace.codeblock: (self, content: str, language: str = '') -> str
    Produces a Markdown codeblock from the given content by wrapping the string in triple-graves ("```"), and optionally specifies a language.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ md.codeblock('This is a codeblock.', language = 'sh') }}").render()
    'sh\nThis is a codeblock.\n'
    md.list
    metadock.env.MetadockMdNamespace.list: (self, *items: str) -> str
    Produces a Markdown list from the given content by prepending each line with a dash ("- "). If any of its arguments are, themselves, formatted as Markdown lists, then they are simply indented as sublists.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ md.list('This is a list.', md.list('This is a sublist,', 'in two pieces.')) }}"
    ... ).render()
    '- This is a list.\n - This is a sublist,\n - in two pieces.'
    md.tablehead
    metadock.env.MetadockMdNamespace.tablehead: (self, *header_cells: str, bold: bool = False) -> str
    Produces a Markdown table header from the given cells by joining each cell with pipes ("|") and wrapping the result in pipes, plus adding a header divider row. Cell contents have their pipes escaped with a backslash ("\"). To bold the header cell contents, supply bold = true.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ md.tablehead('Column 1', 'Column 2', 'Column 3', bold = true) }}"
    ... ).render()
    '| Column 1 | Column 2 | Column 3 |\n| --- | --- | --- |'
    md.tablerow
    metadock.env.MetadockMdNamespace.tablerow: (self, *row_cells: str) -> str
    Produces a Markdown table row from the given cells by joining each cell with pipes ("|") and wrapping the result in pipes. Cell contents have their pipes escaped with a backslash ("\").

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ md.tablehead('Column 1', 'Column 2', 'Column 3') }}\n"
    ... "{{ md.tablerow('Value 1', 'Value 2', 'Value 3') }}"
    ... ).render()
    '| Column 1 | Column 2 | Column 3 |\n| --- | --- | --- |\n| Value 1 | Value 2 | Value 3 |'
    +

    Jinja filters

    +

    The following filters are available in the md namespace:

    +
      +
    • md.convert
    • +
    • md.list
    • +
    +
    + +Jinja filter reference + + + + + + + + + + + + + + + + + + + + +
    FilterSignatureDoc
    md.convert
    metadock.env.MetadockMdNamespace.convert_filter: (self, md_content: str) -> str
    Filter which converts Markdown content to HTML, by invoking marko.convert (using github-flavored md).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ '# This is a heading\n\n> And a block quote.' | md.convert }}").render()
    '

    This is a heading

    \n
    \n

    And a block quote.

    \n
    \n'
    md.list
    metadock.env.MetadockMdNamespace.list_filter: (self, values: str | Iterable[str]) -> str
    Filter which unpacks an iterable of values into a Markdown list, or formats a single value as a Markdown list element.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ ['This is a list.', 'This is a second element'] | md.list }}\n"
    ... ).render()
    '- This is a list.\n- This is a second element\n'
    +

    html namespace

    +

    Jinja namespace which owns HTML-related functions and filters.

    +

    Jinja macros

    +

    The following macros are available in the html namespace:

    +
      +
    • html.bold
    • +
    • html.code
    • +
    • html.details
    • +
    • html.italic
    • +
    • html.pre
    • +
    • html.summary
    • +
    • html.underline
    • +
    +
    + +Jinja macro reference + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    MacroSignatureDoc
    html.bold
    metadock.env.MetadockHtmlNamespace.bold: (self, content: str) -> str
    Wraps a string in HTML bold tags (<b></b>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.bold('This is bold text.') }}").render()
    'This is bold text.'
    html.code
    metadock.env.MetadockHtmlNamespace.code: (self, content: str) -> str
    Wraps a string in HTML code tags (<code></code>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.code('This is code text.') }}").render()
    'This is code text.'
    html.details
    metadock.env.MetadockHtmlNamespace.details: (self, *contents: str) -> str
    Wraps a string in line-broken HTML details tags (<details></details>). Multiple arguments get separated by two line breaks.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.details('This is details text.') }}").render()
    '
    \nThis is details text.\n
    '
    html.italic
    metadock.env.MetadockHtmlNamespace.italic: (self, content: str) -> str
    Wraps a string in HTML italic tags (<i></i>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.italic('This is italic text.') }}").render()
    'This is italic text.'
    html.pre
    metadock.env.MetadockHtmlNamespace.pre: (self, content: str, indent: int = 0) -> str
    Wraps a string in preformatted HTML pre tags (<pre></pre>), and indents the content by the given amount.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.pre('This is code text.', indent = 4) }}").render()
    '
        This is code text.
    '
    html.summary
    metadock.env.MetadockHtmlNamespace.summary: (self, content: str) -> str
    Wraps a string in line-broken HTML summary tags (<summary>\n\n</summary>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.summary('This is summary text.') }}").render()
    '\nThis is summary text.\n'
    html.underline
    metadock.env.MetadockHtmlNamespace.underline: (self, content: str) -> str
    Wraps a string in HTML underline tags (<u></u>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.underline('This is underlined text.') }}").render()
    'This is underlined text.'
    +

    Jinja filters

    +

    The following filters are available in the html namespace:

    +
      +
    • html.escape
    • +
    • html.inline
    • +
    +
    + +Jinja filter reference + + + + + + + + + + + + + + + + + + + + +
    FilterSignatureDoc
    html.escape
    metadock.env.MetadockHtmlNamespace.escape_filter: (self, content: str) -> str
    Filter which escapes a string by replacing all HTML special characters with their HTML entity equivalents.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ '

    This is a paragraph.

    ' | html.escape }}").render()
    '<p>This is a paragraph.</p>'
    html.inline
    metadock.env.MetadockHtmlNamespace.inline_filter: (self, content: str) -> str
    Filter which inlines a string by replacing all newlines with HTML line-breaks <br> singleton tags.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a multi-line string.\nThis is the second line.\nAnd the third.' | html.inline }}").render()
    'This is a multi-line string.
    This is the second line.
    And the third.'

    Acknowledgements

    Author:

      diff --git a/.metadock/generated_documents/README.md b/.metadock/generated_documents/README.md index 1bf7f9b..569cfe0 100644 --- a/.metadock/generated_documents/README.md +++ b/.metadock/generated_documents/README.md @@ -34,7 +34,8 @@ The root of your project is expected to have a `.metadock` folder, which can be ## Basic CLI Usage -The `metadock` CLI, installed using `pip install metadock`, has 5 basic commands, spelled out in the help message: +The `metadock` CLI, installed using `pip install metadock`, has 5 basic commands, +spelled out in the help message: ```sh usage: metadock [-h] [-p PROJECT_DIR] {init,validate,build,list,clean} ... @@ -69,7 +70,7 @@ Each of the commands supports a programmatic invocation from the `metadock.Metad
    • Python interface:
      • Name: metadock.Metadock.init
      • -
      • Signature: (Path | str) -> metadock.Metadock
      • +
      • Signature: (self, working_directory: Path | str = Path.cwd()) -> metadock.Metadock
    @@ -86,7 +87,7 @@ Each of the commands supports a programmatic invocation from the `metadock.Metad
  • Python interface:
    • Name: metadock.Metadock.validate
    • -
    • Signature: () -> metadock.engine.MetadockProjectValidationResult
    • +
    • Signature: (self) -> metadock.engine.MetadockProjectValidationResult
  • @@ -103,7 +104,7 @@ Each of the commands supports a programmatic invocation from the `metadock.Metad
  • Python interface:
    • Name: metadock.Metadock.build
    • -
    • Signature: (list[str], list[str]) -> metadock.engine.MetadockProjectBuildResult
    • +
    • Signature: "(self, schematic_globs: list[str] = [], template_globs: list[str] = []) -> metadock.engine.MetadockProjectBuildResult"
  • @@ -120,7 +121,7 @@ Each of the commands supports a programmatic invocation from the `metadock.Metad
  • Python interface:
    • Name: metadock.Metadock.list
    • -
    • Signature: (list[str], list[str]) -> metadock.engine.MetadockProjectListResult
    • +
    • Signature: (self, schematic_globs: list[str] = [], template_globs: list[str] = []) -> list[str]
  • @@ -137,7 +138,7 @@ Each of the commands supports a programmatic invocation from the `metadock.Metad
  • Python interface:
    • Name: metadock.Metadock.clean
    • -
    • Signature: () -> None
    • +
    • Signature: (self) -> None
  • @@ -179,12 +180,7 @@ For more information, please check out the Jira ticket associated with this MR, This is a very simple MR format which can easily be generalized to allow for quickly generating large sets of docs which meet the same format and style requirements. An example *content schematic* which could service this template could be in `gitlab_mr__feature1.yml`: - ```yml -#... -# yaml anchor definitions -#... - content_schematics: - name: gitlab_mr__feature1 @@ -194,7 +190,8 @@ content_schematics: context: jira: - <<: *JiraProject-IGDP + project_name: "IGDP" + project_id: "12001" ticket_num: "13" merge_request: @@ -205,8 +202,10 @@ content_schematics: breaking_changes: - summary: "Dropping all records which are missing software version." affected_downstream: - - *Stakeholder-Service - - *Stakeholder-Analytics + - id: Service + email: service@company.com + - id: Analytics + email: analytics-data@company.com suggested_remedy: | - Drop all records which are missing software version. - Add software version as a hard requirement for staging. @@ -232,8 +231,8 @@ called `generated_documents/gitlab_mr__feature1.md`: > > For more information, please check out the Jira ticket associated with this MR, IGDP-13. -Because the `target_formats` we chose included `md+html` _and_ `md`, we also get an HTML rendering of the document for free, -located at `generated_documents/gitlab_mr__feature_1.html`: +Because the `target_formats` we chose included `md+html` _and_ `md`, we also get an HTML rendering of the document for +free, located at `generated_documents/gitlab_mr__feature_1.html`: ```html

    [IGDP-13] Adding software version as hard requirement for staging

    @@ -266,9 +265,253 @@ The natively supported values for `target_formats` are: - `md+html`: - Generates the given template, parses it into a markdown document, and then generates HTML from it. -- Anything else, e.g. `txt`, `sql` or `py` -- Generates the given template as plaintext, and adds the given string as a file extension, e.g. `.txt`, `.sql` or - `.py`. +- Anything else, e.g. `txt`, `sql` or `py`: + - Generates the given template as plaintext, and adds the given string as a file extension, e.g. + `.txt`, `.sql` or `.py`. + +## Code splitting with YAML imports + +In order to keep your content schematics DRY, you can use YAML imports to split your content schematics into multiple +YAML files. For example, if you have a set of content schematics responsible for laying out a "knowledge base" of +services maintained by your team, you might have a YAML file for each service, e.g. +`services/airflow/google_forms_scrubber.yml` and `services/pipelines/user_interaction_data_pipeline.yml` which +separately model their respective service specifications. + +A content schematic can import context from a specific YAML key in another YAML file by using the special _import-key_ +object, e.g.: + +```yml +content_schematics: + +- name: alerting_project_proposal + template: airflow_project_proposal_template.md + target_formats: [ md+html, md ] + + context: + + jira: + + # "block" syntax for importing a root-level key "IGDP" + project: + import: jira/projects.yml + key: IGDP + + # "flow" syntax for importing a sub-key, "David_Sillman" inside "eng_identity" + code_owners: + - { import: jira/identities.yml, key: eng_identity.David_Sillman } + + # "flow" syntax for importing a sub-key using a merge key ("<<"), + <<: { import: team_contexts/data.yml, key: resources.alerting_channels } + + # "block" syntax for importing multiple subkeys from multiple files using a merge key, + <<: + - import: team_contexts/data_contacts.yml + key: contacts.email + - import: team_contexts/data_push_api.yml + key: push_api.contracts +``` + +Note that all paths for the `import` field are relative to the `content_schematics` folder for the project. +If you'd like to import the entire content of a file as context, you may omit the `key` field, e.g.: + +```yml +content_schematics: + +- name: confluence_docs_summary + template: confluence/data_docs/confluence_docs_summary_template.md + target_formats: [ md+html, md ] + context: + + # "flow" syntax for a single whole-file import, + all_contracts: { import: confluence/data_docs/contracts.yml } + + # "block" syntax for importing multiple whole files using a merge key, + <<: + - import: confluence/data_docs/projects.yml + - import: confluence/data_docs/sources.yml +``` + +At the moment, no protection against cyclic dependencies are implemented (apart from a recursion depth exception which +will likely be thrown before memory is consumed). Users are responsible for ensuring that their imports do not create +cyclic dependencies. + +## Jinja Templating Helpers + +In the Jinja templating context which is loaded for each templated document, there are a handful of helpful Jinja macros +and filters which can be used to make formatting content easier. The macros and filters are segregated into +3 namespaces, documented below: + +### Global namespace + +Jinja namespace for the global Metadock environment, including all global macros, filters, and namespaces. + + + +#### Jinja macros + +The following macros are available in the global namespace: + +- `debug` + +
    + +Jinja macro reference + + +| Macro | Signature | Doc | +| --- | --- | --- | +|
    debug
    |
    metadock.env.MetadockEnv.debug: (self, message: str) -> None
    | Prints a debug message to stdout, and returns an empty string.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("No changes!{{ debug('This is a debug message.') }}").render()
    This is a debug message.
    'No changes!'
    | + +
    + +#### Jinja filters + +The following filters are available in the global namespace: + +- `chain` +- `inline` +- `with_prefix` +- `with_suffix` +- `wrap` +- `zip` + +
    + +Jinja filter reference + + +| Filter | Signature | Doc | +| --- | --- | --- | +|
    chain
    |
    metadock.env.MetadockEnv.chain_filter: (self, iterables: Sequence[Iterable[Any]]) -> Iterable[Any]
    | Filter which flattens a sequence of iterables into a single iterable.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string('{{ {"first": 1, "second": 2}.items() \| chain \| join(" ") }}').render()
    'first 1 second 2'
    | +|
    inline
    |
    metadock.env.MetadockEnv.inline_filter: (self, value: str) -> str
    | Filter which inlines a string by replacing all newlines with spaces, and all double spaces with single spaces.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a multi-line string.\nThis is the second line.\nAnd the third.' \| inline }}").render()
    'This is a multi-line string. This is the second line. And the third.'
    | +|
    with_prefix
    |
    metadock.env.MetadockEnv.with_prefix_filter: (self, value: str, prefix: str, sep: str = '') -> str
    | Filter which prepends a prefix to a string, with an optional separator.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a string.' \| with_prefix('Prefix') }}").render()
    'PrefixThis is a string.'
    >>> env.from_string("{{ 'This is a string.' \| with_prefix('Prefix: ', sep = ' : ') }}").render()
    'Prefix : This is a string.'
    | +|
    with_suffix
    |
    metadock.env.MetadockEnv.with_suffix_filter: (self, value: str, suffix: str, sep: str = '') -> str
    | Filter which appends a suffix to a string, with an optional separator.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a string' \| with_suffix('Suffix') }}").render()
    'This is a stringSuffix'
    >>> env.from_string("{{ 'This is a string' \| with_suffix('Suffix', sep = ' : ') }}").render()
    'This is a string : Suffix'
    | +|
    wrap
    |
    metadock.env.MetadockEnv.wrap_filter: (self, value: str, wrap: str) -> str
    | Filter which wraps an inner string with a given outer string.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> # Wrap with graves, like md.code(...)
    >>> env.from_string("{{ 'This is a string.' \| wrap('\`') }}").render()
    '\`This is a string.\`'
    | +|
    zip
    |
    metadock.env.MetadockEnv.zip_filter: (self, input_iterable: Iterable[Any], *iterables: Iterable[Any]) -> Iterable[tuple[Any, ...]]
    | Filter which zips an input iterable with one or more iterables.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ ['a', 'b', 'c'] \| zip([1, 2, 3]) \| list }}").render()
    "[('a', 1), ('b', 2), ('c', 3)]"
    | + +
    + +### `md` namespace + +Jinja Namespace for Markdown-related functions and filters. + +**Macros**: + + blockquote + code + codeblock + list + tablehead + tablerow + +**Filters**: + + convert + list + + + +#### Jinja macros + +The following macros are available in the md namespace: + +- `md.blockquote` +- `md.code` +- `md.codeblock` +- `md.list` +- `md.tablehead` +- `md.tablerow` + +
    + +Jinja macro reference + + +| Macro | Signature | Doc | +| --- | --- | --- | +|
    md.blockquote
    |
    metadock.env.MetadockMdNamespace.blockquote: (self, content: str) -> str
    | Produces a Markdown blockquote from the given content by prepending each line with a gt symbol ("> ").

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ md.blockquote('This is a blockquote.') }}").render()
    '> This is a blockquote.'
    | +|
    md.code
    |
    metadock.env.MetadockMdNamespace.code: (self, content: str) -> str
    | Produces a Markdown inline code block from the given content by wrapping the string in graves ("\`").

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ md.code('This is an inline code block.') }}").render()
    '`This is an inline code block.`'
    | +|
    md.codeblock
    |
    metadock.env.MetadockMdNamespace.codeblock: (self, content: str, language: str = '') -> str
    | Produces a Markdown codeblock from the given content by wrapping the string in triple-graves ("\`\`\`"), and optionally specifies a language.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ md.codeblock('This is a codeblock.', language = 'sh') }}").render()
    '```sh\nThis is a codeblock.\n```'
    | +|
    md.list
    |
    metadock.env.MetadockMdNamespace.list: (self, *items: str) -> str
    | Produces a Markdown list from the given content by prepending each line with a dash ("- "). If any of its arguments are, themselves, formatted as Markdown lists, then they are simply indented as sublists.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ md.list('This is a list.', md.list('This is a sublist,', 'in two pieces.')) }}"
    ... ).render()
    '- This is a list.\n - This is a sublist,\n - in two pieces.'
    | +|
    md.tablehead
    |
    metadock.env.MetadockMdNamespace.tablehead: (self, *header_cells: str, bold: bool = False) -> str
    | Produces a Markdown table header from the given cells by joining each cell with pipes ("\|") and wrapping the result in pipes, plus adding a header divider row. Cell contents have their pipes escaped with a backslash ("\\"). To bold the header cell contents, supply `bold = true`.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ md.tablehead('Column 1', 'Column 2', 'Column 3', bold = true) }}"
    ... ).render()
    '\| Column 1 \| Column 2 \| Column 3 \|\n\| --- \| --- \| --- \|'
    | +|
    md.tablerow
    |
    metadock.env.MetadockMdNamespace.tablerow: (self, *row_cells: str) -> str
    | Produces a Markdown table row from the given cells by joining each cell with pipes ("\|") and wrapping the result in pipes. Cell contents have their pipes escaped with a backslash ("\\").

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ md.tablehead('Column 1', 'Column 2', 'Column 3') }}\n"
    ... "{{ md.tablerow('Value 1', 'Value 2', 'Value 3') }}"
    ... ).render()
    '\| Column 1 \| Column 2 \| Column 3 \|\n\| --- \| --- \| --- \|\n\| Value 1 \| Value 2 \| Value 3 \|'
    | + +
    + +#### Jinja filters + +The following filters are available in the md namespace: + +- `md.convert` +- `md.list` + +
    + +Jinja filter reference + + +| Filter | Signature | Doc | +| --- | --- | --- | +|
    md.convert
    |
    metadock.env.MetadockMdNamespace.convert_filter: (self, md_content: str) -> str
    | Filter which converts Markdown content to HTML, by invoking `marko.convert` (using github-flavored md).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ '# This is a heading\n\n> And a block quote.' \| md.convert }}").render()
    '

    This is a heading

    \n
    \n

    And a block quote.

    \n
    \n'
    | +|
    md.list
    |
    metadock.env.MetadockMdNamespace.list_filter: (self, values: str \| Iterable[str]) -> str
    | Filter which unpacks an iterable of values into a Markdown list, or formats a single value as a Markdown list element.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ ['This is a list.', 'This is a second element'] \| md.list }}\n"
    ... ).render()
    '- This is a list.\n- This is a second element\n'
    | + +
    + +### `html` namespace + +Jinja namespace which owns HTML-related functions and filters. + + + +#### Jinja macros + +The following macros are available in the html namespace: + +- `html.bold` +- `html.code` +- `html.details` +- `html.italic` +- `html.pre` +- `html.summary` +- `html.underline` + +
    + +Jinja macro reference + + +| Macro | Signature | Doc | +| --- | --- | --- | +|
    html.bold
    |
    metadock.env.MetadockHtmlNamespace.bold: (self, content: str) -> str
    | Wraps a string in HTML bold tags (<b></b>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.bold('This is bold text.') }}").render()
    'This is bold text.'
    | +|
    html.code
    |
    metadock.env.MetadockHtmlNamespace.code: (self, content: str) -> str
    | Wraps a string in HTML code tags (<code></code>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.code('This is code text.') }}").render()
    'This is code text.'
    | +|
    html.details
    |
    metadock.env.MetadockHtmlNamespace.details: (self, *contents: str) -> str
    | Wraps a string in line-broken HTML details tags (<details></details>). Multiple arguments get separated by two line breaks.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.details('This is details text.') }}").render()
    '
    \nThis is details text.\n
    '
    | +|
    html.italic
    |
    metadock.env.MetadockHtmlNamespace.italic: (self, content: str) -> str
    | Wraps a string in HTML italic tags (<i></i>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.italic('This is italic text.') }}").render()
    'This is italic text.'
    | +|
    html.pre
    |
    metadock.env.MetadockHtmlNamespace.pre: (self, content: str, indent: int = 0) -> str
    | Wraps a string in preformatted HTML pre tags (<pre></pre>), and indents the content by the given amount.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.pre('This is code text.', indent = 4) }}").render()
    '
        This is code text.
    '
    | +|
    html.summary
    |
    metadock.env.MetadockHtmlNamespace.summary: (self, content: str) -> str
    | Wraps a string in line-broken HTML summary tags (<summary>\n\n</summary>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.summary('This is summary text.') }}").render()
    '\nThis is summary text.\n'
    | +|
    html.underline
    |
    metadock.env.MetadockHtmlNamespace.underline: (self, content: str) -> str
    | Wraps a string in HTML underline tags (<u></u>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.underline('This is underlined text.') }}").render()
    'This is underlined text.'
    | + +
    + +#### Jinja filters + +The following filters are available in the html namespace: + +- `html.escape` +- `html.inline` + +
    + +Jinja filter reference + + +| Filter | Signature | Doc | +| --- | --- | --- | +|
    html.escape
    |
    metadock.env.MetadockHtmlNamespace.escape_filter: (self, content: str) -> str
    | Filter which escapes a string by replacing all HTML special characters with their HTML entity equivalents.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ '

    This is a paragraph.

    ' \| html.escape }}").render()
    '<p>This is a paragraph.</p>'
    | +|
    html.inline
    |
    metadock.env.MetadockHtmlNamespace.inline_filter: (self, content: str) -> str
    | Filter which inlines a string by replacing all newlines with HTML line-breaks <br> singleton tags.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a multi-line string.\nThis is the second line.\nAnd the third.' \| html.inline }}").render()
    'This is a multi-line string.
    This is the second line.
    And the third.'
    | + +
    + + ## Acknowledgements diff --git a/.metadock/generated_documents/jinja-md.code-snippets b/.metadock/generated_documents/jinja-md.code-snippets new file mode 100644 index 0000000..d3d2231 --- /dev/null +++ b/.metadock/generated_documents/jinja-md.code-snippets @@ -0,0 +1,215 @@ +{ + + // Metadock snippets for macros and filters in global namespace + + "(macro) Debug message": { + "scope": "jinja-md,md", + "prefix": "debug", + "body": [ + "debug($1)" + ], + "description": "Debug message macro" + }, + "(filter) Chain iterables": { + "scope": "jinja-md,md", + "prefix": "chain", + "body": [ + "chain" + ], + "description": "Chain iterables filter" + }, + "(filter) Inline text": { + "scope": "jinja-md,md", + "prefix": "inline", + "body": [ + "inline" + ], + "description": "Inline text filter" + }, + "(filter) With prefix": { + "scope": "jinja-md,md", + "prefix": "with_prefix", + "body": [ + "with_prefix($1)" + ], + "description": "With prefix filter" + }, + "(filter) With suffix": { + "scope": "jinja-md,md", + "prefix": "with_suffix", + "body": [ + "with_suffix($1)" + ], + "description": "With suffix filter" + }, + "(filter) Wrap text": { + "scope": "jinja-md,md", + "prefix": "wrap", + "body": [ + "wrap($1)" + ], + "description": "Wrap text filter" + }, + "(filter) Zip iterables": { + "scope": "jinja-md,md", + "prefix": "zip", + "body": [ + "zip($1)" + ], + "description": "Zip iterables filter" + }, + + // Metadock snippets for macros and filters in md namespace + + "(macro) Markdown blockquote": { + "scope": "jinja-md,md", + "prefix": "md.blockquote", + "body": [ + "md.blockquote($1)" + ], + "description": "Markdown blockquote macro" + }, + + "(macro) Markdown inline code": { + "scope": "jinja-md,md", + "prefix": "md.code", + "body": [ + "md.code($1)" + ], + "description": "Markdown inline code macro" + }, + + "(macro) Markdown codeblock": { + "scope": "jinja-md,md", + "prefix": "md.codeblock", + "body": [ + "md.codeblock($1)" + ], + "description": "Markdown codeblock macro" + }, + + "(macro) Markdown list": { + "scope": "jinja-md,md", + "prefix": "md.list", + "body": [ + "md.list($1)" + ], + "description": "Markdown list macro" + }, + + "(macro) Markdown table head": { + "scope": "jinja-md,md", + "prefix": "md.tablehead", + "body": [ + "md.tablehead($1)" + ], + "description": "Markdown table head macro" + }, + + "(macro) Markdown table row": { + "scope": "jinja-md,md", + "prefix": "md.tablerow", + "body": [ + "md.tablerow($1)" + ], + "description": "Markdown table row macro" + }, + "(filter) Markdown convert": { + "scope": "jinja-md,md", + "prefix": "md.convert", + "body": [ + "md.convert($1)" + ], + "description": "Markdown convert filter" + }, + "(filter) Markdown list": { + "scope": "jinja-md,md", + "prefix": "md.list", + "body": [ + "md.list" + ], + "description": "Markdown list filter" + }, + + // Metadock snippets for macros and filters in html namespace + + "(macro) HTML bold": { + "scope": "jinja-md,md", + "prefix": "html.bold", + "body": [ + "html.bold($1)" + ], + "description": "HTML bold macro" + }, + + "(macro) HTML code": { + "scope": "jinja-md,md", + "prefix": "html.code", + "body": [ + "html.code($1)" + ], + "description": "HTML code macro" + }, + + "(macro) HTML details": { + "scope": "jinja-md,md", + "prefix": "html.details", + "body": [ + "html.details($1)" + ], + "description": "HTML details macro" + }, + + "(macro) HTML italic": { + "scope": "jinja-md,md", + "prefix": "html.italic", + "body": [ + "html.italic($1)" + ], + "description": "HTML italic macro" + }, + + "(macro) HTML pre": { + "scope": "jinja-md,md", + "prefix": "html.pre", + "body": [ + "html.pre($1)" + ], + "description": "HTML pre macro" + }, + + "(macro) HTML summary": { + "scope": "jinja-md,md", + "prefix": "html.summary", + "body": [ + "html.summary($1)" + ], + "description": "HTML summary macro" + }, + + "(macro) HTML underline": { + "scope": "jinja-md,md", + "prefix": "html.underline", + "body": [ + "html.underline($1)" + ], + "description": "HTML underline macro" + }, + "(filter) HTML escape": { + "scope": "jinja-md,md", + "prefix": "html.escape", + "body": [ + "html.escape" + ], + "description": "HTML escape filter" + }, + "(filter) HTML inline": { + "scope": "jinja-md,md", + "prefix": "html.inline", + "body": [ + "html.inline" + ], + "description": "HTML inline filter" + }, + +} \ No newline at end of file diff --git a/.metadock/metadock-build-docs.sh b/.metadock/metadock-build-docs.sh new file mode 100644 index 0000000..bece6e4 --- /dev/null +++ b/.metadock/metadock-build-docs.sh @@ -0,0 +1,18 @@ +# install poetry project + +echo "Installing poetry project..." +poetry install + +# metadock build README + +echo "Metadock-building README.md..." +poetry run metadock build -s README +cp .metadock/generated_documents/README.md README.md +git add README.md + +# metadock build Intellisense snippets + +echo "Metadock-building Intellisense snippets..." +poetry run metadock build -s jinja-md +cp .metadock/generated_documents/jinja-md.code-snippets .vscode/jinja-md.code-snippets +git add .vscode/jinja-md.code-snippets \ No newline at end of file diff --git a/.metadock/templated_documents/repo_readme_template.md b/.metadock/templated_documents/repo_readme_template.md index a632d22..2948fd4 100644 --- a/.metadock/templated_documents/repo_readme_template.md +++ b/.metadock/templated_documents/repo_readme_template.md @@ -18,7 +18,8 @@ The root of your project is expected to have a `.metadock` folder, which can be ## Basic CLI Usage -The `metadock` CLI, installed using `pip install metadock`, has {{ cli.get("commands") | length }} basic commands, spelled out in the help message: +The `metadock` CLI, installed using `pip install metadock`, has {{ cli.get("commands") | length }} basic commands, +spelled out in the help message: {{ md.codeblock(cli.get("usage_string"), language="sh") }} @@ -33,7 +34,7 @@ Each of the commands supports a programmatic invocation from the `metadock.Metad - Name: {{ md.code(python_interface.get("method_name")) }} - Signature: {{ md.code(python_interface.get("signature")) }} {%- endset -%} -{{ html.details(html.summary(html.code("metadock " ~ command)), (command_details | md.convert)) }} + {{ html.details(html.summary(html.code("metadock " ~ command)), (command_details | md.convert)) }} {% endfor %} ## Example Usage @@ -50,7 +51,6 @@ In the example above, we can imagine the content of our template, `gitlab_mr_tem This is a very simple MR format which can easily be generalized to allow for quickly generating large sets of docs which meet the same format and style requirements. An example *content schematic* which could service this template could be in `gitlab_mr__feature1.yml`: - {{ md.codeblock( example_project.get("content_schematics").get("gitlab_mr__feature1.yml"), @@ -63,8 +63,8 @@ called `generated_documents/gitlab_mr__feature1.md`: {{ md.blockquote(example_project.get("generated_documents").get("gitlab_mr__feature1.md")) }} -Because the `target_formats` we chose included `md+html` _and_ `md`, we also get an HTML rendering of the document for free, -located at `generated_documents/gitlab_mr__feature_1.html`: +Because the `target_formats` we chose included `md+html` _and_ `md`, we also get an HTML rendering of the document for +free, located at `generated_documents/gitlab_mr__feature_1.html`: {{ md.codeblock( @@ -84,16 +84,135 @@ The natively supported values for `target_formats` are: {% for target_format, data in target_formats.items() %} {{ md.list(md.code(target_format) ~ ":", md.list(data.get("description"))) }} {% endfor -%} +- Anything else, e.g. `txt`, `sql` or `py`: + - Generates the given template as plaintext, and adds the given string as a file extension, e.g. + `.txt`, `.sql` or `.py`. + +## Code splitting with YAML imports + +In order to keep your content schematics DRY, you can use YAML imports to split your content schematics into multiple +YAML files. For example, if you have a set of content schematics responsible for laying out a "knowledge base" of +services maintained by your team, you might have a YAML file for each service, e.g. +`services/airflow/google_forms_scrubber.yml` and `services/pipelines/user_interaction_data_pipeline.yml` which +separately model their respective service specifications. + +A content schematic can import context from a specific YAML key in another YAML file by using the special _import-key_ +object, e.g.: + +{{ + md.codeblock( + example_project.get("content_schematics").get("import_key_examples.yml"), + language="yml", + ) +}} + +Note that all paths for the `import` field are relative to the `content_schematics` folder for the project. +If you'd like to import the entire content of a file as context, you may omit the `key` field, e.g.: + +{{ + md.codeblock( + example_project.get("content_schematics").get("import_key_examples_2.yml"), + language="yml", + ) +}} + +At the moment, no protection against cyclic dependencies are implemented (apart from a recursion depth exception which +will likely be thrown before memory is consumed). Users are responsible for ensuring that their imports do not create +cyclic dependencies. + +## Jinja Templating Helpers + +In the Jinja templating context which is loaded for each templated document, there are a handful of helpful Jinja macros +and filters which can be used to make formatting content easier. The macros and filters are segregated into +{{ jinja_helpers.keys() | length }} namespaces, documented below: + +{% for namespace, namespace_spec in jinja_helpers.items() -%} +{%- set namespace_title_prefix = "Global" if namespace == "global" else md.code(namespace) -%} +{%- set ns_code_prefix = (namespace ~ ".") if namespace != "global" else "" -%} +{%- set namespace_intro -%} +### {{ namespace_title_prefix }} namespace + +{{ namespace_spec.get("docstring") }} +{% endset -%} +{%- set ns_macro_intro -%} +{%- if namespace_spec.get("macros", {}).keys() | length %} +The following macros are available in the {{ namespace }} namespace: + {{ - md.list( - "Anything else, e.g. `txt`, `sql` or `py`", - "Generates the given template as plaintext, and adds the given string as a file extension, e.g. `.txt`, `.sql` or - `.py`." - ) + namespace_spec.get("macros", {}).keys() + | map("with_prefix", ns_code_prefix) + | map("wrap", "`") + | md.list +}} +{%- else -%} +There are no macros available in the {{ namespace }} namespace. +{%- endif -%} +{%- endset -%} +{%- set ns_macro_table -%} +{{ md.tablehead("Macro", "Signature", "Doc", bold=true) }} +{% for macro, macro_spec in namespace_spec.get("macros", {}).items() -%} +{{ + md.tablerow( + html.pre(ns_code_prefix ~ macro), + html.pre(macro_spec.get("method_name") ~ ": " ~ macro_spec.get("signature")), + (macro_spec.get("docstring") | inline | html.escape) ~ "

    " ~ (html.pre(macro_spec.get("example")) + | html.inline), + ) }} +{% endfor -%} +{%- endset -%} +{%- set ns_filter_intro -%} +{%- if namespace_spec.get("filters", {}).keys() | length %} +The following filters are available in the {{ namespace }} namespace: + +{{ + namespace_spec.get("filters", {}).keys() + | map("with_prefix", ns_code_prefix) + | map("wrap", "`") + | md.list +}} +{%- else -%} +There are no filters available in the {{ namespace }} namespace. +{%- endif -%} +{%- endset -%} +{%- set ns_filter_table -%} +{{ md.tablehead("Filter", "Signature", "Doc", bold=true) }} +{% for filter, filter_spec in namespace_spec.get("filters", {}).items() -%} +{{ + md.tablerow( + html.pre(ns_code_prefix ~ filter), + html.pre(filter_spec.get("method_name") ~ ": " ~ filter_spec.get("signature")), + ( + filter_spec.get("docstring") + | inline + | html.escape + ) ~ "

    " ~ ( + html.pre(filter_spec.get("example")) + | html.inline + ), + ) +}} +{% endfor -%} +{%- endset -%} +{{ namespace_intro }} + +#### Jinja macros +{{ ns_macro_intro }} + +{% if namespace_spec.get("macros", {}).keys() | length -%} +{{ html.details(html.summary(html.bold("Jinja macro reference")), ns_macro_table) }} +{%- endif %} + +#### Jinja filters +{{ ns_filter_intro }} + +{% if namespace_spec.get("filters", {}).keys() | length -%} +{{ html.details(html.summary(html.bold("Jinja filter reference")), ns_filter_table) }} +{% endif %} +{% endfor %} ## Acknowledgements Author{% if (authors | length) > 1 %}s{% endif %}: -{{ md.list(*authors) }} \ No newline at end of file +{{ authors | md.list }} \ No newline at end of file diff --git a/.metadock/templated_documents/vscode_jinja_md_metadock.code-snippets b/.metadock/templated_documents/vscode_jinja_md_metadock.code-snippets new file mode 100644 index 0000000..1f06bb5 --- /dev/null +++ b/.metadock/templated_documents/vscode_jinja_md_metadock.code-snippets @@ -0,0 +1,31 @@ +{ + {% for ns in jinja_helpers.keys() %} + // Metadock snippets for macros and filters in {{ ns }} namespace + {% for macro_name, macro_spec in jinja_helpers[ns].get("macros", {}).items() -%} + {%- set macro_prefix_prefix = (ns ~ ".") if ns != "global" else "" -%} + {%- set intellisense_spec = macro_spec.get("intellisense") -%} + {%- set snippet_body_lines = intellisense_spec.get("snippet_body", []) | map("with_prefix", '"') | map("with_suffix", '"') %} + "(macro) {{ intellisense_spec.get("snippet_key") }}": { + "scope": "jinja-md,md", + "prefix": "{{ macro_prefix_prefix ~ macro_name }}", + "body": [ + {{ snippet_body_lines | join(",\n") | indent(12) }} + ], + "description": "{{ intellisense_spec.get("snippet_key") }} macro" + }, + {% endfor -%} + {%- for filter_name, filter_spec in jinja_helpers[ns].get("filters", {}).items() -%} + {%- set filter_prefix_prefix = (ns ~ ".") if ns != "global" else "" -%} + {%- set intellisense_spec = filter_spec.get("intellisense") -%} + {%- set snippet_body_lines = intellisense_spec.get("snippet_body", []) | map("with_prefix", '"') | map("with_suffix", '"') -%} + "(filter) {{ intellisense_spec.get("snippet_key") }}": { + "scope": "jinja-md,md", + "prefix": "{{ filter_prefix_prefix ~ filter_name }}", + "body": [ + {{ snippet_body_lines | join(",\n") | indent(12) }} + ], + "description": "{{ intellisense_spec.get("snippet_key") }} filter" + }, + {% endfor -%} + {%- endfor %} +} \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 73dd116..9f75820 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: -- repo: https://github.com/psf/black - rev: 22.10.0 - hooks: - - id: black - args: [--line-length=120] + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black + args: [--line-length=120] -- repo: local - hooks: - - id: metadock-build-README - name: Metadock-build README - entry: bash -c './.git/hooks/pre-commit || true' - language: system \ No newline at end of file + - repo: local + hooks: + - id: metadock-build + name: Metadock-build documents + entry: bash -c "bash .metadock/metadock-build-docs.sh" -- + language: system diff --git a/.vscode/jinja-md.code-snippets b/.vscode/jinja-md.code-snippets new file mode 100644 index 0000000..d3d2231 --- /dev/null +++ b/.vscode/jinja-md.code-snippets @@ -0,0 +1,215 @@ +{ + + // Metadock snippets for macros and filters in global namespace + + "(macro) Debug message": { + "scope": "jinja-md,md", + "prefix": "debug", + "body": [ + "debug($1)" + ], + "description": "Debug message macro" + }, + "(filter) Chain iterables": { + "scope": "jinja-md,md", + "prefix": "chain", + "body": [ + "chain" + ], + "description": "Chain iterables filter" + }, + "(filter) Inline text": { + "scope": "jinja-md,md", + "prefix": "inline", + "body": [ + "inline" + ], + "description": "Inline text filter" + }, + "(filter) With prefix": { + "scope": "jinja-md,md", + "prefix": "with_prefix", + "body": [ + "with_prefix($1)" + ], + "description": "With prefix filter" + }, + "(filter) With suffix": { + "scope": "jinja-md,md", + "prefix": "with_suffix", + "body": [ + "with_suffix($1)" + ], + "description": "With suffix filter" + }, + "(filter) Wrap text": { + "scope": "jinja-md,md", + "prefix": "wrap", + "body": [ + "wrap($1)" + ], + "description": "Wrap text filter" + }, + "(filter) Zip iterables": { + "scope": "jinja-md,md", + "prefix": "zip", + "body": [ + "zip($1)" + ], + "description": "Zip iterables filter" + }, + + // Metadock snippets for macros and filters in md namespace + + "(macro) Markdown blockquote": { + "scope": "jinja-md,md", + "prefix": "md.blockquote", + "body": [ + "md.blockquote($1)" + ], + "description": "Markdown blockquote macro" + }, + + "(macro) Markdown inline code": { + "scope": "jinja-md,md", + "prefix": "md.code", + "body": [ + "md.code($1)" + ], + "description": "Markdown inline code macro" + }, + + "(macro) Markdown codeblock": { + "scope": "jinja-md,md", + "prefix": "md.codeblock", + "body": [ + "md.codeblock($1)" + ], + "description": "Markdown codeblock macro" + }, + + "(macro) Markdown list": { + "scope": "jinja-md,md", + "prefix": "md.list", + "body": [ + "md.list($1)" + ], + "description": "Markdown list macro" + }, + + "(macro) Markdown table head": { + "scope": "jinja-md,md", + "prefix": "md.tablehead", + "body": [ + "md.tablehead($1)" + ], + "description": "Markdown table head macro" + }, + + "(macro) Markdown table row": { + "scope": "jinja-md,md", + "prefix": "md.tablerow", + "body": [ + "md.tablerow($1)" + ], + "description": "Markdown table row macro" + }, + "(filter) Markdown convert": { + "scope": "jinja-md,md", + "prefix": "md.convert", + "body": [ + "md.convert($1)" + ], + "description": "Markdown convert filter" + }, + "(filter) Markdown list": { + "scope": "jinja-md,md", + "prefix": "md.list", + "body": [ + "md.list" + ], + "description": "Markdown list filter" + }, + + // Metadock snippets for macros and filters in html namespace + + "(macro) HTML bold": { + "scope": "jinja-md,md", + "prefix": "html.bold", + "body": [ + "html.bold($1)" + ], + "description": "HTML bold macro" + }, + + "(macro) HTML code": { + "scope": "jinja-md,md", + "prefix": "html.code", + "body": [ + "html.code($1)" + ], + "description": "HTML code macro" + }, + + "(macro) HTML details": { + "scope": "jinja-md,md", + "prefix": "html.details", + "body": [ + "html.details($1)" + ], + "description": "HTML details macro" + }, + + "(macro) HTML italic": { + "scope": "jinja-md,md", + "prefix": "html.italic", + "body": [ + "html.italic($1)" + ], + "description": "HTML italic macro" + }, + + "(macro) HTML pre": { + "scope": "jinja-md,md", + "prefix": "html.pre", + "body": [ + "html.pre($1)" + ], + "description": "HTML pre macro" + }, + + "(macro) HTML summary": { + "scope": "jinja-md,md", + "prefix": "html.summary", + "body": [ + "html.summary($1)" + ], + "description": "HTML summary macro" + }, + + "(macro) HTML underline": { + "scope": "jinja-md,md", + "prefix": "html.underline", + "body": [ + "html.underline($1)" + ], + "description": "HTML underline macro" + }, + "(filter) HTML escape": { + "scope": "jinja-md,md", + "prefix": "html.escape", + "body": [ + "html.escape" + ], + "description": "HTML escape filter" + }, + "(filter) HTML inline": { + "scope": "jinja-md,md", + "prefix": "html.inline", + "body": [ + "html.inline" + ], + "description": "HTML inline filter" + }, + +} \ No newline at end of file diff --git a/README.md b/README.md index 1bf7f9b..569cfe0 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,8 @@ The root of your project is expected to have a `.metadock` folder, which can be ## Basic CLI Usage -The `metadock` CLI, installed using `pip install metadock`, has 5 basic commands, spelled out in the help message: +The `metadock` CLI, installed using `pip install metadock`, has 5 basic commands, +spelled out in the help message: ```sh usage: metadock [-h] [-p PROJECT_DIR] {init,validate,build,list,clean} ... @@ -69,7 +70,7 @@ Each of the commands supports a programmatic invocation from the `metadock.Metad
  • Python interface:
    • Name: metadock.Metadock.init
    • -
    • Signature: (Path | str) -> metadock.Metadock
    • +
    • Signature: (self, working_directory: Path | str = Path.cwd()) -> metadock.Metadock
  • @@ -86,7 +87,7 @@ Each of the commands supports a programmatic invocation from the `metadock.Metad
  • Python interface:
    • Name: metadock.Metadock.validate
    • -
    • Signature: () -> metadock.engine.MetadockProjectValidationResult
    • +
    • Signature: (self) -> metadock.engine.MetadockProjectValidationResult
  • @@ -103,7 +104,7 @@ Each of the commands supports a programmatic invocation from the `metadock.Metad
  • Python interface:
    • Name: metadock.Metadock.build
    • -
    • Signature: (list[str], list[str]) -> metadock.engine.MetadockProjectBuildResult
    • +
    • Signature: "(self, schematic_globs: list[str] = [], template_globs: list[str] = []) -> metadock.engine.MetadockProjectBuildResult"
  • @@ -120,7 +121,7 @@ Each of the commands supports a programmatic invocation from the `metadock.Metad
  • Python interface:
    • Name: metadock.Metadock.list
    • -
    • Signature: (list[str], list[str]) -> metadock.engine.MetadockProjectListResult
    • +
    • Signature: (self, schematic_globs: list[str] = [], template_globs: list[str] = []) -> list[str]
  • @@ -137,7 +138,7 @@ Each of the commands supports a programmatic invocation from the `metadock.Metad
  • Python interface:
    • Name: metadock.Metadock.clean
    • -
    • Signature: () -> None
    • +
    • Signature: (self) -> None
  • @@ -179,12 +180,7 @@ For more information, please check out the Jira ticket associated with this MR, This is a very simple MR format which can easily be generalized to allow for quickly generating large sets of docs which meet the same format and style requirements. An example *content schematic* which could service this template could be in `gitlab_mr__feature1.yml`: - ```yml -#... -# yaml anchor definitions -#... - content_schematics: - name: gitlab_mr__feature1 @@ -194,7 +190,8 @@ content_schematics: context: jira: - <<: *JiraProject-IGDP + project_name: "IGDP" + project_id: "12001" ticket_num: "13" merge_request: @@ -205,8 +202,10 @@ content_schematics: breaking_changes: - summary: "Dropping all records which are missing software version." affected_downstream: - - *Stakeholder-Service - - *Stakeholder-Analytics + - id: Service + email: service@company.com + - id: Analytics + email: analytics-data@company.com suggested_remedy: | - Drop all records which are missing software version. - Add software version as a hard requirement for staging. @@ -232,8 +231,8 @@ called `generated_documents/gitlab_mr__feature1.md`: > > For more information, please check out the Jira ticket associated with this MR, IGDP-13. -Because the `target_formats` we chose included `md+html` _and_ `md`, we also get an HTML rendering of the document for free, -located at `generated_documents/gitlab_mr__feature_1.html`: +Because the `target_formats` we chose included `md+html` _and_ `md`, we also get an HTML rendering of the document for +free, located at `generated_documents/gitlab_mr__feature_1.html`: ```html

    [IGDP-13] Adding software version as hard requirement for staging

    @@ -266,9 +265,253 @@ The natively supported values for `target_formats` are: - `md+html`: - Generates the given template, parses it into a markdown document, and then generates HTML from it. -- Anything else, e.g. `txt`, `sql` or `py` -- Generates the given template as plaintext, and adds the given string as a file extension, e.g. `.txt`, `.sql` or - `.py`. +- Anything else, e.g. `txt`, `sql` or `py`: + - Generates the given template as plaintext, and adds the given string as a file extension, e.g. + `.txt`, `.sql` or `.py`. + +## Code splitting with YAML imports + +In order to keep your content schematics DRY, you can use YAML imports to split your content schematics into multiple +YAML files. For example, if you have a set of content schematics responsible for laying out a "knowledge base" of +services maintained by your team, you might have a YAML file for each service, e.g. +`services/airflow/google_forms_scrubber.yml` and `services/pipelines/user_interaction_data_pipeline.yml` which +separately model their respective service specifications. + +A content schematic can import context from a specific YAML key in another YAML file by using the special _import-key_ +object, e.g.: + +```yml +content_schematics: + +- name: alerting_project_proposal + template: airflow_project_proposal_template.md + target_formats: [ md+html, md ] + + context: + + jira: + + # "block" syntax for importing a root-level key "IGDP" + project: + import: jira/projects.yml + key: IGDP + + # "flow" syntax for importing a sub-key, "David_Sillman" inside "eng_identity" + code_owners: + - { import: jira/identities.yml, key: eng_identity.David_Sillman } + + # "flow" syntax for importing a sub-key using a merge key ("<<"), + <<: { import: team_contexts/data.yml, key: resources.alerting_channels } + + # "block" syntax for importing multiple subkeys from multiple files using a merge key, + <<: + - import: team_contexts/data_contacts.yml + key: contacts.email + - import: team_contexts/data_push_api.yml + key: push_api.contracts +``` + +Note that all paths for the `import` field are relative to the `content_schematics` folder for the project. +If you'd like to import the entire content of a file as context, you may omit the `key` field, e.g.: + +```yml +content_schematics: + +- name: confluence_docs_summary + template: confluence/data_docs/confluence_docs_summary_template.md + target_formats: [ md+html, md ] + context: + + # "flow" syntax for a single whole-file import, + all_contracts: { import: confluence/data_docs/contracts.yml } + + # "block" syntax for importing multiple whole files using a merge key, + <<: + - import: confluence/data_docs/projects.yml + - import: confluence/data_docs/sources.yml +``` + +At the moment, no protection against cyclic dependencies are implemented (apart from a recursion depth exception which +will likely be thrown before memory is consumed). Users are responsible for ensuring that their imports do not create +cyclic dependencies. + +## Jinja Templating Helpers + +In the Jinja templating context which is loaded for each templated document, there are a handful of helpful Jinja macros +and filters which can be used to make formatting content easier. The macros and filters are segregated into +3 namespaces, documented below: + +### Global namespace + +Jinja namespace for the global Metadock environment, including all global macros, filters, and namespaces. + + + +#### Jinja macros + +The following macros are available in the global namespace: + +- `debug` + +
    + +Jinja macro reference + + +| Macro | Signature | Doc | +| --- | --- | --- | +|
    debug
    |
    metadock.env.MetadockEnv.debug: (self, message: str) -> None
    | Prints a debug message to stdout, and returns an empty string.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("No changes!{{ debug('This is a debug message.') }}").render()
    This is a debug message.
    'No changes!'
    | + +
    + +#### Jinja filters + +The following filters are available in the global namespace: + +- `chain` +- `inline` +- `with_prefix` +- `with_suffix` +- `wrap` +- `zip` + +
    + +Jinja filter reference + + +| Filter | Signature | Doc | +| --- | --- | --- | +|
    chain
    |
    metadock.env.MetadockEnv.chain_filter: (self, iterables: Sequence[Iterable[Any]]) -> Iterable[Any]
    | Filter which flattens a sequence of iterables into a single iterable.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string('{{ {"first": 1, "second": 2}.items() \| chain \| join(" ") }}').render()
    'first 1 second 2'
    | +|
    inline
    |
    metadock.env.MetadockEnv.inline_filter: (self, value: str) -> str
    | Filter which inlines a string by replacing all newlines with spaces, and all double spaces with single spaces.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a multi-line string.\nThis is the second line.\nAnd the third.' \| inline }}").render()
    'This is a multi-line string. This is the second line. And the third.'
    | +|
    with_prefix
    |
    metadock.env.MetadockEnv.with_prefix_filter: (self, value: str, prefix: str, sep: str = '') -> str
    | Filter which prepends a prefix to a string, with an optional separator.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a string.' \| with_prefix('Prefix') }}").render()
    'PrefixThis is a string.'
    >>> env.from_string("{{ 'This is a string.' \| with_prefix('Prefix: ', sep = ' : ') }}").render()
    'Prefix : This is a string.'
    | +|
    with_suffix
    |
    metadock.env.MetadockEnv.with_suffix_filter: (self, value: str, suffix: str, sep: str = '') -> str
    | Filter which appends a suffix to a string, with an optional separator.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a string' \| with_suffix('Suffix') }}").render()
    'This is a stringSuffix'
    >>> env.from_string("{{ 'This is a string' \| with_suffix('Suffix', sep = ' : ') }}").render()
    'This is a string : Suffix'
    | +|
    wrap
    |
    metadock.env.MetadockEnv.wrap_filter: (self, value: str, wrap: str) -> str
    | Filter which wraps an inner string with a given outer string.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> # Wrap with graves, like md.code(...)
    >>> env.from_string("{{ 'This is a string.' \| wrap('\`') }}").render()
    '\`This is a string.\`'
    | +|
    zip
    |
    metadock.env.MetadockEnv.zip_filter: (self, input_iterable: Iterable[Any], *iterables: Iterable[Any]) -> Iterable[tuple[Any, ...]]
    | Filter which zips an input iterable with one or more iterables.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ ['a', 'b', 'c'] \| zip([1, 2, 3]) \| list }}").render()
    "[('a', 1), ('b', 2), ('c', 3)]"
    | + +
    + +### `md` namespace + +Jinja Namespace for Markdown-related functions and filters. + +**Macros**: + + blockquote + code + codeblock + list + tablehead + tablerow + +**Filters**: + + convert + list + + + +#### Jinja macros + +The following macros are available in the md namespace: + +- `md.blockquote` +- `md.code` +- `md.codeblock` +- `md.list` +- `md.tablehead` +- `md.tablerow` + +
    + +Jinja macro reference + + +| Macro | Signature | Doc | +| --- | --- | --- | +|
    md.blockquote
    |
    metadock.env.MetadockMdNamespace.blockquote: (self, content: str) -> str
    | Produces a Markdown blockquote from the given content by prepending each line with a gt symbol ("> ").

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ md.blockquote('This is a blockquote.') }}").render()
    '> This is a blockquote.'
    | +|
    md.code
    |
    metadock.env.MetadockMdNamespace.code: (self, content: str) -> str
    | Produces a Markdown inline code block from the given content by wrapping the string in graves ("\`").

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ md.code('This is an inline code block.') }}").render()
    '`This is an inline code block.`'
    | +|
    md.codeblock
    |
    metadock.env.MetadockMdNamespace.codeblock: (self, content: str, language: str = '') -> str
    | Produces a Markdown codeblock from the given content by wrapping the string in triple-graves ("\`\`\`"), and optionally specifies a language.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ md.codeblock('This is a codeblock.', language = 'sh') }}").render()
    '```sh\nThis is a codeblock.\n```'
    | +|
    md.list
    |
    metadock.env.MetadockMdNamespace.list: (self, *items: str) -> str
    | Produces a Markdown list from the given content by prepending each line with a dash ("- "). If any of its arguments are, themselves, formatted as Markdown lists, then they are simply indented as sublists.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ md.list('This is a list.', md.list('This is a sublist,', 'in two pieces.')) }}"
    ... ).render()
    '- This is a list.\n - This is a sublist,\n - in two pieces.'
    | +|
    md.tablehead
    |
    metadock.env.MetadockMdNamespace.tablehead: (self, *header_cells: str, bold: bool = False) -> str
    | Produces a Markdown table header from the given cells by joining each cell with pipes ("\|") and wrapping the result in pipes, plus adding a header divider row. Cell contents have their pipes escaped with a backslash ("\\"). To bold the header cell contents, supply `bold = true`.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ md.tablehead('Column 1', 'Column 2', 'Column 3', bold = true) }}"
    ... ).render()
    '\| Column 1 \| Column 2 \| Column 3 \|\n\| --- \| --- \| --- \|'
    | +|
    md.tablerow
    |
    metadock.env.MetadockMdNamespace.tablerow: (self, *row_cells: str) -> str
    | Produces a Markdown table row from the given cells by joining each cell with pipes ("\|") and wrapping the result in pipes. Cell contents have their pipes escaped with a backslash ("\\").

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ md.tablehead('Column 1', 'Column 2', 'Column 3') }}\n"
    ... "{{ md.tablerow('Value 1', 'Value 2', 'Value 3') }}"
    ... ).render()
    '\| Column 1 \| Column 2 \| Column 3 \|\n\| --- \| --- \| --- \|\n\| Value 1 \| Value 2 \| Value 3 \|'
    | + +
    + +#### Jinja filters + +The following filters are available in the md namespace: + +- `md.convert` +- `md.list` + +
    + +Jinja filter reference + + +| Filter | Signature | Doc | +| --- | --- | --- | +|
    md.convert
    |
    metadock.env.MetadockMdNamespace.convert_filter: (self, md_content: str) -> str
    | Filter which converts Markdown content to HTML, by invoking `marko.convert` (using github-flavored md).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ '# This is a heading\n\n> And a block quote.' \| md.convert }}").render()
    '

    This is a heading

    \n
    \n

    And a block quote.

    \n
    \n'
    | +|
    md.list
    |
    metadock.env.MetadockMdNamespace.list_filter: (self, values: str \| Iterable[str]) -> str
    | Filter which unpacks an iterable of values into a Markdown list, or formats a single value as a Markdown list element.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string(
    ... "{{ ['This is a list.', 'This is a second element'] \| md.list }}\n"
    ... ).render()
    '- This is a list.\n- This is a second element\n'
    | + +
    + +### `html` namespace + +Jinja namespace which owns HTML-related functions and filters. + + + +#### Jinja macros + +The following macros are available in the html namespace: + +- `html.bold` +- `html.code` +- `html.details` +- `html.italic` +- `html.pre` +- `html.summary` +- `html.underline` + +
    + +Jinja macro reference + + +| Macro | Signature | Doc | +| --- | --- | --- | +|
    html.bold
    |
    metadock.env.MetadockHtmlNamespace.bold: (self, content: str) -> str
    | Wraps a string in HTML bold tags (<b></b>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.bold('This is bold text.') }}").render()
    'This is bold text.'
    | +|
    html.code
    |
    metadock.env.MetadockHtmlNamespace.code: (self, content: str) -> str
    | Wraps a string in HTML code tags (<code></code>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.code('This is code text.') }}").render()
    'This is code text.'
    | +|
    html.details
    |
    metadock.env.MetadockHtmlNamespace.details: (self, *contents: str) -> str
    | Wraps a string in line-broken HTML details tags (<details></details>). Multiple arguments get separated by two line breaks.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.details('This is details text.') }}").render()
    '
    \nThis is details text.\n
    '
    | +|
    html.italic
    |
    metadock.env.MetadockHtmlNamespace.italic: (self, content: str) -> str
    | Wraps a string in HTML italic tags (<i></i>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.italic('This is italic text.') }}").render()
    'This is italic text.'
    | +|
    html.pre
    |
    metadock.env.MetadockHtmlNamespace.pre: (self, content: str, indent: int = 0) -> str
    | Wraps a string in preformatted HTML pre tags (<pre></pre>), and indents the content by the given amount.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.pre('This is code text.', indent = 4) }}").render()
    '
        This is code text.
    '
    | +|
    html.summary
    |
    metadock.env.MetadockHtmlNamespace.summary: (self, content: str) -> str
    | Wraps a string in line-broken HTML summary tags (<summary>\n\n</summary>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.summary('This is summary text.') }}").render()
    '\nThis is summary text.\n'
    | +|
    html.underline
    |
    metadock.env.MetadockHtmlNamespace.underline: (self, content: str) -> str
    | Wraps a string in HTML underline tags (<u></u>).

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ html.underline('This is underlined text.') }}").render()
    'This is underlined text.'
    | + +
    + +#### Jinja filters + +The following filters are available in the html namespace: + +- `html.escape` +- `html.inline` + +
    + +Jinja filter reference + + +| Filter | Signature | Doc | +| --- | --- | --- | +|
    html.escape
    |
    metadock.env.MetadockHtmlNamespace.escape_filter: (self, content: str) -> str
    | Filter which escapes a string by replacing all HTML special characters with their HTML entity equivalents.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ '

    This is a paragraph.

    ' \| html.escape }}").render()
    '<p>This is a paragraph.</p>'
    | +|
    html.inline
    |
    metadock.env.MetadockHtmlNamespace.inline_filter: (self, content: str) -> str
    | Filter which inlines a string by replacing all newlines with HTML line-breaks <br> singleton tags.

    >>> from metadock.env import MetadockEnv
    >>> env = MetadockEnv().jinja_environment()
    >>> env.from_string("{{ 'This is a multi-line string.\nThis is the second line.\nAnd the third.' \| html.inline }}").render()
    'This is a multi-line string.
    This is the second line.
    And the third.'
    | + +
    + + ## Acknowledgements diff --git a/metadock/engine.py b/metadock/engine.py index 707bb30..e458821 100644 --- a/metadock/engine.py +++ b/metadock/engine.py @@ -439,9 +439,7 @@ def to_compiled_targets(self, project: MetadockProject) -> dict[str, str | bytes for target_format in self.target_formats: target_format = MetadockTargetFormatFactory.target_format(target_format) templated_document = project.templated_documents[self.template] - rendered_document = templated_document.jinja_template(project).render( - self.context - ) # | MetadockEnv().dict()) + rendered_document = templated_document.jinja_template(project).render(self.context) post_processed_document = target_format.handler(rendered_document) compiled_targets[target_format.identifier] = post_processed_document @@ -451,7 +449,8 @@ def to_compiled_targets(self, project: MetadockProject) -> dict[str, str | bytes @classmethod def collect_from_file(cls, yaml_path: Path | str) -> "list[MetadockContentSchematic]": """ - Collects content schematics from a YAML file. Flattens any merge keys in the YAML specification. + Collects content schematics from a YAML file. Flattens any merge keys in the YAML specification. Also resolves + any external YAML files imported in the context. Args: yaml_path (Path | str): The path to the YAML file. @@ -461,6 +460,7 @@ def collect_from_file(cls, yaml_path: Path | str) -> "list[MetadockContentSchema Raises: MetadockContentSchematicParsingException: If the YAML file is not found or if a required key is missing. + MetadockYamlImportError: If an imported YAML key or file is not found (or is not a file). """ yaml_path = Path(yaml_path) @@ -471,24 +471,37 @@ def collect_from_file(cls, yaml_path: Path | str) -> "list[MetadockContentSchema "Could not find content schematic file %s" % yaml_path ) + """ Read raw content schematics yaml file """ with yaml_path.open("r") as handle: yaml_contents: dict = yaml.load(handle, yaml.BaseLoader) + """ Determine if there are content schematics in the file """ defined_schematics = yaml_contents.get("content_schematics", []) required_keys = ["name", "target_formats", "template"] + """ For each schematic defined in the YAML file, """ for def_schematic in defined_schematics: + """validate that it has all of the required keys,""" for req_key in required_keys: if not def_schematic.get(req_key): raise exceptions.MetadockContentSchematicParsingException( "Missing required key for content schematic in %s: '%s'" % (yaml_path, req_key) ) + + context = def_schematic.get("context", {}) + """ resolve all imports in the context, including those nested in merge keys. """ + context = yaml_utils.resolve_all_imports( + Path(str(yaml_path).split("/content_schematics/")[0]) / "content_schematics", context + ) + """ Then, flatten all of the merge keys. """ + context = yaml_utils.flatten_merge_keys(context) + """ Schematic is now fully determined. Put into pydantic model. """ content_schematics.append( cls( name=def_schematic["name"], template=def_schematic["template"], target_formats=def_schematic["target_formats"], - context=yaml_utils.flatten_merge_keys(def_schematic.get("context", {})), + context=context, ) ) diff --git a/metadock/env.py b/metadock/env.py index dbdb0c5..76c0618 100644 --- a/metadock/env.py +++ b/metadock/env.py @@ -1,9 +1,10 @@ import abc +import html import itertools from typing import Annotated, Any, Iterable, Literal, Sequence import jinja2 -import marko +from marko.ext.gfm import gfm def _is_nonstr_iter(item: Any) -> bool: @@ -69,7 +70,8 @@ def jinja_environment(self) -> jinja2.Environment: class MetadockMdNamespace(MetadockNamespace): """Jinja Namespace for Markdown-related functions and filters. - Exports: + **Macros**: + blockquote code codeblock @@ -77,7 +79,8 @@ class MetadockMdNamespace(MetadockNamespace): tablehead tablerow - Filters: + **Filters**: + convert list """ @@ -86,7 +89,7 @@ class MetadockMdNamespace(MetadockNamespace): filters = ["convert", "list"] def blockquote(self, content: str) -> str: - """Produces a Markdown blockquote from the given content. + """Produces a Markdown blockquote from the given content by prepending each line with a gt symbol ("> "). Args: content (str): The content of the blockquote. @@ -97,20 +100,8 @@ def blockquote(self, content: str) -> str: _blockquoted = content.strip().replace("\n", "\n> ") return f"> {_blockquoted}" - def codeblock(self, content: str, language: str = "") -> str: - """Produces a Markdown codeblock from the given content. - - Args: - content (str): The content of the codeblock. - language (str, optional): Language attribute for the code block. Defaults to empty string. - - Returns: - str: The Markdown codeblock. - """ - return f"```{language}\n{content.strip()}\n```" - def code(self, content: str) -> str: - """Produces a Markdown inline code block from the given content. + """Produces a Markdown inline code block from the given content by wrapping the string in graves ("`"). Args: content (str): The content of the inline code block. @@ -120,34 +111,22 @@ def code(self, content: str) -> str: """ return f"`{content.strip()}`" - def tablerow(self, *cells: str) -> str: - """Produces a Markdown table row from the given cells. - - Args: - *cells (str): The cells of the table row. - - Returns: - str: The Markdown table row. - """ - return "| " + " | ".join(cells) + " |" - - def tablehead(self, *header_cells: str, bold: bool = False) -> str: - """Produces a Markdown table header row from the given header cells. + def codeblock(self, content: str, language: str = "") -> str: + """Produces a Markdown codeblock from the given content by wrapping the string in triple-graves ("```"), + and optionally specifies a language. Args: - *header_cells (str): The header cells of the table header row. - bold (bool, optional): Whether or not to bold the header's contents. Defaults to False. + content (str): The content of the codeblock. + language (str, optional): Language attribute for the code block. Defaults to empty string. Returns: - str: The Markdown table header row. + str: The Markdown codeblock. """ - if bold: - header_cells = tuple(MetadockHtmlNamespace().bold(cell) for cell in header_cells) - return self.tablerow(*header_cells) + "\n" + self.tablerow(*(["---"] * len(header_cells))) + return f"```{language}\n{content.strip()}\n```" def list(self, *items: str) -> str: - """Produces a Markdown list from the given items, even when those items themselves may be formatted as Markdown - lists. + """Produces a Markdown list from the given content by prepending each line with a dash ("- "). If any of its + arguments are, themselves, formatted as Markdown lists, then they are simply indented as sublists. Args: *items (str): The individual items and/or sub-lists which compose the list. @@ -168,8 +147,38 @@ def _is_md_list(item: str): for flat_item, indented_item in zip(flat_items, indented_items) ) + def tablerow(self, *cells: str) -> str: + """Produces a Markdown table row from the given cells by joining each cell with pipes ("|") and wrapping the + result in pipes. Cell contents have their pipes escaped with a backslash ("\\"). + + Args: + *cells (str): The cells of the table row. + + Returns: + str: The Markdown table row. + """ + _pipe_escaped_cells = tuple(map(lambda cell: cell.replace("|", "\\|"), cells)) + return "| " + " | ".join(_pipe_escaped_cells) + " |" + + def tablehead(self, *header_cells: str, bold: bool = False) -> str: + """Produces a Markdown table header from the given cells by joining each cell with pipes ("|") and wrapping the + result in pipes, plus adding a header divider row. Cell contents have their pipes escaped with a backslash + ("\\"). To bold the header cell contents, supply `bold = true`. + + Args: + *header_cells (str): The header cells of the table header row. + bold (bool, optional): Whether or not to bold the header's contents. Defaults to False. + + Returns: + str: The Markdown table header row. + """ + _pipe_escaped_cells = tuple(map(lambda cell: cell.replace("|", "\\|"), header_cells)) + if bold: + _pipe_escaped_cells = tuple(MetadockHtmlNamespace().bold(cell) for cell in _pipe_escaped_cells) + return self.tablerow(*_pipe_escaped_cells) + "\n" + self.tablerow(*(["---"] * len(_pipe_escaped_cells))) + def convert_filter(self, md_content: str) -> str: - """Filter which converts Markdown content to HTML, by invoking `marko.convert`. + """Filter which converts Markdown content to HTML, by invoking `marko.convert` (using github-flavored md). Args: md_content (str): The Markdown content to be converted to HTML. @@ -177,7 +186,7 @@ def convert_filter(self, md_content: str) -> str: Returns: str: The HTML content. """ - return marko.convert(md_content) + return gfm.convert(md_content) def list_filter(self, values: str | Iterable[str]) -> str: """Filter which unpacks an iterable of values into a Markdown list, or formats a single value as a Markdown list @@ -197,17 +206,24 @@ def list_filter(self, values: str | Iterable[str]) -> str: class MetadockHtmlNamespace(MetadockNamespace): """Jinja namespace which owns HTML-related functions and filters. - Exports: + **Macros**: + bold code - codeblock details italic + pre summary underline + + **Filters**: + + escape + inline """ - exports = ["bold", "code", "codeblock", "details", "italic", "summary", "underline"] + exports = ["bold", "code", "details", "italic", "pre", "summary", "underline"] + filters = ["escape", "inline"] def bold(self, content: str) -> str: """Wraps a string in HTML bold tags (). @@ -231,32 +247,18 @@ def code(self, content: str) -> str: """ return f"{content}" - def codeblock(self, content: str, indent: int = 0) -> str: - """Wraps a string in line-broken HTML code tags (\\n\\n), and indents the content by the given - amount. - - Args: - content (str): The content to be formatted as code. - indent (int, optional): Number of spaces which should be used to indent the contents. Defaults to 0. - - Returns: - str: The HTML code block content. - """ - indented_content = content.replace("\n", "\n" + " " * indent) - return f"\n{indented_content}\n" - - def details(self, *contents: str, indent: int = 0) -> str: - """Wraps a string in HTML details tags (
    ), and indents the content by the given amount. + def details(self, *contents: str) -> str: + """Wraps a string in HTML details tags (
    ). Multiple arguments get separated by two line + breaks. Args: *contents (str): The content to be wrapped in details tags. Multiple arguments get separated by two line breaks. - indent (int, optional): Number of spaces which should be used to indent the contents. Defaults to 0. Returns: str: The HTML details content. """ - indented_linesep_contents = "\n\n".join(contents).replace("\n", "\n" + " " * indent) + indented_linesep_contents = "\n\n".join(contents).replace("\n", "\n") return f"
    \n{indented_linesep_contents}\n
    " def italic(self, content: str) -> str: @@ -270,18 +272,30 @@ def italic(self, content: str) -> str: """ return f"{content}" - def summary(self, content: str, indent: int = 0) -> str: - """Wraps a string in HTML summary tags (), and indents the content by the given amount. + def pre(self, content: str, indent: int = 0) -> str: + """Wraps a string in preformatted HTML pre tags (
    ), and indents the content by the
    +        given amount.
    +
    +        Args:
    +            content (str): The content to be formatted as pre-formatted code.
    +            indent (int, optional): Number of spaces which should be used to indent the contents. Defaults to 0.
    +
    +        Returns:
    +            str: The HTML code block content.
    +        """
    +        indented_content = " " * indent + content.replace("\n", "\n" + " " * indent)
    +        return f"
    {indented_content}
    " + + def summary(self, content: str) -> str: + """Wraps a string in HTML summary tags (). Args: content (str): The content to be wrapped in summary tags. - indent (int, optional): Number of spaces to use when indenting the content. Defaults to 0. Returns: str: The HTML summary content. """ - indented_content = content.replace("\n", "\n" + " " * indent) - return f"\n{indented_content}\n" + return f"\n{content}\n" def underline(self, content: str) -> str: """Wraps a string in HTML underline tags (). @@ -294,22 +308,48 @@ def underline(self, content: str) -> str: """ return f"{content}" + def escape_filter(self, content: str) -> str: + """Filter which escapes a string by replacing all HTML special characters with their HTML entity equivalents. + + Args: + content (str): Piped input string to be HTML-escaped. + + Returns: + str: The escaped string. + """ + return html.escape(content) + + def inline_filter(self, content: str) -> str: + """Filter which inlines a string by replacing all newlines with HTML line-break
    singleton tags. + + Args: + content (str): Piped input string to be HTML-inlined. + + Returns: + str: The HTML-inlined string. + """ + return content.replace("\n", "
    ") + class MetadockEnv(MetadockNamespace): - """Jinja namespace for the global Metadock environment, including all global exports, filters, and namespaces. + """Jinja namespace for the global Metadock environment, including all global macros, filters, and namespaces. + + **Macros**: - Exports: debug - Namespaces: + **Namespaces**: + html md - Filters: + **Filters**: + chain inline with_prefix with_suffix + wrap zip """ @@ -317,23 +357,23 @@ class MetadockEnv(MetadockNamespace): html = MetadockHtmlNamespace() exports = ["debug"] namespaces = ["html", "md"] - filters = ["chain", "inline", "with_prefix", "with_suffix", "zip"] + filters = ["chain", "inline", "with_prefix", "with_suffix", "wrap", "zip"] def debug(self, message: str) -> Literal[""]: """Prints a debug message to stdout, and returns an empty string.""" print(message) return "" - def chain_filter(self, values: Sequence[Iterable[Any]]) -> Iterable[Any]: + def chain_filter(self, iterables: Sequence[Iterable[Any]]) -> Iterable[Any]: """Filter which flattens a sequence of iterables into a single iterable. Args: - values (Sequence[Iterable[Any]]): Piped input sequence of iterables to be flattened. + iterables (Sequence[Iterable[Any]]): Piped input sequence of iterables to be flattened. Returns: Iterable[Any]: The flattened iterable. """ - return itertools.chain.from_iterable(values) + return itertools.chain.from_iterable(iterables) def inline_filter(self, value: str) -> str: """Filter which inlines a string by replacing all newlines with spaces, and all double spaces with single @@ -348,7 +388,7 @@ def inline_filter(self, value: str) -> str: return value.replace("\n", " ").replace(" ", " ") def with_prefix_filter(self, value: str, prefix: str, sep: str = "") -> str: - """Filter which concatenates a prefix to a string, with an optional separator. + """Filter which prepends a prefix to a string, with an optional separator. Args: value (str): Piped input string to be prefixed. @@ -361,7 +401,7 @@ def with_prefix_filter(self, value: str, prefix: str, sep: str = "") -> str: return sep.join((prefix, value)) def with_suffix_filter(self, value: str, suffix: str, sep: str = "") -> str: - """Filter which concatenates a suffix to a string, with an optional separator. + """Filter which appends a suffix to a string, with an optional separator. Args: value (str): Piped input string to be suffixed. @@ -373,6 +413,18 @@ def with_suffix_filter(self, value: str, suffix: str, sep: str = "") -> str: """ return sep.join((value, suffix)) + def wrap_filter(self, value: str, wrap: str) -> str: + """Filter which wraps an inner string with a given outer string. + + Args: + value (str): Piped input string to be wrapped. + wrap (str): String to wrap the input string with. + + Returns: + str: The wrapped string. + """ + return wrap + value + wrap + def zip_filter(self, input_iterable: Iterable[Any], *iterables: Iterable[Any]) -> Iterable[tuple[Any, ...]]: """Filter which zips an input iterable with one or more iterables. diff --git a/metadock/exceptions.py b/metadock/exceptions.py index 4f0fe78..7561597 100644 --- a/metadock/exceptions.py +++ b/metadock/exceptions.py @@ -12,3 +12,7 @@ class MetadockTemplateParsingException(MetadockException): class MetadockContentSchematicParsingException(MetadockException): pass + + +class MetadockYamlImportError(MetadockException): + pass diff --git a/metadock/target_formats.py b/metadock/target_formats.py index bcb9cbb..29d3996 100644 --- a/metadock/target_formats.py +++ b/metadock/target_formats.py @@ -2,6 +2,7 @@ from typing import MutableMapping, Protocol, Type import marko +from marko.ext.gfm import gfm from metadock import exceptions @@ -110,4 +111,4 @@ def handler(cls, rendered_document: str | bytes) -> str | bytes: Returns: str | bytes: HTML markup of the original Markdown document """ - return marko.convert(str(rendered_document)) + return gfm.convert(str(rendered_document)) diff --git a/metadock/yaml_utils.py b/metadock/yaml_utils.py index 9c853eb..7bd4cdc 100644 --- a/metadock/yaml_utils.py +++ b/metadock/yaml_utils.py @@ -1,6 +1,11 @@ import operator from functools import reduce -from typing import Any +from pathlib import Path +from typing import Any, Optional + +import yaml + +from metadock import exceptions def flatten_merge_keys(yaml_dict: Any) -> dict: @@ -12,6 +17,9 @@ def flatten_merge_keys(yaml_dict: Any) -> dict: Returns: dict: Flattened representation of the nested dictionary object """ + if isinstance(yaml_dict, list): + return [flatten_merge_keys(el) for el in yaml_dict] + if not isinstance(yaml_dict, dict): return yaml_dict # type: ignore @@ -28,3 +36,60 @@ def flatten_merge_keys(yaml_dict: Any) -> dict: flattened_yaml_dict[key] = flat_yaml_value return flattened_yaml_dict + + +def import_key(root_path: Path, relative_path: Path, key: Optional[str] = None) -> Any: + """Try to import an alias from the root path with the given name. + + Args: + root_path (Path): Absolute path to the Metadock project's content_schematics directory + relative_path (Path): Relative path to the external file + key (Optional[str]): Key path to resolve, or None to return the entire file + + Raises: + exceptions.MetadockYamlImportError: Imported key / file could not be resolved + + Returns: + Any: Fully resolved yaml source from the external file + """ + + if not (root_path / relative_path).exists(): + raise exceptions.MetadockYamlImportError(f"Could not find import path '{root_path}'") + + if not (root_path / relative_path).is_file(): + raise exceptions.MetadockYamlImportError(f"Import path '{root_path}' is not a file") + + contents: dict[str, Any] = yaml.load((root_path / relative_path).read_text(), yaml.BaseLoader) + if key is not None: + contents = reduce(lambda acc, el: acc[el], key.split("."), contents) + return resolve_all_imports(root_path, contents) + + +def resolve_all_imports(root_path: Path, yaml_obj: Any) -> Any: + """Recursively resolve all imports in a yaml object. + + Args: + root_path (Path): Root path to resolve the imports + yaml_obj (Any): Yaml object with imports to resolve + + Raises: + exceptions.MetadockYamlImportError: One or more import could not be resolved + + Returns: + Any: Yaml object with imports resolved + """ + if isinstance(yaml_obj, list): + return [resolve_all_imports(root_path, el) for el in yaml_obj] + + if not isinstance(yaml_obj, dict): + return yaml_obj # type: ignore + + if set(yaml_obj.keys()) in ({"import"}, {"import", "key"}): + return import_key(root_path, yaml_obj["import"], yaml_obj.get("key", None)) + + resolved_subdict: dict[str, Any] = {} + + for key in yaml_obj.keys(): + resolved_subdict[key] = resolve_all_imports(root_path, yaml_obj[key]) + + return resolved_subdict diff --git a/pre_test.md b/pre_test.md new file mode 100644 index 0000000..4133e6b --- /dev/null +++ b/pre_test.md @@ -0,0 +1,5 @@ +Table: + +| Col 1 | Col 2 | +| --- | --- | +| Item | Item explanation

    This is one line,
    This is another.
    | \ No newline at end of file diff --git a/test.md b/test.md new file mode 100644 index 0000000..ae48cb8 --- /dev/null +++ b/test.md @@ -0,0 +1,5 @@ +Table: + +| Col 1 | Col 2 | +| --- | --- | +| Item | Item explanation

    This is one line,
    This is another.
    | \ No newline at end of file diff --git a/tests/test_engine.py b/tests/test_engine.py index d1fdd05..645ef09 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -11,6 +11,24 @@ def metadock_project(empty_metadock_project_dir): # Create some dummy files within the directories (project_dir / "templated_documents" / "template1.md").write_text("Simple plaintext document.") (project_dir / "templated_documents" / "template2.md").write_text("{{ var1 }} is {{ var2 }}.") + (project_dir / "templated_documents" / "imported.md").write_text( + """**Imported identity**: {{ name }} ({{ sem_version }})""" + ) + (project_dir / "content_schematics" / "lib.yml").write_text( + """ + identity: + name: lib + sem_version: 3.0.1 + fix_version: 2023.04.11 + """ + ) + (project_dir / "content_schematics" / "lib2.yml").write_text( + """ + name: lib + sem_version: 3.0.2 + fix_version: 2023.04.21 + """ + ) (project_dir / "content_schematics" / "schematic1.yml").write_text( """ content_schematics: @@ -45,6 +63,25 @@ def metadock_project(empty_metadock_project_dir): var2: a test """ ) + (project_dir / "content_schematics" / "imported.yml").write_text( + """ + content_schematics: + + - name: schematic_import + template: imported.md + target_formats: [ md ] + context: { import: lib.yml, key: identity } + + - name: schematic_import2 + template: imported.md + target_formats: [ md ] + context: + <<: + - import: lib.yml + key: identity + - import: lib2.yml + """ + ) return MetadockProject(project_dir) @@ -54,7 +91,7 @@ def test_metadock_project_templated_documents_directory(metadock_project): def test_metadock_project_templated_documents(metadock_project): - assert len(metadock_project.templated_documents) == 2 + assert len(metadock_project.templated_documents) == 3 def test_metadock_project_content_schematics_directory(metadock_project): @@ -62,7 +99,7 @@ def test_metadock_project_content_schematics_directory(metadock_project): def test_metadock_project_content_schematics(metadock_project): - assert len(metadock_project.content_schematics) == 4 + assert len(metadock_project.content_schematics) == 6 def test_metadock_project_generated_documents_directory(metadock_project): @@ -73,8 +110,8 @@ def test_metadock_project_build(metadock_project): # Test building all schematics build_result = metadock_project.build() - assert len(list(metadock_project.generated_documents_directory.glob("*"))) == 4 - assert len(build_result.generated_documents) == 4 + assert len(list(metadock_project.generated_documents_directory.glob("*"))) == 6 + assert len(build_result.generated_documents) == 6 assert all(gd.status == "new" for gd in build_result.generated_documents) metadock_project.clean() @@ -149,3 +186,23 @@ def test_metadock_content_schematic(metadock_project): assert content_schem_2a.to_compiled_targets(metadock_project) == {"md": " is ."} assert content_schem_2b.to_compiled_targets(metadock_project) == {"md": "This is a test."} + + +def test_metadock_content_schematic__import_key(metadock_project): + content_schem_imported = metadock_project.content_schematics["schematic_import"] + assert content_schem_imported.context + assert content_schem_imported.context == { + "name": "lib", + "sem_version": "3.0.1", + "fix_version": "2023.04.11", + } + + metadock_project.build(["schematic_import", "schematic_import2"]) + + gen_doc = metadock_project.generated_documents_directory / "schematic_import.md" + assert gen_doc.exists() + assert gen_doc.read_text() == "**Imported identity**: lib (3.0.1)" + + gen_doc2 = metadock_project.generated_documents_directory / "schematic_import2.md" + assert gen_doc2.exists() + assert gen_doc2.read_text() == "**Imported identity**: lib (3.0.2)" diff --git a/tests/test_env.py b/tests/test_env.py index 46cfb33..a4e860d 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -375,3 +375,26 @@ def test_env__inline(empty_metadock_project_dir): assert (project_dir / "generated_documents" / "example1.md").read_text() == ( """This is a paragraph with a line break. This is the second line.""" ) + + +def test_env__wrap(empty_metadock_project_dir): + project_dir = empty_metadock_project_dir + (project_dir / "templated_documents" / "simple.md").write_text("""{{ lines | join(", ") | wrap("||") }}""") + (project_dir / "content_schematics" / "schematic1.yml").write_text( + """ + content_schematics: + - name: example1 + template: simple.md + target_formats: [ md ] + context: + lines: + - line a + - line b + - line c + """ + ) + + metadock = MetadockProject(project_dir) + metadock.build() + + assert (project_dir / "generated_documents" / "example1.md").read_text() == ("""||line a, line b, line c||""") diff --git a/tests/test_yaml_utils.py b/tests/test_yaml_utils.py index 963302b..054d9c9 100644 --- a/tests/test_yaml_utils.py +++ b/tests/test_yaml_utils.py @@ -1,4 +1,8 @@ +import string +from pathlib import Path + import pytest +import yaml from metadock import yaml_utils @@ -50,3 +54,60 @@ ) def test_yaml_utils__flatten_merge_keys(yml_dict, flat_dict): assert yaml_utils.flatten_merge_keys(yml_dict) == flat_dict + + +@pytest.mark.parametrize( + "contents,key,value", + [ + pytest.param( + "name: foo", + "name", + "foo", + id="simple single-key dict", + ), + pytest.param( + "name: foo\nvalue: bar", + "value", + "bar", + id="simple 2-key dict", + ), + pytest.param( + "name: \n x: 32\n y: 64", + "name", + {"x": "32", "y": "64"}, + id="simple full dict query", + ), + pytest.param( + "name: \n x: 32\n y: 64", + "name.x", + "32", + id="simple dict key query", + ), + ], +) +def test_yaml_utils__import_key(tmp_path, contents, key, value): + (tmp_path / "test.yml").write_text(contents) + assert yaml_utils.import_key(tmp_path, Path("test.yml"), key) == value + + +def test_yaml_utils__resolve_all_imports(tmp_path): + (tmp_path / "misc.yml").write_text("et_cetera:\n first_value: David\n second_value: Excelsior") + + (tmp_path / "test1.yml").write_text("test:\n <<: { import: misc.yml, key: et_cetera.second_value }") + test_1_contents = yaml_utils.flatten_merge_keys(yaml.safe_load((tmp_path / "test1.yml").read_text())) + + assert yaml_utils.resolve_all_imports(tmp_path, test_1_contents) == {"test": "Excelsior"} + + (tmp_path / "test2.yml").write_text("test:\n <<: { import: misc.yml, key: et_cetera }") + test_2_contents = yaml_utils.flatten_merge_keys(yaml.safe_load((tmp_path / "test2.yml").read_text())) + + assert yaml_utils.resolve_all_imports(tmp_path, test_2_contents) == { + "test": {"first_value": "David", "second_value": "Excelsior"} + } + + (tmp_path / "test3.yml").write_text("test:\n <<: { import: misc.yml }") + test_2_contents = yaml_utils.flatten_merge_keys(yaml.safe_load((tmp_path / "test3.yml").read_text())) + + assert yaml_utils.resolve_all_imports(tmp_path, test_2_contents) == { + "test": {"et_cetera": {"first_value": "David", "second_value": "Excelsior"}} + }