sets up code splitting using yaml imports (#4)

* sets up code splitting using yaml imports

* making `key` optional, fixing order of ops in yaml processing

* doing some introspective metadocking on jinja helpers

* some reorg, metadock-generation of snippets

* fixing precommit

* starting the process of adding jinja docs to readme using metadock

* completing html.summary jinja helper metadock

* fixing pre-commit install

* trying double linebreaking code blocks

* linebreaking code blocks in readme

* reverting to md.converted code blocks

* reverting to html.inlined code blocks in tables

* fleshing out jinja helper metadocks

* minor metadock correction in jinja helper section

* trying converting the tables

* committing test

* committing pre_test

* using html.pre for multiline code blocks

* fixing readme multiline code blocks

* escaping grave in metadock

* adding docs for yaml imports to README

* fixing import key typo in readme ex project

.gitignore

@@ -1,5 +1,5 @@
 poetry.lock
-.vscode/
+.vscode/settings.json
 **/__pycache__/
 .pytest_cache/
 .hypothesis/

.metadock/content_schematics/cli.yml

@@ -0,0 +1,45 @@
+cli:
+  usage_string: |
+    usage: metadock [-h] [-p PROJECT_DIR] {init,validate,build,list,clean} ...
+
+    Generates and formats Jinja documentation templates from yaml sources.
+
+    positional arguments:
+      {init,validate,build,list,clean}
+                            Metadock command
+        init                Initialize a new Metadock project in a folder which does not currently have one.
+        validate            Validate the structure of an existing Metadock project.
+        build               Build a Metadock project, rendering some or all documents.
+        list                List all recognized documents which can be generated from a given selection.
+        clean               Cleans the generated_documents directory for the Metadock project.
+
+    options:
+      -h, --help            show this help message and exit
+      -p PROJECT_DIR, --project-dir PROJECT_DIR
+                            Project directory containing a .metadock directory.
+  
+  commands:
+    init:
+      description: Used to initialize a fresh Metadock project in a folder which does not currently have one.
+      usage: metadock [-p PROJECT_DIR] init
+      python_interface: { import: python_interfaces.yml, key: python_interfaces.init }
+
+    validate:
+      description: Used to validate the structure of an existing Metadock project.
+      usage: metadock [-p PROJECT_DIR] validate
+      python_interface: { import: python_interfaces.yml, key: python_interfaces.validate }
+
+    build:
+      description: Used to build a Metadock project, rendering some or all documents.
+      usage: metadock [-p PROJECT_DIR] build [-s SCHEMATIC_GLOBS [SCHEMATIC_GLOBS ...]] [-t TEMPLATE_GLOBS [TEMPLATE_GLOBS ...]]
+      python_interface: { import: python_interfaces.yml, key: python_interfaces.build }
+
+    list:
+      description: Used to list all recognized documents which can be generated from a given selection.
+      usage: metadock [-p PROJECT_DIR] list [-s SCHEMATIC_GLOBS [SCHEMATIC_GLOBS ...]] [-t TEMPLATE_GLOBS [TEMPLATE_GLOBS ...]]
+      python_interface: { import: python_interfaces.yml, key: python_interfaces.list }
+
+    clean:
+      description: Used to clean the generated_documents directory for the Metadock project.
+      usage: metadock [-p PROJECT_DIR] clean
+      python_interface: { import: python_interfaces.yml, key: python_interfaces.clean }

.metadock/content_schematics/documents/repo_readme.yml

@@ -0,0 +1,26 @@
+content_schematics:
+  - name: README
+    template: repo_readme_template.md
+    target_formats: [md+html, md]
+    context:
+      # Hyperlinks in readme
+      links:
+        Jinja2: "[Jinja2](https://github.com/pallets/jinja)"
+        marko: "[marko](https://github.com/frostming/marko)"
+
+      <<:
+        # Import example project configuration
+        - import: example_project.yml
+        # Import CLI information
+        - import: cli.yml
+        # Import Jinja2 helpers
+        - import: jinja_helpers.yml
+
+      # Target formats
+      target_formats:
+        md+html:
+          description: Generates the given template, parses it into a markdown document, and then generates HTML from it.
+
+      # Project author(s)
+      authors:
+        - David Sillman <[email protected]>

.metadock/content_schematics/documents/vscode_jinja_md_metadock_snippets.yml

@@ -0,0 +1,9 @@
+content_schematics:
+
+  - name: jinja-md
+    template: vscode_jinja_md_metadock.code-snippets
+    target_formats: [ code-snippets ]
+    context:
+
+      # Import jinja helpers context
+      <<: { import: jinja_helpers.yml }

.metadock/content_schematics/example_project.yml

@@ -0,0 +1,159 @@
+example_project:
+  structure: |
+    MyProject/
+      - ...
+      - <my project content>
+      - ...
+      - .metadock/
+          - templated_documents/
+              - gitlab_mr_template.md
+          - content_schematics/
+              - gitlab_mr__feature1.yml
+              - gitlab_mr__otherfeature.yml
+          - generated_documents/
+              - gitlab_mr__feature1.md
+              - gitlab_mr__feature1.html
+              - gitlab_mr__otherfeature.md
+              - gitlab_mr__otherfeature.html
+
+  templated_documents:
+    gitlab_mr_template.md: |
+      {%- set jira_project_name = jira.get('project_name') -%}
+      {%- set jira_project_id = jira.get('project_id') -%}
+      {%- set jira_ticket_num = jira.get('ticket_num') -%}
+      {%- set jira_ticket_id = jira_project_name ~ "-" ~ jira_ticket_num -%}
+      {%- set mr_summary = merge_request.get('summary') -%}
+      # [{{ jira_ticket_id }}] {{ mr_summary }}
+
+      Welcome to my MR. Some of the changes are listed below:
+
+      {% for change in merge_request.get('changes', []) -%}
+      {{ loop.index }}. {{ change }}{{ "\n" if not loop.last else "" }}
+      {%- endfor %}
+
+      {% if merge_request.get('breaking_changes') -%}
+      In addition to the changes above, there are also a few breaking changes introduced in this MR:
+
+      {% for breaking_change in merge_request.get('breaking_changes') -%}
+      - {{ breaking_change.get('summary') }}
+        - **Affected downstream stakeholders**: {{ breaking_change.get('affected_downstream', [{'id': 'None'}]) | map(attribute='id') | join(", ") }}.
+        - **Suggested remedy**: {{ breaking_change.get('suggested_remedy', 'None') }}{{ "\n" if not loop.last else "" }}
+      {%- endfor -%}
+      {%- endif %}
+
+      For more information, please check out the Jira ticket associated with this MR, {{ jira_ticket_id }}.
+
+  content_schematics:
+    gitlab_mr__feature1.yml: |
+      content_schematics:
+
+      - name: gitlab_mr__feature1
+        template: gitlab_mr_template.md
+        target_formats: [ md+html, md ]
+
+        context:
+
+          jira:
+            project_name: "IGDP"
+            project_id: "12001"
+            ticket_num: "13"
+
+          merge_request:
+            summary: Adding software version as hard requirement for staging
+            changes:
+              - "Added software version to staging model."
+              - "Added unit tests for valid software version, invalid software version, missing software version."
+            breaking_changes:
+              - summary: "Dropping all records which are missing software version."
+                affected_downstream:
+                  - id: Service
+                    email: [email protected]
+                  - id: Analytics
+                    email: [email protected]
+                suggested_remedy: |
+                  - Drop all records which are missing software version.
+                  - Add software version as a hard requirement for staging.
+
+    import_key_examples.yml: |
+      content_schematics:
+
+      - name: alerting_project_proposal
+        template: airflow_project_proposal_template.md
+        target_formats: [ md+html, md ]
+
+        context:
+
+          jira:
+
+            # "block" syntax for importing a root-level key "IGDP"
+            project:
+              import: jira/projects.yml
+              key: IGDP
+
+            # "flow" syntax for importing a sub-key, "David_Sillman" inside "eng_identity"
+            code_owners: 
+              - { import: jira/identities.yml, key: eng_identity.David_Sillman }
+
+            # "flow" syntax for importing a sub-key using a merge key ("<<"),
+            <<: { import: team_contexts/data.yml, key: resources.alerting_channels }
+
+            # "block" syntax for importing multiple subkeys from multiple files using a merge key,
+            <<:
+              - import: team_contexts/data_contacts.yml
+                key: contacts.email
+              - import: team_contexts/data_push_api.yml
+                key: push_api.contracts
+
+    import_key_examples_2.yml: |
+      content_schematics:
+
+      - name: confluence_docs_summary
+        template: confluence/data_docs/confluence_docs_summary_template.md
+        target_formats: [ md+html, md ]
+        context:
+
+          # "flow" syntax for a single whole-file import,
+          all_contracts: { import: confluence/data_docs/contracts.yml }
+
+          # "block" syntax for importing multiple whole files using a merge key,
+          <<:
+            - import: confluence/data_docs/projects.yml
+            - import: confluence/data_docs/sources.yml
+
+  generated_documents:
+    gitlab_mr__feature1.md: |
+      # [IGDP-13] Adding software version as hard requirement for staging
+
+      Welcome to my MR. Some of the changes are listed below:
+
+      1. Added software version to staging model.
+      2. Added unit tests for valid software version, invalid software version, missing software version.
+
+      In addition to the changes above, there are also a few breaking changes introduced in this MR:
+
+      - Dropping all records which are missing software version.
+        - **Affected downstream stakeholders**: Service, Analytics.
+        - **Suggested remedy**:
+          - Drop all records which are missing software version.
+          - Add software version as a hard requirement for staging.
+
+      For more information, please check out the Jira ticket associated with this MR, IGDP-13.
+
+    gitlab_mr__feature1.html: |
+      <h1>[IGDP-13] Adding software version as hard requirement for staging</h1>
+      <p>Welcome to my MR. Some of the changes are listed below:</p>
+      <ol>
+      <li>Added software version to staging model.</li>
+      <li>Added unit tests for valid software version, invalid software version, missing software version.</li>
+      </ol>
+      <p>In addition to the changes above, there are also a few breaking changes introduced in this MR:</p>
+      <ul>
+      <li>
+      Dropping all records which are missing software version.<ul>
+      <li><strong>Affected downstream stakeholders</strong>: Service, Analytics.</li>
+      <li><strong>Suggested remedy</strong>: Handle deletions manualy, using the software version column in the exposures to identify source records
+      which will be dropped, and drop them in the target environment after our change is deployed.</li>
+      </ul>
+      </li>
+      </ul>
+      <p>For more information, please check out the Jira ticket associated with this MR, IGDP-13.</p>

.metadock/content_schematics/jinja_helpers.yml

@@ -0,0 +1,10 @@
+jinja_helpers:
+
+  <<:
+    # Import global macros, filters context
+    - import: jinja_helpers/global.yml
+    # TODO: import markdown macros, filters context
+    - import: jinja_helpers/md.yml
+    # TODO: import html macros, filters context
+    - import: jinja_helpers/html.yml
+

.metadock/content_schematics/jinja_helpers/global.yml

@@ -0,0 +1,122 @@
+global:
+  docstring: |
+    Jinja namespace for the global Metadock environment, including all global macros, filters, and namespaces.
+
+  macros:
+    debug:
+      docstring: Prints a debug message to stdout, and returns an empty string.
+      example: |
+        >>> from metadock.env import MetadockEnv
+        >>> env = MetadockEnv().jinja_environment()
+        >>> env.from_string("No changes!{{ debug('This is a debug message.') }}").render()
+        This is a debug message.
+        'No changes!'
+      source_file: metadock/env.py
+      method_name: metadock.env.MetadockEnv.debug
+      signature: "(self, message: str) -> None"
+      intellisense:
+        snippet_key: Debug message
+        snippet_body:
+          - debug($1)
+
+  filters:
+    chain:
+      docstring: |
+        Filter which flattens a sequence of iterables into a single iterable.
+      example: |
+        >>> from metadock.env import MetadockEnv
+        >>> env = MetadockEnv().jinja_environment()
+        >>> env.from_string('{{ {"first": 1, "second": 2}.items() | chain | join(" ") }}').render()
+        'first 1 second 2'
+      source_file: metadock/env.py
+      method_name: metadock.env.MetadockEnv.chain_filter
+      signature: "(self, iterables: Sequence[Iterable[Any]]) -> Iterable[Any]"
+      intellisense:
+        snippet_key: Chain iterables
+        snippet_body:
+          - chain
+
+    inline:
+      docstring: |
+        Filter which inlines a string by replacing all newlines with spaces, and all double spaces with single spaces.
+      example: |
+        >>> from metadock.env import MetadockEnv
+        >>> env = MetadockEnv().jinja_environment()
+        >>> env.from_string("{{ 'This is a multi-line string.\nThis is the second line.\nAnd the third.' | inline }}").render()
+        'This is a multi-line string. This is the second line. And the third.'
+      source_file: metadock/env.py
+      method_name: metadock.env.MetadockEnv.inline_filter
+      signature: "(self, value: str) -> str"
+      intellisense:
+        snippet_key: Inline text
+        snippet_body:
+          - inline
+
+    with_prefix:
+      docstring: |
+        Filter which prepends a prefix to a string, with an optional separator.
+      example: |
+        >>> from metadock.env import MetadockEnv
+        >>> env = MetadockEnv().jinja_environment()
+        >>> env.from_string("{{ 'This is a string.' | with_prefix('Prefix') }}").render()
+        'PrefixThis is a string.'
+        >>> env.from_string("{{ 'This is a string.' | with_prefix('Prefix: ', sep = ' : ') }}").render()
+        'Prefix : This is a string.'
+      source_file: metadock/env.py
+      method_name: metadock.env.MetadockEnv.with_prefix_filter
+      signature: "(self, value: str, prefix: str, sep: str = '') -> str"
+      intellisense:
+        snippet_key: With prefix
+        snippet_body:
+          - with_prefix($1)
+
+    with_suffix:
+      docstring: |
+        Filter which appends a suffix to a string, with an optional separator.
+      example: |
+        >>> from metadock.env import MetadockEnv
+        >>> env = MetadockEnv().jinja_environment()
+        >>> env.from_string("{{ 'This is a string' | with_suffix('Suffix') }}").render()
+        'This is a stringSuffix'
+        >>> env.from_string("{{ 'This is a string' | with_suffix('Suffix', sep = ' : ') }}").render()
+        'This is a string : Suffix'
+      source_file: metadock/env.py
+      method_name: metadock.env.MetadockEnv.with_suffix_filter
+      signature: "(self, value: str, suffix: str, sep: str = '') -> str"
+      intellisense:
+        snippet_key: With suffix
+        snippet_body:
+          - with_suffix($1)
+
+    wrap:
+      docstring: |
+        Filter which wraps an inner string with a given outer string.
+      example: |
+        >>> from metadock.env import MetadockEnv
+        >>> env = MetadockEnv().jinja_environment()
+        >>> # Wrap with graves, like md.code(...)
+        >>> env.from_string("{{ 'This is a string.' | wrap('\`') }}").render()
+        '\`This is a string.\`'
+      source_file: metadock/env.py
+      method_name: metadock.env.MetadockEnv.wrap_filter
+      signature: "(self, value: str, wrap: str) -> str"
+      intellisense:
+        snippet_key: Wrap text
+        snippet_body:
+          - wrap($1)
+
+    zip:
+      docstring: |
+        Filter which zips an input iterable with one or more iterables.
+      example: |
+        >>> from metadock.env import MetadockEnv
+        >>> env = MetadockEnv().jinja_environment()
+        >>> env.from_string("{{ ['a', 'b', 'c'] | zip([1, 2, 3]) | list }}").render()
+        "[('a', 1), ('b', 2), ('c', 3)]"
+      source_file: metadock/env.py
+      method_name: metadock.env.MetadockEnv.zip_filter
+      signature: "(self, input_iterable: Iterable[Any], *iterables: Iterable[Any]) -> Iterable[tuple[Any, ...]]"
+      intellisense:
+        snippet_key: Zip iterables
+        snippet_body:
+          - zip($1)