add support for complex variables (lists, nested objects) (#427)

* * fully remove the no longer functioning "allow_import" variable on the create incarnation endpoint * remove support for fvars file * add support for complex template variables (lists, nested objects) * proper handling of template variables with defaults in incarnations (default values will not be fixed anymore - if a default is not specified explictly when creating an incarnation, it will be changed on an incarnation update, if the default changed in the template) * fixed handling of legacy incarnations and default values for nested variables * fixed incorrect responses from the API * avoid storing additionally provided template variables in the incarnation state ... if they are not defined in the template interface * updated docs and fixed a small bug
Roche · Nov 8, 2023 · 8d3b60e · 8d3b60e
1 parent 0f63368
commit 8d3b60e
Show file tree

Hide file tree

Showing 43 changed files with 1,864 additions and 1,454 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -70,6 +70,7 @@ Once you have all this, add the following environment variables and rerun the te
 ```shell
 # defaults to "https://gitlab.com" if not specified
 export FOXOPS_TESTS_GITLAB_ADDRESS=<address of the Gitlab instance>
+export FOXOPS_TESTS_GITLAB_ROOT_GROUP_ID=<ID of the root group>
 export FOXOPS_TESTS_GITLAB_TOKEN=<access token that can create projects>
 
 # these variables can also be set in a file called `.env.test` in the root folder of the project

diff --git a/alembic/versions/2023_11_04-00ee97d0b7a3_adding_column_for_keeping_track_of_full_.py b/alembic/versions/2023_11_04-00ee97d0b7a3_adding_column_for_keeping_track_of_full_.py
@@ -0,0 +1,32 @@
+"""adding column for keeping track of full template data
+
+Revision ID: 00ee97d0b7a3
+Revises: 001f927357ef
+Create Date: 2023-11-04 16:14:57.823773+00:00
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "00ee97d0b7a3"
+down_revision = "001f927357ef"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("change", sa.Column("template_data_full", sa.String(), nullable=True))
+    op.execute("UPDATE change SET template_data_full = requested_data")
+
+    with op.batch_alter_table("change") as batch_op:
+        batch_op.alter_column("template_data_full", nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("change", "template_data_full")
+    # ### end Alembic commands ###
diff --git a/docs/source/configfile_reference.md b/docs/source/configfile_reference.md
@@ -13,13 +13,65 @@ rendering:
 
 variables:
   application_name:
-    type: str
+    type: string
     description: Name of the application. Don't use spaces.
   author:
-    type: str
+    type: string
     description: Name of the author. Use format "Name <email>".
   index:
-    type: str
+    type: string
     description: The PyPI index
     default: pypi.org
 ```
+
+### Variable Definitions
+
+The `variables` section defines the variables that are available to the template. Each variable has a name, a type and a description.
+
+The following basic types are supported:
+
+* `string`: A string value
+* `integer`: An integer value
+* `boolean`: A boolean value
+
+It is mandatory to give a description for every variable definition. A default value can also be specified as shown in the example above.
+
+#### List Variables
+
+A variable can also be a list of values. For now, only string types are supported. Example:
+
+```yaml
+variables:
+  authors:
+    type: list
+    element_type: string
+    
+    description: List of authors
+    
+    default:
+      - John Doe
+      - Jane Doe
+```
+
+#### Nested Object Variables
+
+Variables can be nested arbitrarily deep. Default values can not be specified on the variables of type object itself - instead specify them on the nested variables. Example:
+
+```yaml
+variables:
+  address:
+    type: object
+    description: Address of the author
+    
+    children:
+      street:
+        type: string
+        description: Street name
+      number:
+        type: integer
+        description: Street number
+      city:
+        type: string
+        description: City name
+        default: Zurich
+```
diff --git a/docs/source/tutorials/backwards-compatible-template-changes.md b/docs/source/tutorials/backwards-compatible-template-changes.md
@@ -10,7 +10,7 @@ Specifying the `default` effectively makes the variable *optional*.
 ```yaml
 variables:
   new_var:
-    type: str
+    type: string
     description: New variable for the template
     default: "Hello"
 ```

diff --git a/docs/source/tutorials/write-template-from-scratch.md b/docs/source/tutorials/write-template-from-scratch.md
@@ -46,7 +46,7 @@ Variables can be defined with the following syntax:
 ```yaml
 variables:
     <variable_1_name>:
-        type: str | int | float
+        type: string | integer | boolean
         description: <some description for this variable>
         default: <an optional default value>
 ```
@@ -57,11 +57,13 @@ with the default `Jane Doe` use the following:
 ```yaml
 variables:
     name:
-        type: str
+        type: string
         description: The name of the author
         default: Jane Doe
 ```
 
+More variable types exist (like complex objects or lists): Read more about this in the [template configuration reference](../configfile_reference).
+
 ## Exclude Files from Rendering
 
 Sometimes you don't want to render every file in `template/`, but only copy&paste them as-is

diff --git a/docs/source/usage.md b/docs/source/usage.md
@@ -105,6 +105,3 @@ You can update the `mycat` incarnation using the following command:
 ```sh
 fengine update mycat/ -u v2.0.0
 ```
-
-You may also provide other values for the template data like the `author` variable.
-```
diff --git a/poetry.lock b/poetry.lock
diff --git a/src/foxops/database/repositories/change.py b/src/foxops/database/repositories/change.py
@@ -54,6 +54,8 @@ class ChangeInDB(BaseModel):
     requested_version: str
     requested_data: str
 
+    template_data_full: str
+
     merge_request_id: str | None
     merge_request_branch_name: str | None
     model_config = ConfigDict(from_attributes=True)
@@ -98,6 +100,7 @@ async def create_change(
         requested_version_hash: str,
         requested_version: str,
         requested_data: str,
+        template_data_full: str,
         merge_request_id: str | None = None,
         merge_request_branch_name: str | None = None,
     ) -> ChangeInDB:
@@ -122,6 +125,7 @@ async def create_change(
                     requested_version_hash=requested_version_hash,
                     requested_version=requested_version,
                     requested_data=requested_data,
+                    template_data_full=template_data_full,
                     commit_sha=commit_sha,
                     commit_pushed=commit_pushed,
                     merge_request_id=merge_request_id,
@@ -148,6 +152,7 @@ async def create_incarnation_with_first_change(
         requested_version_hash: str,
         requested_version: str,
         requested_data: str,
+        template_data_full: str,
     ) -> ChangeInDB:
         async with self.engine.begin() as conn:
             query_insert_incarnation = (
@@ -172,6 +177,7 @@ async def create_incarnation_with_first_change(
                     requested_version_hash=requested_version_hash,
                     requested_version=requested_version,
                     requested_data=requested_data,
+                    template_data_full=template_data_full,
                     commit_sha=commit_sha,
                     commit_pushed=False,
                 )

diff --git a/src/foxops/database/schema.py b/src/foxops/database/schema.py
@@ -34,6 +34,7 @@
     Column("requested_version_hash", String, nullable=False),
     Column("requested_version", String, nullable=False),
     Column("requested_data", String, nullable=False),
+    Column("template_data_full", String, nullable=False),
     Column("commit_sha", String, nullable=False),
     Column("commit_pushed", Boolean, nullable=False),
     # fields for merge request changes

diff --git a/src/foxops/engine/__init__.py b/src/foxops/engine/__init__.py
@@ -4,15 +4,19 @@
 It also handles updates of said templates.
 """
 
-from foxops.engine.fvars import FVARS_FILENAME  # noqa
-from foxops.engine.initialization import initialize_incarnation  # noqa
-from foxops.engine.models import IncarnationState  # noqa
-from foxops.engine.models import TemplateData  # noqa
-from foxops.engine.models import load_incarnation_state  # noqa
-from foxops.engine.models import load_incarnation_state_from_string  # noqa
-from foxops.engine.models import save_incarnation_state  # noqa
-from foxops.engine.patching.git_diff_patch import diff_and_patch  # noqa
-from foxops.engine.update import (  # noqa
+from foxops.engine.initialization import initialize_incarnation
+from foxops.engine.models.incarnation_state import IncarnationState, TemplateData
+from foxops.engine.patching.git_diff_patch import diff_and_patch
+from foxops.engine.update import (
     update_incarnation,
     update_incarnation_from_git_template_repository,
 )
+
+__all__ = [
+    "initialize_incarnation",
+    "update_incarnation",
+    "update_incarnation_from_git_template_repository",
+    "diff_and_patch",
+    "IncarnationState",
+    "TemplateData",
+]
diff --git a/src/foxops/engine/__main__.py b/src/foxops/engine/__main__.py
@@ -1,20 +1,17 @@
 import asyncio
 import copy
 import logging
-from dataclasses import asdict
 from pathlib import Path
 from subprocess import PIPE, check_output
-from typing import Optional
+from typing import Annotated, Any, Optional
 
 import typer
 
+from foxops.engine import IncarnationState, TemplateData
 from foxops.engine.initialization import initialize_incarnation
-from foxops.engine.models import (
-    IncarnationState,
+from foxops.engine.models.template_config import (
+    StringVariableDefinition,
     TemplateConfig,
-    TemplateData,
-    VariableDefinition,
-    load_incarnation_state,
 )
 from foxops.engine.patching.git_diff_patch import diff_and_patch
 from foxops.engine.update import update_incarnation_from_git_template_repository
@@ -48,40 +45,54 @@ def cmd_new(
     # Create sample fengine.yaml
     template_config = TemplateConfig(
         variables={
-            "author": VariableDefinition(
-                type="str",
+            "author": StringVariableDefinition(
                 description="The author of the project",
             )
         }
     )
-    template_config.to_yaml(target_directory / "fengine.yaml")
+    template_config.save(target_directory / "fengine.yaml")
 
     # Create sample README in template directory
     (target_directory / "template").mkdir(0o755)
     (target_directory / "template" / "README.md").write_text("Created by {{ author }}\n")
 
 
+def parse_template_data_arguments(raw_template_data: list[str]) -> TemplateData:
+    template_data: dict[str, Any] = {}
+    for arg in raw_template_data:
+        key, value = arg.split("=", maxsplit=1)
+
+        key_elements = key.split(".")
+
+        # descend into the correct sub-dict of the template data object
+        current_level = template_data
+        for key_element in key_elements[:-1]:
+            current_level = current_level.setdefault(key_element, {})
+
+        # now we can inject the value to the "current level"
+        if key_elements[-1].endswith("[]"):  # in case we need to append to a list
+            current_level.setdefault(key_elements[-1][:-2], []).append(value)
+        else:  # or we just want to add a "flat" value
+            current_level[key_elements[-1]] = value
+
+    return template_data
+
+
 @app.command(name="initialize")
 def cmd_initialize(
-    template_repository: Path = typer.Argument(  # noqa: B008
-        ...,
-        exists=True,
-        file_okay=False,
-        dir_okay=True,
-        readable=True,
-        resolve_path=True,
-    ),
-    incarnation_dir: Path = typer.Argument(  # noqa: B008
-        ...,
-        exists=False,
-        file_okay=False,
-        dir_okay=False,
-    ),
+    template_repository: Annotated[
+        Path, typer.Argument(..., exists=True, file_okay=False, dir_okay=True, readable=True, resolve_path=True)
+    ],
+    incarnation_dir: Annotated[
+        Path, typer.Argument(..., exists=False, file_okay=False, dir_okay=False, resolve_path=True)
+    ],
     raw_template_data: list[str] = typer.Option(  # noqa: B008
         [],
         "--data",
         "-d",
-        help="Template data variables in the format of `key=value`",
+        help="Template data variables in the format of `key=value`. "
+        "To add to list values, use -d key[]=value1 -d key[]=value2. "
+        "To add to nested values, use -d key.subkey=value.",
     ),
     template_repository_version: Optional[str] = typer.Option(  # noqa: B008
         None,
@@ -90,7 +101,7 @@ def cmd_initialize(
     ),
 ):
     """Initialize an incarnation repository with a version of a template and some data."""
-    template_data: TemplateData = dict(tuple(x.split("=", maxsplit=1)) for x in raw_template_data)  # type: ignore
+    template_data: TemplateData = parse_template_data_arguments(raw_template_data)
 
     bind(template_repository=template_repository)
     bind(incarnation_dir=incarnation_dir)
@@ -153,7 +164,9 @@ def cmd_update(
         [],
         "--data",
         "-d",
-        help="Template data variables in the format of `key=value`",
+        help="Template data variables in the format of `key=value`. "
+        "To add to list values, use -d key[]=value1 -d key[]=value2. "
+        "To add to nested values, use -d key.subkey=value.",
     ),
     remove_template_data: list[str] = typer.Option(  # noqa: B008
         [],
@@ -174,11 +187,11 @@ def cmd_update(
     ),
 ):
     """Initialize an incarnation repository with a version of a template and some data."""
-    template_data: dict[str, str] = dict(tuple(x.split("=", maxsplit=1)) for x in raw_template_data)  # type: ignore
+    template_data: TemplateData = parse_template_data_arguments(raw_template_data)
 
     incarnation_state_path = incarnation_dir / ".fengine.yaml"
     logger.debug(f"getting template repository path from incarnation state at {incarnation_state_path}")
-    incarnation_state = load_incarnation_state(incarnation_state_path)
+    incarnation_state = IncarnationState.from_file(incarnation_state_path)
 
     logger.debug(
         f"loaded incarnation state from incarnation repository {incarnation_state_path}",
@@ -187,12 +200,7 @@ def cmd_update(
 
     if overridden_template_repository is not None:
         logger.debug(f"overriding template repository with {overridden_template_repository}")
-        incarnation_state = IncarnationState(
-            **{
-                **asdict(incarnation_state),  # type: ignore
-                "template_repository": str(overridden_template_repository),
-            }
-        )
+        incarnation_state.template_repository = str(overridden_template_repository)
 
     if not Path(incarnation_state.template_repository).is_dir():
         logger.error(

diff --git a/src/foxops/engine/errors.py b/src/foxops/engine/errors.py
@@ -0,0 +1,24 @@
+from pydantic import ValidationError
+
+from foxops.errors import FoxopsUserError
+
+
+class ProvidedTemplateDataInvalidError(FoxopsUserError):
+    def get_readable_error_messages(self) -> list[str]:
+        if not isinstance(self.__cause__, ValidationError):
+            raise RuntimeError(
+                "exception was not chained. Must be raised (raise ... from ...) " "with a ValidationError as cause"
+            )
+
+        validation_error = self.__cause__
+
+        error_messages: list[str] = []
+        for e in validation_error.errors():
+            match e:
+                case {"type": "missing"}:
+                    location = ".".join(map(lambda x: str(x), e["loc"]))
+                    error_messages.append(f"'{location}' - no value was provided for this required template variable")
+                case _:
+                    error_messages.append(str(e))
+
+        return error_messages
-Original file line number
+Diff line change
@@ Expand Up @@
     ```sh
     fengine update mycat/ -u v2.0.0
     ```
-    You may also provide other values for the template data like the `author` variable.
-    ```