adding a patch incarnation endpoint which can update in incarnation o…

…nly partially

src/foxops/engine/update.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -11,20 +12,41 @@
 logger = get_logger(__name__)
 
 
+def _patch_template_data(data: TemplateData, patch: TemplateData) -> None:
+    """Patch the template data with the patch data (in-place).
+
+    Will iterate through the provided patch 1-by-1, deeply merging it into the "data" dict.
+    If a list is encountered in the patch, it will fully replace the list in the data dict (instead of appending).
+    """
+
+    for key, value in patch.items():
+        if isinstance(value, dict):
+            if key not in data:
+                data[key] = {}
+
+            _patch_template_data(data[key], value)
+        else:
+            data[key] = value
+
+
 async def update_incarnation_from_git_template_repository(
     template_git_repository: Path,
     update_template_repository_version: str,
     update_template_data: TemplateData,
     incarnation_root_dir: Path,
     diff_patch_func,
+    patch_data: bool = False,
 ) -> tuple[bool, IncarnationState, PatchResult | None]:
     """
     Update an incarnation with a new version of a template.
 
     The process works roughly like this:
     * create a git worktree from the template repository version ('v1') that is currently in use by the incarnation
     * create a git worktree from the template repository version ('v2') that the incarnation should be updated to
-
+    * initialize two _temporary_ incarnations from both template versions into separate directories:
+      - the old template version with the data that is currently in use by the incarnation
+      - the new template version with the data that was provided for the update
+    * diff the two incarnations - then apply the patch on the actual incarnation that should be updated
     """
     if update_template_repository_version.startswith("-"):
         raise ValueError(
@@ -68,6 +90,7 @@ async def update_incarnation_from_git_template_repository(
             updated_template_data=update_template_data,
             incarnation_root_dir=incarnation_root_dir,
             diff_patch_func=diff_patch_func,
+            patch_data=patch_data,
         )
 
 
@@ -78,13 +101,23 @@ async def update_incarnation(
     updated_template_data: TemplateData,
     incarnation_root_dir: Path,
     diff_patch_func,
+    patch_data: bool = False,
 ) -> tuple[bool, IncarnationState, PatchResult | None]:
-    """Update an incarnation with a new version of a template."""
+    """Update an incarnation with a new version of a template.
+
+    If patch_data is True, the updated_template_data will be merged into the current template data.
+    """
 
     # initialize pristine incarnation from current incarnation state
     incarnation_state_path = incarnation_root_dir / ".fengine.yaml"
     incarnation_state = IncarnationState.from_file(incarnation_state_path)
 
+    if patch_data:
+        template_data = deepcopy(incarnation_state.template_data)
+        _patch_template_data(template_data, updated_template_data)
+    else:
+        template_data = updated_template_data
+
     with TemporaryDirectory() as incarnation_v1_dir, TemporaryDirectory() as incarnation_v2_dir:
         logger.debug(
             "initialize pristine incarnation from current incarnation state",
@@ -115,7 +148,7 @@ async def update_incarnation(
             template_root_dir=updated_template_root_dir,
             template_repository=incarnation_state.template_repository,
             template_repository_version=updated_template_repository_version,
-            template_data=updated_template_data,
+            template_data=template_data,
             incarnation_root_dir=Path(incarnation_v2_dir),
         )
 

src/foxops/routers/incarnations.py

@@ -1,5 +1,7 @@
+from typing import Self
+
 from fastapi import APIRouter, Depends, Response, status
-from pydantic import BaseModel
+from pydantic import BaseModel, model_validator
 
 from foxops.database.repositories.incarnation.errors import IncarnationNotFoundError
 from foxops.dependencies import get_change_service, get_hoster, get_incarnation_service
@@ -232,11 +234,52 @@ async def reset_incarnation(
     )
 
 
+async def _create_change(
+    incarnation_id: int,
+    requested_version: str | None,
+    requested_data: TemplateData,
+    automerge: bool,
+    patch: bool,
+    response: Response,
+    change_service: ChangeService,
+) -> IncarnationWithDetails | ApiError:
+    try:
+        await change_service.create_change_merge_request(
+            incarnation_id=incarnation_id,
+            requested_version=requested_version,
+            requested_data=requested_data,
+            automerge=automerge,
+            patch=patch,
+        )
+    except ProvidedTemplateDataInvalidError as e:
+        response.status_code = status.HTTP_400_BAD_REQUEST
+        error_messages = e.get_readable_error_messages()
+        return ApiError(
+            message=f"could not initialize the incarnation as the provided template data "
+            f"is invalid: {'; '.join(error_messages)}"
+        )
+    except IncarnationNotFoundError as exc:
+        response.status_code = status.HTTP_404_NOT_FOUND
+        return ApiError(message=str(exc))
+    except ChangeRejectedDueToPreviousUnfinishedChange:
+        response.status_code = status.HTTP_409_CONFLICT
+        return ApiError(message="There is a previous change that is still open. Please merge/close it first.")
+    except ChangeRejectedDueToNoChanges:
+        logger.info(
+            "A change was requested, but there were no changes to apply",
+            incarnation_id=incarnation_id,
+            requested_version=requested_version,
+            requested_data=requested_data,
+        )
+
+    return await change_service.get_incarnation_with_details(incarnation_id)
+
+
 class UpdateIncarnationRequest(BaseModel):
     """A DesiredIncarnationStatePatch represents the patch for the desired state of an incarnation."""
 
     template_repository_version: str
-    template_data: TemplateData = {}
+    template_data: TemplateData
 
     automerge: bool
 
@@ -245,7 +288,7 @@ class UpdateIncarnationRequest(BaseModel):
     "/{incarnation_id}",
     responses={
         status.HTTP_200_OK: {
-            "description": "The incarnation was successfully reconciled",
+            "description": "The incarnation was successfully updated",
             "model": IncarnationWithDetails,
         },
         status.HTTP_400_BAD_REQUEST: {
@@ -257,11 +300,11 @@ class UpdateIncarnationRequest(BaseModel):
             "model": ApiError,
         },
         status.HTTP_409_CONFLICT: {
-            "description": "The incarnation already has a reconciliation in progress",
+            "description": "The incarnation already has a change in progress",
             "model": ApiError,
         },
         status.HTTP_500_INTERNAL_SERVER_ERROR: {
-            "description": "The reconciliation failed",
+            "description": "The update failed",
             "model": ApiError,
         },
     },
@@ -279,35 +322,81 @@ async def update_incarnation(
     reusing the previously set variable values), use the PATCH endpoint instead.
     """
 
-    try:
-        await change_service.create_change_merge_request(
-            incarnation_id=incarnation_id,
-            requested_version=request.template_repository_version,
-            requested_data=request.template_data,
-            automerge=request.automerge,
-        )
-    except ProvidedTemplateDataInvalidError as e:
-        response.status_code = status.HTTP_400_BAD_REQUEST
-        error_messages = e.get_readable_error_messages()
-        return ApiError(
-            message=f"could not initialize the incarnation as the provided template data "
-            f"is invalid: {'; '.join(error_messages)}"
-        )
-    except IncarnationNotFoundError as exc:
-        response.status_code = status.HTTP_404_NOT_FOUND
-        return ApiError(message=str(exc))
-    except ChangeRejectedDueToPreviousUnfinishedChange:
-        response.status_code = status.HTTP_409_CONFLICT
-        return ApiError(message="There is a previous change that is still open. Please merge/close it first.")
-    except ChangeRejectedDueToNoChanges:
-        logger.info(
-            "A change was requested, but there were no changes to apply",
-            incarnation_id=incarnation_id,
-            requested_version=request.template_repository_version,
-            requested_data=request.template_data,
-        )
+    return await _create_change(
+        incarnation_id=incarnation_id,
+        requested_version=request.template_repository_version,
+        requested_data=request.template_data,
+        automerge=request.automerge,
+        patch=False,
+        response=response,
+        change_service=change_service,
+    )
 
-    return await change_service.get_incarnation_with_details(incarnation_id)
+
+class PatchIncarnationRequest(BaseModel):
+    """A DesiredIncarnationStatePatch represents the patch for the desired state of an incarnation."""
+
+    requested_version: str | None = None
+    requested_data: TemplateData | None = None
+
+    automerge: bool
+
+    @model_validator(mode="after")
+    def check_either_version_or_data_change_requested(self) -> Self:
+        if self.requested_version is None and self.requested_data is None:
+            raise ValueError("Either requested_version or requested_data must be set")
+
+        return self
+
+
+@router.patch(
+    "/{incarnation_id}",
+    responses={
+        status.HTTP_200_OK: {
+            "description": "The incarnation was successfully updated",
+            "model": IncarnationWithDetails,
+        },
+        status.HTTP_400_BAD_REQUEST: {
+            "description": "The desired incarnation state was not valid",
+            "model": ApiError,
+        },
+        status.HTTP_404_NOT_FOUND: {
+            "description": "The incarnation was not found in the inventory",
+            "model": ApiError,
+        },
+        status.HTTP_409_CONFLICT: {
+            "description": "The incarnation already has a change in progress",
+            "model": ApiError,
+        },
+        status.HTTP_500_INTERNAL_SERVER_ERROR: {
+            "description": "The update failed",
+            "model": ApiError,
+        },
+    },
+)
+async def patch_incarnation(
+    response: Response,
+    incarnation_id: int,
+    request: PatchIncarnationRequest,
+    change_service: ChangeService = Depends(get_change_service),
+):
+    """Updates the incarnation to the given version and data.
+
+    Other than the PUT endpoint, this endpoint allows to only update a subset of the incarnation state (e.g. only
+    the template version (without having to respecify all template data) or only individual template data values
+    """
+
+    requested_data = request.requested_data or {}
+
+    return await _create_change(
+        incarnation_id=incarnation_id,
+        requested_version=request.requested_version,
+        requested_data=requested_data,
+        automerge=request.automerge,
+        patch=True,
+        response=response,
+        change_service=change_service,
+    )
 
 
 @router.delete(

src/foxops/services/change.py

@@ -296,7 +296,9 @@ async def create_change_direct(
 
         # https://youtrack.jetbrains.com/issue/PY-36444
         env: _PreparedChangeEnvironment
-        async with self._prepared_change_environment(incarnation_id, requested_version, requested_data) as env:
+        async with self._prepared_change_environment(
+            incarnation_id, requested_version, requested_data, patch=False
+        ) as env:
             # because we want to apply the change without an MR
             # ... let's merge the change directly into the default branch
             await env.incarnation_repository.checkout_branch(env.incarnation_repository_default_branch)
@@ -326,9 +328,10 @@ async def create_change_direct(
     async def create_change_merge_request(
         self,
         incarnation_id: int,
-        requested_version: str,
+        requested_version: str | None,
         requested_data: TemplateData,
         automerge: bool = False,
+        patch: bool = False,
     ) -> ChangeWithMergeRequest:
         """
         Perform a MERGE_REQUEST change on the given incarnation.
@@ -339,7 +342,9 @@ async def create_change_merge_request(
 
         # https://youtrack.jetbrains.com/issue/PY-36444
         env: _PreparedChangeEnvironment
-        async with self._prepared_change_environment(incarnation_id, requested_version, requested_data) as env:
+        async with self._prepared_change_environment(
+            incarnation_id, requested_version, requested_data, patch=patch
+        ) as env:
             change_in_db = await self._change_repository.create_change(
                 incarnation_id=incarnation_id,
                 revision=env.expected_revision,
@@ -578,19 +583,29 @@ async def get_incarnation_with_details(self, incarnation_id: int) -> Incarnation
 
     @asynccontextmanager
     async def _prepared_change_environment(
-        self, incarnation_id: int, requested_version: str, requested_data: TemplateData
+        self, incarnation_id: int, requested_version: str | None, requested_data: TemplateData, patch: bool
     ) -> AsyncIterator[_PreparedChangeEnvironment]:
         """
         This method checks out the incarnation repository, prepares a branch that contains the update and commits.
+
+        If patch is True:
+        - the requested_version can be None, which results in using the same version that is currently applied
+        - the requested_data can be a subset of the required template variables. The provided values
+          will then be added to those that are currently already in use when rendering the incarnation.
         """
-        incarnation = await self._incarnation_repository.get_by_id(incarnation_id)
 
-        if incarnation.template_repository is None:
-            raise Exception("upgrade failed. Should not happen.")
+        incarnation = await self._incarnation_repository.get_by_id(incarnation_id)
 
         # if the previous change was of type merge request and is still open, we dont want to continue
         last_change = await self.get_latest_change_for_incarnation_if_completed(incarnation_id)
 
+        if patch:
+            to_version = requested_version or last_change.requested_version
+        else:
+            if requested_version is None:
+                raise ValueError("requested_version must be set if patch is False")
+            to_version = requested_version
+
         incarnation_repo_metadata = await self._hoster.get_repository_metadata(incarnation.incarnation_repository)
 
         async with (
@@ -600,7 +615,7 @@ async def _prepared_change_environment(
             branch_name = generate_foxops_branch_name(
                 prefix="update-to",
                 target_directory=incarnation.target_directory,
-                template_repository_version=requested_version,
+                template_repository_version=to_version,
             )
             await local_incarnation_repository.create_and_checkout_branch(branch_name, exist_ok=False)
 
@@ -610,28 +625,27 @@ async def _prepared_change_environment(
                 patch_result,
             ) = await fengine.update_incarnation_from_git_template_repository(
                 template_git_repository=local_template_repository.directory,
-                update_template_repository_version=requested_version,
+                update_template_repository_version=to_version,
                 update_template_data=requested_data,
                 incarnation_root_dir=(local_incarnation_repository.directory / incarnation.target_directory),
                 diff_patch_func=fengine.diff_and_patch,
+                patch_data=patch,
             )
 
             if not update_performed:
                 raise ChangeRejectedDueToNoChanges()
             if patch_result is None:
                 raise ChangeFailed("Patch result was None. That is unexpected at this stage.")
 
-            await local_incarnation_repository.commit_all(
-                f"foxops: updating incarnation to version {requested_version}"
-            )
+            await local_incarnation_repository.commit_all(f"foxops: updating incarnation to version {to_version}")
             commit_sha = await local_incarnation_repository.head()
 
             yield _PreparedChangeEnvironment(
                 incarnation_repository=local_incarnation_repository,
                 incarnation_repository_identifier=incarnation.incarnation_repository,
                 incarnation_repository_default_branch=incarnation_repo_metadata["default_branch"],
                 to_version_hash=await local_template_repository.head(),
-                to_version=requested_version,
+                to_version=to_version,
                 to_data=incarnation_state.template_data,
                 to_data_full=incarnation_state.template_data_full,
                 expected_revision=last_change.revision + 1,