diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ff0be2d..ba9c1fa6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,8 +91,13 @@ jobs: # NOTE: we are unable to solely rely on pip because poetry doesn't export the dev dependencies # as extras nor is it able to install ONLY the dev dependencies ... python -m poetry install - - run: python -m pytest + - run: python -m pytest --ignore=tests/e2e env: + # FoxOps test configuration + FOXOPS_GITLAB_ADDRESS: https://nonsense.com/api/v4 + FOXOPS_GITLAB_TOKEN: nonsense + + # Test runner configuration COVERAGE_FILE: .coverage.${{ matrix.python-version }} - name: Upload coverage data diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..964700ff --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,13 @@ +# Contributing + +This project uses [Poetry](https://python-poetry.org/) +and [poetry-dynamic-versioning](https://pypi.org/project/poetry-dynamic-versioning/) +for dependency management and the package building process. + +## Local Development + +tbd. + +### Documentation + +run `make live` in the `docs/` subfolder to start a web server that hosts a live-build of the documentation. It even auto-reloads in case of changes! \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index cfdc78d3..a833195e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ +# ================= BUILD ================== FROM python:3.10-alpine AS builder # Install the build system -RUN apk add --update git gcc musl-dev +RUN apk add --update git RUN python -m pip install -U pip wheel RUN python -m pip install build @@ -14,12 +15,14 @@ RUN python -m build --wheel . # =============== PRODUCTION =============== -FROM builder AS production +FROM python:3.10-alpine # Copy the build artifact COPY --from=builder /build/dist/*.whl /tmp # Install the application +RUN apk add --update git gcc musl-dev RUN python -m pip install /tmp/*.whl +RUN rm -f /tmp/*.whl ENTRYPOINT [ "foxops" ] \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index df74495b..77cb8ed5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ -# foxops +# foxops 🦊 -coming soon, stay tuned! 🦊 +A modest templating tool to keep your projects up-to-date. + +more coming soon, stay tuned! 🚧 \ No newline at end of file diff --git a/docs/source/advanced_usage.md b/docs/source/advanced_usage.md new file mode 100644 index 00000000..b185ae0d --- /dev/null +++ b/docs/source/advanced_usage.md @@ -0,0 +1,87 @@ +# Advanced Usage + +## Authentication to Git Repositories + +tbd. + +## Deployment of foxops + +Foxops is typically deployed in a GitLab CI pipeline. The fundamental approach here is to also use the GitOps principle for the configuration of your desired template incarnations. + +As a general approach we would recommend to setup the Git repository like this: + +```text +/v1/ +.gitlab-ci.yml +``` + +The `v1` folder is here to indicate that the contained incarnation definition files follow the foxops v1 schema. It will help to future-proof the repository, in case future versions of foxops will bring breaking changes. + +In the GitLab CI pipeline, the `foxops reconcile v1/` command can be used, to ensure that all incarnations match the desired state as defined in the files within the `v1/` directory. + +### Example .gitlab-ci.yaml + +Do note that foxops requires a GitLab API token. For the example above this token must be present in the `GITLAB_API_TOKEN` environment variable. + +```yaml +variables: + FOXOPS_VERSION: "1" + FOXOPS_GITLAB_TOKEN: $GITLAB_API_TOKEN + FILES_TO_RECONCILE: v1/ + +stages: + - run + +reconcile: + stage: run + + # resource_group prevents parallel executions of foxops. + # This can be removed, to allow executing multiple "foxops reconcile" runs in parallel + # => at the risk of undefined behavior, if multiple commits affecting the same files + # are created in short intervals + resource_group: main + + image: + name: ghcr.io/roche/foxops:$FOXOPS_VERSION + entrypoint: [""] + + # only run on the default branch + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + + # logic that collects the files that were added/modified in the commit. + # improves the execution speed of `foxops reconcile` by only running it on those files + before_script: + - | + if [[ -z ${FILES_TO_RECONCILE} ]];then + FILES_TO_RECONCILE=() + + for file in $(git diff-tree -M --diff-filter=AM --no-commit-id --name-only -r "${CI_COMMIT_SHA}"); do + if [[ "${file}" =~ ^(v1|v0.7).*\.(yaml|yml)$ ]] ; then + FILES_TO_RECONCILE[${#FILES_TO_RECONCILE[@]}]="${file}" + fi + done + fi + + echo "Files to reconcile: ${FILES_TO_RECONCILE[@]}" + + # gitlab-runners apparently always use `-e` causing the real `foxops` exit code + # to be ignored and always use `1` instead. We have to manually handle it ourselves. + script: + - | + if [[ ${#FILES_TO_RECONCILE[@]} == 0 ]];then + echo "No incarnation configuration changes, skip running foxops reconcile ..." + + exit 0 + fi + + foxops reconcile ${FILES_TO_RECONCILE[@]} || EXIT_CODE=$? + + exit $EXIT_CODE + + # foxops exists with code `2` if a reconciliation failed in a "known" way, + # and with `1` if it fails with an unhandled exceptions. + allow_failure: + exit_codes: + - 2 +``` diff --git a/docs/source/api.md b/docs/source/api.md new file mode 100644 index 00000000..0cb8e794 --- /dev/null +++ b/docs/source/api.md @@ -0,0 +1,44 @@ +# API + +While foxops was mostly intended to be used via the existing CLI applications, the entire functionality is also available Python functions that can be called from your custom application. + +```{eval-rst} +.. module:: foxops +``` + +## fengine API + +The following interfaces are exposed from the `foxops.fengine` package: + +### Models + +```{eval-rst} +.. autoclass:: foxops.engine.IncarnationState + :members: + +.. autofunction:: foxops.engine.load_incarnation_state + +.. autofunction:: foxops.engine.load_incarnation_state_from_string + +.. autofunction:: foxops.engine.save_incarnation_state +``` + +### Initialization + +```{eval-rst} +.. autofunction:: foxops.engine.initialize_incarnation +``` + +### Update + +```{eval-rst} +.. autofunction:: foxops.engine.update_incarnation + +.. autofunction:: foxops.engine.update_incarnation_from_git_template_repository +``` + +### Diff and Patch + +```{eval-rst} +.. autofunction:: foxops.engine.diff_and_patch +``` diff --git a/docs/source/configfile_reference.md b/docs/source/configfile_reference.md new file mode 100644 index 00000000..b66baab4 --- /dev/null +++ b/docs/source/configfile_reference.md @@ -0,0 +1,25 @@ +## Configuration Files + +### fengine.yaml File Reference + +The `fengine.yaml` file holds metadata that is required by fengine when rendering the template into an incarnation. Most importantly it contains definitions of template variables, but can also override some settings that affect the rendering process. + +An example `fengine.yaml` file looks like this: + +```yaml +rendering: + excluded_files: + - vendor/**/* + +variables: + application_name: + type: str + description: Name of the application. Don't use spaces. + author: + type: str + description: Name of the author. Use format "Name ". + index: + type: str + description: The PyPI index + default: pypi.org +``` diff --git a/docs/source/index.md b/docs/source/index.md index e5251c57..c4613ec6 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -1,3 +1,89 @@ -# foxops +```{toctree} +:hidden: +:maxdepth: 2 -🦊 \ No newline at end of file +installation +usage +tutorials/index +advanced_usage +``` + +```{toctree} +:hidden: +:maxdepth: 2 +:caption: References + +terminology +workflows +configfile_reference +api +``` + +# foxops - Templates for Git Repositories + +**foxops** helps to initialize Git repositories (or a subdirectory of a Git repository) with a file structure coming from a template. + +* It does not only cover initialization, but also helps to keep these "incarnations" up-to-date with any further changes that were applied to the template afterwards +* ... even when the incarnation was customized in the meantime! + +:::{note} +Before continuing to read this documentation, make sure to familiarize yourself with the foxops [terminology](terminology). +::: + +```{mermaid} +graph LR + template-->repo1(repo1) + template-->repo2a(repo2/subdir_a) + template-->repo2b(repo2/subdir_b) + template-->... +``` + +## Components + +The foxops tool is split into two components: + +* **fengine** is the underlying templating engine which works with _local_ Git repositories for templates and incarnations. Internally it uses [Jinja](https://jinja.palletsprojects.com/) to render the templates. +* **foxops** is the tool on top of fengine that supports declarative configuration and reconciliation loops. Also, it extends the functionality to work with GitLab, for example to create merge requests when updates need to happen. + +## Quick Start + +First let's declare an *incarnation* for my `catcam` Python application based on a Python *template*: + +```yaml +# catcam-incarnation.yaml +incarnations: + - gitlab_project: my-org/catcam + template_repository: https://gitlab.com/my-org/templates/python + template_version: v1.0.0 + template_data: + package_name: catcam +``` + +Let's call `foxops` with that configuration file: + +```bash +foxops reconcile catcam-incarnation.yaml +``` + +Once the incarnation initialization finished, you'll have a nice Merge Request in the `my-org/catcam` +Git repository. + +:::{tip} +Checkout the [Write Template from Scratch](tutorials/write-template-from-scratch) and +the general [Usage](usage) guides for more detailed information about how to use +foxops and fengine. +::: + +## Example Usecases + +### A Zoo of Microservices + +Imagine a company creating an application based on a large number of microservices that are all (or mostly) built on top of a single tech stack. Like Python. If these microservices are all living in a multi-repository Git structure, there are a bunch of files that are very similar, but need to be replicated in every Git repository. Namely things like CI configuration, basic directory structure and build scripts. + +For example the CI configuration in all these repositories needs to be kept up-to-date with the latest best-practices inside the company, which typically results in a lot of manual copy & paste or just having vastly different setups for every microservice. + +### GitOps with Multiple Environments + +Classical GitOps has a 1:1 mapping of Git Repositories to pieces of infrastructure. FoxOps is a great match if a number of similar environments needs to be maintained, i.e. a prod and staging environment. Or if a separate environment must be created for every customer. + +In such cases it is often advantageous to have separate Git repositories, one for every environment - as that would allow to quickly deploy hotfixes to individual environments if necessary. diff --git a/docs/source/installation.md b/docs/source/installation.md new file mode 100644 index 00000000..445359a7 --- /dev/null +++ b/docs/source/installation.md @@ -0,0 +1,31 @@ +# Installation + +The **foxops** Python package can be installed from the Roche GitLab Package Registry: + +```bash +pip install foxops +``` + +```{note} +Make sure to replace `` with a valid Personal Access Token. +``` + +The foxops Python package contains multiple console scripts: + +* `foxops` - the main foxops tool to manage the full MegOps lifecycle +* `fengine` - exposes the initialize and update APIs to manually execute the template rendering + +## Docker + +`foxops` is also deployed in the GitHub Container Registry and can be pulled from there: + +```bash +docker pull ghcr.io/roche/foxops: +``` + +```{note} +Make sure to replace the `` with a [valid version](https://github.com/Roche/foxops/releases). +``` + +The default entrypoint for the `foxops` docker image is the `foxops` console script. +However, you may also specify the `fengine` console script as entrypoint. diff --git a/docs/source/terminology.md b/docs/source/terminology.md new file mode 100644 index 00000000..f8515e22 --- /dev/null +++ b/docs/source/terminology.md @@ -0,0 +1,9 @@ +# Terminology + +Throughout this documentation the following terms are often used and you it's good if you are familiar with them: + +**template**: A Git repository which are rendered into *incarnation*. A *template* consists of a `template/` folder and a `fengine.yaml` configuration file. + +**incarnation**: A Git repository which is a concrete instance of a rendered *template*. It may or may not contain *customizations* specific to that *incarnation*. + +**reconcile**: The process of bringing a set of incarnations to a desired state in terms of the template, template version and template variables. diff --git a/docs/source/tutorials/backwards-compatible-template-changes.md b/docs/source/tutorials/backwards-compatible-template-changes.md new file mode 100644 index 00000000..a19db9ec --- /dev/null +++ b/docs/source/tutorials/backwards-compatible-template-changes.md @@ -0,0 +1,33 @@ +## Making backwards-compatible Changes to a Template + +The following list suggests how to implement certain template changes in a backwards-compatible way. + +### Adding a new Variable + +fengine supports `default`s, which are used when no value is specified for the template rendering. +Specifying the `default` effectively makes the variable *optional*. + +```yaml +variables: + new_var: + type: str + description: New variable for the template + default: "Hello" +``` + +### Removing a Variable + +fengine ignores, but warns about additional variables being based for the template rendering. +Therefore, removing a variable is supported out of the box and no active measure needs to be taken. + +### Changing the name of a Variable + +This is has to be implemented as a combination of *Removing a Variable* and *Adding a new Variable*. +fengine doesn't yet support automatic migration of renamed variable as in: fengine knows the old name +of a variable and uses its passed value for the newly named variable. + +### Changing the type of a Variable + +fengine doesn't yet care about the type too much. It's neither enforced nor are the passed values +casted to its respective Python type. +Changing the type of a Variable can therefore be implemented without taking active measures. diff --git a/docs/source/tutorials/index.md b/docs/source/tutorials/index.md new file mode 100644 index 00000000..91cefe4d --- /dev/null +++ b/docs/source/tutorials/index.md @@ -0,0 +1,8 @@ +# Tutorials + +```{toctree} +:maxdepth: 2 + +write-template-from-scratch +backwards-compatible-template-changes +``` diff --git a/docs/source/tutorials/write-template-from-scratch.md b/docs/source/tutorials/write-template-from-scratch.md new file mode 100644 index 00000000..5d3d13f5 --- /dev/null +++ b/docs/source/tutorials/write-template-from-scratch.md @@ -0,0 +1,91 @@ +## Write a Template from Scratch + +A valid template that can be used by fengine only consists of a few components: + +* A Git repository in which the template resides. +* A `fengine.yaml` file that contains the templates metadata. +* A `template/` subdirectory that contains the files that should be rendered into the incarnation. + +### Create a New Template + +To initialize a new, empty template you need to create the elements described in the list above. +If you're lazy, just use the following set of bash commands to do that: + +```shell +# create template scaffolding +fengine new mytemplate + +# initialize git repository +git init +git commit -am 'initial commit' +``` + +Be aware that only the files within the `template/` directory will be rendered into the incarnation. +Therefore you can use the root directory of the template Git repository to place other subdirectories or files for documentation, +tests or a CI configuration file for the template repository itself. + +### Template Configuration + +In the above section you've just created an empty template configuration in the `fengine.yaml` file. +There are a few things you can configure for a template: + +#### Template Variables + +Probably the most important template configuration are *variables* or *template data*. +Every template can have zero or more variables that it must define in the `fengine.yaml` file +and which can be used in files inside the `template/` folder. When an incarnation is being rendered +these variables are being substituted in the defined places. + +:::{tip} +foxops uses [Jinja](https://jinja.palletsprojects.com/) to render the template files, including +the variables. +::: + +Variables can be defined with the following syntax: + +```yaml +variables: + : + type: str | int | float + description: + default: +``` + +The values in `<>` are meant to be replaced, for example to define a variable called `name` of type `string` +with the default `Jane Doe` use the following: + +```yaml +variables: + name: + type: str + description: The name of the author + default: Jane Doe +``` + +### Exclude Files from Rendering + +Sometimes you don't want to render every file in `template/`, but only copy&paste them as-is +to the incarnation. You can use the `exclude_files` field to list the files you don't want +to have render with Jinja. + +For example to exclude the files `template/dummy` and all `*.txt` files in `template/` from rendering +use the following config: + +```yaml +exclude_files: + - 'dummy' + - '*.txt' + +variables: + ... +``` + +### Change an Existing Template + +Nothing special to consider here. Just change the files in the repository and commit to git as normal. + +As best practice we recommend to version your template on the main branch using git tags like `v1.2.0` (for example following [semantic versioning](https://semver.org/) or [calendar versioning](https://calver.org/)). Especially in production environments it's strongly recommended to always specify exact version numbers (= git tags) when reconciling incarnations with foxops. + +:::{tip} +Learn [here](backwards-compatible-template-changes) how to make template changes in a backwards-compatible way. +::: diff --git a/docs/source/usage.md b/docs/source/usage.md new file mode 100644 index 00000000..1cc67719 --- /dev/null +++ b/docs/source/usage.md @@ -0,0 +1,167 @@ +# Usage + +foxops is primarily a command line interface and can be installed by following [these instructions](installation). + +## Command Line Interfaces + +The foxops Python package will install two console scripts: + +* `foxops`: main console script to initialize and update incarnation from a template and interacting with GitLab. +* `fengine`: a lower level console script used by `foxops` to do the heavy lifting around incarnation initialization and updates. + It's primarily used to develop templates. + +## Initialize an Incarnation from a Template + +:::{note} +Checkout the [Write Template from Scratch](tutorials/write-template-from-scratch) tutorial to +learn more about templates. +::: + +Let's assume you have a template at `https://gitlab.com/my-org/templates/python` with two variables, one is called `name` and one `category`. + +The first thing you need to initialize an incarnation is a *desired incarnation state* configuration. +This configuration must be in a YAML file, e.g. `incarnations.yaml`. + +The following example configures the incarnation to an already existing Git repository at `my-org/catcam`: + +```yaml +incarnations: + - gitlab_project: my-org/catcam + template_repository: https://gitlab.com/my-org/templates/python + template_version: v1.0.0 + template_data: + name: catcam + category: utils +``` + +foxops can now be used to *reconcile* the incarnation from the actual to the desired state given the above config: + +```sh +foxops reconcile incarnations.yaml +``` + +## Update an Incarnation + +When an update to the template was made it's sometimes worthwhile to update the incarnation to reflect these changes, too. +The same `foxops reconcile` command as for the [initialization](#initialize-an-incarnation-from-a-template) can be used +for an update. + +The only thing which may change is the `template_version` or `template_data` values. +Using the following *desired incarnation state* configuration to change to the update template version `v2.0.0` and change +the category from `utils` to `extras`: + +```yaml +incarnations: + - gitlab_project: my-org/catcam + template_repository: https://gitlab.com/my-org/templates/python + template_version: v2.0.0 + template_data: + name: catcam + category: extras +``` + +To perform the actual update, run `foxops` again: + +```sh +foxops reconcile incarnations.yaml +``` + +An update will always create a Merge Request on the incarnation repository with the changes, ready to be reviewed and merged. + +In case the update wasn't successful because of conflicts between the template and the incarnation the Merge Request will +reflect that. + +You may add an `automerge: true` option to the incarnation configuration so that the update Merge Request is automatically merged. + +## Local Development + +The `fengine` CLI tool is especially useful when you're working on a template, +as it allows you to quickly create or update an incarnation locally to test your changes. + +### Bootstrap a Template + +fengine is able to bootstrap a new template in a folder you specify, e.g. to create a new template in the `catcam` folder, +run the following command: + +```sh +fengine new catcam +``` + +This will scaffold the following structure: + +* `fengine.yaml`: a basic template configuration file with a predefined `author` variable. +* `template/README.md`: a markdown template file which will render the string `Created by {{ author }}` to the incarnation + whereas `{{ author }}` will be replaced by the actual value fo the incarnation. + +To create a proper template we will initialize the `catcam` directory as Git repository and create a commit and first tag: + +```sh +git init catcam +cd catcam +git add . +git commit -a -m 'Initial commit' +git tag v1.0.0 +``` + +### Initialize an Incarnation + +The `fengine initialize` command can be used to create a local incarnation of a template. +To create an incarnation in the `mycat` folder from the `catcam` template created in [Bootstrap a Template](#bootstrap-a-template) +with the `author` name `Albert Einstein` use the following command: + +```sh +fengine initialize catcam/ --template-version v1.0.0 mycat -d author="Albert Einstein" +``` + +This will create the `mycat/` folder and render the file `README.md` with the contents: + +```txt +Created by Albert Einstein +``` + +In addition, it also render a special file called `.fengine.yaml` which is owned by foxops +and shouldn't be manually edited. It's sole purpose is to calculate updates and have a reference +to the underlying template and its version. +It may look something like this: + +```yaml +# This file is auto-generated and owned by foxops. +# DO NOT EDIT MANUALLY. +template_data: + author: Albert Einstein +template_repository: /Users/.../catcam +template_repository_version: v1.0.0 +template_repository_version_hash: 437b0efc7c6515e1ec613d08ce70a9a1d84fe7dc +``` + +### Update an incarnation + +Let's assume the template as updated and a `v2.0.0` version exists +and is available in the local `catcam` template directory. + +You can update the `mycat` incarnation using the following command: + +```sh +fengine update mycat/ -u v2.0.0 +``` + +You may also provide other values for the template data like the `author` variable. + +## `default.fvars` File + +In addition to configuring the `template_data` in the desired incarnation state configuration, +variable values can be provided in the `default.fvars` file located in the target incarnation directory. + +The file format is in `key=value` pairs (like `.env` files), e.g.: + +```ini +author=Jon +age=42 +``` + +### Behavior details + +Foxops will submit a Merge Request during incarnation initialization in the case the repository +is not empty. +If the incarnation repository only contains a `default.fvars` file, it's considered empty and +no Merge Request will be submitted. diff --git a/docs/source/workflows.md b/docs/source/workflows.md new file mode 100644 index 00000000..7bb6286a --- /dev/null +++ b/docs/source/workflows.md @@ -0,0 +1,53 @@ +# Workflows + +The following chapter provides an overview and details about the main `foxops` workflows, like `initialization` and `update`. + +## Reconciliation Workflow + +```{mermaid} +graph TD + START(reconcile)-->EXISTS{{incarnation repository exists?}} + EXISTS-->|no|END + EXISTS-->|yes|INITIALIZED{{incarnation repository already initialized?}} + INITIALIZED-->|no|INITIALIZE[[fengine init]] + INITIALIZED-->|yes|UPDATE[[fengine update]] + + INITIALIZE-->IS_EMPTY{{is incarnation location empty?}} + IS_EMPTY-->|yes|COMMIT_DEFAULT(commit to default branch) + COMMIT_DEFAULT-->END + + IS_EMPTY-->|no|CREATE_MR(commit to feature branch and create MR) + CREATE_MR-->AUTOMERGE_ENABLED{{is automerge enabled?}} + AUTOMERGE_ENABLED-->|no|END + AUTOMERGE_ENABLED-->|yes|SET_AUTOMERGE(set MR to automerge) + SET_AUTOMERGE-->END + + UPDATE-->CHANGES{{any changes?}} + CHANGES-->|no|END + CHANGES-->|yes|CREATE_MR + + END(done) +``` + +## Initialization Workflow (fengine init) + +```{mermaid} +graph TD + START(reconcile)-->EXISTS{{incarnation repository exists?}} + EXISTS-->|no|LOG_NOT_EXISTS[Log Warning] + EXISTS-->|yes|INITIALIZED{{incarnation repository already initialized?}} + INITIALIZED-->|no|INITIALIZE[[fengine init]] + INITIALIZED-->|yes|UPDATE[[fengine update]] + + INITIALIZE-->IS_EMPTY{{is incarnation location empty?}} + + + INITIALIZE-->RENDER_INIT(render template with incarnation variables) + UPDATE-->IS_UPDATE_REQUIRED{{is template update required?}} + IS_UPDATE_REQUIRED-->|yes|RENDER_OLD(render old template version) + IS_UPDATE_REQUIRED-->|no|NO_UPDATE[End] + + RENDER_OLD-->RENDER_NEW(render new template version) + RENDER_NEW-->CALC_DIFF(calculate diff between outcome) + CALC_DIFF-->APPLY_DIFF(apply patch to incarnation) +``` diff --git a/poetry.lock b/poetry.lock index ec87c0f4..9407c3ab 100644 --- a/poetry.lock +++ b/poetry.lock @@ -90,7 +90,7 @@ python-versions = ">=3.6" [[package]] name = "atomicwrites" -version = "1.4.0" +version = "1.4.1" description = "Atomic file writes." category = "dev" optional = false @@ -208,7 +208,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" -version = "6.4.1" +version = "6.4.2" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -257,7 +257,7 @@ pyflakes = ">=2.4.0,<2.5.0" [[package]] name = "flake8-bugbear" -version = "22.6.22" +version = "22.7.1" description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle." category = "dev" optional = false @@ -310,7 +310,7 @@ python-versions = ">=3.5" [[package]] name = "imagesize" -version = "1.3.0" +version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" category = "dev" optional = false @@ -366,7 +366,7 @@ tornado = {version = "*", markers = "python_version > \"2.7\""} [[package]] name = "mako" -version = "1.2.0" +version = "1.2.1" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." category = "dev" optional = false @@ -690,7 +690,7 @@ testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtuale [[package]] name = "pytest-mock" -version = "3.8.1" +version = "3.8.2" description = "Thin-wrapper around the mock package for easier use with pytest" category = "dev" optional = false @@ -1000,15 +1000,15 @@ python-versions = ">=3.7" [[package]] name = "tornado" -version = "6.1" +version = "6.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." category = "dev" optional = false -python-versions = ">= 3.5" +python-versions = ">= 3.7" [[package]] name = "typer" -version = "0.4.1" +version = "0.4.2" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." category = "main" optional = false @@ -1019,13 +1019,13 @@ click = ">=7.1.1,<9.0.0" [package.extras] all = ["colorama (>=0.4.3,<0.5.0)", "shellingham (>=1.3.0,<2.0.0)"] -dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)"] +dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] doc = ["mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "mdx-include (>=1.4.1,<2.0.0)"] test = ["shellingham (>=1.3.0,<2.0.0)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "coverage (>=5.2,<6.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "mypy (==0.910)", "black (>=22.3.0,<23.0.0)", "isort (>=5.0.6,<6.0.0)"] [[package]] name = "typing-extensions" -version = "4.2.0" +version = "4.3.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false @@ -1033,11 +1033,11 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.9" +version = "1.26.10" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] @@ -1058,8 +1058,8 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" -python-versions = ">=3.10,<3.11" -content-hash = "8236af79dd29aa035d40efdb4cb2e32aa7e3aabfff063505489ef71b93ef896a" +python-versions = ">=3.10,<4.0" +content-hash = "e08dc5cffdafb1445946e0e798071e31ab8af086befbb10aea8324d815879b06" [metadata.files] aiofile = [ @@ -1160,10 +1160,7 @@ async-timeout = [ {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, ] -atomicwrites = [ - {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, - {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, -] +atomicwrites = [] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, @@ -1230,49 +1227,7 @@ colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -coverage = [ - {file = "coverage-6.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f1d5aa2703e1dab4ae6cf416eb0095304f49d004c39e9db1d86f57924f43006b"}, - {file = "coverage-6.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4ce1b258493cbf8aec43e9b50d89982346b98e9ffdfaae8ae5793bc112fb0068"}, - {file = "coverage-6.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c4e737f60c6936460c5be330d296dd5b48b3963f48634c53b3f7deb0f34ec4"}, - {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e65ef149028516c6d64461b95a8dbcfce95cfd5b9eb634320596173332ea84"}, - {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69718750eaae75efe506406c490d6fc5a6161d047206cc63ce25527e8a3adad"}, - {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e57816f8ffe46b1df8f12e1b348f06d164fd5219beba7d9433ba79608ef011cc"}, - {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:01c5615d13f3dd3aa8543afc069e5319cfa0c7d712f6e04b920431e5c564a749"}, - {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ab269400706fab15981fd4bd5080c56bd5cc07c3bccb86aab5e1d5a88dc8f4"}, - {file = "coverage-6.4.1-cp310-cp310-win32.whl", hash = "sha256:a7f3049243783df2e6cc6deafc49ea123522b59f464831476d3d1448e30d72df"}, - {file = "coverage-6.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2ddcac99b2d2aec413e36d7a429ae9ebcadf912946b13ffa88e7d4c9b712d6"}, - {file = "coverage-6.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fb73e0011b8793c053bfa85e53129ba5f0250fdc0392c1591fd35d915ec75c46"}, - {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:106c16dfe494de3193ec55cac9640dd039b66e196e4641fa8ac396181578b982"}, - {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87f4f3df85aa39da00fd3ec4b5abeb7407e82b68c7c5ad181308b0e2526da5d4"}, - {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:961e2fb0680b4f5ad63234e0bf55dfb90d302740ae9c7ed0120677a94a1590cb"}, - {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cec3a0f75c8f1031825e19cd86ee787e87cf03e4fd2865c79c057092e69e3a3b"}, - {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:129cd05ba6f0d08a766d942a9ed4b29283aff7b2cccf5b7ce279d50796860bb3"}, - {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bf5601c33213d3cb19d17a796f8a14a9eaa5e87629a53979a5981e3e3ae166f6"}, - {file = "coverage-6.4.1-cp37-cp37m-win32.whl", hash = "sha256:269eaa2c20a13a5bf17558d4dc91a8d078c4fa1872f25303dddcbba3a813085e"}, - {file = "coverage-6.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f02cbbf8119db68455b9d763f2f8737bb7db7e43720afa07d8eb1604e5c5ae28"}, - {file = "coverage-6.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ffa9297c3a453fba4717d06df579af42ab9a28022444cae7fa605af4df612d54"}, - {file = "coverage-6.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:145f296d00441ca703a659e8f3eb48ae39fb083baba2d7ce4482fb2723e050d9"}, - {file = "coverage-6.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d44996140af8b84284e5e7d398e589574b376fb4de8ccd28d82ad8e3bea13"}, - {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2bd9a6fc18aab8d2e18f89b7ff91c0f34ff4d5e0ba0b33e989b3cd4194c81fd9"}, - {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3384f2a3652cef289e38100f2d037956194a837221edd520a7ee5b42d00cc605"}, - {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9b3e07152b4563722be523e8cd0b209e0d1a373022cfbde395ebb6575bf6790d"}, - {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1480ff858b4113db2718848d7b2d1b75bc79895a9c22e76a221b9d8d62496428"}, - {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:865d69ae811a392f4d06bde506d531f6a28a00af36f5c8649684a9e5e4a85c83"}, - {file = "coverage-6.4.1-cp38-cp38-win32.whl", hash = "sha256:664a47ce62fe4bef9e2d2c430306e1428ecea207ffd68649e3b942fa8ea83b0b"}, - {file = "coverage-6.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:26dff09fb0d82693ba9e6231248641d60ba606150d02ed45110f9ec26404ed1c"}, - {file = "coverage-6.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9c80df769f5ec05ad21ea34be7458d1dc51ff1fb4b2219e77fe24edf462d6df"}, - {file = "coverage-6.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:39ee53946bf009788108b4dd2894bf1349b4e0ca18c2016ffa7d26ce46b8f10d"}, - {file = "coverage-6.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5b66caa62922531059bc5ac04f836860412f7f88d38a476eda0a6f11d4724f4"}, - {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd180ed867e289964404051a958f7cccabdeed423f91a899829264bb7974d3d3"}, - {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84631e81dd053e8a0d4967cedab6db94345f1c36107c71698f746cb2636c63e3"}, - {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8c08da0bd238f2970230c2a0d28ff0e99961598cb2e810245d7fc5afcf1254e8"}, - {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d42c549a8f41dc103a8004b9f0c433e2086add8a719da00e246e17cbe4056f72"}, - {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:309ce4a522ed5fca432af4ebe0f32b21d6d7ccbb0f5fcc99290e71feba67c264"}, - {file = "coverage-6.4.1-cp39-cp39-win32.whl", hash = "sha256:fdb6f7bd51c2d1714cea40718f6149ad9be6a2ee7d93b19e9f00934c0f2a74d9"}, - {file = "coverage-6.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:342d4aefd1c3e7f620a13f4fe563154d808b69cccef415415aece4c786665397"}, - {file = "coverage-6.4.1-pp36.pp37.pp38-none-any.whl", hash = "sha256:4803e7ccf93230accb928f3a68f00ffa80a88213af98ed338a57ad021ef06815"}, - {file = "coverage-6.4.1.tar.gz", hash = "sha256:4321f075095a096e70aff1d002030ee612b65a205a0a0f5b815280d5dc58100c"}, -] +coverage = [] dictdiffer = [ {file = "dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595"}, {file = "dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578"}, @@ -1285,10 +1240,7 @@ flake8 = [ {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, ] -flake8-bugbear = [ - {file = "flake8-bugbear-22.6.22.tar.gz", hash = "sha256:ac3317eba27d79dc19dcdeb7356ca1f656f0cde11d899c4551badf770f05cbef"}, - {file = "flake8_bugbear-22.6.22-py3-none-any.whl", hash = "sha256:ad2b33dbe33a6d4ca1f0037e1d156d0a89107ee63c0600e3b4f7b60e37998ac2"}, -] +flake8-bugbear = [] frozenlist = [ {file = "frozenlist-1.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d2257aaba9660f78c7b1d8fea963b68f3feffb1a9d5d05a18401ca9eb3e8d0a3"}, {file = "frozenlist-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4a44ebbf601d7bac77976d429e9bdb5a4614f9f4027777f9e54fd765196e9d3b"}, @@ -1361,10 +1313,7 @@ idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] -imagesize = [ - {file = "imagesize-1.3.0-py2.py3-none-any.whl", hash = "sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c"}, - {file = "imagesize-1.3.0.tar.gz", hash = "sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d"}, -] +imagesize = [] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, @@ -1380,10 +1329,7 @@ jinja2 = [ livereload = [ {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"}, ] -mako = [ - {file = "Mako-1.2.0-py3-none-any.whl", hash = "sha256:23aab11fdbbb0f1051b93793a58323ff937e98e34aece1c4219675122e57e4ba"}, - {file = "Mako-1.2.0.tar.gz", hash = "sha256:9a7c7e922b87db3686210cf49d5d767033a41d4010b284e747682c92bddd8b39"}, -] +mako = [] markdown-it-py = [ {file = "markdown-it-py-2.1.0.tar.gz", hash = "sha256:cf7e59fed14b5ae17c0006eff14a2d9a00ed5f3a846148153899a0224e2c07da"}, {file = "markdown_it_py-2.1.0-py3-none-any.whl", hash = "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27"}, @@ -1632,10 +1578,7 @@ pytest-cov = [ {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, ] -pytest-mock = [ - {file = "pytest-mock-3.8.1.tar.gz", hash = "sha256:2c6d756d5d3bf98e2e80797a959ca7f81f479e7d1f5f571611b0fdd6d1745240"}, - {file = "pytest_mock-3.8.1-py3-none-any.whl", hash = "sha256:d989f11ca4a84479e288b0cd1e6769d6ad0d3d7743dcc75e460d1416a5f2135a"}, -] +pytest-mock = [] pytz = [ {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, @@ -1782,61 +1725,10 @@ tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -tornado = [ - {file = "tornado-6.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32"}, - {file = "tornado-6.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c"}, - {file = "tornado-6.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b"}, - {file = "tornado-6.1-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675"}, - {file = "tornado-6.1-cp35-cp35m-win32.whl", hash = "sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5"}, - {file = "tornado-6.1-cp35-cp35m-win_amd64.whl", hash = "sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68"}, - {file = "tornado-6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb"}, - {file = "tornado-6.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c"}, - {file = "tornado-6.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c"}, - {file = "tornado-6.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085"}, - {file = "tornado-6.1-cp36-cp36m-win32.whl", hash = "sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575"}, - {file = "tornado-6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795"}, - {file = "tornado-6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f"}, - {file = "tornado-6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102"}, - {file = "tornado-6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01"}, - {file = "tornado-6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d"}, - {file = "tornado-6.1-cp37-cp37m-win32.whl", hash = "sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df"}, - {file = "tornado-6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37"}, - {file = "tornado-6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95"}, - {file = "tornado-6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a"}, - {file = "tornado-6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5"}, - {file = "tornado-6.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288"}, - {file = "tornado-6.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f"}, - {file = "tornado-6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6"}, - {file = "tornado-6.1-cp38-cp38-win32.whl", hash = "sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326"}, - {file = "tornado-6.1-cp38-cp38-win_amd64.whl", hash = "sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c"}, - {file = "tornado-6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5"}, - {file = "tornado-6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe"}, - {file = "tornado-6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea"}, - {file = "tornado-6.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2"}, - {file = "tornado-6.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0"}, - {file = "tornado-6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd"}, - {file = "tornado-6.1-cp39-cp39-win32.whl", hash = "sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c"}, - {file = "tornado-6.1-cp39-cp39-win_amd64.whl", hash = "sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4"}, - {file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"}, -] -typer = [ - {file = "typer-0.4.1-py3-none-any.whl", hash = "sha256:e8467f0ebac0c81366c2168d6ad9f888efdfb6d4e1d3d5b4a004f46fa444b5c3"}, - {file = "typer-0.4.1.tar.gz", hash = "sha256:5646aef0d936b2c761a10393f0384ee6b5c7fe0bb3e5cd710b17134ca1d99cff"}, -] -typing-extensions = [ - {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, - {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, -] -urllib3 = [ - {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, - {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, -] +tornado = [] +typer = [] +typing-extensions = [] +urllib3 = [] yarl = [ {file = "yarl-1.7.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f2a8508f7350512434e41065684076f640ecce176d262a7d54f0da41d99c5a95"}, {file = "yarl-1.7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da6df107b9ccfe52d3a48165e48d72db0eca3e3029b5b8cb4fe6ee3cb870ba8b"}, diff --git a/pyproject.toml b/pyproject.toml index 646a5744..68065e96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ readme = "README.md" foxops = 'foxops.__main__:app' [tool.poetry.dependencies] -python = ">=3.10" +python = ">=3.10,<4.0" aiohttp = "^3.8.0" "ruamel.yaml" = "^0.17.20" pydantic = "^1.9.0" diff --git a/src/foxops/__init__.py b/src/foxops/__init__.py index ce783926..ff3fae54 100644 --- a/src/foxops/__init__.py +++ b/src/foxops/__init__.py @@ -1,2 +1,3 @@ -# FIXME: change to take version from package -__version__ = "0.0.1" +from importlib.metadata import version + +__version__ = version("foxops") diff --git a/src/foxops/__main__.py b/src/foxops/__main__.py index c27fd2f3..36536c4b 100644 --- a/src/foxops/__main__.py +++ b/src/foxops/__main__.py @@ -1,14 +1,140 @@ +import asyncio +import itertools +import logging +import warnings +from pathlib import Path + import typer +from pydantic import ValidationError +from ruamel.yaml import YAML +from structlog.stdlib import BoundLogger + +from foxops.logging import get_logger, setup_logging +from foxops.models import DesiredIncarnationStateConfig +from foxops.reconciliation import AsyncGitlabClient, ReconciliationState, reconcile +from foxops.settings import Settings app = typer.Typer() +yaml = YAML(typ="safe") + +logger: BoundLogger + + +def get_settings(): + try: + warnings.simplefilter("ignore", category=UserWarning) + settings = Settings() + except ValidationError as exc: + for error in exc.errors(): + error_loc = " -> ".join(str(e) for e in error["loc"]) + logger.error( + f"error when creating Settings due to the field '{error_loc}': {error['msg']}", + settings_field=error_loc, + field_error_reason=error["msg"], + ) + raise typer.Exit(1) + else: + return settings + + +@app.command(name="reconcile") +def cmd_reconcile( + parallelism: int = typer.Option( # noqa: B008 + 10, "--parallelism", "-p", help="number of parallel reconciliations" + ), + config_paths: list[str] = typer.Argument( # noqa: B008 + None, + help="Path to the configuration file(s) or folder(s) containing configuration file(s) to use. The configuration files define the desired incarnation states.", + ), +): + if not config_paths: + logger.error("no configuration file(s) or folder(s) specified") + raise typer.Exit(1) + + logger.debug("configuring settings for reconciliation") + settings = get_settings() + logger.debug("configured settings", settings=settings) + + logger.debug( + "loading desired incarnation states configurations ...", + desired_incarnation_state_config_paths=config_paths, + ) -@app.command() -def main(): + def expand_dir(path: Path) -> list[Path]: + if path.is_dir(): + return list(path.glob("**/*.yml")) + list(path.glob("**/*.yaml")) + return [path] + + desired_incarnation_states: list[DesiredIncarnationStateConfig] = [] + flattened_config_files = itertools.chain( + *(expand_dir(Path(c)) for c in config_paths) + ) + for config_file in flattened_config_files: + try: + parsed_config = yaml.load(config_file) + desired_incarnation_states.extend( + DesiredIncarnationStateConfig.parse_obj(c) + for c in parsed_config["incarnations"] + ) + except Exception as exc: + logger.error( + f"Project definition config at {config_file} is not valid: {exc}" + ) + raise typer.Exit(1) + + logger.debug( + f"loaded {len(desired_incarnation_states)} desired incarnation states", + desired_incarnation_states=desired_incarnation_states, + ) + + async def main(): + async with AsyncGitlabClient( + base_url=settings.gitlab_address, + token=settings.gitlab_token.get_secret_value(), + ) as gitlab: + reconciliation_states = await reconcile( + gitlab, desired_incarnation_states, parallelism=parallelism + ) + return reconciliation_states + + logger.info("starting reconciliation") + reconciliation_states = asyncio.run(main()) + logger.info("finished reconciliation") + + for desired_incarnation_state, reconciliation_state in sorted( + zip(desired_incarnation_states, reconciliation_states), + key=lambda x: (x[1], x[0].gitlab_project), + ): + logger.info( + f"project '{desired_incarnation_state.gitlab_project}' reconciled with state '{reconciliation_state.name.lower()}'", + reconciliation_state=reconciliation_state, + category="summary", + ) + + if any(s == ReconciliationState.FAILED for s in reconciliation_states): + raise typer.Exit(2) + + +@app.callback() +def main( + verbose: bool = typer.Option( # noqa: B008 + False, "--verbose", "-v", help="turn on verbose logging" + ), + logs_as_json: bool = typer.Option( # noqa: B008 + False, "--json-logs", "-j", help="render logs as JSON" + ), +): """ Foxops for MegOps, or something. """ - print("🦊") + if verbose: + setup_logging(logging.DEBUG, logs_as_json) + else: + setup_logging(logging.INFO, logs_as_json) + + global logger + logger = get_logger("app") if __name__ == "__main__": diff --git a/src/foxops/engine/__init__.py b/src/foxops/engine/__init__.py new file mode 100644 index 00000000..8d054cd0 --- /dev/null +++ b/src/foxops/engine/__init__.py @@ -0,0 +1,18 @@ +""" +foxops engine aka. fengine is a Python package responsible for +rendering a template into a particular incarnation instance. +It also handles updates of said templates. +""" + +from foxops.engine.fvars import FVARS_FILENAME # noqa +from foxops.engine.initialization import initialize_incarnation # noqa +from foxops.engine.models import IncarnationState # noqa +from foxops.engine.models import TemplateData # noqa +from foxops.engine.models import load_incarnation_state # noqa +from foxops.engine.models import load_incarnation_state_from_string # noqa +from foxops.engine.models import save_incarnation_state # noqa +from foxops.engine.patching.git_diff_patch import diff_and_patch # noqa +from foxops.engine.update import ( # noqa + update_incarnation, + update_incarnation_from_git_template_repository, +) diff --git a/src/foxops/engine/__main__.py b/src/foxops/engine/__main__.py new file mode 100644 index 00000000..824693d2 --- /dev/null +++ b/src/foxops/engine/__main__.py @@ -0,0 +1,301 @@ +import asyncio +import logging +from dataclasses import asdict +from pathlib import Path +from subprocess import PIPE, check_output +from typing import Optional + +import typer +from structlog.stdlib import BoundLogger + +from foxops.engine.initialization import initialize_incarnation +from foxops.engine.models import ( + IncarnationState, + TemplateConfig, + TemplateData, + VariableDefinition, + load_incarnation_state, +) +from foxops.engine.patching.git_diff_patch import diff_and_patch +from foxops.engine.update import update_incarnation_from_git_template_repository +from foxops.logging import get_logger, setup_logging + +app = typer.Typer() + +logger: BoundLogger + + +@app.command(name="new", help="Creates a new template scaffold in the given directory") +def cmd_new( + target_directory: Path = typer.Argument( # noqa: B008 + ..., + exists=False, + file_okay=False, + dir_okay=True, + writable=True, + readable=True, + resolve_path=True, + ) +): + # Make sure the target_directory exists and is empty. Create it if it doesn't exist. + if target_directory.exists(): + if any(target_directory.iterdir()): + raise typer.Abort("Target directory is not empty.") + else: + target_directory.mkdir(0o755, parents=True) + + # Create sample fengine.yaml + template_config = TemplateConfig( + variables={ + "author": VariableDefinition( + type="str", + description="The author of the project", + ) + } + ) + template_config.to_yaml(target_directory / "fengine.yaml") + + # Create sample README in template directory + (target_directory / "template").mkdir(0o755) + (target_directory / "template" / "README.md").write_text( + "Created by {{ author }}\n" + ) + + +@app.command(name="initialize") +def cmd_initialize( + template_repository: Path = typer.Argument( # noqa: B008 + ..., + exists=True, + file_okay=False, + dir_okay=True, + readable=True, + resolve_path=True, + ), + incarnation_dir: Path = typer.Argument( # noqa: B008 + ..., + exists=False, + file_okay=False, + dir_okay=False, + ), + raw_template_data: list[str] = typer.Option( # noqa: B008 + [], + "--data", + "-d", + help="Template data variables in the format of `key=value`", + ), + template_repository_version: Optional[str] = typer.Option( # noqa: B008 + None, + "--template-version", + help="Template repository version to use", + ), +): + """Initialize an incarnation repository with a version of a template and some data.""" + template_data: TemplateData = dict( + tuple(x.split("=", maxsplit=1)) for x in raw_template_data # type: ignore + ) + + log = logger.bind( + template_repository=template_repository, + incarnation_dir=incarnation_dir, + template_data=template_data, + ) + + log.debug("creating empty incarnation directory") + incarnation_dir.mkdir(parents=True, exist_ok=False) + + if template_repository_version: + log.debug( + f"checking out template repository version {template_repository_version}" + ) + check_output( + ["git", "checkout", template_repository_version], + cwd=template_repository, + stderr=PIPE, + ) + + repository_version = ( + check_output(["git", "rev-parse", "HEAD"], cwd=template_repository) + .decode() + .strip() + ) + + log.info( + "starting initialization incarnation ...", + template_repository=str(template_repository), + template_repository_version=repository_version, + template_data=template_data, + ) + + try: + asyncio.run( + initialize_incarnation( + template_root_dir=template_repository, + template_repository=str(template_repository), + template_repository_version=repository_version, + template_data=template_data, + incarnation_root_dir=incarnation_dir, + logger=log, + ) + ) + except Exception as exc: + log.exception(f"initialization failed: {exc}") + else: + log.info("successfully initialized incarnation") + finally: + if template_repository_version: + check_output( + ["git", "checkout", "-"], + cwd=template_repository, + stderr=PIPE, + ) + + +@app.command(name="update") +def cmd_update( + incarnation_dir: Path = typer.Argument( # noqa: B008 + ..., + exists=True, + file_okay=False, + dir_okay=True, + readable=True, + writable=True, + ), + raw_template_data: list[str] = typer.Option( # noqa: B008 + [], + "--data", + "-d", + help="Template data variables in the format of `key=value`", + ), + remove_template_data: list[str] = typer.Option( # noqa: B008 + [], + "--remove-data", + help="Template data variables to remove", + ), + update_repository_version: Optional[str] = typer.Option( # noqa: B008 + None, + "--update-repository-version", + "-u", + help="the version of the template repository to update to", + ), + overridden_template_repository: Optional[Path] = typer.Option( # noqa: B008 + None, + "--template-repository", + "-r", + help="Override the template repository with a local path recorded in the incarnation state", + ), +): + """Initialize an incarnation repository with a version of a template and some data.""" + template_data: dict[str, str] = dict( + tuple(x.split("=", maxsplit=1)) for x in raw_template_data # type: ignore + ) + + incarnation_state_path = incarnation_dir / ".fengine.yaml" + logger.debug( + f"getting template repository path from incarnation state at {incarnation_state_path}" + ) + incarnation_state = load_incarnation_state(incarnation_state_path) + + logger.debug( + f"loaded incarnation state from incarnation repository {incarnation_state_path}", + incarnation_state=incarnation_state, + ) + + if overridden_template_repository is not None: + logger.debug( + f"overriding template repository with {overridden_template_repository}" + ) + incarnation_state = IncarnationState( + **{ + **asdict(incarnation_state), # type: ignore + "template_repository": str(overridden_template_repository), + } + ) + + if not Path(incarnation_state.template_repository).is_dir(): + logger.error( + f"template repository at {incarnation_state.template_repository} is not a local directory. " + "Might it be an URL? If, so, use `-r` to override the template repository from the " + "incarnation state with a path to the local clone of that template repository." + ) + raise typer.Exit(1) + + logger.debug( + "updating template data from incarnation state with given data from user", + incarnation_state_data=incarnation_state.template_data, + user_template_data=template_data, + ) + merged_template_data = incarnation_state.template_data.copy() + merged_template_data.update(template_data) + merged_template_data = { + k: v for k, v in merged_template_data.items() if k not in remove_template_data + } + + log = logger.bind( + template_repository=incarnation_state.template_repository, + incarnation_dir=incarnation_dir, + template_data=merged_template_data, + ) + + if update_repository_version is None: + update_repository_version = ( + check_output( + ["git", "rev-parse", "HEAD"], cwd=incarnation_state.template_repository + ) + .decode("utf-8") + .strip() + ) + + log.info( + f"starting update of incarnation to version {update_repository_version}...", + template_repository=str(incarnation_state.template_repository), + template_repository_version=update_repository_version, + template_data=merged_template_data, + ) + + try: + files_with_conflicts = asyncio.run( + update_incarnation_from_git_template_repository( + template_git_root_dir=Path(incarnation_state.template_repository), + update_template_repository=str(incarnation_state.template_repository), + update_template_repository_version=update_repository_version, + update_template_data=merged_template_data, + incarnation_root_dir=incarnation_dir, + diff_patch_func=diff_and_patch, + logger=log, + ) + ) + + if files_with_conflicts: + log.error( + f"update failed, there were conflicts while updating the following files: {', '.join([str(f) for f in files_with_conflicts])}" + ) + else: + log.info("successfully updated incarnation") + except Exception as exc: + log.exception(f"update failed: {exc}") + + +@app.callback() +def main( + verbose: bool = typer.Option( # noqa: B008 + False, "--verbose", "-v", help="turn on verbose logging" + ), + logs_as_json: bool = typer.Option( # noqa: B008 + False, "--json-logs", "-j", help="render logs as JSON" + ), +): + """ + Foxops engine ... use it to initialize or update template incarnations. + """ + if verbose: + setup_logging(logging.DEBUG, logs_as_json) + else: + setup_logging(logging.INFO, logs_as_json) + + global logger + logger = get_logger("app") + + +if __name__ == "__main__": + app() diff --git a/src/foxops/engine/fvars.py b/src/foxops/engine/fvars.py new file mode 100644 index 00000000..ef200bcc --- /dev/null +++ b/src/foxops/engine/fvars.py @@ -0,0 +1,39 @@ +from pathlib import Path + +from structlog.stdlib import BoundLogger + +from foxops.engine.models import TemplateData + +#: Holds the filename for the fvars file. +FVARS_FILENAME = "default.fvars" + + +def merge_template_data_with_fvars( + template_data: TemplateData, + fvars_directory: Path, + logger: BoundLogger, +) -> TemplateData: + """Merge the given Template data with fvars. + + The fvars filename is hardcoded as `default.fvars`. + + The `template_data` takes precedence over fvars. + If no fvars file in `fvars_directory` exists, the `template_data` is returned. + """ + fvars_path = fvars_directory / FVARS_FILENAME + fvars = read_variables_from_fvars_file(fvars_path, logger) + return {**fvars, **template_data} + + +def read_variables_from_fvars_file(path: Path, logger: BoundLogger) -> TemplateData: + """Read variables from a fvars file. + + If the file does not exist an empty `TemplateData` is returned. + """ + if not path.exists(): + return {} + + raw_variables = path.read_text().strip() + variables: TemplateData = dict(line.strip().split("=", maxsplit=1) for line in raw_variables.splitlines()) # type: ignore + logger.debug(f"read fvars from {path}: {variables}") + return variables diff --git a/src/foxops/engine/initialization.py b/src/foxops/engine/initialization.py new file mode 100644 index 00000000..455c1bad --- /dev/null +++ b/src/foxops/engine/initialization.py @@ -0,0 +1,106 @@ +from pathlib import Path + +from structlog.stdlib import BoundLogger + +from foxops.engine.fvars import merge_template_data_with_fvars +from foxops.engine.models import ( + IncarnationState, + TemplateData, + fill_missing_optionals_with_defaults, + load_template_config, + save_incarnation_state, +) +from foxops.engine.rendering import render_template +from foxops.external.git import GitRepository + + +async def initialize_incarnation( + template_root_dir: Path, + template_repository: str, + template_repository_version: str, + template_data: TemplateData, + incarnation_root_dir: Path, + logger: BoundLogger, +) -> IncarnationState: + """Initialize an incarnation repository with a version of a template. + + The initialization process consists of the following steps: + * ensure incarnation directory exists + * render template directory file system contents into incarnation directory + """ + template_data = merge_template_data_with_fvars( + template_data=template_data, + fvars_directory=incarnation_root_dir, + logger=logger, + ) + return await _initialize_incarnation( + template_root_dir=template_root_dir, + template_repository=template_repository, + template_repository_version=template_repository_version, + template_data=template_data, + incarnation_root_dir=incarnation_root_dir, + logger=logger, + ) + + +async def _initialize_incarnation( + template_root_dir: Path, + template_repository: str, + template_repository_version: str, + template_data: TemplateData, + incarnation_root_dir: Path, + logger: BoundLogger, +) -> IncarnationState: + # verify that the template data in the desired incarnation state match the required template variables + template_config = load_template_config(template_root_dir / "fengine.yaml") + logger.debug( + f"load template config from {template_config} to initialize incarnation at {incarnation_root_dir}" + ) + required_variable_names = set(template_config.required_variables.keys()) + provided_variable_names = set(template_data.keys()) + if not required_variable_names.issubset(provided_variable_names): + raise ValueError( + f"the template required the variables {sorted(required_variable_names)} " + "but the provided template data for the incarnation " + f"where {sorted(provided_variable_names)}. " + "Please make sure that the provided ones are a superset of the required ones." + ) + + # log additional template data passed for the incarnation + config_variable_names = set(template_config.variables.keys()) + if additional_values := provided_variable_names - config_variable_names: + logger.warn( + f"got additional template data for the incarnation: {sorted(additional_values)}" + ) + + # fill defaults in passed data + template_data_with_defaults = fill_missing_optionals_with_defaults( + provided_template_data=template_data, + template_config=template_config, + logger=logger, + ) + + await render_template( + template_root_dir / "template", + incarnation_root_dir, + template_data_with_defaults, + rendering_filename_exclude_patterns=template_config.rendering.excluded_files, + logger=logger, + ) + + template_repository_version_hash = await GitRepository( + template_root_dir, logger + ).head() + incarnation_state = IncarnationState( + template_repository=template_repository, + template_repository_version=template_repository_version, + template_repository_version_hash=template_repository_version_hash, + template_data=template_data_with_defaults, + ) + + incarnation_config_path = Path(incarnation_root_dir, ".fengine.yaml") + save_incarnation_state(incarnation_config_path, incarnation_state) + logger.debug( + f"save incarnation state to {incarnation_config_path} after template initialization" + ) + return incarnation_state diff --git a/src/foxops/engine/models.py b/src/foxops/engine/models.py new file mode 100644 index 00000000..569b75a4 --- /dev/null +++ b/src/foxops/engine/models.py @@ -0,0 +1,154 @@ +from dataclasses import asdict, dataclass +from pathlib import Path + +from pydantic import BaseModel, Field +from ruamel.yaml import YAML +from structlog.stdlib import BoundLogger + +yaml = YAML(typ="safe") +yaml.default_flow_style = False + + +#: Holds the type for all `template_data` dictionary values +TemplateDataValue = str | int | float +#: Holds the type for all `template_data` dictionaries +TemplateData = dict[str, TemplateDataValue] + + +@dataclass(frozen=True) +class IncarnationState: + """Represents an Incarnation State record. + + The incarnation state is recorded for every incarnation in `.fengine.yaml` + relative to the incarnation target directory. + + Use the `load_incarnation_state` and `save_incarnation_state` below to + load and save the incarnation from and to the file system. + """ + + #: Holds a reference to the template repository. + #: This may be a local path or a Git URL. + template_repository: str + #: Holds a version of the template repository + template_repository_version: str + #: Holds a git sha of the template repository the `template_repository_version` points to + template_repository_version_hash: str + #: Holds a dict of string key value pairs which have been used to + #: as template render data. + template_data: TemplateData + + +def save_incarnation_state( + incarnation_state_path: Path, incarnation_state: IncarnationState +) -> None: + with incarnation_state_path.open("w") as f: + f.write("# This file is auto-generated and owned by foxops.\n") + f.write("# DO NOT EDIT MANUALLY.\n") + yaml.dump(asdict(incarnation_state), f) + + +def load_incarnation_state(incarnation_state_path: Path) -> IncarnationState: + with incarnation_state_path.open("r") as f: + raw_state = yaml.load(f) + return IncarnationState(**raw_state) + + +def load_incarnation_state_from_string(incarnation_state: str) -> IncarnationState: + raw_state = yaml.load(incarnation_state) + return IncarnationState(**raw_state) + + +class VariableDefinition(BaseModel): + type: str = Field(..., description="The type of the variable") + description: str = Field(..., help="The description for this variable") + default: TemplateDataValue | None = Field( + None, + help="The default value for this variable (setting this makes the variable optional)", + ) + + def is_required(self) -> bool: + return self.default is None + + class Config: + allow_mutation = False + + +class TemplateRenderingConfig(BaseModel): + excluded_files: list[str] = Field( + default_factory=list, + description="List of filenames (relative to template/ directory) of which the content should not be rendered using Jinja. " + "Glob patterns are supported, but must match the individual files. Use '**/*' to match everything recursively.", + ) + + class Config: + allow_mutation = False + + +class TemplateConfig(BaseModel): + """Represents the configuration of a template. + + Example: + + ```yaml + required_foxops_version: v1.0.0 + + variables: + name: + description: the name of the project + type: str + + number: + description: same number + type: int + ``` + """ + + required_foxops_version: str = "v0.0.0" + rendering: TemplateRenderingConfig = Field(default_factory=TemplateRenderingConfig) + + variables: dict[str, VariableDefinition] = Field(default_factory=dict) + + @property + def required_variables(self) -> dict[str, VariableDefinition]: + return {k: v for k, v in self.variables.items() if v.is_required()} + + @property + def optional_variables_defaults(self) -> TemplateData: + return { + k: v.default for k, v in self.variables.items() if v.default is not None + } + + def to_yaml(self, target: Path) -> None: + with target.open("w") as f: + yaml.dump(self.dict(), f) + + class Config: + allow_mutation = False + + +def load_template_config(template_config_path: Path) -> TemplateConfig: + try: + with template_config_path.open("r") as f: + raw_state = yaml.load(f) + return TemplateConfig(**raw_state) + except FileNotFoundError: + return TemplateConfig() + + +def fill_missing_optionals_with_defaults( + provided_template_data: TemplateData, + template_config: TemplateConfig, + logger: BoundLogger, +) -> TemplateData: + provided_variable_names = set(provided_template_data.keys()) + config_variable_names = set(template_config.variables.keys()) + template_data_with_defaults = provided_template_data.copy() + optional_vars = template_config.optional_variables_defaults + if need_default := config_variable_names.difference(provided_variable_names): + logger.debug( + f"Given template data is missing the values for the optional variables {need_default}, using defaults for those" + ) + for key in (k for k in need_default if k in optional_vars): + template_data_with_defaults[key] = optional_vars[key] + + return template_data_with_defaults diff --git a/src/foxops/engine/patching/__init__.py b/src/foxops/engine/patching/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/foxops/engine/patching/git_diff_patch.py b/src/foxops/engine/patching/git_diff_patch.py new file mode 100644 index 00000000..fd581004 --- /dev/null +++ b/src/foxops/engine/patching/git_diff_patch.py @@ -0,0 +1,202 @@ +import filecmp +import re +import shutil +import typing +from contextlib import asynccontextmanager +from pathlib import Path +from tempfile import TemporaryDirectory, mkstemp + +from structlog.stdlib import BoundLogger + +from foxops.external.git import GitRepository +from foxops.settings import Settings +from foxops.utils import CalledProcessError, check_call + + +async def diff_and_patch( + diff_a_directory: Path, + diff_b_directory: Path, + patch_directory: Path, + logger: BoundLogger, +) -> list[Path]: + patch_path = await diff(diff_a_directory, diff_b_directory, logger=logger) + try: + return await patch( + patch_path, + patch_directory, + diff_a_directory, + diff_b_directory, + logger=logger, + ) + finally: + patch_path.unlink() + + +@asynccontextmanager +async def setup_diff_git_repository( + old_directory: Path, new_directory: Path +) -> typing.AsyncGenerator[Path, None]: + # FIXME(TF): in case the *git way* provides as the viable long-term solution we could directly + # bootstrap that intermediate git repository instead of this copy-paste roundtrip. + settings = Settings() + git_tmpdir: str + with TemporaryDirectory() as git_tmpdir: + await check_call("git", "init", ".", "--initial-branch", "main", cwd=git_tmpdir) + await check_call( + "git", + "config", + "user.name", + settings.git_commit_author_name, + cwd=git_tmpdir, + ) + await check_call( + "git", + "config", + "user.email", + settings.git_commit_author_email, + cwd=git_tmpdir, + ) + await check_call( + "git", + "commit", + "--allow-empty", + "-m", + "empty initial commit", + cwd=git_tmpdir, + ) + await check_call("git", "switch", "--create", "old", "main", cwd=git_tmpdir) + shutil.copytree(old_directory, git_tmpdir, dirs_exist_ok=True) + await check_call("git", "add", ".", cwd=git_tmpdir) + await check_call("git", "commit", "-m", "old template status", cwd=git_tmpdir) + await check_call("git", "switch", "--create", "new", "main", cwd=git_tmpdir) + shutil.copytree(new_directory, git_tmpdir, dirs_exist_ok=True) + await check_call("git", "add", ".", cwd=git_tmpdir) + await check_call("git", "commit", "-m", "new template status", cwd=git_tmpdir) + + yield Path(git_tmpdir) + + +async def diff(old_directory: Path, new_directory: Path, logger: BoundLogger) -> Path: + async with setup_diff_git_repository(old_directory, new_directory) as git_tmpdir: + logger.debug(f"create git diff between branch old and new in {git_tmpdir}") + + repo = GitRepository(git_tmpdir, logger) + diff_output = await repo.diff("old", "new") + + patch_path: str + _, patch_path = mkstemp(prefix="fengine-update-", suffix=".patch") + + (p := Path(patch_path)).write_text(diff_output) + return p + + +async def patch( + patch_path: Path, + incarnation_root_dir: Path, + diff_a_directory: Path, + diff_b_directory: Path, + logger: BoundLogger, +) -> list[Path]: + # NOTE(TF): it's crucial that the paths are fully resolved here, + # because we are going to fiddle around how they + # are relative to each other. + resolved_incarnation_root_dir = incarnation_root_dir.resolve() + proc = await check_call( + "git", "rev-parse", "--show-toplevel", cwd=str(resolved_incarnation_root_dir) + ) + incarnation_git_root_dir = Path((await proc.stdout.read()).decode("utf-8").strip()).resolve() # type: ignore + incarnation_git_dir = ( + resolved_incarnation_root_dir.relative_to(incarnation_git_root_dir) + if resolved_incarnation_root_dir != incarnation_git_root_dir + else "" + ) + + # FIXME(TF): may check git status to check if something has been modified or not ... + logger.debug( + f"applying patch {patch_path} to {incarnation_git_dir} inside {incarnation_root_dir}" + ) + try: + await check_call( + "git", + "apply", + "--reject", # This option makes it apply the parts of the patch that are applicable, and leave the rejected hunks in corresponding *.rej files. + "--verbose", + "--directory", + str(incarnation_git_dir), + str(patch_path), + cwd=str(incarnation_git_root_dir), + ) + except CalledProcessError as exc: + logger.debug( + "detected conflicts with patch, analyzing rejections ...", + patch_path=patch_path, + ) + apply_rejection_output = exc.stderr + files_with_conflicts = await analyze_patch_rejections( + apply_rejection_output, + resolved_incarnation_root_dir, + diff_a_directory, + diff_b_directory, + logger=logger, + ) + return files_with_conflicts + else: + return [] + + +async def analyze_patch_rejections( + apply_rejection_output: bytes, + patch_directory: Path, + diff_a_directory: Path, + diff_b_directory: Path, + logger: BoundLogger, +) -> list[Path]: + reject_file_regex = re.compile(rb"Applying patch (.*?) with (\d+) reject...") + + files_with_rejections = [] + for line in apply_rejection_output.splitlines(): + if match := reject_file_regex.match(line): + files_with_rejections.append(Path(match.group(1).decode())) + + logger.debug( + f"detected {len(files_with_rejections)} files with rejections", + files_with_rejections=files_with_rejections, + ) + files_with_conflicts: list[Path] = [] + for file_with_rejection in files_with_rejections: + conflict_fixed = await attempt_fixing_rejection( + file_with_rejection, + patch_directory, + diff_a_directory, + diff_b_directory, + logger=logger, + ) + if not conflict_fixed: + logger.debug(f"file {file_with_rejection} still has conflicts") + files_with_conflicts.append(file_with_rejection) + return files_with_conflicts + + +async def attempt_fixing_rejection( + file_with_rejection: Path, + patch_directory: Path, + diff_a_directory: Path, + diff_b_directory: Path, + logger: BoundLogger, +) -> bool: + # diff_a_file = diff_a_directory / file_with_rejection + diff_b_file = diff_b_directory / file_with_rejection + patch_file = patch_directory / file_with_rejection + + logger.debug(f"attempting to fix rejection for file {file_with_rejection} ...") + + if filecmp.cmp(diff_b_file, patch_file, shallow=True): + # the rejected hunk tried to apply a change which was already applied, + # we can safely remove the rejection file. + (patch_file.with_suffix(patch_file.suffix + ".rej")).unlink() + logger.debug( + f"the rejection was caused because the two files are identical, mark {file_with_rejection} as fixed" + ) + return True + + return False diff --git a/src/foxops/engine/rendering.py b/src/foxops/engine/rendering.py new file mode 100644 index 00000000..7d8287b7 --- /dev/null +++ b/src/foxops/engine/rendering.py @@ -0,0 +1,244 @@ +import functools +import os +import typing +from pathlib import Path + +from aiopath import AsyncPath +from jinja2 import FileSystemLoader, StrictUndefined +from jinja2.sandbox import SandboxedEnvironment +from structlog.stdlib import BoundLogger + +from foxops.engine.models import TemplateData + + +def create_template_environment(template_root_dir: Path) -> SandboxedEnvironment: + """Create a virtual environment to render a template into an incarnation. + + As of now the environment is an untouched jinja2 sandboxed environment + which only has access to the template root directory. + """ + paths = [template_root_dir] + loader = FileSystemLoader(paths) + # NOTE(TF): add extensions to the loader if necessary. + env = SandboxedEnvironment( + loader=loader, + enable_async=True, + keep_trailing_newline=True, + undefined=StrictUndefined, + ) + return env + + +async def render_template( + template_root_dir: Path, + incarnation_root_dir: Path, + template_data: TemplateData, + rendering_filename_exclude_patterns: list[str], + logger: BoundLogger, +) -> None: + """Render a template into an incarnation. + + As of now a very simplistic approach is used to find and render the files + and folders in a template. + + :param rendering_filename_exclude_patterns: A list of glob patterns matching files which contents should not be rendered. Can be empty. + """ + if not template_root_dir.is_absolute(): + raise ValueError( + f"template_root_dir must be an absolute path, got {template_root_dir}" + ) + + files_to_render = set(template_root_dir.glob("**/*")) + for pattern in rendering_filename_exclude_patterns: + files_to_render -= set(template_root_dir.glob(pattern)) + + environment = create_template_environment(template_root_dir) + + logger.debug( + "start rendering template", + template_root_dir=template_root_dir, + incarnation_root_dir=incarnation_root_dir, + template_data=template_data, + rendering_filename_exclude_patterns=rendering_filename_exclude_patterns, + ) + + async def _render_template_symlink(template_symlink_path): + return await render_template_symlink( + environment, + template_symlink_path, + incarnation_root_dir, + template_data, + logger=logger, + ) + + async def _render_template_dir(template_dir_path): + return await render_template_dir( + environment, + template_dir_path, + incarnation_root_dir, + template_data, + logger=logger, + ) + + async def _render_template_file(template_file_path, render_content: bool): + return await render_template_file( + environment, + template_file_path, + incarnation_root_dir, + template_data, + render_content=render_content, + logger=logger, + ) + + for root_dir, dirs, files in os.walk(template_root_dir): + for d in dirs: + template_dir_path = Path(root_dir) / d + if template_dir_path.is_symlink(): + await _render_template_symlink(template_dir_path) + else: + await _render_template_dir(template_dir_path) + + for f in files: + template_file_path = Path(root_dir) / f + if template_file_path.is_symlink(): + await _render_template_symlink(template_file_path) + else: + await _render_template_file( + template_file_path, + render_content=template_file_path in files_to_render, + ) + + +async def render_template_file( + environment: SandboxedEnvironment, + template_file_path: Path, + incarnation_root_dir: Path, + template_data: TemplateData, + render_content: bool, + logger: BoundLogger, +) -> Path: + """Render a template file into an incarnation file. + + The template file content and file name are rendered if rendering_enabled is True. Otherwise, rendering of the file + content is skipped. + """ + loader: FileSystemLoader = typing.cast(FileSystemLoader, environment.loader) + relative_template_path = template_file_path.relative_to(loader.searchpath[0]) + + if render_content: + # get and render template file contents + content_template = environment.get_template(str(relative_template_path)) + rendered_content = await content_template.render_async(**template_data) + else: + rendered_content = template_file_path.read_text() + + # get and render template file path + # NOTE (AH): Even when file content rendering is disabled, we still need to render the file path. + # This is because we always render folder names - so the file wouldn't end up in the correct location + # within the incarnation. + path_template = environment.from_string(str(relative_template_path)) + rendered_path = Path(await path_template.render_async(**template_data)) + + logger.debug( + "rendering file in incarnation", + content_rendered=render_content, + path=rendered_path, + ) + + template_file_stat = template_file_path.stat(follow_symlinks=False) # type: ignore + incarnation_file_path = AsyncPath(incarnation_root_dir, rendered_path) + await incarnation_file_path.parent.mkdir(parents=True, exist_ok=True) + await incarnation_file_path.write_text(rendered_content) + apply_path_stats(Path(incarnation_file_path), template_file_stat) + return incarnation_file_path + + +async def render_template_dir( + environment: SandboxedEnvironment, + template_dir_path: Path, + incarnation_root_dir: Path, + template_data: TemplateData, + logger: BoundLogger, +) -> Path: + """Render a template directory path into an incarnation directory path.""" + loader: FileSystemLoader = typing.cast(FileSystemLoader, environment.loader) + relative_template_dir_path = template_dir_path.relative_to(loader.searchpath[0]) + + # get and render template file path + path_template = environment.from_string(str(relative_template_dir_path)) + rendered_path = Path(await path_template.render_async(**template_data)) + + logger.debug("rendering directory in incarnation", path=rendered_path) + + template_dir_stat = template_dir_path.stat(follow_symlinks=False) # type: ignore + incarnation_dir_path = AsyncPath(incarnation_root_dir, rendered_path) + await incarnation_dir_path.mkdir(parents=True, exist_ok=True) + apply_path_stats(Path(incarnation_dir_path), template_dir_stat) + return incarnation_dir_path + + +async def render_template_symlink( + environment: SandboxedEnvironment, + template_symlink_path: Path, + incarnation_root_dir: Path, + template_data: TemplateData, + logger: BoundLogger, +) -> Path: + """Render a template symlink path into an incarnation symlink path.""" + loader: FileSystemLoader = typing.cast(FileSystemLoader, environment.loader) + relative_template_symlink_path = template_symlink_path.relative_to( + loader.searchpath[0] + ) + + # get and render template file path + path_template = environment.from_string(str(relative_template_symlink_path)) + rendered_path = Path(await path_template.render_async(**template_data)) + # get and render template symlink target + symlink_target_template = environment.from_string( + str(template_symlink_path.readlink()) + ) + rendered_symlink_target_path = Path( + await symlink_target_template.render_async(**template_data) + ) + + logger.debug( + "rendering symlink in incarnation", + source_path=rendered_path, + target_path=rendered_symlink_target_path, + ) + + template_symlink_stat = template_symlink_path.stat(follow_symlinks=False) # type: ignore + incarnation_symlink_path = incarnation_root_dir / rendered_path + incarnation_symlink_path.parent.mkdir(parents=True, exist_ok=True) + incarnation_symlink_path.symlink_to(rendered_symlink_target_path) + apply_path_stats(incarnation_symlink_path, template_symlink_stat) + return incarnation_symlink_path + + +def apply_path_stats(path: Path, target_stat: os.stat_result) -> None: + """Apply the stats obtained from one path to another path. + + Insights: + + fengine mainly operates within Git repositories. + Git doesn't store information about the owners and also ONLY + tracks the executable bit of a UNIX file permissions, meaning that + only the modes `100755` and `100644` are supported. + + However, this function still applies the entire mode (reported by `stat`), + to keep things simple. + It also doesn't affect the ownership of the file. + + This function doesn't follow symlinks. + """ + chmod = functools.partial(path.chmod, target_stat.st_mode) + if path.is_symlink(): + try: + chmod(follow_symlinks=False) + except NotImplementedError: + # NOTE(TF): some UNIX platforms (like Linux) don't allow to change permissions + # on symlinks. They ALWAYS get 0o777. + # Thus, we don't do nothing here. + pass + else: + chmod() diff --git a/src/foxops/engine/update.py b/src/foxops/engine/update.py new file mode 100644 index 00000000..3f7bd65f --- /dev/null +++ b/src/foxops/engine/update.py @@ -0,0 +1,153 @@ +from pathlib import Path +from tempfile import TemporaryDirectory + +from dictdiffer import diff +from structlog.stdlib import BoundLogger + +from foxops import utils +from foxops.engine.fvars import merge_template_data_with_fvars +from foxops.engine.initialization import _initialize_incarnation +from foxops.engine.models import ( + IncarnationState, + TemplateData, + fill_missing_optionals_with_defaults, + load_incarnation_state, + load_template_config, +) + + +def get_data_mismatch( + desired_template_data: TemplateData, + actual_template_data: TemplateData, + template_root_dir: Path, + logger: BoundLogger, +) -> list: + template_config = load_template_config(template_root_dir / "fengine.yaml") + + # fill defaults in passed data + template_data_with_defaults = fill_missing_optionals_with_defaults( + provided_template_data=desired_template_data, + template_config=template_config, + logger=logger, + ) + + return list( + diff( + actual_template_data, + template_data_with_defaults, + ) + ) + + +async def update_incarnation_from_git_template_repository( + template_git_root_dir: Path, + update_template_repository: str, + update_template_repository_version: str, + update_template_data: TemplateData, + incarnation_root_dir: Path, + diff_patch_func, + logger: BoundLogger, +) -> tuple[IncarnationState, list[Path]]: + # initialize pristine incarnation from current incarnation state + current_incarnation_state_path = incarnation_root_dir / ".fengine.yaml" + current_incarnation_state = load_incarnation_state(current_incarnation_state_path) + + with TemporaryDirectory() as template_root_dir, TemporaryDirectory() as updated_template_root_dir: + logger.debug( + f"creating git worktree from current template repository (version: {current_incarnation_state.template_repository_version_hash}) at {template_root_dir}" + ) + await utils.check_call( + "git", + "worktree", + "add", + template_root_dir, + current_incarnation_state.template_repository_version_hash, + cwd=template_git_root_dir, + ) + logger.debug( + f"creating git worktree from updated template repository (version: {update_template_repository_version}) at {updated_template_root_dir}" + ) + await utils.check_call( + "git", + "worktree", + "add", + updated_template_root_dir, + update_template_repository_version, + cwd=template_git_root_dir, + ) + + return await update_incarnation( + template_root_dir=Path(template_root_dir), + update_template_root_dir=Path(updated_template_root_dir), + update_template_repository=update_template_repository, + update_template_repository_version=update_template_repository_version, + update_template_data=update_template_data, + incarnation_root_dir=incarnation_root_dir, + diff_patch_func=diff_patch_func, + logger=logger, + ) + + +async def update_incarnation( + template_root_dir: Path, + update_template_root_dir: Path, + update_template_repository: str, + update_template_repository_version: str, + update_template_data: TemplateData, + incarnation_root_dir: Path, + diff_patch_func, + logger: BoundLogger, +) -> tuple[IncarnationState, list[Path]]: + """Update an incarnation with a new version of a template.""" + # initialize pristine incarnation from current incarnation state + current_incarnation_state_path = incarnation_root_dir / ".fengine.yaml" + current_incarnation_state = load_incarnation_state(current_incarnation_state_path) + + with TemporaryDirectory() as tmp_pristine_incarnation_dir, TemporaryDirectory() as tmp_updated_incarnation_dir: + logger.debug( + "initialize pristine incarnation from current incarnation state", + template_dir=template_root_dir, + incarnation_dir=tmp_pristine_incarnation_dir, + ) + await _initialize_incarnation( + template_root_dir=template_root_dir, + template_repository=current_incarnation_state.template_repository, + template_repository_version=current_incarnation_state.template_repository_version, + template_data=current_incarnation_state.template_data, + incarnation_root_dir=Path(tmp_pristine_incarnation_dir), + logger=logger, + ) + + logger.debug( + "initialize new incarnation from update incarnation state", + template_dir=update_template_root_dir, + incarnation_dir=tmp_updated_incarnation_dir, + ) + updated_incarnation_state = await _initialize_incarnation( + template_root_dir=update_template_root_dir, + template_repository=update_template_repository, + template_repository_version=update_template_repository_version, + template_data=merge_template_data_with_fvars( + update_template_data, + incarnation_root_dir, + logger, + ), + incarnation_root_dir=Path(tmp_updated_incarnation_dir), + logger=logger, + ) + + # diff pristine and new incarnations + # apply patch on incarnation to update + logger.debug( + "applying patch on pristine and new incarnations", + diff_a_directory=tmp_pristine_incarnation_dir, + diff_b_directory=tmp_updated_incarnation_dir, + patch_directory=incarnation_root_dir, + ) + files_with_conflicts = await diff_patch_func( + diff_a_directory=tmp_pristine_incarnation_dir, + diff_b_directory=tmp_updated_incarnation_dir, + patch_directory=incarnation_root_dir, + logger=logger, + ) + return updated_incarnation_state, files_with_conflicts diff --git a/src/foxops/errors.py b/src/foxops/errors.py new file mode 100644 index 00000000..50a7dd48 --- /dev/null +++ b/src/foxops/errors.py @@ -0,0 +1,6 @@ +class FoxopsError(Exception): + """Base Exception for all Foxops specific errors.""" + + +class RetryableError(FoxopsError): + """Exception raised when an error occurs for which a retry usually helps.""" diff --git a/src/foxops/external/__init__.py b/src/foxops/external/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/foxops/external/git.py b/src/foxops/external/git.py new file mode 100644 index 00000000..0ef80567 --- /dev/null +++ b/src/foxops/external/git.py @@ -0,0 +1,206 @@ +import asyncio +import shutil +from pathlib import Path +from tempfile import mkdtemp +from urllib.parse import quote, urlparse, urlunparse + +from structlog.stdlib import BoundLogger + +from foxops.errors import FoxopsError, RetryableError +from foxops.settings import Settings +from foxops.utils import CalledProcessError, check_call + +GIT_REBASE_REQUIRED_ERROR_MESSAGE = ( + "hint: Updates were rejected because the remote contains work that you do\n" + "hint: not have locally." +) + + +class GitError(FoxopsError): + """Error raised when a call to git fails.""" + + def __init__(self, message=None): + super().__init__( + message if message else "Git failed with an unexpected non-zero exit code." + ) + + +async def git_exec(*args, **kwargs) -> asyncio.subprocess.Process: + try: + return await check_call("git", *args, **kwargs) + except CalledProcessError as exc: + raise GitError() from exc + + +def add_authentication_to_git_clone_url(source: str, username: str, password: str): + if not source.startswith("https://"): + raise ValueError("only https:// repository URLs are allowed") + + url_parts = urlparse(source) + if url_parts.username is not None or url_parts.password is not None: + raise ValueError( + f"the repository URL ({source}) must not contain a username/password. " + "pass them in as separate variables instead." + ) + + netloc = f"{quote(username, safe='')}:{quote(password, safe='')}@{url_parts.netloc}" + url_parts = url_parts._replace(netloc=netloc) + + return urlunparse(url_parts) + + +class GitRepository: + def __init__(self, directory: Path, logger: BoundLogger): + if not directory.exists(): + raise ValueError("the given path doesn't exist") + if not directory.is_dir(): + raise ValueError("the given path is not a directory") + + self.directory = directory + self.logger = logger.bind(directory=directory.name) + + async def _run( + self, *args, timeout: int | float | None = 10, **kwargs + ) -> asyncio.subprocess.Process: + return await git_exec(*args, cwd=self.directory, timeout=timeout, **kwargs) + + async def has_any_commits(self) -> bool: + result = await self._run("rev-list", "-n", "1", "--all") + stdout = await result.stdout.read() if result.stdout is not None else b"" + + return len(stdout.strip()) > 0 + + async def create_and_checkout_branch(self, branch: str, exist_ok=False): + try: + await self._run("checkout", "-b", branch) + except GitError: + if exist_ok: + await self._run("checkout", branch) + else: + raise + + async def commit_all(self, message: str): + await self._run("add", ".") + return await self._run("commit", "-m", message) + + async def diff(self, ref_old: str, ref_new: str) -> str: + proc = await self._run( + "--no-pager", + "diff", + f"{ref_old}..{ref_new}", + expected_returncodes=frozenset({0, 1}), + timeout=30, + ) + + output = "" + if proc.stdout is not None: + output = (await proc.stdout.read()).decode() + + return output + + async def push(self): + proc = await self._run("branch", "--show-current") + current_branch = (await proc.stdout.read()).strip() + + self.logger.debug("pushing branch", current_branch=current_branch) + + proc = await self._run("push", "--porcelain", "-u", "origin", current_branch) + stderr = (await proc.stderr.read()).decode() + + # exclude remote messages from stderr + stderr_non_remote_lines = list( + filter(lambda line: not line.startswith("remote:"), stderr.splitlines()) + ) + if len(stderr_non_remote_lines) > 0: + self.logger.error( + "got unexpected error output while reading branch", stderr=stderr + ) + raise GitError() + + self.logger.debug("push output", stdout=await proc.stdout.read()) + + async def push_with_potential_retry(self) -> None: + try: + await self.push() + except GitError as exc: + if isinstance(exc.__cause__, CalledProcessError): + if GIT_REBASE_REQUIRED_ERROR_MESSAGE in exc.__cause__.stderr: + raise RetryableError( + "new commits on the target branch, need to retry" + ) from exc + raise exc + + async def head(self) -> str: + proc = await self._run("rev-parse", "HEAD") + if proc.stdout is None: + raise GitError("unable to determine the current git HEAD") + return (await proc.stdout.read()).decode().strip() + + async def fetch(self, refspec: str) -> None: + await self._run("fetch", "origin", refspec) + + +class TemporaryGitRepository: + GIT_HISTORY_DEPTH = 1 + + def __init__( + self, + logger: BoundLogger, + source: str, + username: str, + password: str, + refspec: str | None = None, + ): + self.settings = Settings() + self.logger = logger + self.source = add_authentication_to_git_clone_url(source, username, password) + self.refspec = refspec + + self.tempdir: Path | None = None + + async def __aenter__(self) -> GitRepository: + self.tempdir = Path(mkdtemp()) + + if self.refspec is None: + await git_exec( + "clone", + f"--depth={self.GIT_HISTORY_DEPTH}", + self.source, + self.tempdir, + cwd=Path.home(), + ) + else: + # NOTE(TF): this only works for git hosters which have enabled `uploadpack.allowReachableSHA1InWant` on the server side. + # It seems to be the case for GitHub and GitLab. + # In addition it seems that if the refspec is a tag, it won't be created locally and we later on + # cannot address it in e.g. a `switch`. So we need to fetch all tag refs, which should be fine. + await git_exec("init", self.tempdir, cwd=Path.home()) + await git_exec("remote", "add", "origin", self.source, cwd=self.tempdir) + await git_exec( + "fetch", + f"--depth={self.GIT_HISTORY_DEPTH}", + "origin", + "--tags", + self.refspec, + cwd=self.tempdir, + ) + await git_exec("reset", "--hard", "FETCH_HEAD", cwd=self.tempdir) + + # NOTE(TF): set author data + await git_exec( + "config", + "user.name", + self.settings.git_commit_author_name, + cwd=self.tempdir, + ) + await git_exec( + "config", + "user.email", + self.settings.git_commit_author_email, + cwd=self.tempdir, + ) + + return GitRepository(self.tempdir, logger=self.logger) + + async def __aexit__(self, exc_type, exc_val, exc_tb): + shutil.rmtree(self.tempdir) diff --git a/src/foxops/external/gitlab.py b/src/foxops/external/gitlab.py new file mode 100644 index 00000000..e9eeafcd --- /dev/null +++ b/src/foxops/external/gitlab.py @@ -0,0 +1,198 @@ +import base64 +import typing +from datetime import timedelta +from pathlib import Path +from typing import Any +from urllib.parse import quote_plus + +import aiohttp +from tenacity import retry, retry_if_exception_type, stop_after_delay, wait_fixed + +ProjectIdentifier = typing.Union[str, Path, int] + + +class GitlabException(Exception): + def __init__(self, code: int, message: str): + self.code = code + self.message = message + + +class GitlabNotFoundException(GitlabException): + def __init__(self, message: str): + super().__init__(404, message) + + +class AsyncGitlabClient: + def __init__(self, token: str, base_url: str): + self.base_url = base_url.rstrip("/") + self.token = token + self.session = aiohttp.ClientSession(headers={"PRIVATE-TOKEN": token}) + + async def __aenter__(self) -> "AsyncGitlabClient": + return self + + async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + await self.session.close() + + async def _request( + self, + method: str, + url: str, + json: Any = None, + data: Any = None, + params: dict[str, str] | None = None, + ) -> Any: + url = self.base_url + url + async with self.session.request( + method, url, params=params, json=json, data=data + ) as response: + if 200 <= response.status < 300: + if response.content_type == "application/json": + return await response.json() + + data = await response.json() + if response.status == 404: + message = data.get("message", data.get("error", "Unknown error")) + raise GitlabNotFoundException(message) + raise GitlabException(response.status, data["message"]) + + async def _get(self, url: str, params: dict[str, str] | None = None) -> Any: + return await self._request("GET", url, params=params) + + async def _post( + self, + url: str, + json: Any = None, + data: Any = None, + params: dict[str, str] | None = None, + ) -> Any: + return await self._request("POST", url, json=json, data=data, params=params) + + async def _put( + self, + url: str, + json: Any = None, + data: Any = None, + params: dict[str, str] | None = None, + ) -> Any: + return await self._request("PUT", url, json=json, data=data, params=params) + + async def _delete(self, url: str, params: dict[str, str] | None = None) -> Any: + return await self._request("DELETE", url, params=params) + + async def group_get(self, id_: str | int): + group_id = quote_plus(str(id_)) + return await self._get(f"/groups/{group_id}") + + async def project_get(self, id_: ProjectIdentifier): + project_id = quote_plus(str(id_)) + return await self._get(f"/projects/{project_id}") + + async def project_merge_requests_create( + self, + id_: ProjectIdentifier, + source_branch: str, + target_branch: str, + title: str, + description: str, + ): + return await self._post( + f"/projects/{quote_plus(str(id_))}/merge_requests", + json={ + "source_branch": source_branch, + "target_branch": target_branch, + "title": title, + "remove_source_branch": True, + "description": description, + }, + ) + + async def automerge_merge_request( + self, + id_: ProjectIdentifier, + merge_request_iid: int, + timeout: timedelta | None = None, + ): + """Automerge the given Merge Request. + + It will immediately merge if possible, otherwise when the pipeline succeeds. + There won't be made any rebases or similar nor wait for the actual merge to happen. + """ + if timeout is None: + timeout = timedelta(minutes=5) + + @retry( + retry=retry_if_exception_type(GitlabException), + stop=stop_after_delay(timeout.total_seconds()), + wait=wait_fixed(1), + ) + async def __merge(): + merge_request = await self._get( + f"/projects/{quote_plus(str(id_))}/merge_requests/{merge_request_iid}" + ) + has_pipeline = merge_request.get("pipeline", None) is not None + data = {"merge_when_pipeline_succeeds": True} if has_pipeline else None + return await self._put( + f"/projects/{quote_plus(str(id_))}/merge_requests/{merge_request_iid}/merge", + json=data, + ) + + return await __merge() + + async def project_merge_requests_list( + self, + id_: ProjectIdentifier, + state: str | None = None, + source_branch: str | None = None, + ): + params = {} + if state is not None: + params["state"] = state + if source_branch is not None: + params["source_branch"] = source_branch + + return await self._get( + f"/projects/{quote_plus(str(id_))}/merge_requests", params=params + ) + + async def project_repository_branches_create( + self, id_: ProjectIdentifier, branch: str, ref: str + ): + await self._post( + f"/projects/{quote_plus(str(id_))}/repository/branches", + params={"branch": branch, "ref": ref}, + ) + + async def project_repository_branches_list( + self, id_: ProjectIdentifier, search: str | None = None + ): + params = {} + if search is not None: + params["search"] = search + + return await self._get( + f"/projects/{quote_plus(str(id_))}/repository/branches", params=params + ) + + async def project_repository_branches_delete( + self, id_: ProjectIdentifier, branch: str + ): + branch = quote_plus(branch) + await self._delete( + f"/projects/{quote_plus(str(id_))}/repository/branches/{branch}" + ) + + async def project_repository_files_get_content( + self, id_: ProjectIdentifier, branch: str, filepath: str | Path + ) -> bytes: + response = await self._get( + f"/projects/{quote_plus(str(id_))}/repository/files/{quote_plus(str(filepath))}", + params={"ref": branch}, + ) + + if (encoding := response.get("encoding")) != "base64": + raise ValueError( + f"Received file content was not base64 encoded. Instead: {encoding}" + ) + + return base64.b64decode(response["content"]) diff --git a/src/foxops/logging.py b/src/foxops/logging.py new file mode 100644 index 00000000..4a4e503c --- /dev/null +++ b/src/foxops/logging.py @@ -0,0 +1,60 @@ +import logging +import sys + +import structlog + + +def setup_logging(log_level: int = logging.INFO, as_json: bool = False): + processors: list[structlog.types.Processor] = [ + # If log level is too low, abort pipeline and throw away log entry. + structlog.stdlib.filter_by_level, + # Add the name of the logger to event dict. + structlog.stdlib.add_logger_name, + # Add log level to event dict. + structlog.stdlib.add_log_level, + # Perform %-style formatting. + structlog.stdlib.PositionalArgumentsFormatter(), + # Add a timestamp in ISO 8601 format. + structlog.processors.TimeStamper(fmt="iso"), + # If the "stack_info" key in the event dict is true, remove it and + # render the current stack trace in the "stack" key. + structlog.processors.StackInfoRenderer(), + # If the "exc_info" key in the event dict is either true or a + # sys.exc_info() tuple, remove "exc_info" and render the exception + # with traceback into the "exception" key. + structlog.processors.format_exc_info, + # If some value is in bytes, decode it to a unicode str. + structlog.processors.UnicodeDecoder(), + ] + + if as_json: + # Render the final event dict as JSON. + processors.append(structlog.processors.JSONRenderer()) + else: + # Render the final event for the console. + processors.append(structlog.dev.ConsoleRenderer()) + + structlog.configure( + processors=processors, + # `wrapper_class` is the bound logger that you get back from + # get_logger(). This one imitates the API of `logging.Logger`. + wrapper_class=structlog.stdlib.BoundLogger, + # `logger_factory` is used to create wrapped loggers that are used for + # OUTPUT. This one returns a `logging.Logger`. The final value (a JSON + # string) from the final processor (`JSONRenderer`) will be passed to + # the method of the same name as that you've called on the bound logger. + logger_factory=structlog.stdlib.LoggerFactory(), + # Effectively freeze configuration after creating the first bound + # logger. + cache_logger_on_first_use=True, + ) + + logging.basicConfig( + format="%(message)s", + stream=sys.stdout, + level=log_level, + ) + + +def get_logger(name: str) -> structlog.stdlib.BoundLogger: + return structlog.get_logger(name) diff --git a/src/foxops/models.py b/src/foxops/models.py new file mode 100644 index 00000000..73076872 --- /dev/null +++ b/src/foxops/models.py @@ -0,0 +1,49 @@ +from pathlib import Path +from typing import Optional + +from pydantic import BaseModel, Field +from ruamel.yaml import YAML + +from foxops.engine import IncarnationState, TemplateData + +yaml = YAML(typ="safe") + + +class DesiredIncarnationStateConfig(BaseModel): + gitlab_project: Path = Field( + ..., description="The full GitLab path to the incarnation project" + ) + target_directory: Path = Field( + Path("."), description="The target incarnation directory within the repository" + ) + template_repository: str = Field( + ..., description="The URL to the template repository" + ) + template_repository_version: str = Field( + ..., description="The version of the template repository as a Git revision" + ) + template_data: TemplateData = Field( + ..., description="The template data to use when rendering the template" + ) + automerge: bool = Field( + False, + description="Automatically merge the created Merge Request. It won't wait until the merge is complete.", + ) + + class Config: + allow_mutation = False + + +class IncarnationRemoteGitRepositoryState(BaseModel): + """Represents a remote Git repository that contains one or multiple incarnations.""" + + # FIXME(TF): this is the only thing GitLab specific here. + # We might want to get rid of that ... + gitlab_project_id: int + remote_url: str + default_branch: str + incarnation_directory: Path + incarnation_state: Optional[IncarnationState] + + class Config: + allow_mutation = False diff --git a/src/foxops/reconciliation.py b/src/foxops/reconciliation.py new file mode 100644 index 00000000..c59b78d2 --- /dev/null +++ b/src/foxops/reconciliation.py @@ -0,0 +1,492 @@ +import asyncio +import glob +import hashlib +import os +import uuid +from enum import IntEnum, auto +from pathlib import Path + +from structlog.stdlib import BoundLogger +from tenacity import retry, retry_if_exception_type, stop_after_attempt + +from foxops.engine import ( + FVARS_FILENAME, + diff_and_patch, + initialize_incarnation, + load_incarnation_state_from_string, + update_incarnation_from_git_template_repository, +) +from foxops.engine.update import get_data_mismatch +from foxops.errors import RetryableError +from foxops.external.git import GitRepository, TemporaryGitRepository +from foxops.external.gitlab import AsyncGitlabClient, GitlabNotFoundException +from foxops.logging import get_logger +from foxops.models import ( + DesiredIncarnationStateConfig, + IncarnationRemoteGitRepositoryState, +) + +logger = get_logger("reconciliation") + + +class ReconciliationState(IntEnum): + UNCHANGED = auto() + CHANGED = auto() + CHANGED_WITH_CONFLICT = auto() + FAILED = auto() + UNSUPPORTED = auto() + + +async def get_actual_incarnation_state( + gitlab: AsyncGitlabClient, + desired_incarnation_state: DesiredIncarnationStateConfig, +) -> IncarnationRemoteGitRepositoryState | None: + try: + gitlab_project = await gitlab.project_get( + desired_incarnation_state.gitlab_project + ) + except GitlabNotFoundException: + return None + + default_branch = gitlab_project.get("default_branch") or "main" + fengine_path = os.path.normpath( + desired_incarnation_state.target_directory / ".fengine.yaml" + ) + + try: + # TODO(TF): we should make the default branch configurable. + actual_incarnation_state_raw = ( + await gitlab.project_repository_files_get_content( + gitlab_project["id"], + default_branch, + fengine_path, + ) + ) + except GitlabNotFoundException: + return IncarnationRemoteGitRepositoryState( + gitlab_project_id=gitlab_project["id"], + remote_url=gitlab_project["http_url_to_repo"], + default_branch=default_branch, + incarnation_directory=desired_incarnation_state.target_directory, + ) + else: + actual_incarnation_state = load_incarnation_state_from_string( + actual_incarnation_state_raw.decode("utf-8") + ) + + return IncarnationRemoteGitRepositoryState( + gitlab_project_id=gitlab_project["id"], + remote_url=gitlab_project["http_url_to_repo"], + default_branch=default_branch, + incarnation_directory=desired_incarnation_state.target_directory, + incarnation_state=actual_incarnation_state, + ) + + +async def reconcile( + gitlab: AsyncGitlabClient, + desired_incarnation_states: list[DesiredIncarnationStateConfig], + parallelism: int, +) -> list[ReconciliationState]: + """Reconcile the given projects to a desired incarnation state. + + The `parallelism` parameter controls how many projects are reconciled in parallel. + However, it's not a real parallelism "pool", but basically batches the reconciliations + in `parallelism` sized batches. + """ + reconciliation_states = [] + + async def __reconcile_project( + desired_incarnation_state: DesiredIncarnationStateConfig, + ): + log = logger.bind(desired_incarnation_state=desired_incarnation_state) + try: + reconciliation_state = await reconcile_project( + gitlab, desired_incarnation_state + ) + except Exception as exc: + log.exception( + f"failed to reconcile project {desired_incarnation_state.gitlab_project}: {exc}" + ) + reconciliation_state = ReconciliationState.FAILED + return reconciliation_state + + for group in ( + desired_incarnation_states[i : i + parallelism] + for i in range(0, len(desired_incarnation_states), parallelism) + ): + logger.info(f"scheduling reconciliation for {len(group)} projects") + group_reconciliation_states = await asyncio.gather( + *[ + __reconcile_project(desired_incarnation_state) + for desired_incarnation_state in group + ] + ) + reconciliation_states.extend(group_reconciliation_states) + + return reconciliation_states + + +@retry( + retry=retry_if_exception_type(RetryableError), + # NOTE: "why retry 4 times?" ... well, go figure ;) + stop=stop_after_attempt(4), +) +async def reconcile_project( + gitlab: AsyncGitlabClient, + desired_incarnation_state: DesiredIncarnationStateConfig, +) -> ReconciliationState: + reconciliation_uuid = str(uuid.uuid4())[:8] + log = logger.bind( + reconciliation_uuid=reconciliation_uuid, + gitlab_project=desired_incarnation_state.gitlab_project, + target_directory=desired_incarnation_state.target_directory, + ) + actual_incarnation_state = await get_actual_incarnation_state( + gitlab, desired_incarnation_state + ) + + if actual_incarnation_state is None: + log.error( + "couldn't find the incarnation project on Gitlab. Make sure it exists before running foxops" + ) + return ReconciliationState.FAILED + + # NOTE(TF): clone incarnation and updated template remote git repository to a local folder for further operations + async with TemporaryGitRepository( + logger=log, + source=actual_incarnation_state.remote_url, + username="__token__", + password=gitlab.token, + ) as local_incarnation_git_repository, TemporaryGitRepository( + logger=log, + source=desired_incarnation_state.template_repository, + username="__token__", + password=gitlab.token, + refspec=desired_incarnation_state.template_repository_version, + ) as local_template_git_repository: + # check for incarnation is still empty and an initialization is required + if actual_incarnation_state.incarnation_state is None: + log.info( + "incarnation has not yet been initialized, initializing it now ..." + ) + return await initialize_incarnation_from_template( + gitlab, + actual_incarnation_state=actual_incarnation_state, + desired_incarnation_state=desired_incarnation_state, + local_template_repository=local_template_git_repository, + local_incarnation_repository=local_incarnation_git_repository, + logger=log, + ) + + # check if the existing incarnation requires an update + if ( + actual_incarnation_state.incarnation_state.template_repository + != desired_incarnation_state.template_repository + ): + log.error("changing the template repository is not supported") + return ReconciliationState.UNSUPPORTED + + # check if an existing incarnation has already been updated and pushed, but apparently not merged yet. + update_branch_name = generate_foxops_branch_name( + "update-to", + desired_incarnation_state.target_directory, + desired_incarnation_state.template_repository_version, + ) + update_branches = await gitlab.project_repository_branches_list( + actual_incarnation_state.gitlab_project_id, f"^{update_branch_name}" + ) + if len(update_branches) > 0: + log.info( + f"branch for an update to {desired_incarnation_state.template_repository_version} already exists. Skipping update" + ) + return ReconciliationState.UNCHANGED + + needs_update = False + # check if the existing incarnation requires an update because of a template version mismatch + if ( + actual_incarnation_state.incarnation_state.template_repository_version + != desired_incarnation_state.template_repository_version + ): + needs_update = True + log.info( + "reconciliation update is required because of a version mismatch", + actual_template_version=actual_incarnation_state.incarnation_state.template_repository_version, + desired_template_version=desired_incarnation_state.template_repository_version, + ) + + # check if it really(tm) is the same version by checking the exact version hash + if not needs_update: + if ( + actual_incarnation_state.incarnation_state.template_repository_version_hash + != await local_template_git_repository.head() + ): + needs_update = True + log.info( + "reconciliation update is required because of a version hash mismatch even though the version is the same - it's most likely a branch", + actual_template_version_hash=actual_incarnation_state.incarnation_state.template_repository_version_hash, + desired_template_version_hash=await local_template_git_repository.head(), + ) + else: + log.debug( + "the template is already in the desired version", + version=desired_incarnation_state.template_repository_version, + version_hash=await local_template_git_repository.head(), + ) + + # check if the existing incarnation requires an update because of a template data mismatch + if data_diff := get_data_mismatch( + desired_incarnation_state.template_data, + actual_incarnation_state.incarnation_state.template_data, + local_template_git_repository.directory, + logger, + ): + needs_update = True + log.info( + "reconciliation update is required because of template data mismatch", + actual_data=actual_incarnation_state.incarnation_state.template_data, + desired_data=desired_incarnation_state.template_data, + diff=data_diff, + ) + else: + log.debug( + "the template data is the same", + data=desired_incarnation_state.template_data, + ) + + if not needs_update: + log.info("no reconciliation update is required, see debug logs for details") + return ReconciliationState.UNCHANGED + + log.debug( + f"fetching actual template repository version {actual_incarnation_state.incarnation_state.template_repository_version_hash} for update" + ) + await local_template_git_repository.fetch( + refspec=actual_incarnation_state.incarnation_state.template_repository_version_hash + ) + + # create new branch in incarnation repository to operate in + await local_incarnation_git_repository.create_and_checkout_branch( + update_branch_name + ) + + ( + updated_incarnation_state, + files_with_conflicts, + ) = await update_incarnation_from_git_template_repository( + template_git_root_dir=local_template_git_repository.directory, + update_template_repository=desired_incarnation_state.template_repository, + update_template_repository_version=desired_incarnation_state.template_repository_version, + update_template_data=desired_incarnation_state.template_data, + incarnation_root_dir=local_incarnation_git_repository.directory + / desired_incarnation_state.target_directory, + diff_patch_func=diff_and_patch, + logger=log, + ) + log.info( + "successfully updated incarnation with new template version", + updated_incarnation_state=updated_incarnation_state, + ) + + mr_title = f"Update to {desired_incarnation_state.template_repository_version}" + mr_description = ( + f"Update to {desired_incarnation_state.template_repository_version}" + ) + reconciliation_state = ReconciliationState.CHANGED + if files_with_conflicts: + reconciliation_state = ReconciliationState.CHANGED_WITH_CONFLICT + log.info( + f"detected conflicts for files: {', '.join([str(f) for f in files_with_conflicts])} after update to new template" + ) + mr_title = f"🚧 - CONFLICT: {mr_title}" + mr_description = f"""{mr_description} + +There are conflicts in this Merge Request. Please check the rejection files +of the following files from your repository: + +{os.linesep.join([f"- {f}" for f in files_with_conflicts])} +""" + + automerge = desired_incarnation_state.automerge and reconciliation_state in { + ReconciliationState.CHANGED + } + + await local_incarnation_git_repository.commit_all( + f"foxops: updating to template version {desired_incarnation_state.template_repository_version}" + ) + await local_incarnation_git_repository.push() + + await ensure_merge_request_is_submitted( + gitlab=gitlab, + gitlab_project_id=actual_incarnation_state.gitlab_project_id, + source_branch_name=update_branch_name, + target_branch_name=actual_incarnation_state.default_branch, + title=mr_title, + description=mr_description, + automerge=automerge, + logger=log, + ) + return reconciliation_state + + +async def initialize_incarnation_from_template( + gitlab: AsyncGitlabClient, + actual_incarnation_state: IncarnationRemoteGitRepositoryState, + desired_incarnation_state: DesiredIncarnationStateConfig, + local_template_repository: GitRepository, + local_incarnation_repository: GitRepository, + logger: BoundLogger, +) -> ReconciliationState: + target_directory = ( + local_incarnation_repository.directory + / desired_incarnation_state.target_directory + ) + + should_create_mr = ( + await local_incarnation_repository.has_any_commits() + and target_directory.exists() + and list(glob.glob("*", root_dir=target_directory)) != [FVARS_FILENAME] + ) + + branch_name = actual_incarnation_state.default_branch + merge_request_required = False + if should_create_mr: + logger.debug( + "incarnation repository is not empty, creating a branch for initialization" + ) + merge_request_required = True + + # check if an incarnation initialization branch exists already + branch_name = generate_foxops_branch_name( + "initialize-to", + desired_incarnation_state.target_directory, + desired_incarnation_state.template_repository_version, + ) + existing_initialization_branches = ( + await gitlab.project_repository_branches_list( + actual_incarnation_state.gitlab_project_id, f"^{branch_name}" + ) + ) + if len(existing_initialization_branches) > 0: + logger.info( + f"branch for an initialization to {desired_incarnation_state.template_repository_version} already exists. Skipping initialization" + ) + return ReconciliationState.UNCHANGED + + # create new branch in incarnation repository where the template will be rendered + await local_incarnation_repository.create_and_checkout_branch(branch_name) + await local_incarnation_repository.push() + else: + logger.debug("incarnation repository is empty") + await local_incarnation_repository.create_and_checkout_branch( + branch_name, + exist_ok=True, + ) + + incarnation_state = await initialize_incarnation( + template_root_dir=local_template_repository.directory, + template_repository=desired_incarnation_state.template_repository, + template_repository_version=desired_incarnation_state.template_repository_version, + template_data=desired_incarnation_state.template_data, + incarnation_root_dir=local_incarnation_repository.directory + / desired_incarnation_state.target_directory, + logger=logger, + ) + logger.debug( + "existing files in directory", + files=list(local_incarnation_repository.directory.rglob("*")), + ) + result = await local_incarnation_repository._run("status") + logger.debug( + "git status", + result=(await result.stdout.read()) + if result.stdout is not None + else "no output", + ) + + result = await local_incarnation_repository.commit_all( + f"foxops: initializing incarnation from template {desired_incarnation_state.template_repository} " + f"@ {desired_incarnation_state.template_repository_version}" + ) + logger.debug( + "git commit", + result=(await result.stdout.read()) + if result.stdout is not None + else "no output", + ) + result = await local_incarnation_repository._run("status") + logger.debug( + "git status", + result=(await result.stdout.read()) + if result.stdout is not None + else "no output", + ) + + await local_incarnation_repository.push_with_potential_retry() + + if merge_request_required: + await ensure_merge_request_is_submitted( + gitlab=gitlab, + gitlab_project_id=actual_incarnation_state.gitlab_project_id, + source_branch_name=branch_name, + target_branch_name=actual_incarnation_state.default_branch, + title=f"Initialize to {desired_incarnation_state.template_repository_version}", + description=f"Initialize to {desired_incarnation_state.template_repository_version}", + automerge=desired_incarnation_state.automerge, + logger=logger, + ) + + logger.info( + "successfully initialized incarnation from template", + incarnation_state=incarnation_state, + ) + return ReconciliationState.CHANGED + + +async def ensure_merge_request_is_submitted( + gitlab: AsyncGitlabClient, + gitlab_project_id: int, + source_branch_name: str, + target_branch_name: str, + title: str, + description: str, + automerge: bool, + logger: BoundLogger, +) -> bool: + """Ensure that a Merge Request has been submitted. + + If no Merge Request exists, submit a new one. + """ + existing_merge_requests = await gitlab.project_merge_requests_list( + gitlab_project_id, "opened", source_branch_name + ) + if len(existing_merge_requests) > 0: + logger.info( + f"a Merge Request for the update branch {source_branch_name} already exists." + ) + return False + + merge_request = await gitlab.project_merge_requests_create( + gitlab_project_id, + source_branch_name, + target_branch_name, + title, + description, + ) + logger.info(f"created a new Merge Request at {merge_request['web_url']}") + + if automerge: + logger.info( + f"Triggering automerge for the new Merge Request {merge_request['web_url']}" + ) + await gitlab.automerge_merge_request(gitlab_project_id, merge_request["iid"]) + return True + + +def generate_foxops_branch_name( + prefix: str, target_directory: Path, template_version: str +) -> str: + target_directory_hash = hashlib.sha1( + str(target_directory).encode("utf-8") + ).hexdigest()[:7] + return f"foxops/{prefix}-{target_directory_hash}-{template_version}" diff --git a/src/foxops/settings.py b/src/foxops/settings.py new file mode 100644 index 00000000..a6759c43 --- /dev/null +++ b/src/foxops/settings.py @@ -0,0 +1,12 @@ +from pydantic import BaseSettings, SecretStr + + +class Settings(BaseSettings): + gitlab_address: str + gitlab_token: SecretStr + git_commit_author_name: str = "foxops" + git_commit_author_email: str = "noreply@foxops.io" + + class Config: + env_prefix = "foxops_" + secrets_dir = "/var/run/secrets/foxops" diff --git a/src/foxops/utils.py b/src/foxops/utils.py new file mode 100644 index 00000000..72b04c39 --- /dev/null +++ b/src/foxops/utils.py @@ -0,0 +1,65 @@ +import asyncio +import subprocess + +from .errors import FoxopsError +from .logging import get_logger + +logger = get_logger("utils") + + +class CalledProcessError(subprocess.CalledProcessError, FoxopsError): + """Error raised when copier fails.""" + + def __str__(self): + """Extend so that the string prints stdout and stderr by default.""" + return f"{super().__str__()} with stdout '{self.stdout}' and stderr '{self.stderr}'" + + +async def check_call( + program: str, + *args, + expected_returncodes: frozenset = frozenset({0}), + timeout: int | float | None = None, + **kwargs, +) -> asyncio.subprocess.Process: + """Execute the given executable and raise error on non-zero exit code. + + This function is simply wrapping `asyncio.create_subprocess_exec()` + and raises and exception in case the exit code is non-zero, + similar to what `subprocess.check_call()` does. + + The timeout parameter can be used to specify a maximum wait time in seconds. If the timeout expires before the + called process completes, the subprocess will be killed. + -> Setting the timeout to None (default) will allow the child process to take forever. + """ + proc = await asyncio.create_subprocess_exec( + program, + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + **kwargs, + ) + + try: + await asyncio.wait_for(proc.wait(), timeout=timeout) + except asyncio.TimeoutError: + proc.kill() + stdout_buffer = None if proc.stdout is None else bytes(proc.stdout._buffer) # type: ignore + stderr_buffer = None if proc.stderr is None else bytes(proc.stderr._buffer) # type: ignore + + logger.error( + "killed process as it exceeded the timeout", + stdout_buffer=stdout_buffer, + stderr_buffer=stderr_buffer, + ) + raise + + if proc.returncode is not None and proc.returncode not in expected_returncodes: + raise CalledProcessError( + proc.returncode, + [program] + list(args), + await proc.stdout.read() if proc.stdout else None, + await proc.stderr.read() if proc.stderr else None, + ) + + return proc diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..4d692e3d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +import pytest +from structlog.stdlib import BoundLogger + +from foxops import logging + + +@pytest.fixture(name="logger", scope="session") +def get_logger() -> BoundLogger: + logger = logging.get_logger("test") + return logger diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/features/Reconciliation.feature b/tests/e2e/features/Reconciliation.feature new file mode 100644 index 00000000..af86085b --- /dev/null +++ b/tests/e2e/features/Reconciliation.feature @@ -0,0 +1,22 @@ +@e2e +Feature: Foxops Reconciliation + In order to keep GitOps repositories at scale in + sync, we need to be able to reconcile changes + from a template to the real GitOps repositories. + + Scenario: Update incarnation repository when new template version exists + Given I have a template repository at "template" + And I want an incarnation repository at "incarnation" + And I reconcile + When I update the template repository at "template" + And I want the updated template for the repository at "incarnation" + And I reconcile + Then I should see a new Merge Request with the updates on GitLab at "incarnation" + + Scenario: Update incarnation repository when template data changed + Given I have a template repository at "template" + And I want an incarnation repository at "incarnation" + And I reconcile + When I change the template data for the "incarnation" repository + And I reconcile + Then I should see a new Merge Request with the changes on GitLab at "incarnation" diff --git a/tests/e2e/helpers.py b/tests/e2e/helpers.py new file mode 100644 index 00000000..b9cb882f --- /dev/null +++ b/tests/e2e/helpers.py @@ -0,0 +1,55 @@ +from collections import namedtuple + +from foxops.external.gitlab import AsyncGitlabClient, ProjectIdentifier, quote_plus + +#: Holds the GitLab base group id +GITLAB_BASE_GROUP_ID = 18289 + + +GitlabTestGroup = namedtuple("GitlabTestGroup", ["id", "path"]) + + +class ExtendedAsyncGitlabClient(AsyncGitlabClient): + async def project_create( + self, group_id: int, path: str, initialize_with_readme: bool + ): + return await self._post( + "/projects", + json={ + "path": path, + "namespace_id": group_id, + "initialize_with_readme": initialize_with_readme, + "default_branch": "main", + }, + ) + + async def project_delete(self, project_id: int): + return await self._delete(f"/projects/{project_id}") + + async def tag_create(self, project_id: int, tag_name: str, ref: str): + return await self._post( + f"/projects/{project_id}/repository/tags", + json={"tag_name": tag_name, "ref": ref}, + ) + + async def group_create(self, group_name: str, group_path: str, parent_id: int): + return await self._post( + "/groups", + json={"name": group_name, "path": group_path, "parent_id": parent_id}, + ) + + async def group_delete(self, group_id: int): + return await self._delete(f"/groups/{group_id}") + + async def project_merge_request_changes( + self, id_: ProjectIdentifier, merge_request_iid: int + ): + return await self._get( + f"/projects/{quote_plus(str(id_))}/merge_requests/{merge_request_iid}/changes" + ) + + async def get_branch_revision(self, id_: ProjectIdentifier, branch: str) -> str: + branch_data = await self._get( + f"/projects/{quote_plus(str(id_))}/repository/branches/{branch}" + ) + return branch_data["commit"]["id"] diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py new file mode 100644 index 00000000..86633b0a --- /dev/null +++ b/tests/e2e/test_e2e.py @@ -0,0 +1,1642 @@ +import os +import shutil +import subprocess +import textwrap +import uuid +from contextlib import asynccontextmanager +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, AsyncGenerator + +import pytest +from aiopath import AsyncPath +from dictdiffer import diff +from ruamel.yaml import YAML + +from foxops.external.git import GitRepository, TemporaryGitRepository, git_exec +from foxops.external.gitlab import GitlabNotFoundException +from foxops.logging import get_logger +from foxops.reconciliation import generate_foxops_branch_name +from tests.e2e.helpers import ( + GITLAB_BASE_GROUP_ID, + ExtendedAsyncGitlabClient, + GitlabTestGroup, +) + +yaml = YAML(typ="safe") + + +logger = get_logger(__name__) + + +@dataclass(frozen=True) +class TemplateRepo: + gitlab_project: dict[str, Any] + version: str + temporary_git_repository: TemporaryGitRepository + local_temporary_git_repository: GitRepository + + +@pytest.fixture(scope="function") +async def gitlab_token_test(): + return os.environ["GITLAB_TEST_TOKEN"] + + +@pytest.fixture(scope="function") +async def gitlab_address_test(): + return os.environ["FOXOPS_GITLAB_ADDRESS"] + + +@pytest.fixture(scope="function") +async def gitlab_client(gitlab_token_test, gitlab_address_test): + try: + + client = ExtendedAsyncGitlabClient( + token=gitlab_token_test, base_url=gitlab_address_test + ) + yield client + + await client.session.close() + except Exception as exc: + print(exc) + + +@pytest.fixture(scope="function") +async def unique_test_id(): + return str(uuid.uuid4()) + + +@pytest.fixture(scope="function") +async def gitlab_test_group( + gitlab_client: ExtendedAsyncGitlabClient, + unique_test_id: str, +): + test_group = await gitlab_client.group_create( + group_name=unique_test_id, + group_path=unique_test_id, + parent_id=GITLAB_BASE_GROUP_ID, + ) + + logger.info( + f"created test group {unique_test_id} (GitLab Group Id: {test_group['id']})" + ) + + yield GitlabTestGroup(test_group["id"], test_group["full_path"]) + + await gitlab_client.group_delete(test_group["id"]) + logger.info( + f"deleted test group {unique_test_id} (GitLab Group Id: {test_group['id']})" + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_initialize_template_in_root_of_empty_incarnation( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that a template can be initialized as incarnation in the root of an incarnation repository.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + + # WHEN + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + await assert_files_in_repository( + branch=incarnation_repository_project["default_branch"], + repository=incarnation_repository_project["path_with_namespace"], + expected_files=[(Path("README.md"), """Jon is of age 18""")], + gitlab_client=gitlab_client, + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_initialize_template_with_fvars( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that a template can be initialized as incarnation with fvars.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + async with incarnation_customization( + incarnation_repository_project, gitlab_token_test + ) as local_temporary_incarnation_repository: + (local_temporary_incarnation_repository.directory / "default.fvars").write_text( + "author=Jon" + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"age": "18"}, + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + + # WHEN + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + await assert_files_in_repository( + branch=incarnation_repository_project["default_branch"], + repository=incarnation_repository_project["path_with_namespace"], + expected_files=[(Path("README.md"), """Jon is of age 18""")], + gitlab_client=gitlab_client, + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_initialize_template_in_root_of_nonempty_incarnation( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that a template can be initialized as incarnation in the root of an incarnation repository.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=True, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + + rev_before_reconcile = await gitlab_client.get_branch_revision( + incarnation_repository_project["path_with_namespace"], + incarnation_repository_project["default_branch"], + ) + + # WHEN + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + rev_after_reconcile = await gitlab_client.get_branch_revision( + incarnation_repository_project["path_with_namespace"], + incarnation_repository_project["default_branch"], + ) + + # THEN + expected_initialization_branch = generate_foxops_branch_name( + "initialize-to", Path("."), "v1.0.0" + ) + existing_branches = await gitlab_client.project_repository_branches_list( + incarnation_repository_project["path_with_namespace"], + expected_initialization_branch, + ) + assert len(existing_branches) == 1 + assert existing_branches[0]["name"] == expected_initialization_branch + await assert_files_in_repository( + branch=expected_initialization_branch, + repository=incarnation_repository_project["path_with_namespace"], + expected_files=[(Path("README.md"), """Jon is of age 18""")], + gitlab_client=gitlab_client, + ) + + assert rev_before_reconcile == rev_after_reconcile + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_initialize_template_with_automerge_in_root_of_nonempty_incarnation( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that a template can be initialized as incarnation in the root of an incarnation repository.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=True, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("."), + automerge=True, + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + + rev_before_reconcile = await gitlab_client.get_branch_revision( + incarnation_repository_project["path_with_namespace"], + incarnation_repository_project["default_branch"], + ) + + # WHEN + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + rev_after_reconcile = await gitlab_client.get_branch_revision( + incarnation_repository_project["path_with_namespace"], + incarnation_repository_project["default_branch"], + ) + + # THEN + await assert_files_in_repository( + branch=incarnation_repository_project["default_branch"], + repository=incarnation_repository_project["path_with_namespace"], + expected_files=[(Path("README.md"), """Jon is of age 18""")], + gitlab_client=gitlab_client, + ) + + assert rev_before_reconcile != rev_after_reconcile + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_initialize_template_in_root_of_nonempty_incarnation_with_fvars( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that a template can be initialized as incarnation in the root of an incarnation repository.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=True, + ) + async with incarnation_customization( + incarnation_repository_project, gitlab_token_test + ) as local_temporary_incarnation_repository: + (local_temporary_incarnation_repository.directory / "default.fvars").write_text( + "author=Jon" + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"age": "18"}, + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + + rev_before_reconcile = await gitlab_client.get_branch_revision( + incarnation_repository_project["path_with_namespace"], + incarnation_repository_project["default_branch"], + ) + + # WHEN + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + rev_after_reconcile = await gitlab_client.get_branch_revision( + incarnation_repository_project["path_with_namespace"], + incarnation_repository_project["default_branch"], + ) + + # THEN + expected_initialization_branch = generate_foxops_branch_name( + "initialize-to", Path("."), "v1.0.0" + ) + existing_branches = await gitlab_client.project_repository_branches_list( + incarnation_repository_project["path_with_namespace"], + expected_initialization_branch, + ) + assert len(existing_branches) == 1 + assert existing_branches[0]["name"] == expected_initialization_branch + await assert_files_in_repository( + branch=expected_initialization_branch, + repository=incarnation_repository_project["path_with_namespace"], + expected_files=[(Path("README.md"), """Jon is of age 18""")], + gitlab_client=gitlab_client, + ) + + assert rev_before_reconcile == rev_after_reconcile + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_initialize_template_with_wrong_variables_errors( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that a template can not be initialized as incarnation if the template data doesn't match.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon"}, # missing `age` + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + + # THEN + with pytest.raises(subprocess.CalledProcessError) as excinfo: + # WHEN + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + assert ( + "the template required the variables ['age', 'author'] but the provided template data for the incarnation where ['author']." + in excinfo.value.output.decode("utf-8") + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_initialize_template_in_subdir_incarnation( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that a template can be initialized as incarnation in a sub directory within an incarnation repository.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("subdir"), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + + # WHEN + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + await assert_files_in_repository( + branch=incarnation_repository_project["default_branch"], + repository=incarnation_repository_project["path_with_namespace"], + expected_files=[(Path("subdir/README.md"), """Jon is of age 18""")], + gitlab_client=gitlab_client, + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_initialize_template_in_subdirs_incarnation( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that a template can be initialized as incarnation in multiple sub directories within an incarnation repository.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + + # there's an existing subdir incarnation in the repo already + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("subdir1"), + ) + ) + + with TemporaryDirectory() as tmp_path2: + tmp_path2 = Path(tmp_path2) + + desired_incarnation_state_config_path = Path(tmp_path2, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path2, + ) + + # this is the subdir incarnation which should be added + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Ygritte", "age": "17"}, + incarnation_target_directory=Path("subdir2"), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + + # WHEN + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + await assert_files_in_repository( + branch=incarnation_repository_project["default_branch"], + repository=incarnation_repository_project["path_with_namespace"], + expected_files=[ + (Path("subdir2/README.md"), """Ygritte is of age 17"""), + ], + gitlab_client=gitlab_client, + ) + await assert_files_in_repository( + branch=incarnation_repository_project["default_branch"], + repository=incarnation_repository_project["path_with_namespace"], + expected_files=[ + (Path("subdir1/README.md"), """Jon is of age 18"""), + ], + gitlab_client=gitlab_client, + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_update_incarnation_change_single_file( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that an incarnation can be updated when single file in template changed.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # WHEN + ( + template_repository.local_temporary_git_repository.directory + / "template" + / "README.md" + ).write_text("{{ author }} is of age {{ age }}. UPDATED, fuck yeah.") + await create_template_version( + template_repository=template_repository, + new_template_version="v2.0.0", + gitlab_client=gitlab_client, + ) + # FIXME(TF): oh my ... don't @ me. + desired_incarnation_state_config_path.write_text( + desired_incarnation_state_config_path.read_text().replace( + "template_repository_version: v1.0.0", + "template_repository_version: v2.0.0", + ) + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + await assert_merge_request_with_file_changes_in_incarnation( + incarnation_repository=incarnation_repository_project["path_with_namespace"], + expected_incarnation_merge_request_changes=[ + { + "old_path": "README.md", + "new_path": "README.md", + "diff": """ +@@ -1 +1 @@ +-Jon is of age 18 +\\ No newline at end of file ++Jon is of age 18. UPDATED, fuck yeah. +\\ No newline at end of file +""".lstrip(), + }, + ], + gitlab_client=gitlab_client, + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_update_incarnation_change_single_file_with_fvars_change( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that an incarnation can be updated when single file changed because of fvars update.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + async with incarnation_customization( + incarnation_repository_project, gitlab_token_test + ) as local_temporary_incarnation_repository: + (local_temporary_incarnation_repository.directory / "default.fvars").write_text( + "author=Jon" + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"age": "18"}, + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # WHEN + async with incarnation_customization( + incarnation_repository_project, gitlab_token_test + ) as local_temporary_incarnation_repository: + (local_temporary_incarnation_repository.directory / "default.fvars").write_text( + "author=Jon the overridden" + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + await assert_merge_request_with_file_changes_in_incarnation( + incarnation_repository=incarnation_repository_project["path_with_namespace"], + expected_incarnation_merge_request_changes=[ + { + "old_path": "README.md", + "new_path": "README.md", + "diff": """ +@@ -1 +1 @@ +-Jon is of age 18 +\\ No newline at end of file ++Jon the overridden is of age 18 +\\ No newline at end of file +""".lstrip(), + }, + ], + gitlab_client=gitlab_client, + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_update_incarnation_delete_single_file( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that an incarnation can be updated when single file in template was deleted.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # WHEN + ( + template_repository.local_temporary_git_repository.directory + / "template" + / "README.md" + ).unlink() + await create_template_version( + template_repository=template_repository, + new_template_version="v2.0.0", + gitlab_client=gitlab_client, + ) + # FIXME(TF): oh my ... don't @ me. + desired_incarnation_state_config_path.write_text( + desired_incarnation_state_config_path.read_text().replace( + "template_repository_version: v1.0.0", + "template_repository_version: v2.0.0", + ) + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + await assert_merge_request_with_file_changes_in_incarnation( + incarnation_repository=incarnation_repository_project["path_with_namespace"], + expected_incarnation_merge_request_changes=[ + { + "old_path": "README.md", + "deleted_file": True, + "diff": """ +@@ -1 +0,0 @@ +-Jon is of age 18 +\\ No newline at end of file +""".lstrip(), + }, + ], + gitlab_client=gitlab_client, + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_update_incarnation_rename_single_file( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that an incarnation can be updated when single file in template was renamed.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # WHEN + ( + template_repository.local_temporary_git_repository.directory + / "template" + / "README.md" + ).rename( + template_repository.local_temporary_git_repository.directory + / "template" + / "NEW_README.md" + ) + await create_template_version( + template_repository=template_repository, + new_template_version="v2.0.0", + gitlab_client=gitlab_client, + ) + # FIXME(TF): oh my ... don't @ me. + desired_incarnation_state_config_path.write_text( + desired_incarnation_state_config_path.read_text().replace( + "template_repository_version: v1.0.0", + "template_repository_version: v2.0.0", + ) + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + await assert_merge_request_with_file_changes_in_incarnation( + incarnation_repository=incarnation_repository_project["path_with_namespace"], + expected_incarnation_merge_request_changes=[ + { + "old_path": "README.md", + "new_path": "NEW_README.md", + "diff": "", + }, + ], + gitlab_client=gitlab_client, + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_update_incarnation_change_same_file_template_incarnation( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that an incarnation can be updated when single file in template changed which also changed the same way in the incarnation.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # WHEN + ( + template_repository.local_temporary_git_repository.directory + / "template" + / "README.md" + ).write_text("{{ author }} is of age {{ age }}. UPDATED, fuck yeah.") + await create_template_version( + template_repository=template_repository, + new_template_version="v2.0.0", + gitlab_client=gitlab_client, + ) + async with incarnation_customization( + incarnation_repository_project, gitlab_token_test + ) as local_temporary_incarnation_repository: + (local_temporary_incarnation_repository.directory / "README.md").write_text( + "Jon is of age 18. UPDATED, fuck yeah." + ) + + # FIXME(TF): oh my ... don't @ me. + desired_incarnation_state_config_path.write_text( + desired_incarnation_state_config_path.read_text().replace( + "template_repository_version: v1.0.0", + "template_repository_version: v2.0.0", + ) + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + changes = await get_merge_request_changes_in_incarnation( + incarnation_repository=incarnation_repository_project["path_with_namespace"], + gitlab_client=gitlab_client, + ) + assert all(x["old_path"] != "README.md" for x in changes) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_update_incarnation_moving_branch_name( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that an incarnation can be updated when template version is a moving branch.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + + # NOTE(TF): actually make the incarnation point to the branch instead of the tag + patched_template_repository = TemplateRepo( + gitlab_project=template_repository.gitlab_project, + version=template_repository.gitlab_project["default_branch"], + temporary_git_repository=template_repository.temporary_git_repository, + local_temporary_git_repository=template_repository.local_temporary_git_repository, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=patched_template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # WHEN + ( + patched_template_repository.local_temporary_git_repository.directory + / "template" + / "README.md" + ).write_text("{{ author }} is of age {{ age }}. UPDATED, fuck yeah.") + await create_template_version( + template_repository=patched_template_repository, + new_template_version=None, # we don't want a new tag + gitlab_client=gitlab_client, + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + await assert_merge_request_with_file_changes_in_incarnation( + incarnation_repository=incarnation_repository_project["path_with_namespace"], + expected_incarnation_merge_request_changes=[ + { + "old_path": "README.md", + "new_path": "README.md", + "diff": """ +@@ -1 +1 @@ +-Jon is of age 18 +\\ No newline at end of file ++Jon is of age 18. UPDATED, fuck yeah. +\\ No newline at end of file +""".lstrip(), + }, + ], + gitlab_client=gitlab_client, + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_update_incarnation_with_conflict_is_correctly_detected_and_presented( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that an incarnation update with a conflict is correctly detected and presented.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("."), + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # WHEN + ( + template_repository.local_temporary_git_repository.directory + / "template" + / "README.md" + ).write_text("{{ author }} is of age '{{ age }}'") + await create_template_version( + template_repository=template_repository, + new_template_version="v2.0.0", + gitlab_client=gitlab_client, + ) + async with incarnation_customization( + incarnation_repository_project, gitlab_token_test + ) as local_temporary_incarnation_repository: + (local_temporary_incarnation_repository.directory / "README.md").write_text( + "Jon is of age 30" + ) + + # FIXME(TF): oh my ... don't @ me. + desired_incarnation_state_config_path.write_text( + desired_incarnation_state_config_path.read_text().replace( + "template_repository_version: v1.0.0", + "template_repository_version: v2.0.0", + ) + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + merge_request = await get_merge_request_in_incarnation( + incarnation_repository=incarnation_repository_project["path_with_namespace"], + gitlab_client=gitlab_client, + ) + assert merge_request["state"] != "merged" + assert merge_request["merge_when_pipeline_succeeds"] is False + assert "CONFLICT" in merge_request["title"] + assert "README.md" in merge_request["description"] + assert "conflict" in merge_request["description"] + assert "rejection files" in merge_request["description"] + + await assert_merge_request_with_file_changes_in_incarnation( + incarnation_repository=incarnation_repository_project["path_with_namespace"], + expected_incarnation_merge_request_changes=[ + { + "old_path": "README.md.rej", + "new_path": "README.md.rej", + }, + ], + gitlab_client=gitlab_client, + ) + + +@pytest.mark.non_gherkin +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_update_incarnation_mr_is_automerged( + gitlab_client, gitlab_test_group, gitlab_address_test, gitlab_token_test, tmp_path +): + """Verify that an incarnation Merge Request is automerged if configured as such.""" + # GIVEN + template_repository = await setup_template_repository( + template_repository_name="template", + template_version="v1.0.0", + template_files=[(Path("README.md"), """{{ author }} is of age {{ age }}""")], + template_variables={"author": "str", "age": "int"}, + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + gitlab_token_test=gitlab_token_test, + ) + + incarnation_repository_project = await setup_incarnation_repository( + incarnation_repository_name="incarnation", + gitlab_client=gitlab_client, + gitlab_test_group=gitlab_test_group, + initialize_with_readme=False, + ) + + desired_incarnation_state_config = ( + await setup_incarnation_from_template_in_repository( + template_repository=template_repository, + incarnation_repository_project=incarnation_repository_project, + incarnation_variables={"author": "Jon", "age": "18"}, + incarnation_target_directory=Path("."), + automerge=True, + ) + ) + desired_incarnation_state_config_path = Path(tmp_path, "incarnations.yaml") + desired_incarnation_state_config_path.write_text( + "incarnations:\n" + textwrap.indent(desired_incarnation_state_config, " ") + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # WHEN + ( + template_repository.local_temporary_git_repository.directory + / "template" + / "README.md" + ).write_text("{{ author }} is of age {{ age }}. UPDATED, fuck yeah.") + await create_template_version( + template_repository=template_repository, + new_template_version="v2.0.0", + gitlab_client=gitlab_client, + ) + # FIXME(TF): oh my ... don't @ me. + desired_incarnation_state_config_path.write_text( + desired_incarnation_state_config_path.read_text().replace( + "template_repository_version: v1.0.0", + "template_repository_version: v2.0.0", + ) + ) + reconcile( + desired_incarnation_state_config_path, + gitlab_address_test, + gitlab_token_test, + tmp_path, + ) + + # THEN + assert await get_merge_request_in_incarnation( + incarnation_repository=incarnation_repository_project["path_with_namespace"], + gitlab_client=gitlab_client, + state="merged", + ) + + +async def setup_template_repository( + template_repository_name: str, + template_version: str, + template_files: list[tuple[Path, str]], + template_variables: dict[str, str], + gitlab_client, + gitlab_test_group, + gitlab_token_test, +) -> TemplateRepo: + logger.info("setup template repository") + + project = await gitlab_client.project_create( + group_id=gitlab_test_group.id, + path=template_repository_name, + initialize_with_readme=True, + ) + logger.debug(f"created template repository at {project['path_with_namespace']}") + + logger.debug("clone template repository to add template files") + temporary_git_repository = TemporaryGitRepository( + logger=logger, + source=project["http_url_to_repo"], + username="__token__", + password=gitlab_token_test, + ) + local_temporary_git_repository = await temporary_git_repository.__aenter__() + await git_exec( + "switch", + project["default_branch"], + cwd=local_temporary_git_repository.directory, + ) + logger.debug( + f"cloned template repository into {local_temporary_git_repository.directory}" + ) + + for template_file_path, template_file_contents in template_files: + path = AsyncPath( + local_temporary_git_repository.directory / "template" / template_file_path + ) + await path.parent.mkdir(parents=True, exist_ok=True) + await path.write_text(template_file_contents) + + foxops_config_path = local_temporary_git_repository.directory / "fengine.yaml" + foxops_config = { + "required_foxops_version": "v1.0.0", + "variables": { + k: {"type": v, "description": "nope"} for k, v in template_variables.items() + }, + } + with foxops_config_path.open("w") as f: + yaml.dump(foxops_config, f) + + await local_temporary_git_repository.commit_all(message="Initial commit") + await local_temporary_git_repository.push() + logger.debug("committed and pushed template repository") + + await gitlab_client.tag_create( + project["id"], template_version, project["default_branch"] + ) + await git_exec("push", "--tags", cwd=local_temporary_git_repository.directory) + logger.debug(f"created and pushed tag {template_version} in template repository") + + return TemplateRepo( + gitlab_project=project, + version=template_version, + temporary_git_repository=temporary_git_repository, + local_temporary_git_repository=local_temporary_git_repository, + ) + + +async def create_template_version( + template_repository: TemplateRepo, + new_template_version: str | None, + gitlab_client, +) -> TemplateRepo: + logger.info("create template version") + + await template_repository.local_temporary_git_repository.commit_all( + message="Initial commit" + ) + await template_repository.local_temporary_git_repository.push() + logger.debug("committed and pushed template repository") + + if new_template_version is not None: + await gitlab_client.tag_create( + template_repository.gitlab_project["id"], + new_template_version, + template_repository.gitlab_project["default_branch"], + ) + await git_exec( + "push", + "--tags", + cwd=template_repository.local_temporary_git_repository.directory, + ) + logger.debug( + f"created and pushed tag {new_template_version} in template repository" + ) + + return TemplateRepo( + gitlab_project=template_repository.gitlab_project, + version=( + new_template_version + if new_template_version is not None + else template_repository.version + ), + temporary_git_repository=template_repository.temporary_git_repository, + local_temporary_git_repository=template_repository.local_temporary_git_repository, + ) + + +async def setup_incarnation_repository( + incarnation_repository_name, + gitlab_client, + gitlab_test_group, + initialize_with_readme: bool, +): + logger.info("setup incarnation repository") + + project = await gitlab_client.project_create( + group_id=gitlab_test_group.id, + path=incarnation_repository_name, + initialize_with_readme=initialize_with_readme, + ) + return project + + +async def setup_incarnation_from_template_in_repository( + template_repository: TemplateRepo, + incarnation_repository_project, + incarnation_target_directory: Path, + incarnation_variables: dict[str, str], + automerge: bool = False, +) -> str: + logger.info("setup incarnation from template in repository") + + desired_incarnation_state_config = f""" +- gitlab_project: {incarnation_repository_project['path_with_namespace']} + target_directory: {incarnation_target_directory} + automerge: {'true' if automerge else 'false'} + template_repository: {template_repository.gitlab_project["http_url_to_repo"]} + template_repository_version: {template_repository.version} + template_data: +{os.linesep.join(f"{' ' * 4}{key}: {value}" for key, value in incarnation_variables.items())} + """ + + logger.debug( + f"create incarnation repository at {incarnation_repository_project['path_with_namespace']}" + ) + return desired_incarnation_state_config + + +@asynccontextmanager +async def incarnation_customization( + incarnation_repository_project, gitlab_token_test +) -> AsyncGenerator[GitRepository, None]: + logger.info("incarnation customization") + + async with TemporaryGitRepository( + logger=logger, + source=incarnation_repository_project["http_url_to_repo"], + username="__token__", + password=gitlab_token_test, + ) as local_temporary_incarnation_repository: + yield local_temporary_incarnation_repository + + await local_temporary_incarnation_repository.commit_all(message="customization") + await local_temporary_incarnation_repository.push() + logger.debug("committed and pushed incarnation repository customization") + + +def reconcile( + desired_incarnation_state_config_path: Path, + gitlab_address_test: str, + gitlab_token_test: str, + tmp_path: Path, +): + logger.info("reconcile incarnations") + try: + reconcile_output = subprocess.check_output( + [ + "coverage", + "run", + "--parallel-mode", + "--branch", + "--source", + "foxops", + "--module", + "foxops", + "--verbose", + "--json-logs", + "reconcile", + "--parallelism", + "1", + str(desired_incarnation_state_config_path), + ], + env={ + "FOXOPS_GITLAB_ADDRESS": gitlab_address_test, + "FOXOPS_GITLAB_TOKEN": gitlab_token_test, + "PATH": os.environ[ + "PATH" + ], # foxops, coverage etc. might not be installed system wide + }, + cwd=tmp_path, + ) + except subprocess.CalledProcessError as exc: + logger.error(f"reconcile failed with {exc.returncode} logs:") + print(exc.output.decode("utf-8")) + raise + + reconciliation_result = reconcile_output.decode("utf-8") + logger.debug("Reconciliation logs:") + print(reconciliation_result) + + # copy coverage files to current test execution directory + coverage_file_target_path = Path.cwd() + for coverage_file in tmp_path.glob(".coverage.*"): + logger.debug( + "copy coverage file to test execution directory", + filename=coverage_file.name, + target=coverage_file_target_path, + ) + shutil.move(coverage_file, coverage_file_target_path) + + return reconciliation_result + + +async def assert_files_in_repository( + branch: str, + repository: Path, + expected_files: list[tuple[Path, str]], + gitlab_client, +): + logger.info("asserting files in repository") + + for ( + expected_file_path, + expected_file_contents, + ) in expected_files: + logger.debug(f"checking if file contents of `{expected_file_path}` match ...") + try: + actual_file_contents: bytes = ( + await gitlab_client.project_repository_files_get_content( + id_=repository, + branch=branch, + filepath=expected_file_path, + ) + ) + except GitlabNotFoundException: + logger.error( + f"file `{expected_file_path}` not found in incarnation on branch `{branch}` of repository `{repository}`" + ) + raise + else: + assert actual_file_contents.decode("utf-8") == expected_file_contents + + +async def get_merge_request_in_incarnation( + incarnation_repository: Path, + gitlab_client: ExtendedAsyncGitlabClient, + state="opened", +): + merge_requests = await gitlab_client.project_merge_requests_list( + incarnation_repository, + state=state, + ) + assert len(merge_requests) == 1, "Only single merge request expected" + return merge_requests[0] + + +async def get_merge_request_changes_in_incarnation( + incarnation_repository: Path, + gitlab_client: ExtendedAsyncGitlabClient, +) -> list: + merge_request = await get_merge_request_in_incarnation( + incarnation_repository=incarnation_repository, gitlab_client=gitlab_client + ) + + changes = ( + await gitlab_client.project_merge_request_changes( + incarnation_repository, + merge_request["iid"], + ) + )["changes"] + return changes + + +async def assert_merge_request_with_file_changes_in_incarnation( + incarnation_repository: Path, + expected_incarnation_merge_request_changes, + gitlab_client, +): + changes = await get_merge_request_changes_in_incarnation( + incarnation_repository, gitlab_client + ) + for expected_change in expected_incarnation_merge_request_changes: + actual_change = next( + (c for c in changes if c["old_path"] == expected_change["old_path"]), None + ) + assert ( + actual_change is not None + ), f"Change `{expected_change['old_path']}` not found. Changes are {changes=}" + + diffs = list( + diff( + actual_change, + expected_change, + ) + ) + diffs = [d for d in diffs if d[0] != "remove"] + assert diffs == [], f"Diff is {diffs=}. Changes are {changes=}" diff --git a/tests/e2e/test_reconciliation_gherkin.py b/tests/e2e/test_reconciliation_gherkin.py new file mode 100644 index 00000000..98d58cc7 --- /dev/null +++ b/tests/e2e/test_reconciliation_gherkin.py @@ -0,0 +1,443 @@ +import asyncio +import os +import shutil +import subprocess +import uuid +from dataclasses import dataclass +from io import StringIO +from pathlib import Path +from typing import Any + +import pytest +from pytest_bdd import given, parsers, scenario, then, when +from ruamel.yaml import YAML + +from foxops.external.git import GitRepository, TemporaryGitRepository, git_exec +from tests.e2e.helpers import ( + GITLAB_BASE_GROUP_ID, + ExtendedAsyncGitlabClient, + GitlabTestGroup, +) + +yaml = YAML(typ="safe") + + +@pytest.fixture(scope="session") +def loop(): + _loop = asyncio.new_event_loop() + asyncio.set_event_loop(_loop) + yield _loop + _loop.close() + + +@pytest.fixture(scope="session") +def gitlab_token_test(): + return os.environ["GITLAB_TEST_TOKEN"] + + +@pytest.fixture(scope="session") +def gitlab_address_test(): + return os.environ["FOXOPS_GITLAB_ADDRESS"] + + +@pytest.fixture(scope="session") +def gitlab_client( + loop: asyncio.AbstractEventLoop, gitlab_token_test, gitlab_address_test +): + try: + + async def create_client(): + return ExtendedAsyncGitlabClient( + token=gitlab_token_test, base_url=gitlab_address_test + ) + + client = loop.run_until_complete(create_client()) + yield client + loop.run_until_complete(client.session.close()) + except Exception as exc: + print(exc) + + +@pytest.fixture(scope="function") +def unique_test_id(): + return str(uuid.uuid4()) + + +@pytest.fixture(scope="function") +def gitlab_test_group( + loop: asyncio.AbstractEventLoop, + gitlab_client: ExtendedAsyncGitlabClient, + unique_test_id: str, + logger, +): + test_group = loop.run_until_complete( + gitlab_client.group_create( + group_name=unique_test_id, + group_path=unique_test_id, + parent_id=GITLAB_BASE_GROUP_ID, + ) + ) + logger.info( + f"created test group {unique_test_id} (GitLab Group Id: {test_group['id']})" + ) + + yield GitlabTestGroup(test_group["id"], test_group["full_path"]) + + loop.run_until_complete(gitlab_client.group_delete(test_group["id"])) + logger.info( + f"deleted test group {unique_test_id} (GitLab Group Id: {test_group['id']})" + ) + + +@dataclass +class TemplateRepo: + gitlab_project: dict[str, Any] + tmp_local_git_repository: TemporaryGitRepository + local_git_repository: GitRepository + + +@scenario( + "Reconciliation.feature", + "Update incarnation repository when new template version exists", +) +def test_update_incarnation_repository_when_new_template_version_exists(): + """Update incarnation repository when new template version exists.""" + pass + + +@scenario( + "Reconciliation.feature", "Update incarnation repository when template data changed" +) +def test_update_incarnation_repository_when_template_data_changed(): + """Update incarnation repository when template data changed.""" + pass + + +@given( + parsers.parse('I have a template repository at "{gitops_template_repo_name:S}"'), + target_fixture="template_repo", +) +def i_have_a_template_repository_at_gitopstemplate( + loop: asyncio.AbstractEventLoop, + gitops_template_repo_name, + gitlab_client, + gitlab_test_group, + gitlab_token_test, + logger, +): + """I have a template repository at "gitops-template".""" + project_create_task = gitlab_client.project_create( + group_id=gitlab_test_group.id, + path=gitops_template_repo_name, + initialize_with_readme=True, + ) + project = loop.run_until_complete(project_create_task) + logger.debug(f"created template repo project at {project['id']}") + + tmp_local_git_repository = TemporaryGitRepository( + logger=logger, + source=project["http_url_to_repo"], + username="__token__", + password=gitlab_token_test, + ) + local_git_repository = loop.run_until_complete( + tmp_local_git_repository.__aenter__() + ) + loop.run_until_complete( + git_exec( + "switch", + project["default_branch"], + cwd=local_git_repository.directory, + ) + ) + logger.debug(f"cloned template repository into {local_git_repository.directory}") + + shutil.copytree( + "tests/e2e/testdata/template-repo", + local_git_repository.directory, + dirs_exist_ok=True, + ) + logger.debug("copied template-repo template into local template-repo directory") + + loop.run_until_complete(local_git_repository.commit_all(message="Initial commit")) + loop.run_until_complete(local_git_repository.push()) + logger.debug("committed and pushed template-repo contents to GitLab") + + loop.run_until_complete( + gitlab_client.tag_create(project["id"], "v0.0.1", project["default_branch"]) + ) + loop.run_until_complete( + git_exec("push", "--tags", cwd=local_git_repository.directory) + ) + logger.debug("created tag v0.0.1") + + return TemplateRepo( + gitlab_project=project, + tmp_local_git_repository=tmp_local_git_repository, + local_git_repository=local_git_repository, + ) + + +@given( + parsers.parse('I want an incarnation repository at "{gitops_repo_name:S}"'), + target_fixture="gitops_desired_incarnation_states", +) +def i_want_an_incarnation_repository_at_gitopsrepo( + gitops_repo_name, + gitlab_client, + gitlab_test_group, + loop: asyncio.AbstractEventLoop, + template_repo, +): + """I want an incarnation repository at "gitops-repo".""" + + project_create_task = gitlab_client.project_create( + group_id=gitlab_test_group.id, + path=gitops_repo_name, + initialize_with_readme=False, + ) + loop.run_until_complete(project_create_task) + + desired_incarnation_states = f""" +incarnations: + - gitlab_project: {gitlab_test_group.path}/{gitops_repo_name} + template_repository: {template_repo.gitlab_project['http_url_to_repo']} + template_repository_version: v0.0.1 + template_data: + name: {gitops_repo_name} + version: v1.0.0 +""" + return desired_incarnation_states + + +@given("I reconcile", target_fixture="reconciliation_result") +@when("I reconcile", target_fixture="reconciliation_result") +def i_reconcile( + gitlab_address_test, + gitlab_token_test, + gitops_desired_incarnation_states, + tmp_path, + logger, +): + """I reconcile.""" + project_definition_config = tmp_path / "incarnaions.yaml" + project_definition_config.write_text(gitops_desired_incarnation_states) + + try: + reconcile_output = subprocess.check_output( + [ + "coverage", + "run", + "--parallel-mode", + "--branch", + "--source", + "foxops", + "--module", + "foxops", + "--verbose", + "--json-logs", + "reconcile", + "--parallelism", + "1", + str(project_definition_config), + ], + env={ + "FOXOPS_GITLAB_ADDRESS": gitlab_address_test, + "FOXOPS_GITLAB_TOKEN": gitlab_token_test, + # FIXME(TF): "foxops" and "copier" might not be installed into the system, + # and therefore not in shells default `$PATH`, but in a virtualenv + # which's bin path is injected in `$PATH` in the currently running shell, + # therefore we pass along the `$PATH`. + "PATH": os.environ["PATH"], + }, + ) + except subprocess.CalledProcessError as exc: + logger.error(f"reconcile failed with {exc.returncode} logs:") + print(exc.output.decode("utf-8")) + raise + + reconciliation_result = reconcile_output.decode("utf-8") + logger.debug("Reconciliation logs:") + print(reconciliation_result) + return reconciliation_result + + +@when( + parsers.parse( + 'I update the template repository at "{gitops_template_repo_name:S}"' + ), + target_fixture="update_in_template_repo", +) +def i_update_the_template_repository_at_gitopstemplate( + loop: asyncio.AbstractEventLoop, + gitops_template_repo_name: str, + template_repo: TemplateRepo, + gitlab_client, + logger, +): + """I update the template repository at "gitops-template".""" + subprocess.check_call( + "echo 'NEW FILE CONTENT' > template/UPDATE.md", + shell=True, + cwd=template_repo.local_git_repository.directory, + ) + logger.debug( + f"added UPDATE.md file to template repository at {template_repo.local_git_repository.directory}" + ) + loop.run_until_complete( + template_repo.local_git_repository.commit_all(message="Update") + ) + loop.run_until_complete(template_repo.local_git_repository.push()) + loop.run_until_complete( + gitlab_client.tag_create( + template_repo.gitlab_project["id"], + "v0.0.2", + template_repo.gitlab_project["default_branch"], + ) + ) + loop.run_until_complete( + git_exec("push", "--tags", cwd=template_repo.local_git_repository.directory) + ) + logger.debug("created tag v0.0.2") + return {"new_version": "v0.0.2", "new_file": Path("UPDATE.md")} + + +@when( + parsers.parse( + 'I want the updated template for the repository at "{gitops_repo_name:S}"' + ), + target_fixture="gitops_desired_incarnation_states", +) +def i_want_the_updated_template_for_the_repository_at_gitopsrepo( + gitops_repo_name, + gitlab_test_group, + template_repo, + update_in_template_repo, +): + """I want the updated template for the repository at "gitops-repo".""" + desired_incarnation_states = f""" +incarnations: + - gitlab_project: {gitlab_test_group.path}/{gitops_repo_name} + template_repository: {template_repo.gitlab_project['http_url_to_repo']} + template_repository_version: {update_in_template_repo["new_version"]} + template_data: + name: {gitops_repo_name} + version: v1.0.0 +""" + return desired_incarnation_states + + +@then( + parsers.parse( + 'I should see a new Merge Request with the updates on GitLab at "{gitops_repo_name:S}"' + ) +) +def i_should_see_a_new_merge_request_with_the_updates_on_gitlab_at_gitopsrepo( + loop: asyncio.AbstractEventLoop, + gitops_desired_incarnation_states, + gitlab_client, + gitlab_token_test, + update_in_template_repo, + logger, +): + """I should see a new Merge Request with the updates on GitLab at "gitops-repo".""" + # FIXME(TF): limitation that only one project definition is supported + gitops_repo_path = Path( + yaml.load(gitops_desired_incarnation_states)["incarnations"][0][ + "gitlab_project" + ] + ) + gitops_repo_project = loop.run_until_complete( + gitlab_client.project_get(gitops_repo_path) + ) + merge_requests = loop.run_until_complete( + gitlab_client.project_merge_requests_list( + gitops_repo_project["id"], state="opened" + ) + ) + assert len(merge_requests) == 1 + merge_request = merge_requests[0] + + tmp_local_gitops_repository = TemporaryGitRepository( + logger=logger, + source=gitops_repo_project["http_url_to_repo"], + username="__token__", + password=gitlab_token_test, + refspec=merge_request["source_branch"], + ) + local_gitops_repository = loop.run_until_complete( + tmp_local_gitops_repository.__aenter__() + ) + + try: + assert ( + local_gitops_repository.directory / update_in_template_repo["new_file"] + ).is_file() + finally: + loop.run_until_complete(tmp_local_gitops_repository.__aexit__(None, None, None)) + + +@when( + parsers.parse( + 'I change the template data for the "{gitops_repo_name:S}" repository' + ), + target_fixture="gitops_desired_incarnation_states", +) +def i_change_the_template_data_for_the_incarnation_repository( + gitops_repo_name, gitops_desired_incarnation_states +): + """I change the template data for the "incarnation" repository.""" + # load current project definition + incarnations = yaml.load(gitops_desired_incarnation_states) + # change the template data + incarnations["incarnations"][0]["template_data"]["name"] += "_CHANGED" + string_stream = StringIO() + yaml.dump(incarnations, string_stream) + return string_stream.getvalue() + + +@then( + parsers.parse( + 'I should see a new Merge Request with the changes on GitLab at "{gitops_repo_name:S}"' + ) +) +def i_should_see_a_new_merge_request_with_the_changes_on_gitlab_at_incarnation( + gitops_repo_name, + gitops_desired_incarnation_states, + loop: asyncio.AbstractEventLoop, + gitlab_client, + gitlab_token_test, + logger, +): + """I should see a new Merge Request with the changes on GitLab at "incarnation".""" + gitops_repo_path = Path( + yaml.load(gitops_desired_incarnation_states)["incarnations"][0][ + "gitlab_project" + ] + ) + gitops_repo_project = loop.run_until_complete( + gitlab_client.project_get(gitops_repo_path) + ) + merge_requests = loop.run_until_complete( + gitlab_client.project_merge_requests_list( + gitops_repo_project["id"], state="opened" + ) + ) + assert len(merge_requests) == 1 + merge_request = merge_requests[0] + + tmp_local_gitops_repository = TemporaryGitRepository( + logger=logger, + source=gitops_repo_project["http_url_to_repo"], + username="__token__", + password=gitlab_token_test, + refspec=merge_request["source_branch"], + ) + local_gitops_repository = loop.run_until_complete( + tmp_local_gitops_repository.__aenter__() + ) + try: + with Path(local_gitops_repository.directory, "README.md").open() as readme_file: + assert f"# Application {gitops_repo_name}_CHANGED" in readme_file.read() + finally: + loop.run_until_complete(tmp_local_gitops_repository.__aexit__(None, None, None)) diff --git a/tests/e2e/testdata/template-repo/fengine.yaml b/tests/e2e/testdata/template-repo/fengine.yaml new file mode 100644 index 00000000..cf47c580 --- /dev/null +++ b/tests/e2e/testdata/template-repo/fengine.yaml @@ -0,0 +1,10 @@ +required_foxops_version: v1.0.0 + +variables: + name: + type: str + description: The name of the application + + version: + type: int + description: The version of the application diff --git a/tests/e2e/testdata/template-repo/template/README.md b/tests/e2e/testdata/template-repo/template/README.md new file mode 100644 index 00000000..504d5a84 --- /dev/null +++ b/tests/e2e/testdata/template-repo/template/README.md @@ -0,0 +1,3 @@ +# Application {{ name }} + +Initialized with version {{ version }}. diff --git a/tests/engine/test_cli.py b/tests/engine/test_cli.py new file mode 100644 index 00000000..85253b36 --- /dev/null +++ b/tests/engine/test_cli.py @@ -0,0 +1,538 @@ +import json +import logging +from dataclasses import asdict +from pathlib import Path +from subprocess import check_output +from textwrap import dedent + +import pytest +from typer.testing import CliRunner + +from foxops.engine.__main__ import app +from foxops.engine.models import ( + IncarnationState, + load_incarnation_state, + save_incarnation_state, +) + + +@pytest.fixture(scope="module") +def cli_runner() -> CliRunner: + runner = CliRunner() + return runner + + +def init_repository(repository_dir): + check_output(["git", "init", str(repository_dir)]) + check_output(["git", "config", "user.name", "test"], cwd=repository_dir) + check_output(["git", "config", "user.email", "test@test.com"], cwd=repository_dir) + + +def commit_version(repository_dir, version): + check_output(["git", "add", "."], cwd=repository_dir) + check_output(["git", "commit", "-m", f"version: {version}"], cwd=repository_dir) + check_output(["git", "tag", version], cwd=repository_dir) + + +@pytest.fixture() +def template_repository_without_variables(tmp_path: Path): + template_repository_dir = tmp_path / "template-repository" + init_repository(template_repository_dir) + (template_repository_dir / "fengine.yaml").write_text("variables: {}") + (template_repository_dir / "template").mkdir() + (template_repository_dir / "template" / "README.md").write_text("# Hello World") + commit_version(template_repository_dir, "v1") + + yield template_repository_dir + + +@pytest.fixture() +def template_repository(tmp_path: Path): + template_repository_dir = tmp_path / "template-repository" + init_repository(template_repository_dir) + (template_repository_dir / "fengine.yaml").write_text( + dedent( + """ + variables: + name: + type: str + description: the name + + age: + type: int + description: the age + """ + ) + ) + (template_repository_dir / "template").mkdir() + (template_repository_dir / "template" / "README.md").write_text( + "# Hello, {{ name }} of age {{ age }}!" + ) + commit_version(template_repository_dir, "v1") + + yield template_repository_dir + + +@pytest.fixture() +def template_repository_with_two_versions(template_repository: Path): + (template_repository / "template" / "info.txt").write_text( + "some info for {{ name }}." + ) + commit_version(template_repository, "v2") + + yield template_repository + + +@pytest.fixture() +def template_repository_with_two_versions_different_variables( + template_repository: Path, +): + (template_repository / "fengine.yaml").write_text( + (template_repository / "fengine.yaml").read_text() + + """ + new: + type: str + description: the new +""" + ) + (template_repository / "template" / "info.txt").write_text( + "some info for {{ name }} with {{ new }}." + ) + commit_version(template_repository, "v2") + + yield template_repository + + +@pytest.mark.parametrize("command", ["new", "initialize", "update"]) +def test_app_has_commands(command, cli_runner: CliRunner): + # WHEN + result = cli_runner.invoke(app, [command, "--help"]) + + # THEN + assert result.exit_code == 0 + assert result.stdout.startswith("Usage: ") + + +def test_app_should_create_template( + cli_runner: CliRunner, + tmp_path: Path, +): + # WHEN + result = cli_runner.invoke(app, ["new", str(tmp_path)]) + + # THEN + assert result.exit_code == 0 + assert (tmp_path / "fengine.yaml").exists() + assert (tmp_path / "template" / "README.md").exists() + + +def test_app_should_initialize_incarnation_from_template_without_variables( + cli_runner: CliRunner, + template_repository_without_variables: Path, + tmp_path: Path, +): + # GIVEN + incarnation_dir = tmp_path / "incarnation" + + # WHEN + result = cli_runner.invoke( + app, + [ + "initialize", + str(template_repository_without_variables), + str(incarnation_dir), + ], + ) + + # THEN + assert result.exit_code == 0 + assert (incarnation_dir / "README.md").read_text() == "# Hello World" + + +def test_app_should_initialize_incarnation_from_template_with_variables( + cli_runner: CliRunner, + template_repository: Path, + tmp_path: Path, +): + # GIVEN + incarnation_dir = tmp_path / "incarnation" + + # WHEN + result = cli_runner.invoke( + app, + [ + "initialize", + str(template_repository), + str(incarnation_dir), + "-d", + "name=jon", + "-d", + "age=42", + ], + ) + + # THEN + assert result.exit_code == 0 + assert (incarnation_dir / "README.md").read_text() == "# Hello, jon of age 42!" + + +def test_app_should_initialize_incarnation_of_specific_template_version( + cli_runner: CliRunner, + template_repository_with_two_versions: Path, + tmp_path: Path, +): + # GIVEN + incarnation_dir = tmp_path / "incarnation" + + # WHEN + result = cli_runner.invoke( + app, + [ + "initialize", + str(template_repository_with_two_versions), + str(incarnation_dir), + "-d", + "name=jon", + "-d", + "age=42", + "--template-version", + "v1", + ], + ) + + # THEN + assert result.exit_code == 0 + assert not (incarnation_dir / "info.txt").exists() + + +def test_app_should_update_incarnation_to_head_in_template_repository( + cli_runner: CliRunner, + template_repository_with_two_versions: Path, + tmp_path: Path, +): + # GIVEN + incarnation_dir = tmp_path / "incarnation" + + cli_runner.invoke( + app, + [ + "initialize", + str(template_repository_with_two_versions), + str(incarnation_dir), + "-d", + "name=jon", + "-d", + "age=42", + "--template-version", + "v1", + ], + ) + init_repository(incarnation_dir) + commit_version(incarnation_dir, "v1") + + # WHEN + result = cli_runner.invoke( + app, + [ + "update", + str(incarnation_dir), + ], + ) + + # THEN + assert result.exit_code == 0 + assert (incarnation_dir / "info.txt").read_text() == "some info for jon." + + +def test_app_should_update_incarnation_to_specific_version_in_template_repository( + cli_runner: CliRunner, + template_repository_with_two_versions: Path, + tmp_path: Path, +): + # GIVEN + incarnation_dir = tmp_path / "incarnation" + + cli_runner.invoke( + app, + [ + "initialize", + str(template_repository_with_two_versions), + str(incarnation_dir), + "-d", + "name=jon", + "-d", + "age=42", + "--template-version", + "v1", + ], + ) + init_repository(incarnation_dir) + commit_version(incarnation_dir, "v1") + + # WHEN + result = cli_runner.invoke( + app, + [ + "update", + str(incarnation_dir), + "-u", + "v2", + ], + ) + + # THEN + assert result.exit_code == 0 + assert (incarnation_dir / "info.txt").read_text() == "some info for jon." + incarnation_state = load_incarnation_state(incarnation_dir / ".fengine.yaml") + assert incarnation_state.template_repository_version == "v2" + assert incarnation_state.template_repository_version_hash != "v2" + + +def test_app_should_update_incarnation_to_version_with_new_variable( + cli_runner: CliRunner, + template_repository_with_two_versions_different_variables: Path, + tmp_path: Path, +): + # GIVEN + incarnation_dir = tmp_path / "incarnation" + + cli_runner.invoke( + app, + [ + "initialize", + str(template_repository_with_two_versions_different_variables), + str(incarnation_dir), + "-d", + "name=jon", + "-d", + "age=42", + "--template-version", + "v1", + ], + ) + init_repository(incarnation_dir) + commit_version(incarnation_dir, "v1") + + # WHEN + result = cli_runner.invoke( + app, + [ + "update", + str(incarnation_dir), + "-d", + "new=foobar", + "-u", + "v2", + ], + ) + + # THEN + assert result.exit_code == 0 + assert ( + incarnation_dir / "info.txt" + ).read_text() == "some info for jon with foobar." + + +def test_app_should_update_incarnation_to_version_with_removed_variable( + cli_runner: CliRunner, + template_repository_with_two_versions_different_variables: Path, + tmp_path: Path, +): + # GIVEN + incarnation_dir = tmp_path / "incarnation" + + cli_runner.invoke( + app, + [ + "initialize", + str(template_repository_with_two_versions_different_variables), + str(incarnation_dir), + "-d", + "name=jon", + "-d", + "age=42", + "-d", + "new=foobar", + "--template-version", + "v2", + ], + ) + init_repository(incarnation_dir) + commit_version(incarnation_dir, "v1") + + # WHEN + result = cli_runner.invoke( + app, + [ + "update", + str(incarnation_dir), + "--remove-data", + "new", + "-u", + "v1", + ], + ) + + # THEN + assert result.exit_code == 0 + assert not (incarnation_dir / "info.txt").exists() + + +def test_app_should_update_incarnation_with_new_variable_data( + cli_runner: CliRunner, + template_repository: Path, + tmp_path: Path, +): + # GIVEN + incarnation_dir = tmp_path / "incarnation" + + cli_runner.invoke( + app, + [ + "initialize", + str(template_repository), + str(incarnation_dir), + "-d", + "name=jon", + "-d", + "age=42", + "--template-version", + "v1", + ], + ) + init_repository(incarnation_dir) + commit_version(incarnation_dir, "v1") + + # WHEN + result = cli_runner.invoke( + app, + [ + "update", + str(incarnation_dir), + "-d", + "name=ygritte", + "-u", + "v1", + ], + ) + + # THEN + assert result.exit_code == 0 + assert (incarnation_dir / "README.md").read_text() == "# Hello, ygritte of age 42!" + + +def test_app_update_incarnation_should_complain_if_template_repository_is_not_a_local_path( + cli_runner: CliRunner, + template_repository_with_two_versions: Path, + tmp_path: Path, + caplog, +): + # GIVEN + incarnation_dir = tmp_path / "incarnation" + + cli_runner.invoke( + app, + [ + "initialize", + str(template_repository_with_two_versions), + str(incarnation_dir), + "-d", + "name=jon", + "-d", + "age=42", + "--template-version", + "v1", + ], + ) + + # NOTE(TF): patch the `.fengine.yaml` file to point to a remote Git template repository. + # This would be the case if the template wasn't initialized locally, but with + # foxops and later on cloned to develop locally using the `fengine` cli tool. + incarnation_state = load_incarnation_state(incarnation_dir / ".fengine.yaml") + patched_incarnation_state = IncarnationState( + **{ + **asdict(incarnation_state), # type: ignore + "template_repository": "https://any-remote-repository.com/any-repository.git", + } + ) + save_incarnation_state(incarnation_dir / ".fengine.yaml", patched_incarnation_state) + + init_repository(incarnation_dir) + commit_version(incarnation_dir, "v1") + + # WHEN + with caplog.at_level(logging.INFO): + result = cli_runner.invoke( + app, + [ + "--json-logs", + "update", + str(incarnation_dir), + "-u", + "v2", + ], + ) + + # THEN + assert result.exit_code == 1 + assert caplog.records[-1].levelname == "ERROR" + assert ( + "is not a local directory. Might it be an URL?" + in json.loads(caplog.records[-1].message)["event"] + ) + + +def test_app_should_update_incarnation_with_overridden_template_repository( + cli_runner: CliRunner, + template_repository_with_two_versions: Path, + tmp_path: Path, +): + # GIVEN + incarnation_dir = tmp_path / "incarnation" + + cli_runner.invoke( + app, + [ + "initialize", + str(template_repository_with_two_versions), + str(incarnation_dir), + "-d", + "name=jon", + "-d", + "age=42", + "--template-version", + "v1", + ], + ) + + # NOTE(TF): patch the `.fengine.yaml` file to point to a remote Git template repository. + # This would be the case if the template wasn't initialized locally, but with + # foxops and later on cloned to develop locally using the `fengine` cli tool. + incarnation_state = load_incarnation_state(incarnation_dir / ".fengine.yaml") + patched_incarnation_state = IncarnationState( + **{ + **asdict(incarnation_state), # type: ignore + "template_repository": "https://any-remote-repository.com/any-repository.git", + } + ) + save_incarnation_state(incarnation_dir / ".fengine.yaml", patched_incarnation_state) + + init_repository(incarnation_dir) + commit_version(incarnation_dir, "v1") + + # WHEN + result = cli_runner.invoke( + app, + [ + "update", + str(incarnation_dir), + "-u", + "v2", + "-r", + str(template_repository_with_two_versions), + ], + ) + + # THEN + assert result.exit_code == 0 + assert (incarnation_dir / "info.txt").read_text() == "some info for jon." diff --git a/tests/engine/test_initialization.py b/tests/engine/test_initialization.py new file mode 100644 index 00000000..fc19c1cc --- /dev/null +++ b/tests/engine/test_initialization.py @@ -0,0 +1,439 @@ +from pathlib import Path + +import pytest + +from foxops import utils +from foxops.engine import initialize_incarnation + + +async def init_repository(repository_dir: Path) -> None: + await utils.check_call("git", "init", cwd=repository_dir) + await utils.check_call("git", "config", "user.name", "test", cwd=repository_dir) + await utils.check_call( + "git", "config", "user.email", "test@test.com", cwd=repository_dir + ) + await utils.check_call("git", "add", ".", cwd=repository_dir) + await utils.check_call("git", "commit", "-m", "initial commit", cwd=repository_dir) + proc = await utils.check_call("git", "rev-parse", "HEAD", cwd=repository_dir) + return (await proc.stdout.read()).decode().strip() # type: ignore + + +@pytest.mark.asyncio +async def test_initialize_template_at_root_of_incarnation_repository( + tmp_path: Path, logger +): + # GIVEN + (tmp_path / "fengine.yaml").write_text( + """ +variables: + author: + type: str + description: dummy + three: + type: int + description: dummy""" + ) + + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "README.md").write_text("{{ author }} knows that 1+2 = {{ three }}") + repository_head = await init_repository(tmp_path) + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + # WHEN + await initialize_incarnation( + template_root_dir=tmp_path, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={"author": "John Doe", "three": "3"}, + incarnation_root_dir=incarnation_dir, + logger=logger, + ) + + # THEN + assert (incarnation_dir / "README.md").read_text() == "John Doe knows that 1+2 = 3" + assert ( + (incarnation_dir / ".fengine.yaml").read_text() + == f"""# This file is auto-generated and owned by foxops. +# DO NOT EDIT MANUALLY. +template_data: + author: John Doe + three: '3' +template_repository: any-repository-url +template_repository_version: any-version +template_repository_version_hash: {repository_head} +""" + ) + + +@pytest.mark.asyncio +async def test_initialize_template_at_root_of_incarnation_repository_with_existing_file( + tmp_path: Path, logger +): + # GIVEN + (tmp_path / "fengine.yaml").write_text( + """ +variables: + author: + type: str + description: dummy + three: + type: int + description: dummy""" + ) + + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "README.md").write_text("{{ author }} knows that 1+2 = {{ three }}") + repository_head = await init_repository(tmp_path) + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + (incarnation_dir / "README.md").write_text("I exist already") + (incarnation_dir / "config.yaml").write_text("existing: true") + + # WHEN + await initialize_incarnation( + template_root_dir=tmp_path, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={"author": "John Doe", "three": "3"}, + incarnation_root_dir=incarnation_dir, + logger=logger, + ) + + # THEN + assert (incarnation_dir / "config.yaml").read_text() == "existing: true" + assert (incarnation_dir / "README.md").read_text() == "John Doe knows that 1+2 = 3" + assert ( + (incarnation_dir / ".fengine.yaml").read_text() + == f"""# This file is auto-generated and owned by foxops. +# DO NOT EDIT MANUALLY. +template_data: + author: John Doe + three: '3' +template_repository: any-repository-url +template_repository_version: any-version +template_repository_version_hash: {repository_head} +""" + ) + + +@pytest.mark.asyncio +async def test_initialize_template_fails_when_variables_are_not_set(tmp_path, logger): + # GIVEN + (tmp_path / "fengine.yaml").write_text( + """ +variables: + author: + type: str + description: dummy + three: + type: int + description: dummy""" + ) + + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "README.md").write_text("{{ author }} knows that 1+2 = {{ three }}") + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + # THEN + with pytest.raises(ValueError): + await initialize_incarnation( + template_root_dir=tmp_path, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={"author": "John Doe"}, + incarnation_root_dir=incarnation_dir, + logger=logger, + ) + + +@pytest.mark.asyncio +async def test_initialize_template_used_passed_value_instead_default_for_optional_variables( + tmp_path, logger +): + # GIVEN + (tmp_path / "fengine.yaml").write_text( + """ +variables: + author: + type: str + description: dummy + three: + type: int + description: dummy + default: 42""" + ) + + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "README.md").write_text("{{ author }} knows that 1+2 = {{ three }}") + await init_repository(tmp_path) + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + # WHEN + await initialize_incarnation( + template_root_dir=tmp_path, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={"author": "John Doe", "three": 3}, + incarnation_root_dir=incarnation_dir, + logger=logger, + ) + + # THEN + assert (incarnation_dir / "README.md").read_text() == "John Doe knows that 1+2 = 3" + + +@pytest.mark.asyncio +async def test_initialize_template_allows_optional_variables(tmp_path, logger): + # GIVEN + (tmp_path / "fengine.yaml").write_text( + """ +variables: + author: + type: str + description: dummy + three: + type: int + description: dummy + default: 42""" + ) + + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "README.md").write_text("{{ author }} knows that 1+2 = {{ three }}") + await init_repository(tmp_path) + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + # WHEN + await initialize_incarnation( + template_root_dir=tmp_path, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={"author": "John Doe"}, + incarnation_root_dir=incarnation_dir, + logger=logger, + ) + + # THEN + assert (incarnation_dir / "README.md").read_text() == "John Doe knows that 1+2 = 42" + + +@pytest.mark.asyncio +async def test_initialize_template_ignores_but_warns_about_additional_variables( + tmp_path, mocker +): + # GIVEN + (tmp_path / "fengine.yaml").write_text( + """ +variables: + author: + type: str + description: dummy + three: + type: int + description: dummy + default: 42""" + ) + + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "README.md").write_text("{{ author }} knows that 1+2 = {{ three }}") + await init_repository(tmp_path) + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + template_data_with_additional_values = { + "author": "John Doe", + "additional_variable_1": "any value", + "additional_variable_2": 42, + } + logger_mock = mocker.MagicMock() + + # WHEN + await initialize_incarnation( + template_root_dir=tmp_path, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data=template_data_with_additional_values, + incarnation_root_dir=incarnation_dir, + logger=logger_mock, + ) + + # THEN + logger_mock.warn.assert_called_once_with( + "got additional template data for the incarnation: ['additional_variable_1', 'additional_variable_2']" + ) + + +@pytest.mark.asyncio +async def test_initialize_template_with_variables_from_fvars_file( + tmp_path: Path, logger +): + # GIVEN + (tmp_path / "fengine.yaml").write_text( + """ +variables: + author: + type: str + description: dummy + three: + type: int + description: dummy""" + ) + + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "README.md").write_text("{{ author }} knows that 1+2 = {{ three }}") + repository_head = await init_repository(tmp_path) + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + (incarnation_dir / "default.fvars").write_text( + """ + author=John Doe + """ + ) + + # WHEN + await initialize_incarnation( + template_root_dir=tmp_path, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={"three": "3"}, + incarnation_root_dir=incarnation_dir, + logger=logger, + ) + + # THEN + assert (incarnation_dir / "README.md").read_text() == "John Doe knows that 1+2 = 3" + assert ( + (incarnation_dir / ".fengine.yaml").read_text() + == f"""# This file is auto-generated and owned by foxops. +# DO NOT EDIT MANUALLY. +template_data: + author: John Doe + three: '3' +template_repository: any-repository-url +template_repository_version: any-version +template_repository_version_hash: {repository_head} +""" + ) + + +@pytest.mark.asyncio +async def test_initialize_template_template_data_precedence_over_fvars( + tmp_path: Path, logger +): + # GIVEN + (tmp_path / "fengine.yaml").write_text( + """ +variables: + author: + type: str + description: dummy + three: + type: int + description: dummy""" + ) + + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "README.md").write_text("{{ author }} knows that 1+2 = {{ three }}") + repository_head = await init_repository(tmp_path) + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + (incarnation_dir / "default.fvars").write_text( + """ + author=John Doe + """ + ) + + # WHEN + await initialize_incarnation( + template_root_dir=tmp_path, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={"author": "Overridden John Doe", "three": "3"}, + incarnation_root_dir=incarnation_dir, + logger=logger, + ) + + # THEN + assert ( + incarnation_dir / "README.md" + ).read_text() == "Overridden John Doe knows that 1+2 = 3" + assert ( + (incarnation_dir / ".fengine.yaml").read_text() + == f"""# This file is auto-generated and owned by foxops. +# DO NOT EDIT MANUALLY. +template_data: + author: Overridden John Doe + three: '3' +template_repository: any-repository-url +template_repository_version: any-version +template_repository_version_hash: {repository_head} +""" + ) + + +@pytest.mark.asyncio +async def test_initialize_template_empty_fvars_file(tmp_path: Path, logger): + # GIVEN + (tmp_path / "fengine.yaml").write_text( + """ +variables: + author: + type: str + description: dummy + three: + type: int + description: dummy""" + ) + + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "README.md").write_text("{{ author }} knows that 1+2 = {{ three }}") + repository_head = await init_repository(tmp_path) + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + (incarnation_dir / "default.fvars").write_text("") + + # WHEN + await initialize_incarnation( + template_root_dir=tmp_path, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={"author": "John Doe", "three": "3"}, + incarnation_root_dir=incarnation_dir, + logger=logger, + ) + + # THEN + assert (incarnation_dir / "README.md").read_text() == "John Doe knows that 1+2 = 3" + assert ( + (incarnation_dir / ".fengine.yaml").read_text() + == f"""# This file is auto-generated and owned by foxops. +# DO NOT EDIT MANUALLY. +template_data: + author: John Doe + three: '3' +template_repository: any-repository-url +template_repository_version: any-version +template_repository_version_hash: {repository_head} +""" + ) diff --git a/tests/engine/test_rendering.py b/tests/engine/test_rendering.py new file mode 100644 index 00000000..872aa9b9 --- /dev/null +++ b/tests/engine/test_rendering.py @@ -0,0 +1,410 @@ +import stat +from pathlib import Path +from tempfile import TemporaryDirectory + +import jinja2 +import pytest + +from foxops.engine.rendering import ( + create_template_environment, + render_template, + render_template_file, + render_template_symlink, +) + + +def supports_symlink_permissions(): + """Check if the current platform supports setting permissions on symlinks.""" + with TemporaryDirectory() as tmpdir: + try: + Path(tmpdir).chmod(0o755, follow_symlinks=False) + except NotImplementedError: + return False + else: + return True + + +@pytest.mark.asyncio +async def test_rendering_a_template_file_renders_data_in_file_content( + tmp_path: Path, logger +): + # GIVEN + template_file = tmp_path / "template.txt" + template_file.write_text("{{ data }}") + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + env = create_template_environment(tmp_path) + + # WHEN + await render_template_file( + env, + template_file, + incarnation_dir, + {"data": "Hello World"}, + render_content=True, + logger=logger, + ) + + # THEN + assert (incarnation_dir / "template.txt").read_text() == "Hello World" + + +@pytest.mark.asyncio +async def test_rendering_a_template_file_with_invalid_templating_syntax_raises_exception( + tmp_path: Path, + logger, +): + # GIVEN + template_file = tmp_path / "template.txt" + template_file.write_text("{{ data }") + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + env = create_template_environment(tmp_path) + + # THEN + with pytest.raises(jinja2.TemplateSyntaxError): + # WHEN + await render_template_file( + env, + template_file, + incarnation_dir, + {"data": "Hello World"}, + render_content=True, + logger=logger, + ) + + +@pytest.mark.asyncio +async def test_rendering_a_template_file_renders_data_in_filename( + tmp_path: Path, logger +): + # GIVEN + template_file = tmp_path / "template-{{ idx }}.txt" + template_file.touch() + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + env = create_template_environment(tmp_path) + + # WHEN + await render_template_file( + env, + template_file, + incarnation_dir, + {"idx": "42"}, + render_content=True, + logger=logger, + ) + + # THEN + assert (incarnation_dir / "template-42.txt").exists() + + +@pytest.mark.asyncio +async def test_rendering_a_template_file_renders_data_in_entire_filepath( + tmp_path: Path, logger +): + # GIVEN + template_file = tmp_path / "project-{{ name }}/{{ subdir }}/template-{{ idx }}.txt" + template_file.parent.mkdir(parents=True) + template_file.touch() + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + env = create_template_environment(tmp_path) + + # WHEN + await render_template_file( + env, + template_file, + incarnation_dir, + {"name": "jon", "subdir": "tests", "idx": "42"}, + render_content=True, + logger=logger, + ) + + # THEN + assert (incarnation_dir / "project-jon" / "tests" / "template-42.txt").exists() + + +@pytest.mark.asyncio +async def test_rendering_a_template_symlink_renders_data_in_filename( + tmp_path: Path, logger +): + # GIVEN + template_symlink = tmp_path / "template-symlink-{{ idx }}" + template_symlink.symlink_to("template.txt") + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + env = create_template_environment(tmp_path) + + # WHEN + await render_template_symlink( + env, template_symlink, incarnation_dir, {"idx": "42"}, logger=logger + ) + + # THEN + assert (incarnation_dir / "template-symlink-42").is_symlink() + assert (incarnation_dir / "template-symlink-42").readlink() == Path("template.txt") + + +@pytest.mark.asyncio +async def test_rendering_a_template_symlink_renders_data_in_target_filename( + tmp_path: Path, logger +): + # GIVEN + template_symlink = tmp_path / "template-symlink" + template_symlink.symlink_to("symlink-target-{{ idx }}.txt") + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + env = create_template_environment(tmp_path) + + # WHEN + await render_template_symlink( + env, template_symlink, incarnation_dir, {"idx": "42"}, logger=logger + ) + + # THEN + assert (incarnation_dir / "template-symlink").is_symlink() + assert (incarnation_dir / "template-symlink").readlink() == Path( + "symlink-target-42.txt" + ) + + +@pytest.mark.asyncio +async def test_rendering_an_entire_template_directory_with_excluded_file( + tmp_path: Path, logger +): + # GIVEN + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "README.md").write_text("{{ invalid syntax } {%") + (template_dir / "code.c").write_text("{{ data }}") + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + excludes = [ + "README.md", + ] + + # WHEN + await render_template( + template_dir, + incarnation_dir, + {"data": "Hello World"}, + rendering_filename_exclude_patterns=excludes, + logger=logger, + ) + + # THEN + assert (incarnation_dir / "README.md").read_text() == "{{ invalid syntax } {%" + assert (incarnation_dir / "code.c").read_text() == "Hello World" + + +@pytest.mark.asyncio +async def test_rendering_an_entire_template_directory_with_excluded_file_in_rendered_subdir( + tmp_path: Path, logger +): + # GIVEN + template_dir = tmp_path / "template" + template_dir.mkdir() + + (template_dir / "{{ package_name }}").mkdir() + (template_dir / "{{ package_name }}" / "README.md").write_text( + "{{ invalid syntax } {%" + ) + (template_dir / "code.c").write_text("{{ data }}") + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + excludes = [ + "{{ package_name }}/*", + ] + + # WHEN + await render_template( + template_dir, + incarnation_dir, + {"data": "Hello World", "package_name": "test"}, + rendering_filename_exclude_patterns=excludes, + logger=logger, + ) + + # THEN + assert ( + incarnation_dir / "test" / "README.md" + ).read_text() == "{{ invalid syntax } {%" + assert (incarnation_dir / "code.c").read_text() == "Hello World" + + +@pytest.mark.asyncio +async def test_rendering_an_entire_template_directory(tmp_path: Path, logger): + # GIVEN + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "README.md").write_text("README: {{ data }}") + (template_dir / "{{ name }}").mkdir() + (template_dir / "{{ name }}" / "code.c").write_text("{{ data }}") + (template_dir / "tests" / "{{ name }}").mkdir(parents=True) + (template_dir / "tests" / "{{ name }}" / "test_code.c").write_text( + "Test: {{ data }}" + ) + (template_dir / "README-symlink").symlink_to("README.md") + (template_dir / "test_code-symlink").symlink_to("tests/{{ name }}/test_code.c") + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + # WHEN + await render_template( + template_dir, + incarnation_dir, + {"name": "jon", "data": "Hello World"}, + [], + logger=logger, + ) + + # THEN + assert (incarnation_dir / "README.md").read_text() == "README: Hello World" + assert (incarnation_dir / "jon").exists() + assert (incarnation_dir / "jon" / "code.c").read_text() == "Hello World" + assert ( + incarnation_dir / "tests" / "jon" / "test_code.c" + ).read_text() == "Test: Hello World" + assert (incarnation_dir / "README-symlink").is_symlink() + assert (incarnation_dir / "README-symlink").exists() + assert (incarnation_dir / "README-symlink").readlink() == Path("README.md") + assert (incarnation_dir / "test_code-symlink").is_symlink() + assert (incarnation_dir / "test_code-symlink").exists() + assert (incarnation_dir / "test_code-symlink").readlink() == Path( + "tests/jon/test_code.c" + ) + + +@pytest.mark.asyncio +async def test_rendering_a_template_file_inherits_file_permissions( + tmp_path: Path, logger +): + # GIVEN + template_file = tmp_path / "template.txt" + template_file.touch() + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + # change permissions on template file + mode_before_xusr = template_file.stat().st_mode + template_file.chmod(template_file.stat().st_mode | stat.S_IXUSR) + expected_mode = stat.S_IMODE(mode_before_xusr | stat.S_IXUSR) + + env = create_template_environment(tmp_path) + + # WHEN + await render_template_file( + env, template_file, incarnation_dir, {}, render_content=True, logger=logger + ) + + # THEN + assert (incarnation_dir / "template.txt").exists() + assert ( + stat.S_IMODE((incarnation_dir / "template.txt").stat().st_mode) == expected_mode + ) + + +@pytest.mark.asyncio +@pytest.mark.skipif( + not supports_symlink_permissions(), + reason="Platform doesn't support setting symlink permissions", +) +async def test_rendering_a_template_symlink_inherits_file_permissions( + tmp_path: Path, logger +): + # GIVEN + template_file = tmp_path / "template.txt" + template_file.write_text("test") + template_symlink = tmp_path / "template-symlink" + template_symlink.symlink_to(template_file) + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + # change permissions on template file + template_file.chmod(template_file.stat().st_mode | stat.S_IXUSR) + + # change permissions on template symlink + mode_before_woth = template_symlink.stat(follow_symlinks=False).st_mode # type: ignore + template_symlink.chmod(mode_before_woth | stat.S_IWOTH, follow_symlinks=False) # type: ignore + expected_symlink_mode = stat.S_IMODE(mode_before_woth | stat.S_IWOTH) + + env = create_template_environment(tmp_path) + + # WHEN + await render_template_file( + env, + template_file, + incarnation_dir, + {}, + render_content=True, + logger=logger, + ) + await render_template_file( + env, + template_symlink, + incarnation_dir, + {}, + render_content=True, + logger=logger, + ) + + # THEN + assert (incarnation_dir / "template-symlink").exists() + assert ( + stat.S_IMODE( + (incarnation_dir / "template-symlink").stat(follow_symlinks=False).st_mode # type: ignore + ) + == expected_symlink_mode + ) + + +@pytest.mark.asyncio +async def test_rendering_a_template_directory_inherits_file_permissions( + tmp_path: Path, logger +): + # GIVEN + template_dir = tmp_path / "template" + template_subdir = template_dir / "subdir" + template_file = template_subdir / "template.txt" + template_dir.mkdir() + template_subdir.mkdir() + template_file.touch() + + incarnation_dir = tmp_path / "incarnation" + incarnation_dir.mkdir() + + # change permissions on template file + subdir_mode_before_woth = template_subdir.stat().st_mode + template_subdir.chmod(template_subdir.stat().st_mode | stat.S_IWOTH) + expected_subdir_mode = stat.S_IMODE(subdir_mode_before_woth | stat.S_IWOTH) + + file_mode_before_xusr = template_file.stat().st_mode + template_file.chmod(template_file.stat().st_mode | stat.S_IXUSR) + expected_file_mode = stat.S_IMODE(file_mode_before_xusr | stat.S_IXUSR) + + # WHEN + await render_template(template_dir, incarnation_dir, {}, [], logger=logger) + + # THEN + assert (incarnation_dir / "subdir").exists() + assert ( + stat.S_IMODE((incarnation_dir / "subdir").stat().st_mode) + == expected_subdir_mode + ) + assert (incarnation_dir / "subdir" / "template.txt").exists() + assert ( + stat.S_IMODE((incarnation_dir / "subdir" / "template.txt").stat().st_mode) + == expected_file_mode + ) diff --git a/tests/engine/test_update.py b/tests/engine/test_update.py new file mode 100644 index 00000000..995f00b5 --- /dev/null +++ b/tests/engine/test_update.py @@ -0,0 +1,479 @@ +import shutil +from pathlib import Path + +import pytest + +from foxops import utils +from foxops.engine import diff_and_patch, initialize_incarnation, update_incarnation + + +async def init_repository(repository_dir: Path) -> None: + await utils.check_call("git", "init", cwd=repository_dir) + await utils.check_call("git", "config", "user.name", "test", cwd=repository_dir) + await utils.check_call( + "git", "config", "user.email", "test@test.com", cwd=repository_dir + ) + await utils.check_call("git", "add", ".", cwd=repository_dir) + await utils.check_call("git", "commit", "-m", "initial commit", cwd=repository_dir) + proc = await utils.check_call("git", "rev-parse", "HEAD", cwd=repository_dir) + return (await proc.stdout.read()).decode().strip() # type: ignore + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "diff_patch_func", + [diff_and_patch], +) +async def test_diff_and_patch_update_single_file_without_conflict( + diff_patch_func, tmp_path, logger +): + # GIVEN + old_directory = tmp_path / "old" + old_directory.mkdir() + (old_directory / "file.txt").write_text("old content") + new_directory = tmp_path / "new" + to_patch_directory = tmp_path / "to_patch" + shutil.copytree(old_directory, to_patch_directory) + await init_repository(to_patch_directory) + shutil.copytree(old_directory, new_directory) + (new_directory / "file.txt").write_text("new content") + + # WHEN + await diff_patch_func( + diff_a_directory=old_directory, + diff_b_directory=new_directory, + patch_directory=to_patch_directory, + logger=logger, + ) + + # THEN + assert (to_patch_directory / "file.txt").read_text() == "new content" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("diff_patch_func", [diff_and_patch]) +async def test_diff_and_patch_adding_new_file_without_conflict( + diff_patch_func, tmp_path, logger +): + # GIVEN + old_directory = tmp_path / "old" + old_directory.mkdir() + (old_directory / "file.txt").write_text("any content") + new_directory = tmp_path / "new" + to_patch_directory = tmp_path / "to_patch" + shutil.copytree(old_directory, to_patch_directory) + await init_repository(to_patch_directory) + shutil.copytree(old_directory, new_directory) + (new_directory / "new-file.txt").write_text("new content") + + # WHEN + await diff_patch_func( + diff_a_directory=old_directory, + diff_b_directory=new_directory, + patch_directory=to_patch_directory, + logger=logger, + ) + + # THEN + assert (to_patch_directory / "new-file.txt").read_text() == "new content" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("diff_patch_func", [diff_and_patch]) +async def test_diff_and_patch_removing_file_without_conflict( + diff_patch_func, tmp_path, logger +): + # GIVEN + old_directory = tmp_path / "old" + old_directory.mkdir() + (old_directory / "file.txt").write_text("any content") + (old_directory / "deprecated-file.txt").write_text("deprecated content") + new_directory = tmp_path / "new" + to_patch_directory = tmp_path / "to_patch" + shutil.copytree(old_directory, to_patch_directory) + await init_repository(to_patch_directory) + shutil.copytree(old_directory, new_directory) + (new_directory / "deprecated-file.txt").unlink() + + # WHEN + await diff_patch_func( + diff_a_directory=old_directory, + diff_b_directory=new_directory, + patch_directory=to_patch_directory, + logger=logger, + ) + + # THEN + assert not (to_patch_directory / "deprecated-file.txt").exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("diff_patch_func", [diff_and_patch]) +async def test_diff_and_patch_no_change_when_updating_to_template_version_with_identical_change( + diff_patch_func, + tmp_path, + logger, +): + """ + Verify that no change is being made to the incarnation repository when updating to a template version + which contains the same changes as the incarnation. + """ + # GIVEN + template_directory = tmp_path / "template" + template_directory.mkdir() + (template_directory / "myfile.txt").write_text( + """r1 {...} + +r2 {...} +""" + ) + await init_repository(tmp_path) + incarnation_directory = tmp_path / "incarnation" + incarnation_directory.mkdir() + + incarnation_state = await initialize_incarnation( + template_root_dir=template_directory, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={}, + incarnation_root_dir=incarnation_directory, + logger=logger, + ) + + # WHEN + # same change in template and incarnation + updated_template_directory = tmp_path / "updated-template" + shutil.copytree(template_directory, updated_template_directory) + (updated_template_directory / "myfile.txt").write_text( + """r1 {...} + +rnew {...} + +r2 {...} +""" + ) + (incarnation_directory / "myfile.txt").write_text( + """r1 {...} + +rnew {...} + +r2 {...} +""" + ) + await init_repository(incarnation_directory) + + await update_incarnation( + template_root_dir=template_directory, + update_template_root_dir=updated_template_directory, + update_template_repository=incarnation_state.template_repository, + update_template_repository_version=incarnation_state.template_repository_version, + update_template_data=incarnation_state.template_data, + incarnation_root_dir=incarnation_directory, + diff_patch_func=diff_patch_func, + logger=logger, + ) + + # THEN + assert (incarnation_directory / "myfile.txt").read_text() == ( + """r1 {...} + +rnew {...} + +r2 {...} +""" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("diff_patch_func", [diff_and_patch]) +async def test_diff_and_patch_conflict_for_nearby_changes_in_template_and_incarnation( + diff_patch_func, + tmp_path, + logger, +): + """ + Verify that a conflict is detected when updating to a template version + which contains a change nearby a change in the incarnation. + """ + # GIVEN + template_directory = tmp_path / "template" + template_directory.mkdir() + + (template_directory / "template").mkdir() + (template_directory / "template" / "myfile.txt").write_text( + """a +b +c +""" + ) + await init_repository(tmp_path) + incarnation_directory = tmp_path / "incarnation" + incarnation_directory.mkdir() + + incarnation_state = await initialize_incarnation( + template_root_dir=template_directory, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={}, + incarnation_root_dir=incarnation_directory, + logger=logger, + ) + + # WHEN + # nearby change in template and incarnation + updated_template_directory = tmp_path / "updated-template" + shutil.copytree(template_directory, updated_template_directory) + (updated_template_directory / "template" / "myfile.txt").write_text( + """a +b +a +""" + ) + (incarnation_directory / "myfile.txt").write_text( + """c +b +c +""" + ) + await init_repository(incarnation_directory) + + # WHEN + _, files_with_conflicts = await update_incarnation( + template_root_dir=template_directory, + update_template_root_dir=updated_template_directory, + update_template_repository=incarnation_state.template_repository, + update_template_repository_version=incarnation_state.template_repository_version, + update_template_data=incarnation_state.template_data, + incarnation_root_dir=incarnation_directory, + diff_patch_func=diff_patch_func, + logger=logger, + ) + + # THEN + assert Path("myfile.txt") in files_with_conflicts + + +@pytest.mark.asyncio +@pytest.mark.parametrize("diff_patch_func", [diff_and_patch]) +async def test_diff_and_patch_success_when_changes_in_different_places_in_template_and_incarnation( + diff_patch_func, + tmp_path, + logger, +): + """ + Verify that a incarnation can successfully be updated to a new template version + if both contain locality unrelated changes. + """ + # GIVEN + template_directory = tmp_path / "template" + template_directory.mkdir() + + (template_directory / "template").mkdir() + (template_directory / "template" / "myfile.txt").write_text( + """a +b +c + +### +### Add custom changes to this file below +### +""" + ) + await init_repository(tmp_path) + incarnation_directory = tmp_path / "incarnation" + incarnation_directory.mkdir() + + incarnation_state = await initialize_incarnation( + template_root_dir=template_directory, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={}, + incarnation_root_dir=incarnation_directory, + logger=logger, + ) + + # WHEN + # same change in template and incarnation + updated_template_directory = tmp_path / "updated-template" + shutil.copytree(template_directory, updated_template_directory) + (updated_template_directory / "template" / "myfile.txt").write_text( + """a +b1 +b2 +c + +### +### Add custom changes to this file below +### +""" + ) + (incarnation_directory / "myfile.txt").write_text( + """a +b +c + +### +### Add custom changes to this file below +### +mychange +""" + ) + await init_repository(incarnation_directory) + + await update_incarnation( + template_root_dir=template_directory, + update_template_root_dir=updated_template_directory, + update_template_repository=incarnation_state.template_repository, + update_template_repository_version=incarnation_state.template_repository_version, + update_template_data=incarnation_state.template_data, + incarnation_root_dir=incarnation_directory, + diff_patch_func=diff_patch_func, + logger=logger, + ) + + # THEN + assert ( + (incarnation_directory / "myfile.txt").read_text() + == """a +b1 +b2 +c + +### +### Add custom changes to this file below +### +mychange +""" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("diff_patch_func", [diff_and_patch]) +async def test_diff_and_patch_success_when_update_in_fvars_file( + diff_patch_func, + tmp_path: Path, + logger, +): + """ + Verify that a incarnation can successfully be updated with new variable + values from fvars file. + """ + # GIVEN + template_directory = tmp_path / "template" + template_directory.mkdir() + (template_directory / "fengine.yaml").write_text( + """ +variables: + author: + type: str + description: dummy""" + ) + + (template_directory / "template").mkdir() + (template_directory / "template" / "myfile.txt").write_text("From: {{ author }}") + await init_repository(tmp_path) + incarnation_directory = tmp_path / "incarnation" + incarnation_directory.mkdir() + (incarnation_directory / "default.fvars").write_text( + """ + author=John Doe + """ + ) + + incarnation_state = await initialize_incarnation( + template_root_dir=template_directory, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={}, + incarnation_root_dir=incarnation_directory, + logger=logger, + ) + + # WHEN + # update fvars in incarnation + (incarnation_directory / "default.fvars").write_text( + """ + author=Updated John Doe + """ + ) + await init_repository(incarnation_directory) + + await update_incarnation( + template_root_dir=template_directory, + update_template_root_dir=template_directory, + update_template_repository=incarnation_state.template_repository, + update_template_repository_version=incarnation_state.template_repository_version, + update_template_data={}, + incarnation_root_dir=incarnation_directory, + diff_patch_func=diff_patch_func, + logger=logger, + ) + + # THEN + assert ( + incarnation_directory / "myfile.txt" + ).read_text() == "From: Updated John Doe" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("diff_patch_func", [diff_and_patch]) +async def test_diff_and_patch_success_when_deleting_file_in_template( + diff_patch_func, + tmp_path, + logger, +): + """ + Verify that a file is successfully deleted from the incarnation if it has been deleted in the template. + """ + # GIVEN + template_directory = tmp_path / "template" + template_directory.mkdir() + + (template_directory / "template").mkdir() + (template_directory / "template" / "myfile1.txt").write_text("some content") + (template_directory / "template" / "myfile2.txt").write_text("more content") + await init_repository(tmp_path) + + incarnation_directory = tmp_path / "incarnation" + incarnation_directory.mkdir() + incarnation_state = await initialize_incarnation( + template_root_dir=template_directory, + template_repository="any-repository-url", + template_repository_version="any-version", + template_data={}, + incarnation_root_dir=incarnation_directory, + logger=logger, + ) + + # WHEN + # same change in template and incarnation + updated_template_directory = tmp_path / "updated-template" + shutil.copytree(template_directory, updated_template_directory) + (updated_template_directory / "template" / "myfile2.txt").unlink() + + await utils.check_call("git", "init", ".", cwd=str(incarnation_directory)) + await utils.check_call( + "git", "config", "user.name", "test", cwd=str(incarnation_directory) + ) + await utils.check_call( + "git", "config", "user.email", "test@test.com", cwd=str(incarnation_directory) + ) + await utils.check_call("git", "add", ".", cwd=str(incarnation_directory)) + await utils.check_call( + "git", "commit", "-am", "Initial commit", cwd=str(incarnation_directory) + ) + + await update_incarnation( + template_root_dir=template_directory, + update_template_root_dir=updated_template_directory, + update_template_repository=incarnation_state.template_repository, + update_template_repository_version=incarnation_state.template_repository_version, + update_template_data=incarnation_state.template_data, + incarnation_root_dir=incarnation_directory, + diff_patch_func=diff_patch_func, + logger=logger, + ) + + # THEN + assert (incarnation_directory / "myfile1.txt").exists() + assert not (incarnation_directory / "myfile2.txt").exists() diff --git a/tests/external/test_git.py b/tests/external/test_git.py new file mode 100644 index 00000000..28659114 --- /dev/null +++ b/tests/external/test_git.py @@ -0,0 +1,62 @@ +import pytest + +from foxops.external.git import ( + GitError, + GitRepository, + add_authentication_to_git_clone_url, + git_exec, +) + + +@pytest.mark.asyncio +async def test_git_exec_throws_exception_on_nonzero_exit_code(): + # WHEN + git_args = ["--invalid-flag"] + + # THEN + with pytest.raises(GitError): + await git_exec(*git_args) + + +@pytest.mark.asyncio +async def test_has_any_commits_returns_false_if_there_are_no_commits(tmp_path, logger): + # GIVEN + repo = GitRepository(tmp_path, logger=logger) + await repo._run("init") + + # WHEN + result = await repo.has_any_commits() + + # THEN + assert result is False + + +@pytest.mark.asyncio +async def test_has_any_commits_returns_true_if_there_are_commits(tmp_path, logger): + # GIVEN + (tmp_path / "testfile").write_text("hello") + + repo = GitRepository(tmp_path, logger=logger) + await repo._run("init") + await repo._run("config", "user.name", "Test User") + await repo._run("config", "user.email", "testuser@local") + await repo.commit_all("initial commit") + + # WHEN + result = await repo.has_any_commits() + + # THEN + assert result is True + + +def test_add_authentication_to_git_clone_url_includes_username_password_in_output(): + # GIVEN + source = "https://myrepo/test.git" + username = "us:er" + password = "p@ssword" + + # WHEN + result = add_authentication_to_git_clone_url(source, username, password) + + # THEN + assert result == "https://us%3Aer:p%40ssword@myrepo/test.git" diff --git a/tests/test_cli.py b/tests/test_cli.py index 9353a4a7..9ed6daa0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,7 +1,12 @@ +import json +import logging +from pathlib import Path + import pytest from typer.testing import CliRunner from foxops.__main__ import app +from foxops.reconciliation import ReconciliationState @pytest.fixture(scope="module") @@ -10,10 +15,133 @@ def cli_runner() -> CliRunner: return runner -def test_app_can_be_called(cli_runner): +def test_app_has_reconcile_command(cli_runner): + # WHEN + result = cli_runner.invoke(app, ["reconcile", "--help"]) + + # THEN + assert result.exit_code == 0 + assert result.stdout.startswith("Usage: ") + + +def test_app_fails_to_reconcile_when_gitlab_token_missing_in_env(cli_runner): + # WHEN + result = cli_runner.invoke(app, ["reconcile", "any-project-definition-config"]) + + # THEN + assert result.exit_code == 1 + + +def test_app_exits_with_exit_code_1_for_unhandled_exceptions( + cli_runner, tmp_path, mocker +): + # GIVEN + project_definition_config = tmp_path / "project-definition-config.yaml" + project_definition_config.write_text( + """ +incarnations: + - gitlab_project: some/project1 + template_repository: any-repository + template_repository_version: any-version + template_data: {} + - gitlab_project: some/project2 + template_repository: any-repository + template_repository_version: any-version + template_data: {} +""" + ) + + mocker.patch( + "foxops.__main__.reconcile", + side_effect=Exception("any-exception"), + ) + + # WHEN + result = cli_runner.invoke( + app, + ["reconcile", str(project_definition_config)], + env={"FOXOPS_GITLAB_TOKEN": "any-token"}, + ) + + # THEN + assert result.exit_code == 1 + + +def test_app_exits_with_exit_code_2_if_a_project_fails_to_reconcile( + cli_runner, tmp_path, mocker +): + # GIVEN + project_definition_config = tmp_path / "project-definition-config.yaml" + project_definition_config.write_text( + """ +incarnations: + - gitlab_project: some/project1 + template_repository: any-repository + template_repository_version: any-version + template_data: {} + - gitlab_project: some/project2 + template_repository: any-repository + template_repository_version: any-version + template_data: {} +""" + ) + + mocker.patch( + "foxops.__main__.reconcile", + return_value=[ReconciliationState.FAILED, ReconciliationState.UNCHANGED], + ) + + # WHEN + result = cli_runner.invoke( + app, + ["reconcile", str(project_definition_config)], + env={"FOXOPS_GITLAB_TOKEN": "any-token"}, + ) + + # THEN + assert result.exit_code == 2 + + +def test_app_prints_reconciliation_summary_for_all_projects( + cli_runner: CliRunner, + tmp_path: Path, + mocker, + caplog, +): + # GIVEN + project_definition_config = tmp_path / "incarnations.yaml" + project_definition_config.write_text( + """ +incarnations: + - gitlab_project: test1 + template_repository: any-repository + template_repository_version: any-repository-version + template_data: + any: any + - gitlab_project: test2 + template_repository: any-repository + template_repository_version: any-repository-version + template_data: + any: any +""" + ) + + mocker.patch( + "foxops.__main__.reconcile", + return_value=[ReconciliationState.UNSUPPORTED, ReconciliationState.UNCHANGED], + ) + # WHEN - result = cli_runner.invoke(app) + with caplog.at_level(logging.INFO): + result = cli_runner.invoke( + app, + ["--json-logs", "reconcile", str(project_definition_config)], + env={"FOXOPS_GITLAB_TOKEN": "any-token"}, + ) # THEN + summary_log_records = [ + r for r in caplog.records if json.loads(r.message).get("category") == "summary" + ] assert result.exit_code == 0 - assert result.stdout.startswith("🦊") + assert len(summary_log_records) == 2 diff --git a/tests/test_reconciliation.py b/tests/test_reconciliation.py new file mode 100644 index 00000000..d59d176b --- /dev/null +++ b/tests/test_reconciliation.py @@ -0,0 +1,308 @@ +from pathlib import Path + +import pytest +import tenacity + +from foxops.engine import IncarnationState +from foxops.engine.models import TemplateConfig +from foxops.errors import RetryableError +from foxops.external.gitlab import AsyncGitlabClient, GitlabNotFoundException +from foxops.models import ( + DesiredIncarnationStateConfig, + IncarnationRemoteGitRepositoryState, + yaml, +) +from foxops.reconciliation import ( + ReconciliationState, + get_actual_incarnation_state, + reconcile, + reconcile_project, +) + + +@pytest.fixture() +def gitlab_client_mock(mocker): + gitlab_client = mocker.AsyncMock( + name="GitLab Client AsyncMock", spec=AsyncGitlabClient + ) + gitlab_client.token = None + return gitlab_client + + +@pytest.mark.asyncio +async def test_project_state_if_gitlab_project_does_not_exist( + mocker, gitlab_client_mock +): + # GIVEN + gitlab_client_mock.project_get.side_effect = GitlabNotFoundException(message="any") + + # WHEN + actual_project_state = await get_actual_incarnation_state( + gitlab_client_mock, mocker.Mock() + ) + + # THEN + assert actual_project_state is None + + +@pytest.mark.asyncio +async def test_project_state_if_gitlab_project_exists_but_no_template( + mocker, gitlab_client_mock +): + # GIVEN + gitlab_client_mock.project_get.return_value = { + "id": 1, + "default_branch": "main", + "http_url_to_repo": "some.url", + } + gitlab_client_mock.project_repository_files_get_content.side_effect = ( + GitlabNotFoundException(message="any") + ) + + # WHEN + actual_project_state = await get_actual_incarnation_state( + gitlab_client_mock, mocker.Mock(target_directory=Path("dir")) + ) + + # THEN + assert actual_project_state.gitlab_project_id == 1 + assert actual_project_state.remote_url == "some.url" + assert actual_project_state.default_branch == "main" + assert actual_project_state.incarnation_directory == Path("dir") + + +@pytest.mark.asyncio +async def test_project_state_with_content_template_if_template_initialized( + mocker, gitlab_client_mock +): + # GIVEN + gitlab_client_mock.project_get.return_value = { + "id": 1, + "default_branch": "main", + "http_url_to_repo": "some.url", + } + gitlab_client_mock.project_repository_files_get_content.return_value = b""" +template_data: + author: John Doe + three: '3' +template_repository: any-repository-url +template_repository_version: any-version +template_repository_version_hash: any-version-hash +""" + + # WHEN + actual_project_state = await get_actual_incarnation_state( + gitlab_client_mock, mocker.Mock(target_directory=Path("dir")) + ) + + # THEN + assert ( + actual_project_state.incarnation_state.template_repository + == "any-repository-url" + ) + assert ( + actual_project_state.incarnation_state.template_repository_version + == "any-version" + ) + assert actual_project_state.incarnation_state.template_data == { + "author": "John Doe", + "three": "3", + } + + +@pytest.mark.asyncio +async def test_reconcile_project_should_fail_if_project_does_not_exist( + mocker, gitlab_client_mock +): + # GIVEN + desired_incarnation_state_config = DesiredIncarnationStateConfig( + gitlab_project="any-project", + template_repository="any-repository", + template_repository_version="any-version", + template_data={}, + ) + mocker.patch( + "foxops.reconciliation.get_actual_incarnation_state", + return_value=None, + ) + + # WHEN + reconciliation_state = await reconcile_project( + gitlab_client_mock, desired_incarnation_state_config + ) + + # THEN + assert reconciliation_state is ReconciliationState.FAILED + + +@pytest.mark.asyncio +async def test_reconciliation_should_warn_that_templates_cannot_be_changed( + mocker, + gitlab_client_mock, +): + # GIVEN + actual_incarnation_state = IncarnationRemoteGitRepositoryState( + gitlab_project_id=1, + remote_url="any-url", + default_branch="any-branch", + incarnation_directory=Path("dir"), + incarnation_state=IncarnationState( + template_repository="any-repository", + template_repository_version="any-version", + template_repository_version_hash="any-version-hash", + template_data={}, + ), + ) + desired_incarnation_state = DesiredIncarnationStateConfig( + gitlab_project="any-project", + target_directory=Path("dir"), + template_repository="ANOTHER-repository", + template_repository_version="any-version", + template_data={}, + ) + mocker.patch( + "foxops.reconciliation.get_actual_incarnation_state", + return_value=actual_incarnation_state, + ) + mocker.patch("foxops.reconciliation.TemporaryGitRepository") + + # WHEN + reconciliation_state = await reconcile_project( + gitlab_client_mock, desired_incarnation_state + ) + + # THEN + assert reconciliation_state is ReconciliationState.UNSUPPORTED + + +@pytest.mark.asyncio +async def test_reconcile_projects_does_not_abort_if_single_project_fails( + mocker, gitlab_client_mock +): + # GIVEN + reconcile_project_mock = mocker.patch("foxops.reconciliation.reconcile_project") + desired_incarnation_state_configs = [ + DesiredIncarnationStateConfig( + gitlab_project="first-project", + target_directory=Path("dir"), + template_repository="any-repository", + template_repository_version="any-version", + template_data={}, + ), + DesiredIncarnationStateConfig( + gitlab_project="second-project", + target_directory=Path("dir"), + template_repository="any-repository", + template_repository_version="any-version", + template_data={}, + ), + DesiredIncarnationStateConfig( + gitlab_project="third-project", + target_directory=Path("dir"), + template_repository="any-repository", + template_repository_version="any-version", + template_data={}, + ), + ] + + reconcile_project_mock.side_effect = [ + Exception("boohoo"), + ReconciliationState.CHANGED, + ReconciliationState.UNCHANGED, + ] + + # WHEN + reconciliation_states = await reconcile( + gitlab_client_mock, + desired_incarnation_state_configs, + parallelism=1, + ) + + # THEN + assert reconciliation_states == [ + ReconciliationState.FAILED, + ReconciliationState.CHANGED, + ReconciliationState.UNCHANGED, + ] + + +@pytest.mark.asyncio +async def test_reconcile_project_should_not_require_change_when_default_variable_is_used( + mocker, gitlab_client_mock +): + """ + Verify that no change is required when reconciliation is performed without specifying + an optional variable which is recorded in the actual state with the default value. + """ + # GIVEN + desired_incarnation_state_config = DesiredIncarnationStateConfig( + gitlab_project="any-project", + template_repository="any-repository", + template_repository_version="any-version", + template_data={}, + ) + actual_incarnation_state_config = IncarnationRemoteGitRepositoryState( + gitlab_project_id=1, + remote_url="any-url", + default_branch="any-branch", + incarnation_directory=Path("dir"), + incarnation_state=IncarnationState( + template_repository="any-repository", + template_repository_version="any-version", + template_repository_version_hash="any-version-hash", + template_data={"optional": "default"}, + ), + ) + template_config = """ +optional: + type: str + description: "Optional variable" + default: "default" + """ + mocker.patch( + "foxops.reconciliation.get_actual_incarnation_state", + return_value=actual_incarnation_state_config, + ) + mocker.patch( + "foxops.engine.update.load_template_config", + return_value=TemplateConfig(variables=yaml.load(template_config)), + ) + git_repository_mock = mocker.patch("foxops.reconciliation.TemporaryGitRepository") + git_repository_mock.return_value.__aenter__.return_value.head.return_value = ( + "any-version-hash" + ) + + # WHEN + reconciliation_state = await reconcile_project( + gitlab_client_mock, desired_incarnation_state_config + ) + + # THEN + assert reconciliation_state is ReconciliationState.UNCHANGED + + +async def test_project_reconciliation_is_retried_for_retryable_errors(mocker): + # GIVEN + uuid_mock = mocker.patch("uuid.uuid4", side_effect=RetryableError) + + # WHEN + with pytest.raises(tenacity.RetryError): + await reconcile_project(mocker.MagicMock(), mocker.MagicMock()) + + # THEN + assert uuid_mock.call_count == 4 + + +async def test_project_reconciliation_is_not_retried_for_non_retryable_errors(mocker): + # GIVEN + class _TestException(Exception): + ... + + uuid_mock = mocker.patch("uuid.uuid4", side_effect=_TestException) + + # WHEN + with pytest.raises(_TestException): + await reconcile_project(mocker.MagicMock(), mocker.MagicMock()) + + # THEN + assert uuid_mock.call_count == 1 diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 00000000..48d7d66b --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,16 @@ +import pytest +from _pytest.monkeypatch import MonkeyPatch + +from foxops.settings import Settings + + +@pytest.mark.filterwarnings('ignore:directory "/var/run/secrets/foxops" does not exist') +def test_settings_can_load_config_from_env(monkeypatch: MonkeyPatch): + # GIVEN + monkeypatch.setenv("FOXOPS_GITLAB_TOKEN", "dummy") + + # WHEN + settings = Settings() + + # THEN + assert settings.gitlab_token.get_secret_value() == "dummy" diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..5acac775 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,64 @@ +import asyncio +from subprocess import CalledProcessError + +import pytest + +from foxops.utils import check_call + + +@pytest.mark.asyncio +async def test_check_call_should_raise_exception_on_non_zero_exit_code(): + # GIVEN + program = "false" + expected_error_msg = r"Command '\['false'\]' returned non-zero exit status 1. with stdout 'b''' and stderr 'b'''" + + # THEN + with pytest.raises(CalledProcessError, match=expected_error_msg): + await check_call(program) + + +@pytest.mark.asyncio +async def test_check_call_should_raise_exception_on_non_zero_exit_code_with_stderr( + tmp_path, +): + # GIVEN + program = "git" + args = ["status"] + expected_error_msg = r"not a git repository" + + # THEN + with pytest.raises(CalledProcessError, match=expected_error_msg): + await check_call(program, *args, cwd=tmp_path) + + +@pytest.mark.asyncio +async def test_check_call_should_pass_for_zero_exit_code(): + # GIVEN + program = "true" + + # WHEN & THEN + await check_call(program) + + +@pytest.mark.asyncio +async def test_check_call_should_forward_args(): + # GIVEN + program = "echo" + args = ["Hello", "World"] + + # WHEN + proc = await check_call(program, *args) + + # THEN + assert await proc.stdout.read() == b"Hello World\n" + + +@pytest.mark.asyncio +async def test_check_call_should_kill_process_when_timeout_is_exceeded(): + # GIVEN + program = "sleep" + args = ["5"] + + # WHEN & THEN + with pytest.raises(asyncio.TimeoutError): + await check_call(program, *args, timeout=0.5)