Improve GitLab support

project/cookiecutter.json

@@ -16,6 +16,7 @@
   "devops_cache": ["1", "0"],
   "devops_ansible": ["1", "0"],
   "devops_gha_deploy": ["1", "0"],
+  "devops_gitlab_deploy": ["1", "0"],
   "__feature_headless": "1",
   "__npm_package_name": "{{ cookiecutter.frontend_addon_name }}",
   "__folder_name": "{{ cookiecutter.project_slug }}",
@@ -99,6 +100,11 @@
       "__prompt__": "Add GitHub Action to Deploy this project?",
       "1": "Yes",
       "0": "No"
+    },
+    "devops_gitlab_deploy": {
+      "__prompt__": "Add GitLab CI/CD to deploy this project?",
+      "1": "Yes",
+      "0": "No"
     }
   },
   "_copy_without_render": [

project/hooks/post_gen_project.py

@@ -36,6 +36,10 @@
         "devops/.env_gha",
         "devops/README-GHA.md",
     ],
+    "gitlab": [
+        ".gitlab-ci.yml",
+        "devops/README-GITLAB.md",
+    ],
 }
 
 
@@ -45,10 +49,15 @@ def handle_devops_ansible(context: OrderedDict, output_dir: Path):
 
 
 def handle_devops_gha_deploy(context: OrderedDict, output_dir: Path):
-    """Clean up ansible."""
+    """Clean up GitHub Actions deploy files but not docker image generation files."""
     files.remove_files(output_dir, DEVOPS_TO_REMOVE["gha"])
 
 
+def handle_remove_devops_gitlab(context: OrderedDict, output_dir: Path):
+    """clean up GitLab deployment files"""
+    files.remove_files(output_dir, DEVOPS_TO_REMOVE["gitlab"])
+
+
 def handle_git_initialization(context: OrderedDict, output_dir: Path):
     """Initialize a GIT repository for the project codebase."""
     git.initialize_repository(output_dir)
@@ -86,6 +95,7 @@ def generate_sub_project_settings(context: OrderedDict, output_dir: Path):
     # Use the same base folder
     folder_name = output_dir.name
     output_dir = output_dir.parent
+
     generator.generate_subtemplate(
         "sub/project_settings", output_dir, folder_name, context
     )
@@ -115,6 +125,13 @@ def main():
                 context.get("devops_gha_deploy")
             ),  # {{ cookiecutter.devops_gha_deploy }}
         ],
+        [
+            handle_remove_devops_gitlab,
+            "Remove GitLab CI/CD deployment files",
+            not int(
+                context.get("devops_gitlab_deploy")
+            ),  # {{ cookiecutter.devops_gha_deploy }}
+        ],
         [
             handle_git_initialization,
             "Initialize Git repository",

project/{{ cookiecutter.__folder_name }}/.gitlab-ci.yml

@@ -0,0 +1,173 @@
+default:
+  image: docker:24.0.5
+  services:
+    - docker:24.0.5-dind
+  before_script:
+    - apk add --no-cache make bash python3 ncurses openssh-client-common
+    - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY -u $CI_REGISTRY_USER --password-stdin
+
+stages:
+  - check
+  - test
+  - build
+  - deploy
+
+variables:
+  # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled
+  # DOCKER_HOST: tcp://docker:2376
+  DOCKER_TLS_CERTDIR: "/certs"
+  CONTAINER_BACKEND_RELEASE_IMAGE: $CI_REGISTRY_IMAGE/backend
+  CONTAINER_FRONTEND_RELEASE_IMAGE: $CI_REGISTRY_IMAGE/frontend
+  {%- if cookiecutter.devops_cache == '1' %}
+  CONTAINER_VARNISH_RELEASE_IMAGE: $CI_REGISTRY_IMAGE/varnish
+  {%- endif %}
+
+check-backend:
+  stage: check
+  before_script:
+    - apk add --no-cache make bash python3 ncurses openssh-client-common git
+  script:
+    - cd backend
+    - make check
+
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+
+check-frontend:
+  image: node:20-alpine
+  stage: check
+  before_script:
+    - apk add --no-cache make bash python3 ncurses openssh-client-common git
+    - corepack enable
+  script:
+    - cd frontend
+    - make install
+    - make lint
+    - make ci-i18n
+
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+
+
+test-backend:
+  stage: test
+  image: python:3.11-bookworm
+  before_script:
+    - apt update
+    - apt install build-essential python3 python3-dev git -y
+  script:
+    - cd backend
+    - make install
+    - make test
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+
+test-frontend:
+  image: node:20-alpine
+  stage: test
+  before_script:
+    - apk add --no-cache make bash python3 ncurses openssh-client-common git
+    - corepack enable
+
+  script:
+    - cd frontend
+    - make install
+    - make ci-test
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+
+build-backend:
+  stage: build
+  before_script:
+    - apk add --no-cache make bash python3 ncurses openssh-client-common
+  script:
+    - cd backend
+    - make build-image
+    - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY -u $CI_REGISTRY_USER --password-stdin
+    - docker push $CONTAINER_BACKEND_RELEASE_IMAGE:latest
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+
+build-frontend:
+  stage: build
+  before_script:
+    - apk add --no-cache make bash python3 ncurses openssh-client-common
+
+  script:
+    - cd frontend
+    - make build-image
+    - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY -u $CI_REGISTRY_USER --password-stdin
+    - docker push $CONTAINER_FRONTEND_RELEASE_IMAGE:latest
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+
+{%- if cookiecutter.devops_cache == '1' %}
+build-varnish:
+  stage: build
+  before_script:
+    - apk add --no-cache make bash python3 ncurses openssh-client-common
+  script:
+    - cd devops/varnish
+    - docker build . -t $CONTAINER_VARNISH_RELEASE_IMAGE:latest
+    - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY -u $CI_REGISTRY_USER --password-stdin
+    - docker push $CONTAINER_VARNISH_RELEASE_IMAGE:latest
+
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+{%- endif %}
+
+deploy:
+  stage: deploy
+  rules:
+    - if: $CI_COMMIT_REF_NAME == "main"
+  script:
+    - mkdir -p ~/.ssh/
+    - touch ~/.ssh/known_hosts
+    - echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+    - cd devops
+    - eval `ssh-agent -s`
+    - echo "${DEPLOY_SSH_PRIVATE_KEY}" | tr -d '\r' | ssh-add - > /dev/null # add ssh key
+    - export TERM=dumb
+    - touch .env
+    - 'echo "DEPLOY_ENV: ${DEPLOY_ENV}" >> .env'
+    - 'echo "DEPLOY_HOST: ${DEPLOY_HOST}" >> .env'
+    - 'echo "DEPLOY_PORT: ${DEPLOY_PORT}" >> .env'
+    - 'echo "DEPLOY_USER: ${DEPLOY_USER}" >> .env'
+    - 'echo "DOCKER_CONFIG: ${DOCKER_CONFIG}" >> .env'
+    - 'echo "STACK_NAME: ${STACK_NAME}" >> .env'
+    - 'echo "STACK_PARAM: ${CI_COMMIT_TAG}" >> .env'
+    - make docker-setup
+    - make stack-deploy
+  environment: production
+# deploy-with-auxiliary-docker-image:
+#   stage: deploy
+#   variables:
+#     REGISTRY: ${CI_REGISTRY}
+#     USERNAME: ${CI_REGISTRY_USER}
+#     PASSWORD: ${CI_REGISTRY_PASSWORD}
+#     REMOTE_HOST: ${DEPLOY_HOST}
+#     REMOTE_PORT: ${DEPLOY_PORT}
+#     REMOTE_USER: ${DEPLOY_USER}
+#     REMOTE_PRIVATE_KEY: "${DEPLOY_SSH_PRIVATE_KEY}"
+#     STACK_FILE: devops/stacks/${DEPLOY_HOST}.yml
+#     STACK_NAME: ${STACK_NAME}
+#     DEPLOY_IMAGE: ghcr.io/kitconcept/docker-stack-deploy:latest
+#   script:
+#     - docker pull ${DEPLOY_IMAGE}
+#     - docker run --rm
+#       -v "$(pwd)":/github/workspace
+#       -v /var/run/docker.sock:/var/run/docker.sock
+#       -e REGISTRY=${REGISTRY}
+#       -e USERNAME=${USERNAME}
+#       -e PASSWORD=${PASSWORD}
+#       -e REMOTE_HOST=${REMOTE_HOST}
+#       -e REMOTE_PORT=${REMOTE_PORT}
+#       -e REMOTE_USER=${REMOTE_USER}
+#       -e REMOTE_PRIVATE_KEY="${REMOTE_PRIVATE_KEY}"
+#       -e STACK_FILE=${STACK_FILE}
+#       -e STACK_NAME=${STACK_NAME}
+#       ${DEPLOY_IMAGE}
+#   only:
+#     - main
+#   environment: production

project/{{ cookiecutter.__folder_name }}/devops/README-GITLAB.md

@@ -4,6 +4,51 @@ Welcome to the DevOps pipelines guide for deploying your project using GitLab pi
 This README provides step-by-step instructions to set up your GitLab repository and initiate manual deployment pipelines.
 Follow each step carefully to correctly configure your environment and secrets.
 
+## Runner setup 🛠️
+
+If you use your own runners, you will need to configure your runners to do "Docker-in-Docker". You can read about it in the [GitLab Documentation](https://docs.gitlab.com/ee/ci/docker/using_docker_build.html).
+
+A GitLab runner configuration like this works out of the box. Perhaps this is not the most secure setup, but it works.
+
+[Register a runner](https://docs.gitlab.com/runner/register/) in one of your servers.
+Then modify its configuration file `/etc/gitlab-runner/config.toml` as needed.
+
+```toml
+concurrent = 1
+check_interval = 0
+connection_max_age = "15m0s"
+shutdown_timeout = 0
+
+[session_server]
+  session_timeout = 1800
+
+[[runners]]
+  name = "runner1"
+  url = "https://gitlab.com"
+  id = <REDACTED>
+  token = "<REDACTED>"
+  token_obtained_at = 2024-09-17T08:51:36Z
+  token_expires_at = 0001-01-01T00:00:00Z
+  executor = "docker"
+  [runners.custom_build_dir]
+  [runners.cache]
+    MaxUploadedArchiveSize = 0
+    [runners.cache.s3]
+    [runners.cache.gcs]
+    [runners.cache.azure]
+  [runners.docker]
+    tls_verify = false
+    image = "docker:24.0.5"
+    privileged = true
+    disable_entrypoint_overwrite = false
+    oom_kill_disable = false
+    disable_cache = false
+    volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/certs/client", "/cache"]
+    shm_size = 0
+    network_mtu = 0
+
+```
+
 ## Repository setup 🛠️
 
 See [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/).
@@ -17,6 +62,8 @@ See [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/).
 4.  Expand the `Variables`.
 5.  Add all variables and their corresponding values, copying them from the `.env` file.
     This file is not commited into the repository, and that's why we need to add them here.
+    > [!NOTE]
+    > Do not mark any of the variables as *protected*.
 
 ### Step 2: Add deployment's host SSH key as a known host
 
@@ -39,16 +86,14 @@ See [Use SSH keys to communicate with GitLab](https://docs.gitlab.com/ee/user/ss
 Check the section "Server Setup" in the file `README.md` in the same folder as this file for details.
 Be aware that the key generated by this template will not be added to your git repository because it is explicitly excluded by the `.gitignore` file.
 
-## Automatic deployment 🚀
+## Development and deployment 🚀
 
-The deployment is executed automatically on each commit to `main` branch.
+Development should be done in a separate branch, such as `develop`.
 
-You may want to adapt your development workflow to that, and use a `develop` branch to develop and create merge requests for deployments.
+When opening a merge request from your development branch to `main`, CI/CD runs the checks and tests, and, if they pass, then builds the Docker images.
 
-## Manual deployment 🚀
+When merging the merge request, the project will be deployed.
 
-If you want to do manual deployments, you can enable the `when: manual` option of the `.gitlab-ci.yml` file.
-By default it is commented.
 
 ## Deployment with an auxiliary docker image
 

project/{{ cookiecutter.__folder_name }}/devops/stacks/{{ cookiecutter.hostname }}.yml

@@ -99,7 +99,11 @@ services:
         order: start-first
 
   varnish:
+  {%- if cookiecutter.devops_gitlab_deploy %}
+    image: {{ cookiecutter.__container_image_prefix }}/varnish:${STACK_PARAM:-latest}
+  {%- else %}
     image: {{ cookiecutter.__container_image_prefix }}-varnish:${STACK_PARAM:-latest}
+  {%- endif %}
     command:
       - '-p'
       - 'nuke_limit=2000'
@@ -135,7 +139,11 @@ services:
 {%- endif %}
 
   frontend:
+  {%- if cookiecutter.devops_gitlab_deploy %}
+    image: {{ cookiecutter.__container_image_prefix }}/frontend:${STACK_PARAM:-latest}
+  {%- else %}
     image: {{ cookiecutter.__container_image_prefix }}-frontend:${STACK_PARAM:-latest}
+  {%- endif %}
     environment:
       RAZZLE_INTERNAL_API_PATH: http://backend:8080/Plone
       RAZZLE_API_PATH: https://{{ cookiecutter.hostname }}
@@ -172,7 +180,11 @@ services:
 {%- endif %}
 
   backend:
+  {%- if cookiecutter.devops_gitlab_deploy %}
+    image: {{ cookiecutter.__container_image_prefix }}/backend:${STACK_PARAM:-latest}
+  {% else %}
     image: {{ cookiecutter.__container_image_prefix }}-backend:${STACK_PARAM:-latest}
+  {% endif %}
     environment:
       RELSTORAGE_DSN: "dbname='${DB_NAME:-plone}' user='${DB_USER:-plone}' host='${DB_HOST:-db}' password='${DB_PASSWORD:-{{ cookiecutter.__devops_db_password }}}' port='${DB_PORT:-5432}'"
     depends_on:

sub/project_settings/cookiecutter.json

@@ -9,7 +9,12 @@
   "language_code": "en",
   "plone_version": "{{ 'No' | latest_plone }}",
   "github_organization": "collective",
-  "container_registry": ["github", "docker_hub", "gitlab"],
+  "container_registry": [
+    "github",
+    "docker_hub",
+    "gitlab"
+  ],
+  "devops_gitlab_deploy": "0",
   "__version_package": "1.0.0a0",
   "__npm_package_name": "{{ cookiecutter.frontend_addon_name }}",
   "__container_registry_prefix": "{{ cookiecutter.container_registry | image_prefix }}",
@@ -36,4 +41,4 @@
   ],
   "__cookieplone_repository_path": "",
   "__cookieplone_template": ""
-}
+}