From 84c73b6da9c2ca3feb12791662d7b978d212a5e0 Mon Sep 17 00:00:00 2001 From: andrea rota Date: Fri, 12 May 2023 12:38:40 +0100 Subject: [PATCH 1/3] build and push to GHCR the OCI images for test jobs --- .../tests-backend-build-oci-images.yml | 253 ++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 .github/workflows/tests-backend-build-oci-images.yml diff --git a/.github/workflows/tests-backend-build-oci-images.yml b/.github/workflows/tests-backend-build-oci-images.yml new file mode 100644 index 0000000000..461d14efde --- /dev/null +++ b/.github/workflows/tests-backend-build-oci-images.yml @@ -0,0 +1,253 @@ +name: Build and publish container images for test jobs to GitHub Container Registry + +on: + push: + branches: + - 'main' + - 'develop' + - 'staging' + - 'production' + paths: + - 'api/**' + - 'postgresql/**' + - 'webshot/**' + - '.github/**' + pull_request: + workflow_dispatch: + +env: + REGISTRY: ghcr.io + +jobs: + + build_and_push_api: + name: Build and push API service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + REPOSITORY: marxan-api + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./api + push: true + file: api/api.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} + + build_and_push_geoprocesing: + name: Build and push Geoprocessing service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + REPOSITORY: marxan-geoprocessing + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./api + push: true + file: api/geo.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} + + build_and_push_apidb: + name: Build and push apidb service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + REPOSITORY: marxan-postgresql-api + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./postgresql + push: true + file: postgresql/apidb.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} + + build_and_push_geodb: + name: Build and push geodb service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + REPOSITORY: marxan-postgresql-geo-api + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./postgresql + push: true + file: postgresql/geodb.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} + + build_and_push_redis: + name: Build and push Redis service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + REPOSITORY: marxan-redis + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./redis + push: true + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} + + build_and_push_webshot: + name: Build and push Webshot service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + REPOSITORY: marxan-webshot + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./webshot + push: true + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} From 69d00f125c51b1eb72b55c2ca6f63d69c984908d Mon Sep 17 00:00:00 2001 From: andrea rota Date: Fri, 12 May 2023 18:56:09 +0100 Subject: [PATCH 2/3] link workflows --- .github/workflows/tests-api-e2e.yml | 13 +++++++ .../tests-backend-build-oci-images.yml | 37 +++++++++++++++---- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests-api-e2e.yml b/.github/workflows/tests-api-e2e.yml index 285aa8a51a..9c45ee290a 100644 --- a/.github/workflows/tests-api-e2e.yml +++ b/.github/workflows/tests-api-e2e.yml @@ -10,10 +10,14 @@ on: jobs: + build_and_push: + uses: vizzuality/marxan-cloud/.github/workflows/tests-backend-build-oci-images.yml@v1 + test-api-e2e: name: E2E tests - API runs-on: ubuntu-22.04 timeout-minutes: 60 + needs: build_and_push strategy: fail-fast: false matrix: @@ -49,5 +53,14 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 + - name: Render Compose file with image tags + env: + API_IMAGE_TAG: ${{ needs.build_and_push.build_and_push_api.outputs.image_tag }} + GEOPROCESSING_IMAGE_TAG: ${{ needs.build_and_push.build_and_push_geoprocessing.outputs.image_tag }} + APIDB_IMAGE_TAG: ${{ needs.build_and_push.build_and_push_apidb.outputs.image_tag }} + GEODB_IMAGE_TAG: ${{ needs.build_and_push.build_and_push_geodb.outputs.image_tag }} + REDIS_IMAGE_TAG: ${{ needs.build_and_push.build_and_push_redis.outputs.image_tag }} + run: | + envsubst < docker-compose-test-e2e.ci.yml.tpl > docker-compose-test-e2e.ci.yml - name: Run CI tests via make task run: TEST_SUITE_PATH="${{ matrix.test-suite }}" make test-e2e-api environment=ci diff --git a/.github/workflows/tests-backend-build-oci-images.yml b/.github/workflows/tests-backend-build-oci-images.yml index 461d14efde..26a7b634d4 100644 --- a/.github/workflows/tests-backend-build-oci-images.yml +++ b/.github/workflows/tests-backend-build-oci-images.yml @@ -14,6 +14,11 @@ on: - '.github/**' pull_request: workflow_dispatch: + workflow_call: + outputs: + api-service-tag: + description: OCI image tag for the marxan-api image + value: {{ jobs.build_and_push_api.outputs.image_tag }} env: REGISTRY: ghcr.io @@ -23,7 +28,9 @@ jobs: build_and_push_api: name: Build and push API service container image to registry runs-on: ubuntu-22.04 - timeout-minutes: 30 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} env: REPOSITORY: marxan-api steps: @@ -49,6 +56,7 @@ jobs: id: extract_branch - name: Build and push + id: build_and_push uses: docker/build-push-action@v4 env: IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} @@ -59,10 +67,12 @@ jobs: file: api/api.Dockerfile tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} - build_and_push_geoprocesing: + build_and_push_geoprocessing: name: Build and push Geoprocessing service container image to registry runs-on: ubuntu-22.04 - timeout-minutes: 30 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} env: REPOSITORY: marxan-geoprocessing steps: @@ -88,6 +98,7 @@ jobs: id: extract_branch - name: Build and push + id: build_and_push uses: docker/build-push-action@v4 env: IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} @@ -101,7 +112,9 @@ jobs: build_and_push_apidb: name: Build and push apidb service container image to registry runs-on: ubuntu-22.04 - timeout-minutes: 30 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} env: REPOSITORY: marxan-postgresql-api steps: @@ -127,6 +140,7 @@ jobs: id: extract_branch - name: Build and push + id: build_and_push uses: docker/build-push-action@v4 env: IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} @@ -140,7 +154,9 @@ jobs: build_and_push_geodb: name: Build and push geodb service container image to registry runs-on: ubuntu-22.04 - timeout-minutes: 30 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} env: REPOSITORY: marxan-postgresql-geo-api steps: @@ -166,6 +182,7 @@ jobs: id: extract_branch - name: Build and push + id: build_and_push uses: docker/build-push-action@v4 env: IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} @@ -179,7 +196,9 @@ jobs: build_and_push_redis: name: Build and push Redis service container image to registry runs-on: ubuntu-22.04 - timeout-minutes: 30 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} env: REPOSITORY: marxan-redis steps: @@ -205,6 +224,7 @@ jobs: id: extract_branch - name: Build and push + id: build_and_push uses: docker/build-push-action@v4 env: IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} @@ -217,7 +237,9 @@ jobs: build_and_push_webshot: name: Build and push Webshot service container image to registry runs-on: ubuntu-22.04 - timeout-minutes: 30 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} env: REPOSITORY: marxan-webshot steps: @@ -243,6 +265,7 @@ jobs: id: extract_branch - name: Build and push + id: build_and_push uses: docker/build-push-action@v4 env: IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} From 0912c81d35b1bae4b872e0caf98cf333301c5c98 Mon Sep 17 00:00:00 2001 From: andrea rota Date: Fri, 12 May 2023 18:57:09 +0100 Subject: [PATCH 3/3] render compose file for CI from template --- Makefile | 4 +--- docker-compose-test-e2e.ci.yml.tpl | 16 ++++++++++++++++ docker-compose-test-e2e.local.yml | 15 +++++++++++++++ docker-compose-test-e2e.yml | 25 ++++++++++--------------- 4 files changed, 42 insertions(+), 18 deletions(-) create mode 100644 docker-compose-test-e2e.ci.yml.tpl diff --git a/Makefile b/Makefile index 13d0f49cec..a044233c94 100644 --- a/Makefile +++ b/Makefile @@ -12,12 +12,11 @@ endif include lib/make/compose-detection.mk -CIENV := $(if $(filter $(environment), local), -f docker-compose-test-e2e.yml -f docker-compose-test-e2e.local.yml, -f docker-compose-test-e2e.yml) +DOCKER_COMPOSE_FILE := $(if $(filter $(environment), local), -f docker-compose-test-e2e.yml -f docker-compose-test-e2e.local.yml, -f docker-compose-test-e2e.yml -f docker-compose-test-e2e.ci.yml) API_DB_INSTANCE := $(if $(environment), test-e2e-postgresql-api, postgresql-api) GEO_DB_INSTANCE := $(if $(environment), test-e2e-postgresql-geo-api, postgresql-geo-api) REDIS_INSTANCE := $(if $(environment), test-e2e-redis, redis) -DOCKER_COMPOSE_FILE := $(CIENV) DOCKER_CLEAN_VOLUMES := $(if $(environment), , \ docker volume rm -f marxan-cloud_marxan-cloud-postgresql-api-data && \ docker volume rm -f marxan-cloud_marxan-cloud-postgresql-geo-data && \ @@ -34,7 +33,6 @@ NC :=\033[0m # No Color test-commands: @echo $(ENVFILE) @echo $(DOCKER_COMPOSE_FILE) - @echo $(CIENV) @echo $(API_POSTGRES_DB) @echo $(GEO_POSTGRES_USER) diff --git a/docker-compose-test-e2e.ci.yml.tpl b/docker-compose-test-e2e.ci.yml.tpl new file mode 100644 index 0000000000..15bbb0b448 --- /dev/null +++ b/docker-compose-test-e2e.ci.yml.tpl @@ -0,0 +1,16 @@ +version: "3.8" +services: + api: + image: ${API_IMAGE_TAG} + + geoprocessing: + image: ${GEOPROCESSING_IMAGE_TAG} + + test-e2e-postgresql-api: + image: ${APIDB_IMAGE_TAG} + + test-e2e-postgresql-geo-api: + image: ${GEODB_IMAGE_TAG} + + test-e2e-redis: + image: ${REDIS_IMAGE_TAG} diff --git a/docker-compose-test-e2e.local.yml b/docker-compose-test-e2e.local.yml index f378807ded..9e330c981b 100644 --- a/docker-compose-test-e2e.local.yml +++ b/docker-compose-test-e2e.local.yml @@ -1,16 +1,31 @@ version: "3.8" services: api: + build: + context: ./api + dockerfile: api.Dockerfile env_file: .env-test-e2e geoprocessing: + build: + context: ./api + dockerfile: geo.Dockerfile env_file: .env-test-e2e test-e2e-postgresql-api: + build: + context: ./postgresql + dockerfile: apidb.Dockerfile env_file: .env-test-e2e test-e2e-postgresql-geo-api: + build: + context: ./postgresql + dockerfile: geodb.Dockerfile env_file: .env-test-e2e test-e2e-redis: + build: + context: ./redis + dockerfile: Dockerfile env_file: .env-test-e2e diff --git a/docker-compose-test-e2e.yml b/docker-compose-test-e2e.yml index 9b049e38a6..39be1d1005 100644 --- a/docker-compose-test-e2e.yml +++ b/docker-compose-test-e2e.yml @@ -1,9 +1,8 @@ version: "3.8" services: api: - build: - context: ./api - dockerfile: api.Dockerfile + # no build section: when running in CI, we use `image` and when running + # locally we define the build in docker-compose-test-e2e.local.yml ports: - "3000:3000" container_name: marxan-test-e2e-api @@ -33,9 +32,8 @@ services: - test-e2e-redis geoprocessing: - build: - context: ./api - dockerfile: geo.Dockerfile + # no build section: when running in CI, we use `image` and when running + # locally we define the build in docker-compose-test-e2e.local.yml ports: - "3001:3000" container_name: marxan-test-e2e-geoprocessing @@ -66,9 +64,8 @@ services: - api test-e2e-postgresql-api: - build: - context: ./postgresql - dockerfile: apidb.Dockerfile + # no build section: when running in CI, we use `image` and when running + # locally we define the build in docker-compose-test-e2e.local.yml container_name: marxan-test-e2e-postgresql-api volumes: - "./api/test/fixtures:/opt" @@ -81,9 +78,8 @@ services: restart: on-failure test-e2e-postgresql-geo-api: - build: - context: ./postgresql - dockerfile: geodb.Dockerfile + # no build section: when running in CI, we use `image` and when running + # locally we define the build in docker-compose-test-e2e.local.yml container_name: marxan-test-e2e-postgresql-geo-api ports: - "3541:5432" @@ -94,9 +90,8 @@ services: restart: on-failure test-e2e-redis: - build: - context: ./redis - dockerfile: Dockerfile + # no build section: when running in CI, we use `image` and when running + # locally we define the build in docker-compose-test-e2e.local.yml container_name: marxan-test-e2e-redis command: [ "redis-server" ] ports: