diff --git a/.github/workflows/tests-api-e2e.yml b/.github/workflows/tests-api-e2e.yml index 285aa8a51a..9c45ee290a 100644 --- a/.github/workflows/tests-api-e2e.yml +++ b/.github/workflows/tests-api-e2e.yml @@ -10,10 +10,14 @@ on: jobs: + build_and_push: + uses: vizzuality/marxan-cloud/.github/workflows/tests-backend-build-oci-images.yml@v1 + test-api-e2e: name: E2E tests - API runs-on: ubuntu-22.04 timeout-minutes: 60 + needs: build_and_push strategy: fail-fast: false matrix: @@ -49,5 +53,14 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 + - name: Render Compose file with image tags + env: + API_IMAGE_TAG: ${{ needs.build_and_push.build_and_push_api.outputs.image_tag }} + GEOPROCESSING_IMAGE_TAG: ${{ needs.build_and_push.build_and_push_geoprocessing.outputs.image_tag }} + APIDB_IMAGE_TAG: ${{ needs.build_and_push.build_and_push_apidb.outputs.image_tag }} + GEODB_IMAGE_TAG: ${{ needs.build_and_push.build_and_push_geodb.outputs.image_tag }} + REDIS_IMAGE_TAG: ${{ needs.build_and_push.build_and_push_redis.outputs.image_tag }} + run: | + envsubst < docker-compose-test-e2e.ci.yml.tpl > docker-compose-test-e2e.ci.yml - name: Run CI tests via make task run: TEST_SUITE_PATH="${{ matrix.test-suite }}" make test-e2e-api environment=ci diff --git a/.github/workflows/tests-backend-build-oci-images.yml b/.github/workflows/tests-backend-build-oci-images.yml new file mode 100644 index 0000000000..26a7b634d4 --- /dev/null +++ b/.github/workflows/tests-backend-build-oci-images.yml @@ -0,0 +1,276 @@ +name: Build and publish container images for test jobs to GitHub Container Registry + +on: + push: + branches: + - 'main' + - 'develop' + - 'staging' + - 'production' + paths: + - 'api/**' + - 'postgresql/**' + - 'webshot/**' + - '.github/**' + pull_request: + workflow_dispatch: + workflow_call: + outputs: + api-service-tag: + description: OCI image tag for the marxan-api image + value: {{ jobs.build_and_push_api.outputs.image_tag }} + +env: + REGISTRY: ghcr.io + +jobs: + + build_and_push_api: + name: Build and push API service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} + env: + REPOSITORY: marxan-api + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + id: build_and_push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./api + push: true + file: api/api.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} + + build_and_push_geoprocessing: + name: Build and push Geoprocessing service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} + env: + REPOSITORY: marxan-geoprocessing + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + id: build_and_push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./api + push: true + file: api/geo.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} + + build_and_push_apidb: + name: Build and push apidb service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} + env: + REPOSITORY: marxan-postgresql-api + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + id: build_and_push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./postgresql + push: true + file: postgresql/apidb.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} + + build_and_push_geodb: + name: Build and push geodb service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} + env: + REPOSITORY: marxan-postgresql-geo-api + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + id: build_and_push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./postgresql + push: true + file: postgresql/geodb.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} + + build_and_push_redis: + name: Build and push Redis service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} + env: + REPOSITORY: marxan-redis + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + id: build_and_push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./redis + push: true + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} + + build_and_push_webshot: + name: Build and push Webshot service container image to registry + runs-on: ubuntu-22.04 + timeout-minutes: 10 + outputs: + image-tag: ${{ steps.build_and_push.outputs.imageid }} + env: + REPOSITORY: marxan-webshot + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Login to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Prepare ids + shell: bash + run: | + echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "registry_namespace=$(echo ${{ github.repository_owner }} | tr A-Z a-z)" >> $GITHUB_OUTPUT + id: extract_branch + + - name: Build and push + id: build_and_push + uses: docker/build-push-action@v4 + env: + IMAGE_TAG: ${{ steps.extract_branch.outputs.shortsha }} + REGISTRY_NAMESPACE: ${{ steps.extract_branch.outputs.registry_namespace }} + with: + context: ./webshot + push: true + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }} diff --git a/Makefile b/Makefile index 13d0f49cec..a044233c94 100644 --- a/Makefile +++ b/Makefile @@ -12,12 +12,11 @@ endif include lib/make/compose-detection.mk -CIENV := $(if $(filter $(environment), local), -f docker-compose-test-e2e.yml -f docker-compose-test-e2e.local.yml, -f docker-compose-test-e2e.yml) +DOCKER_COMPOSE_FILE := $(if $(filter $(environment), local), -f docker-compose-test-e2e.yml -f docker-compose-test-e2e.local.yml, -f docker-compose-test-e2e.yml -f docker-compose-test-e2e.ci.yml) API_DB_INSTANCE := $(if $(environment), test-e2e-postgresql-api, postgresql-api) GEO_DB_INSTANCE := $(if $(environment), test-e2e-postgresql-geo-api, postgresql-geo-api) REDIS_INSTANCE := $(if $(environment), test-e2e-redis, redis) -DOCKER_COMPOSE_FILE := $(CIENV) DOCKER_CLEAN_VOLUMES := $(if $(environment), , \ docker volume rm -f marxan-cloud_marxan-cloud-postgresql-api-data && \ docker volume rm -f marxan-cloud_marxan-cloud-postgresql-geo-data && \ @@ -34,7 +33,6 @@ NC :=\033[0m # No Color test-commands: @echo $(ENVFILE) @echo $(DOCKER_COMPOSE_FILE) - @echo $(CIENV) @echo $(API_POSTGRES_DB) @echo $(GEO_POSTGRES_USER) diff --git a/docker-compose-test-e2e.ci.yml.tpl b/docker-compose-test-e2e.ci.yml.tpl new file mode 100644 index 0000000000..15bbb0b448 --- /dev/null +++ b/docker-compose-test-e2e.ci.yml.tpl @@ -0,0 +1,16 @@ +version: "3.8" +services: + api: + image: ${API_IMAGE_TAG} + + geoprocessing: + image: ${GEOPROCESSING_IMAGE_TAG} + + test-e2e-postgresql-api: + image: ${APIDB_IMAGE_TAG} + + test-e2e-postgresql-geo-api: + image: ${GEODB_IMAGE_TAG} + + test-e2e-redis: + image: ${REDIS_IMAGE_TAG} diff --git a/docker-compose-test-e2e.local.yml b/docker-compose-test-e2e.local.yml index f378807ded..9e330c981b 100644 --- a/docker-compose-test-e2e.local.yml +++ b/docker-compose-test-e2e.local.yml @@ -1,16 +1,31 @@ version: "3.8" services: api: + build: + context: ./api + dockerfile: api.Dockerfile env_file: .env-test-e2e geoprocessing: + build: + context: ./api + dockerfile: geo.Dockerfile env_file: .env-test-e2e test-e2e-postgresql-api: + build: + context: ./postgresql + dockerfile: apidb.Dockerfile env_file: .env-test-e2e test-e2e-postgresql-geo-api: + build: + context: ./postgresql + dockerfile: geodb.Dockerfile env_file: .env-test-e2e test-e2e-redis: + build: + context: ./redis + dockerfile: Dockerfile env_file: .env-test-e2e diff --git a/docker-compose-test-e2e.yml b/docker-compose-test-e2e.yml index 9b049e38a6..39be1d1005 100644 --- a/docker-compose-test-e2e.yml +++ b/docker-compose-test-e2e.yml @@ -1,9 +1,8 @@ version: "3.8" services: api: - build: - context: ./api - dockerfile: api.Dockerfile + # no build section: when running in CI, we use `image` and when running + # locally we define the build in docker-compose-test-e2e.local.yml ports: - "3000:3000" container_name: marxan-test-e2e-api @@ -33,9 +32,8 @@ services: - test-e2e-redis geoprocessing: - build: - context: ./api - dockerfile: geo.Dockerfile + # no build section: when running in CI, we use `image` and when running + # locally we define the build in docker-compose-test-e2e.local.yml ports: - "3001:3000" container_name: marxan-test-e2e-geoprocessing @@ -66,9 +64,8 @@ services: - api test-e2e-postgresql-api: - build: - context: ./postgresql - dockerfile: apidb.Dockerfile + # no build section: when running in CI, we use `image` and when running + # locally we define the build in docker-compose-test-e2e.local.yml container_name: marxan-test-e2e-postgresql-api volumes: - "./api/test/fixtures:/opt" @@ -81,9 +78,8 @@ services: restart: on-failure test-e2e-postgresql-geo-api: - build: - context: ./postgresql - dockerfile: geodb.Dockerfile + # no build section: when running in CI, we use `image` and when running + # locally we define the build in docker-compose-test-e2e.local.yml container_name: marxan-test-e2e-postgresql-geo-api ports: - "3541:5432" @@ -94,9 +90,8 @@ services: restart: on-failure test-e2e-redis: - build: - context: ./redis - dockerfile: Dockerfile + # no build section: when running in CI, we use `image` and when running + # locally we define the build in docker-compose-test-e2e.local.yml container_name: marxan-test-e2e-redis command: [ "redis-server" ] ports: