From 79accf4a36f9681fa7f65e8dd4cf7f494ef5bf96 Mon Sep 17 00:00:00 2001 From: Daniel Kelemen Date: Thu, 12 Oct 2023 12:41:43 +0200 Subject: [PATCH 1/5] chore(docker): use native arm runners related to camunda/camunda-bpm-platform#3600 --- .github/workflows/build-and-test-ee.yml | 13 +------ .../workflows/build-test-and-publish-ce.yml | 39 ++----------------- pipeline.sh | 24 ++++++------ release.sh | 15 ++++--- test/docker-compose.yml | 18 +++------ 5 files changed, 29 insertions(+), 80 deletions(-) diff --git a/.github/workflows/build-and-test-ee.yml b/.github/workflows/build-and-test-ee.yml index ae9bf0f0..7c445f98 100644 --- a/.github/workflows/build-and-test-ee.yml +++ b/.github/workflows/build-and-test-ee.yml @@ -12,27 +12,19 @@ on: jobs: build-and-test-ee: if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.PLATFORM }} strategy: matrix: DISTRO: [tomcat, wildfly, run] - PLATFORM: [amd64, arm64] + PLATFORM: [ubuntu-22.04, macos-13] steps: - name: Checkout uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - if: ${{ matrix.PLATFORM != 'amd64' }} - with: - platforms: ${{ matrix.PLATFORM }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - name: Build run: ./pipeline.sh env: DISTRO: ${{ matrix.DISTRO }} EE: true - PLATFORM: ${{ matrix.PLATFORM }} NEXUS_PASS: ${{ secrets.NEXUS_PASS }} NEXUS_USER: ${{ secrets.NEXUS_USER }} VERSION: ${{ github.event.inputs.version }} @@ -43,6 +35,5 @@ jobs: env: DISTRO: ${{ matrix.DISTRO }} EE: true - PLATFORM: ${{ matrix.PLATFORM }} VERSION: ${{ github.event.inputs.version }} SNAPSHOT: ${{ github.event.inputs.snapshot }} diff --git a/.github/workflows/build-test-and-publish-ce.yml b/.github/workflows/build-test-and-publish-ce.yml index 2c15ffea..1e5ae6dd 100644 --- a/.github/workflows/build-test-and-publish-ce.yml +++ b/.github/workflows/build-test-and-publish-ce.yml @@ -2,38 +2,20 @@ name: Build, test, and publish CE on: [pull_request, push, workflow_dispatch] jobs: build-and-test: - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.PLATFORM }} strategy: fail-fast: false matrix: DISTRO: [tomcat, wildfly, run] - PLATFORM: [amd64, arm64] + PLATFORM: [ubuntu-22.04, macos-13] steps: - name: Checkout uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - if: ${{ matrix.PLATFORM != 'amd64' }} - with: - platforms: ${{ matrix.PLATFORM }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Expose GitHub Runtime - uses: actions/github-script@v7 - with: - script: | - Object.keys(process.env).forEach(function (key) { - if (key.startsWith('ACTIONS_')) { - core.info(`${key}=${process.env[key]}`); - core.exportVariable(key, process.env[key]); - } - }); - name: Build run: ./pipeline.sh env: DISTRO: ${{ matrix.DISTRO }} EE: false - PLATFORM: ${{ matrix.PLATFORM }} NEXUS_PASS: ${{ secrets.NEXUS_PASS }} NEXUS_USER: ${{ secrets.NEXUS_USER }} - name: Test @@ -42,9 +24,8 @@ jobs: env: DISTRO: ${{ matrix.DISTRO }} EE: false - PLATFORM: ${{ matrix.PLATFORM }} publish: - runs-on: ubuntu-20.04 + runs-on: arm needs: build-and-test if: ${{ github.ref == 'refs/heads/next' || startsWith(github.ref, 'refs/heads/7.') }} strategy: @@ -53,22 +34,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64,amd64 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Expose GitHub Runtime - uses: actions/github-script@v7 - with: - script: | - Object.keys(process.env).forEach(function (key) { - if (key.startsWith('ACTIONS_')) { - core.info(`${key}=${process.env[key]}`); - core.exportVariable(key, process.env[key]); - } - }); - name: Publish multi-platform images run: ./release.sh env: diff --git a/pipeline.sh b/pipeline.sh index 23bcad77..cdb9b7de 100755 --- a/pipeline.sh +++ b/pipeline.sh @@ -14,19 +14,17 @@ else VERSION_ARGUMENT="--build-arg VERSION=${VERSION}" fi -IMAGE_NAME=camunda/camunda-bpm-platform:${DISTRO}-${PLATFORM} +IMAGE_NAME=camunda/camunda-bpm-platform:${DISTRO} -docker buildx build . \ - -t "${IMAGE_NAME}" \ - --platform linux/${PLATFORM} \ - --build-arg DISTRO=${DISTRO} \ - --build-arg EE=${EE} \ - --build-arg USER=${NEXUS_USER} \ - --build-arg PASSWORD=${NEXUS_PASS} \ - ${VERSION_ARGUMENT} \ - ${SNAPSHOT_ARGUMENT} \ - --cache-to type=gha,scope="$GITHUB_REF_NAME-$DISTRO-image" \ - --cache-from type=gha,scope="$GITHUB_REF_NAME-$DISTRO-image" \ - --load +echo "Runner architecture: $(uname -m)" + +docker build . \ + -t "${IMAGE_NAME}" \ + --build-arg DISTRO=${DISTRO} \ + --build-arg EE=${EE} \ + --build-arg USER=${NEXUS_USER} \ + --build-arg PASSWORD=${NEXUS_PASS} \ + ${VERSION_ARGUMENT} \ + ${SNAPSHOT_ARGUMENT} docker inspect "${IMAGE_NAME}" | grep "Architecture" -A2 diff --git a/release.sh b/release.sh index 915cb646..740ade91 100755 --- a/release.sh +++ b/release.sh @@ -13,14 +13,13 @@ IMAGE=camunda/camunda-bpm-platform function build_and_push { local tags=("$@") printf -v tag_arguments -- "--tag $IMAGE:%s " "${tags[@]}" - docker buildx build . \ - $tag_arguments \ - --build-arg DISTRO=${DISTRO} \ - --build-arg EE=${EE} \ - --build-arg USER=${NEXUS_USER} \ - --build-arg PASSWORD=${NEXUS_PASS} \ - --cache-from type=gha,scope="$GITHUB_REF_NAME-$DISTRO-image" \ - --platform $PLATFORMS \ + docker buildx build . \ + $tag_arguments \ + --build-arg DISTRO=${DISTRO} \ + --build-arg EE=${EE} \ + --build-arg USER=${NEXUS_USER} \ + --build-arg PASSWORD=${NEXUS_PASS} \ + --platform $PLATFORMS \ --push echo "Tags released:" >> $GITHUB_STEP_SUMMARY diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 91fd8585..9ad33f06 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -2,15 +2,13 @@ version: '2.4' services: camunda: - image: camunda/camunda-bpm-platform:${DISTRO}-${PLATFORM} - platform: linux/${PLATFORM} + image: camunda/camunda-bpm-platform:${DISTRO} ports: - "8080:8080" restart: unless-stopped camunda-mysql: - image: camunda/camunda-bpm-platform:${DISTRO}-${PLATFORM} - platform: linux/${PLATFORM} + image: camunda/camunda-bpm-platform:${DISTRO} environment: - DB_DRIVER=com.mysql.cj.jdbc.Driver - DB_URL=jdbc:mysql://mysql:3306/process-engine?sendFractionalSeconds=false @@ -24,8 +22,7 @@ services: restart: unless-stopped camunda-postgres: - image: camunda/camunda-bpm-platform:${DISTRO}-${PLATFORM} - platform: linux/${PLATFORM} + image: camunda/camunda-bpm-platform:${DISTRO} environment: - DB_DRIVER=org.postgresql.Driver - DB_URL=jdbc:postgresql://postgres:5432/camunda @@ -39,8 +36,7 @@ services: restart: unless-stopped camunda-debug: - image: camunda/camunda-bpm-platform:${DISTRO}-${PLATFORM} - platform: linux/${PLATFORM} + image: camunda/camunda-bpm-platform:${DISTRO} environment: - DEBUG=true ports: @@ -49,8 +45,7 @@ services: restart: unless-stopped camunda-prometheus-jmx: - image: camunda/camunda-bpm-platform:${DISTRO}-${PLATFORM} - platform: linux/${PLATFORM} + image: camunda/camunda-bpm-platform:${DISTRO} environment: - JMX_PROMETHEUS=true ports: @@ -59,8 +54,7 @@ services: restart: unless-stopped camunda-password-file: - image: camunda/camunda-bpm-platform:${DISTRO}-${PLATFORM} - platform: linux/${PLATFORM} + image: camunda/camunda-bpm-platform:${DISTRO} environment: - DB_DRIVER=com.mysql.cj.jdbc.Driver - DB_URL=jdbc:mysql://mysql:3306/process-engine?sendFractionalSeconds=false From 00e7827b76cf4fc46aceb2ae995d975cd87d647a Mon Sep 17 00:00:00 2001 From: Daniel Kelemen Date: Thu, 12 Oct 2023 18:25:37 +0200 Subject: [PATCH 2/5] - use `aws-arm-core-2-default` runner - add logging improvements --- .github/workflows/build-and-test-ee.yml | 2 +- .../workflows/build-test-and-publish-ce.yml | 4 ++-- pipeline.sh | 2 ++ release.sh | 6 ++++-- test/test.sh | 18 ++++++++++++------ 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-test-ee.yml b/.github/workflows/build-and-test-ee.yml index 7c445f98..0a336dd6 100644 --- a/.github/workflows/build-and-test-ee.yml +++ b/.github/workflows/build-and-test-ee.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: DISTRO: [tomcat, wildfly, run] - PLATFORM: [ubuntu-22.04, macos-13] + PLATFORM: [ubuntu-22.04, aws-arm-core-2-default] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/build-test-and-publish-ce.yml b/.github/workflows/build-test-and-publish-ce.yml index 1e5ae6dd..9d5d1d64 100644 --- a/.github/workflows/build-test-and-publish-ce.yml +++ b/.github/workflows/build-test-and-publish-ce.yml @@ -7,7 +7,7 @@ jobs: fail-fast: false matrix: DISTRO: [tomcat, wildfly, run] - PLATFORM: [ubuntu-22.04, macos-13] + PLATFORM: [ubuntu-22.04, aws-arm-core-2-default] steps: - name: Checkout uses: actions/checkout@v4 @@ -25,7 +25,7 @@ jobs: DISTRO: ${{ matrix.DISTRO }} EE: false publish: - runs-on: arm + runs-on: aws-arm-core-2-default needs: build-and-test if: ${{ github.ref == 'refs/heads/next' || startsWith(github.ref, 'refs/heads/7.') }} strategy: diff --git a/pipeline.sh b/pipeline.sh index cdb9b7de..d1ea0973 100755 --- a/pipeline.sh +++ b/pipeline.sh @@ -18,6 +18,7 @@ IMAGE_NAME=camunda/camunda-bpm-platform:${DISTRO} echo "Runner architecture: $(uname -m)" +echo "::group::Docker build" docker build . \ -t "${IMAGE_NAME}" \ --build-arg DISTRO=${DISTRO} \ @@ -26,5 +27,6 @@ docker build . \ --build-arg PASSWORD=${NEXUS_PASS} \ ${VERSION_ARGUMENT} \ ${SNAPSHOT_ARGUMENT} +echo "::endgroup::" docker inspect "${IMAGE_NAME}" | grep "Architecture" -A2 diff --git a/release.sh b/release.sh index 740ade91..d7ced7d0 100755 --- a/release.sh +++ b/release.sh @@ -11,6 +11,7 @@ NEXUS_PASS=${NEXUS_PASS:-} IMAGE=camunda/camunda-bpm-platform function build_and_push { + echo "::group::Docker build push" local tags=("$@") printf -v tag_arguments -- "--tag $IMAGE:%s " "${tags[@]}" docker buildx build . \ @@ -21,9 +22,10 @@ function build_and_push { --build-arg PASSWORD=${NEXUS_PASS} \ --platform $PLATFORMS \ --push + echo "::endgroup::" - echo "Tags released:" >> $GITHUB_STEP_SUMMARY - printf -- "- $IMAGE:%s\n" "${tags[@]}" >> $GITHUB_STEP_SUMMARY + echo "Tags released:" >> $GITHUB_STEP_SUMMARY + printf -- "- $IMAGE:%s\n" "${tags[@]}" >> $GITHUB_STEP_SUMMARY } if [ "${EE}" = "true" ]; then diff --git a/test/test.sh b/test/test.sh index 9bee51a8..5d1ae9dc 100755 --- a/test/test.sh +++ b/test/test.sh @@ -2,13 +2,19 @@ DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +function run_test { + echo "::group::Test '$2'" + "$@" + echo "::endgroup::" +} + cd ${DIR} docker-compose up --force-recreate -d postgres mysql -./test-${DISTRO}.sh camunda -./test-${DISTRO}.sh camunda-mysql -./test-${DISTRO}.sh camunda-postgres -./test-${DISTRO}.sh camunda-password-file -./test-prometheus-jmx-${DISTRO}.sh camunda-prometheus-jmx -./test-debug.sh camunda-debug +run_test ./test-${DISTRO}.sh camunda +run_test ./test-${DISTRO}.sh camunda-mysql +run_test ./test-${DISTRO}.sh camunda-postgres +run_test ./test-${DISTRO}.sh camunda-password-file +run_test ./test-prometheus-jmx-${DISTRO}.sh camunda-prometheus-jmx +run_test ./test-debug.sh camunda-debug docker-compose down -v cd - From 4488b0e3f6de0b840ad66a703d6be3e3bf216bdd Mon Sep 17 00:00:00 2001 From: Daniel Kelemen Date: Mon, 16 Oct 2023 13:27:23 +0200 Subject: [PATCH 3/5] drop me: test publish & buildx --- .github/workflows/build-test-and-publish-ce.yml | 6 +++--- release.sh | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-test-and-publish-ce.yml b/.github/workflows/build-test-and-publish-ce.yml index 9d5d1d64..31cc1d7f 100644 --- a/.github/workflows/build-test-and-publish-ce.yml +++ b/.github/workflows/build-test-and-publish-ce.yml @@ -6,7 +6,7 @@ jobs: strategy: fail-fast: false matrix: - DISTRO: [tomcat, wildfly, run] + DISTRO: [tomcat] PLATFORM: [ubuntu-22.04, aws-arm-core-2-default] steps: - name: Checkout @@ -27,10 +27,10 @@ jobs: publish: runs-on: aws-arm-core-2-default needs: build-and-test - if: ${{ github.ref == 'refs/heads/next' || startsWith(github.ref, 'refs/heads/7.') }} + #if: ${{ github.ref == 'refs/heads/next' || startsWith(github.ref, 'refs/heads/7.') }} strategy: matrix: - DISTRO: [tomcat, wildfly, run] + DISTRO: [tomcat] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/release.sh b/release.sh index d7ced7d0..e5bbbea2 100755 --- a/release.sh +++ b/release.sh @@ -20,8 +20,7 @@ function build_and_push { --build-arg EE=${EE} \ --build-arg USER=${NEXUS_USER} \ --build-arg PASSWORD=${NEXUS_PASS} \ - --platform $PLATFORMS \ - --push + --platform $PLATFORMS echo "::endgroup::" echo "Tags released:" >> $GITHUB_STEP_SUMMARY @@ -39,7 +38,7 @@ if [ $(docker manifest inspect $IMAGE:${DISTRO}-${VERSION} > /dev/null ; echo $? exit 0 fi -docker login -u "${DOCKER_HUB_USERNAME}" -p "${DOCKER_HUB_PASSWORD}" +#docker login -u "${DOCKER_HUB_USERNAME}" -p "${DOCKER_HUB_PASSWORD}" tags=() From d03985939baf2f2f805cb69d81cb3e8749f94bd2 Mon Sep 17 00:00:00 2001 From: Daniel Kelemen Date: Mon, 16 Oct 2023 16:04:21 +0200 Subject: [PATCH 4/5] - add platforms to buildx setup - revert buildx cache --- .../workflows/build-test-and-publish-ce.yml | 18 ++++++++++++++++++ release.sh | 2 ++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/build-test-and-publish-ce.yml b/.github/workflows/build-test-and-publish-ce.yml index 31cc1d7f..1098e6f9 100644 --- a/.github/workflows/build-test-and-publish-ce.yml +++ b/.github/workflows/build-test-and-publish-ce.yml @@ -11,6 +11,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Login to Docker Hub + uses: docker/login-action@v3 + if: ${{ matrix.PLATFORM == 'aws-arm-core-2-default' }} + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} - name: Build run: ./pipeline.sh env: @@ -36,6 +42,18 @@ jobs: uses: actions/checkout@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + platforms: linux/arm64,linux/amd64 + - name: Expose GitHub Runtime + uses: actions/github-script@v6 + with: + script: | + Object.keys(process.env).forEach(function (key) { + if (key.startsWith('ACTIONS_')) { + core.info(`${key}=${process.env[key]}`); + core.exportVariable(key, process.env[key]); + } + }); - name: Publish multi-platform images run: ./release.sh env: diff --git a/release.sh b/release.sh index e5bbbea2..b2a008c8 100755 --- a/release.sh +++ b/release.sh @@ -20,6 +20,8 @@ function build_and_push { --build-arg EE=${EE} \ --build-arg USER=${NEXUS_USER} \ --build-arg PASSWORD=${NEXUS_PASS} \ + --cache-to type=gha,scope="$GITHUB_REF_NAME-$DISTRO-image" \ + --cache-from type=gha,scope="$GITHUB_REF_NAME-$DISTRO-image" \ --platform $PLATFORMS echo "::endgroup::" From ba96a1feca0b4759c91b60f01ec663a5df80c62d Mon Sep 17 00:00:00 2001 From: Daniel Kelemen Date: Mon, 1 Jul 2024 12:51:03 +0200 Subject: [PATCH 5/5] core4 runner, ee login, all distros --- .github/workflows/build-and-test-ee.yml | 8 +++++++- .github/workflows/build-test-and-publish-ce.yml | 10 +++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-test-ee.yml b/.github/workflows/build-and-test-ee.yml index 0a336dd6..bd8d3596 100644 --- a/.github/workflows/build-and-test-ee.yml +++ b/.github/workflows/build-and-test-ee.yml @@ -16,10 +16,16 @@ jobs: strategy: matrix: DISTRO: [tomcat, wildfly, run] - PLATFORM: [ubuntu-22.04, aws-arm-core-2-default] + PLATFORM: [ubuntu-22.04, aws-arm-core-4-default] steps: - name: Checkout uses: actions/checkout@v4 + - name: Login to Docker Hub + uses: docker/login-action@v3 + if: ${{ matrix.PLATFORM == 'aws-arm-core-4-default' }} + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} - name: Build run: ./pipeline.sh env: diff --git a/.github/workflows/build-test-and-publish-ce.yml b/.github/workflows/build-test-and-publish-ce.yml index 1098e6f9..9a01948d 100644 --- a/.github/workflows/build-test-and-publish-ce.yml +++ b/.github/workflows/build-test-and-publish-ce.yml @@ -6,14 +6,14 @@ jobs: strategy: fail-fast: false matrix: - DISTRO: [tomcat] - PLATFORM: [ubuntu-22.04, aws-arm-core-2-default] + DISTRO: [tomcat, wildfly, run] + PLATFORM: [ubuntu-22.04, aws-arm-core-4-default] steps: - name: Checkout uses: actions/checkout@v4 - name: Login to Docker Hub uses: docker/login-action@v3 - if: ${{ matrix.PLATFORM == 'aws-arm-core-2-default' }} + if: ${{ matrix.PLATFORM == 'aws-arm-core-4-default' }} with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_PASSWORD }} @@ -31,12 +31,12 @@ jobs: DISTRO: ${{ matrix.DISTRO }} EE: false publish: - runs-on: aws-arm-core-2-default + runs-on: aws-arm-core-4-default needs: build-and-test #if: ${{ github.ref == 'refs/heads/next' || startsWith(github.ref, 'refs/heads/7.') }} strategy: matrix: - DISTRO: [tomcat] + DISTRO: [tomcat, wildfly, run] steps: - name: Checkout uses: actions/checkout@v4