From dfaa3c1db17104012bf44dfcf031efdd721686e1 Mon Sep 17 00:00:00 2001 From: lokey Date: Wed, 30 Aug 2023 13:03:26 +0530 Subject: [PATCH 01/12] Run build docker image action on self hosted runners --- .github/workflows/build-push-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 7e36beba76..af15bf8d46 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -31,7 +31,7 @@ env: jobs: build-transformer-image: name: Build Transformer Docker Image - runs-on: ubuntu-latest + runs-on: [self-hosted, Linux, ARM64] steps: - name: Checkout uses: actions/checkout@v3.5.3 From 4f1f8e5f11f1e3f680aea287cff32d58e5381390 Mon Sep 17 00:00:00 2001 From: lokey Date: Wed, 30 Aug 2023 13:23:05 +0530 Subject: [PATCH 02/12] Remove cache to test --- .github/workflows/build-push-docker-image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index af15bf8d46..ac025e857f 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -55,8 +55,8 @@ jobs: target: ${{ inputs.load_target }} load: true tags: ${{ inputs.build_tag }} - cache-from: type=gha - cache-to: type=gha,mode=max + # cache-from: type=gha + # cache-to: type=gha,mode=max - name: Run Tests run: | @@ -77,5 +77,5 @@ jobs: build-args: | version=${{ inputs.img_tag }} GIT_COMMIT_SHA=${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max + # cache-from: type=gha + # cache-to: type=gha,mode=max From 5d6ffd5fa8f469ecbd56135ad3b50689116eba14 Mon Sep 17 00:00:00 2001 From: lokey Date: Wed, 30 Aug 2023 14:52:29 +0530 Subject: [PATCH 03/12] test commit From 0113a59d381d9236701dbf09f83818a2fd982d6c Mon Sep 17 00:00:00 2001 From: lokey Date: Wed, 30 Aug 2023 15:11:03 +0530 Subject: [PATCH 04/12] Test --- .github/workflows/build-push-docker-image.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index ac025e857f..a4d2140c70 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -72,7 +72,6 @@ jobs: push: true tags: ${{ inputs.push_tags }} platforms: | - linux/amd64 linux/arm64 build-args: | version=${{ inputs.img_tag }} From 41d95fecfc74fcfa500b8f27972f4ee8c8edb05d Mon Sep 17 00:00:00 2001 From: lokey Date: Wed, 30 Aug 2023 15:26:42 +0530 Subject: [PATCH 05/12] Build only amd64 image --- .github/workflows/build-push-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index a4d2140c70..2f056ef431 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -72,7 +72,7 @@ jobs: push: true tags: ${{ inputs.push_tags }} platforms: | - linux/arm64 + linux/amd64 build-args: | version=${{ inputs.img_tag }} GIT_COMMIT_SHA=${{ github.sha }} From 819c5ab5c8007e60495c955f96938fa70e071247 Mon Sep 17 00:00:00 2001 From: lokey Date: Wed, 30 Aug 2023 17:10:19 +0530 Subject: [PATCH 06/12] build multi step multi arch image --- .github/workflows/build-push-docker-image.yml | 92 ++++++++++++++++--- 1 file changed, 81 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 2f056ef431..073d342b30 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -29,8 +29,8 @@ env: DOCKERHUB_USERNAME: rudderlabs jobs: - build-transformer-image: - name: Build Transformer Docker Image + build-transformer-image-arm64: + name: Build Transformer Docker Image ARM64 runs-on: [self-hosted, Linux, ARM64] steps: - name: Checkout @@ -47,21 +47,70 @@ jobs: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} - - name: Build Docker Image + # - name: Build Docker Image + # uses: docker/build-push-action@v4.1.1 + # with: + # context: . + # file: ${{ inputs.dockerfile }} + # target: ${{ inputs.load_target }} + # load: true + # tags: ${{ inputs.build_tag }} + # # cache-from: type=gha + # # cache-to: type=gha,mode=max + + # - name: Run Tests + # run: | + # docker run ${{ inputs.build_tag }} npm run test:js:ci + # docker run ${{ inputs.build_tag }} npm run test:ts:ci + + - name: Build and Push Multi-platform Images uses: docker/build-push-action@v4.1.1 with: context: . file: ${{ inputs.dockerfile }} - target: ${{ inputs.load_target }} - load: true - tags: ${{ inputs.build_tag }} + target: ${{ inputs.push_target }} + push: true + tags: ${{ inputs.push_tags }} + platforms: | + linux/arm64 + build-args: | + version=${{ inputs.img_tag }}-arm64 + GIT_COMMIT_SHA=${{ github.sha }} # cache-from: type=gha # cache-to: type=gha,mode=max + build-transformer-image-amd64: + name: Build Transformer Docker Image AMD64 + runs-on: [self-hosted, Linux, X64] + steps: + - name: Checkout + uses: actions/checkout@v3.5.3 + with: + fetch-depth: 1 - - name: Run Tests - run: | - docker run ${{ inputs.build_tag }} npm run test:js:ci - docker run ${{ inputs.build_tag }} npm run test:ts:ci + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2.9.1 + + - name: Login to DockerHub + uses: docker/login-action@v2.1.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} + + # - name: Build Docker Image + # uses: docker/build-push-action@v4.1.1 + # with: + # context: . + # file: ${{ inputs.dockerfile }} + # target: ${{ inputs.load_target }} + # load: true + # tags: ${{ inputs.build_tag }} + # # cache-from: type=gha + # # cache-to: type=gha,mode=max + + # - name: Run Tests + # run: | + # docker run ${{ inputs.build_tag }} npm run test:js:ci + # docker run ${{ inputs.build_tag }} npm run test:ts:ci - name: Build and Push Multi-platform Images uses: docker/build-push-action@v4.1.1 @@ -74,7 +123,28 @@ jobs: platforms: | linux/amd64 build-args: | - version=${{ inputs.img_tag }} + version=${{ inputs.img_tag }}-amd64 GIT_COMMIT_SHA=${{ github.sha }} # cache-from: type=gha # cache-to: type=gha,mode=max + + create-manifest: + name: Create multi-arch manifest + runs-on: ubuntu-latest + needs: [build-transformer-image-amd64, build-transformer-image-arm64] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Create multi-arch manifest + run: | + echo '{"experimental":"enabled"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + docker manifest create ${{ inputs.img_tag }} \ + ${{ inputs.img_tag }}-amd64 \ + ${{ inputs.img_tag }}-arm64 + docker manifest push ${{ inputs.img_tag }} From 325163cde690fefc4c9677958f112acca2169dba Mon Sep 17 00:00:00 2001 From: lokey Date: Wed, 30 Aug 2023 17:29:24 +0530 Subject: [PATCH 07/12] test --- .github/workflows/build-push-docker-image.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 073d342b30..5de0292f77 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -70,7 +70,7 @@ jobs: file: ${{ inputs.dockerfile }} target: ${{ inputs.push_target }} push: true - tags: ${{ inputs.push_tags }} + tags: ${{ inputs.push_tags[0] }}-arm64 platforms: | linux/arm64 build-args: | @@ -119,7 +119,7 @@ jobs: file: ${{ inputs.dockerfile }} target: ${{ inputs.push_target }} push: true - tags: ${{ inputs.push_tags }} + tags: ${{ inputs.push_tags[0] }}-amd64 platforms: | linux/amd64 build-args: | @@ -142,9 +142,7 @@ jobs: - name: Create multi-arch manifest run: | - echo '{"experimental":"enabled"}' | sudo tee /etc/docker/daemon.json - sudo systemctl restart docker - docker manifest create ${{ inputs.img_tag }} \ - ${{ inputs.img_tag }}-amd64 \ - ${{ inputs.img_tag }}-arm64 - docker manifest push ${{ inputs.img_tag }} + docker manifest create ${{ inputs.push_tags[0] }} \ + ${{ inputs.push_tags[0] }}-amd64 \ + ${{ inputs.push_tags[0] }}-arm64 + docker manifest push ${{ inputs.push_tags[0] }} From 6000283ccc3e604e20c181a76b893343f14cad6d Mon Sep 17 00:00:00 2001 From: lokey Date: Wed, 30 Aug 2023 17:57:15 +0530 Subject: [PATCH 08/12] Remove rudderlabs docker hub repo for testing --- .github/workflows/build-pr-artifacts.yml | 4 ++-- .github/workflows/build-push-docker-image.yml | 12 ++++++------ .github/workflows/prepare-for-dev-deploy.yml | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-pr-artifacts.yml b/.github/workflows/build-pr-artifacts.yml index b0187beb85..022afcd9f4 100644 --- a/.github/workflows/build-pr-artifacts.yml +++ b/.github/workflows/build-pr-artifacts.yml @@ -43,7 +43,7 @@ jobs: uses: ./.github/workflows/build-push-docker-image.yml with: build_tag: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }} - push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }},rudderlabs/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }} + push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }} img_tag: ${{ needs.generate-tag-names.outputs.tag_name }} dockerfile: Dockerfile load_target: development @@ -59,7 +59,7 @@ jobs: uses: ./.github/workflows/build-push-docker-image.yml with: build_tag: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }} - push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }},rudderlabs/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }} + push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }} img_tag: ${{ needs.generate-tag-names.outputs.tag_name_ut }} dockerfile: Dockerfile-ut-func load_target: development diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 5de0292f77..9f8ea27bb6 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -70,7 +70,7 @@ jobs: file: ${{ inputs.dockerfile }} target: ${{ inputs.push_target }} push: true - tags: ${{ inputs.push_tags[0] }}-arm64 + tags: ${{ inputs.push_tags }}-arm64 platforms: | linux/arm64 build-args: | @@ -119,7 +119,7 @@ jobs: file: ${{ inputs.dockerfile }} target: ${{ inputs.push_target }} push: true - tags: ${{ inputs.push_tags[0] }}-amd64 + tags: ${{ inputs.push_tags }}-amd64 platforms: | linux/amd64 build-args: | @@ -142,7 +142,7 @@ jobs: - name: Create multi-arch manifest run: | - docker manifest create ${{ inputs.push_tags[0] }} \ - ${{ inputs.push_tags[0] }}-amd64 \ - ${{ inputs.push_tags[0] }}-arm64 - docker manifest push ${{ inputs.push_tags[0] }} + docker manifest create ${{ inputs.push_tags }} \ + ${{ inputs.push_tags }}-amd64 \ + ${{ inputs.push_tags }}-arm64 + docker manifest push ${{ inputs.push_tags }} diff --git a/.github/workflows/prepare-for-dev-deploy.yml b/.github/workflows/prepare-for-dev-deploy.yml index 636ba5a6d8..d45f760c66 100644 --- a/.github/workflows/prepare-for-dev-deploy.yml +++ b/.github/workflows/prepare-for-dev-deploy.yml @@ -51,7 +51,7 @@ jobs: uses: ./.github/workflows/build-push-docker-image.yml with: build_tag: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }} - push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }},rudderlabs/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }} + push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name }} img_tag: ${{ needs.generate-tag-names.outputs.tag_name }} dockerfile: Dockerfile load_target: development @@ -67,7 +67,7 @@ jobs: uses: ./.github/workflows/build-push-docker-image.yml with: build_tag: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }} - push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }},rudderlabs/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }} + push_tags: rudderstack/develop-rudder-transformer:${{ needs.generate-tag-names.outputs.tag_name_ut }} img_tag: ${{ needs.generate-tag-names.outputs.tag_name_ut }} dockerfile: Dockerfile-ut-func load_target: development From 316d60950c13169f94b4e51df33886d8a5ef3bea Mon Sep 17 00:00:00 2001 From: lokey Date: Sun, 3 Sep 2023 13:19:03 +0530 Subject: [PATCH 09/12] Update the manifest command to use imagetools --- .github/workflows/build-push-docker-image.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 9f8ea27bb6..f8869d33f9 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -142,7 +142,4 @@ jobs: - name: Create multi-arch manifest run: | - docker manifest create ${{ inputs.push_tags }} \ - ${{ inputs.push_tags }}-amd64 \ - ${{ inputs.push_tags }}-arm64 - docker manifest push ${{ inputs.push_tags }} + docker buildx imagetools create -t ${{ inputs.push_tags }} ${{ inputs.push_tags }}-amd64 ${{ inputs.push_tags }}-arm64 From fb63550d113af64406674b169a9f9891804337a6 Mon Sep 17 00:00:00 2001 From: lokey Date: Sun, 3 Sep 2023 13:36:38 +0530 Subject: [PATCH 10/12] Login to dockerhub --- .github/workflows/build-push-docker-image.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index f8869d33f9..df42facdf4 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -140,6 +140,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v2.1.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} + - name: Create multi-arch manifest run: | docker buildx imagetools create -t ${{ inputs.push_tags }} ${{ inputs.push_tags }}-amd64 ${{ inputs.push_tags }}-arm64 From f31c2a04231d7b3d9f6cd5b2fa546aa812099106 Mon Sep 17 00:00:00 2001 From: lokey Date: Sun, 3 Sep 2023 14:30:08 +0530 Subject: [PATCH 11/12] Remove unnecessary step --- .github/workflows/build-push-docker-image.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index df42facdf4..1a67d6dde0 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -134,11 +134,8 @@ jobs: needs: [build-transformer-image-amd64, build-transformer-image-arm64] steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2.9.1 - name: Login to DockerHub uses: docker/login-action@v2.1.0 From 8f092e5961509de9e087d19d3cae2b583308d882 Mon Sep 17 00:00:00 2001 From: krishnachaitanya Date: Wed, 13 Sep 2023 17:04:07 +0530 Subject: [PATCH 12/12] chore: uncommented test runs --- .github/workflows/build-push-docker-image.yml | 61 ++++++++++--------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 1a67d6dde0..f65aa0c8ce 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -47,21 +47,21 @@ jobs: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} - # - name: Build Docker Image - # uses: docker/build-push-action@v4.1.1 - # with: - # context: . - # file: ${{ inputs.dockerfile }} - # target: ${{ inputs.load_target }} - # load: true - # tags: ${{ inputs.build_tag }} - # # cache-from: type=gha - # # cache-to: type=gha,mode=max - - # - name: Run Tests - # run: | - # docker run ${{ inputs.build_tag }} npm run test:js:ci - # docker run ${{ inputs.build_tag }} npm run test:ts:ci + - name: Build Docker Image + uses: docker/build-push-action@v4.1.1 + with: + context: . + file: ${{ inputs.dockerfile }} + target: ${{ inputs.load_target }} + load: true + tags: ${{ inputs.build_tag }} + # cache-from: type=gha + # cache-to: type=gha,mode=max + + - name: Run Tests + run: | + docker run ${{ inputs.build_tag }} npm run test:js:ci + docker run ${{ inputs.build_tag }} npm run test:ts:ci - name: Build and Push Multi-platform Images uses: docker/build-push-action@v4.1.1 @@ -78,6 +78,7 @@ jobs: GIT_COMMIT_SHA=${{ github.sha }} # cache-from: type=gha # cache-to: type=gha,mode=max + build-transformer-image-amd64: name: Build Transformer Docker Image AMD64 runs-on: [self-hosted, Linux, X64] @@ -96,21 +97,21 @@ jobs: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PROD_TOKEN }} - # - name: Build Docker Image - # uses: docker/build-push-action@v4.1.1 - # with: - # context: . - # file: ${{ inputs.dockerfile }} - # target: ${{ inputs.load_target }} - # load: true - # tags: ${{ inputs.build_tag }} - # # cache-from: type=gha - # # cache-to: type=gha,mode=max - - # - name: Run Tests - # run: | - # docker run ${{ inputs.build_tag }} npm run test:js:ci - # docker run ${{ inputs.build_tag }} npm run test:ts:ci + - name: Build Docker Image + uses: docker/build-push-action@v4.1.1 + with: + context: . + file: ${{ inputs.dockerfile }} + target: ${{ inputs.load_target }} + load: true + tags: ${{ inputs.build_tag }} + # cache-from: type=gha + # cache-to: type=gha,mode=max + + - name: Run Tests + run: | + docker run ${{ inputs.build_tag }} npm run test:js:ci + docker run ${{ inputs.build_tag }} npm run test:ts:ci - name: Build and Push Multi-platform Images uses: docker/build-push-action@v4.1.1