From 34c3004a7067a18b5d464ccd06cdabdb8c118665 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Thu, 12 Dec 2024 19:24:52 +0800 Subject: [PATCH 01/40] Remove push-image and tag-image script --- .github/workflows/ci.yaml | 12 ++++-------- Makefile | 39 +++++++++++++++++++-------------------- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 277ebefd83..4cef2fa90f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -122,10 +122,9 @@ jobs: env: TARGET: portal IMAGE_NAME: authgear-portal + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" steps: - uses: actions/checkout@v4 - - run: make build-image TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - - run: make tag-image IMAGE_NAME=$IMAGE_NAME - name: docker login if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} env: @@ -133,8 +132,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} run: | printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make push-image IMAGE_NAME=$IMAGE_NAME - if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} + - run: make build-image PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} run: | @@ -223,10 +221,9 @@ jobs: env: TARGET: authgear IMAGE_NAME: authgear-server + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" steps: - uses: actions/checkout@v4 - - run: make build-image TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - - run: make tag-image IMAGE_NAME=$IMAGE_NAME - name: docker login if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} env: @@ -234,8 +231,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} run: | printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make push-image IMAGE_NAME=$IMAGE_NAME - if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} + - run: make build-image PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} run: | diff --git a/Makefile b/Makefile index 1a34abdd44..24511fa775 100644 --- a/Makefile +++ b/Makefile @@ -162,26 +162,25 @@ check-tidy: .PHONY: build-image build-image: - # Add --pull so that we are using the latest base image. - docker build --pull --file ./cmd/$(TARGET)/Dockerfile --tag $(IMAGE_NAME) --build-arg GIT_HASH=$(GIT_HASH) . - -.PHONY: tag-image -tag-image: DOCKER_IMAGE ::= quay.io/theauthgear/$(IMAGE_NAME) -tag-image: - docker tag $(IMAGE_NAME) $(DOCKER_IMAGE):latest - docker tag $(IMAGE_NAME) $(DOCKER_IMAGE):$(GIT_HASH) - if [ ! -z $(GIT_NAME) ]; then docker tag $(IMAGE_NAME) $(DOCKER_IMAGE):$(GIT_NAME); fi - -.PHONY: push-image -push-image: DOCKER_IMAGE ::= quay.io/theauthgear/$(IMAGE_NAME) -push-image: - docker manifest inspect $(DOCKER_IMAGE):$(GIT_HASH) > /dev/null; if [ $$? -eq 0 ]; then \ - echo "$(DOCKER_IMAGE):$(GIT_HASH) exists. Skip push"; \ - else \ - docker push $(DOCKER_IMAGE):latest ;\ - docker push $(DOCKER_IMAGE):$(GIT_HASH) ;\ - if [ ! -z $(GIT_NAME) ]; then docker push $(DOCKER_IMAGE):$(GIT_NAME); fi ;\ - fi + $(eval DOCKER_IMAGE ::= quay.io/theauthgear/$(IMAGE_NAME)) + $(eval BUILD_OPTS ::= --tag $(DOCKER_IMAGE)) +ifeq (${TAG_IMAGE},true) # if TAG_IMAGE + $(eval BUILD_OPTS += --tag $(DOCKER_IMAGE):latest) +ifneq (${GIT_HASH},) + $(eval BUILD_OPTS += --tag $(DOCKER_IMAGE):$(GIT_HASH)) +endif +ifneq (${GIT_NAME},) + $(eval BUILD_OPTS += --tag $(DOCKER_IMAGE):$(GIT_NAME)) +endif +endif # endif TAG_IMAGE +ifeq ($(PUSH_IMAGE),true) + $(eval BUILD_OPTS += --push) +endif + @# Add --pull so that we are using the latest base image. + docker buildx build --pull \ + --file ./cmd/$(TARGET)/Dockerfile \ + $(BUILD_OPTS) \ + --build-arg GIT_HASH=$(GIT_HASH) . .PHONY: html-email html-email: From bf8041a195a1f72815a0f20c95d95dc0486bfed9 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Thu, 12 Dec 2024 19:51:03 +0800 Subject: [PATCH 02/40] Remove push-image and tag-image script in custom build --- .github/workflows/custom-build.yaml | 17 ++++++++-------- custombuild/Makefile | 30 ++++++++++++++++++----------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.github/workflows/custom-build.yaml b/.github/workflows/custom-build.yaml index 2773fb1779..efd50a61c8 100644 --- a/.github/workflows/custom-build.yaml +++ b/.github/workflows/custom-build.yaml @@ -53,7 +53,6 @@ jobs: printf "$AUTHGEAR_PRIVATE_DEPLOY_KEY" | base64 --decode | ssh-add - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - run: make -C custombuild build-image TARGET=authgearx IMAGE_NAME=authgear-server-custom - name: Clean up SSH key if: ${{ always() }} run: | @@ -74,10 +73,10 @@ jobs: gcloud auth configure-docker "$REPO" - name: Push to HK env: + REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} run: | - make -C custombuild tag-image IMAGE_NAME=authgear-server-custom REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-server" - make -C custombuild push-image REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + make -C custombuild build-image TARGET=authgearx TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 with: @@ -91,10 +90,10 @@ jobs: gcloud auth configure-docker "$REPO" - name: Push to US env: + REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} run: | - make -C custombuild tag-image IMAGE_NAME=authgear-server-custom REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-server" - make -C custombuild push-image REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + make -C custombuild build-image TARGET=authgearx TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" portal-image-custom: runs-on: ubuntu-24.04 @@ -134,9 +133,9 @@ jobs: - name: Push to HK env: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} + REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} run: | - make -C custombuild tag-image IMAGE_NAME=authgear-portal-custom REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-portal" - make -C custombuild push-image REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + make -C custombuild build-image TARGET=portalx TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 with: @@ -151,6 +150,6 @@ jobs: - name: Push to US env: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} + REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} run: | - make -C custombuild tag-image IMAGE_NAME=authgear-portal-custom REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-portal" - make -C custombuild push-image REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + make -C custombuild build-image TARGET=portalx TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" diff --git a/custombuild/Makefile b/custombuild/Makefile index 71b8860bce..5e8fca6592 100644 --- a/custombuild/Makefile +++ b/custombuild/Makefile @@ -47,14 +47,22 @@ build: .PHONY: build-image build-image: - # Add --pull so that we are using the latest base image. - # The build context is the parent directory - docker build --pull --ssh=default --file ./cmd/$(TARGET)/Dockerfile --tag $(IMAGE_NAME) --build-arg GIT_HASH=$(GIT_HASH) ../ - -.PHONY: tag-image -tag-image: - docker tag $(IMAGE_NAME) $(REMOTE_IMAGE_NAME):$(GIT_HASH) - -.PHONY: push-image -push-image: - docker push $(REMOTE_IMAGE_NAME):$(GIT_HASH) + $(eval BUILD_OPTS ::= --tag $(IMAGE_NAME)) +ifeq (${TAG_IMAGE},true) # if TAG_IMAGE + $(eval BUILD_OPTS += --tag $(IMAGE_NAME):latest) +ifneq (${GIT_HASH},) + $(eval BUILD_OPTS += --tag $(IMAGE_NAME):$(GIT_HASH)) +endif +ifneq (${GIT_NAME},) + $(eval BUILD_OPTS += --tag $(IMAGE_NAME):$(GIT_NAME)) +endif +endif # endif TAG_IMAGE +ifeq ($(PUSH_IMAGE),true) + $(eval BUILD_OPTS += --push) +endif + @# Add --pull so that we are using the latest base image. + @# The build context is the parent directory + docker build --pull --ssh=default \ + --file ./cmd/$(TARGET)/Dockerfile \ + $(BUILD_OPTS) \ + --build-arg GIT_HASH=$(GIT_HASH) ../ From aac6c1892581dce60c59a47a5090d12a44367159 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Thu, 12 Dec 2024 19:55:59 +0800 Subject: [PATCH 03/40] Build multi platform images --- .github/workflows/ci.yaml | 8 ++++++-- .github/workflows/custom-build.yaml | 12 ++++++++---- Makefile | 3 +++ custombuild/Makefile | 3 +++ 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4cef2fa90f..a7dd8d40db 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -125,6 +125,8 @@ jobs: PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" steps: - uses: actions/checkout@v4 + - name: Install qemu for multi arch build + run: docker run --privileged --rm tonistiigi/binfmt --install all - name: docker login if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} env: @@ -132,7 +134,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} run: | printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make build-image PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME + - run: make build-image BUILD_MULTI_PLATFORMS=true PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} run: | @@ -224,6 +226,8 @@ jobs: PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" steps: - uses: actions/checkout@v4 + - name: Install qemu for multi arch build + run: docker run --privileged --rm tonistiigi/binfmt --install all - name: docker login if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} env: @@ -231,7 +235,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} run: | printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make build-image PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME + - run: make build-image BUILD_MULTI_PLATFORMS=true PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} run: | diff --git a/.github/workflows/custom-build.yaml b/.github/workflows/custom-build.yaml index efd50a61c8..503085f17c 100644 --- a/.github/workflows/custom-build.yaml +++ b/.github/workflows/custom-build.yaml @@ -42,6 +42,8 @@ jobs: if: ${{ github.repository == 'authgear/authgear-server' }} steps: - uses: actions/checkout@v4 + - name: Install qemu for multi arch build + run: docker run --privileged --rm tonistiigi/binfmt --install all # https://aran.dev/posts/github-actions-go-private-modules/ - name: Set up SSH key env: @@ -76,7 +78,7 @@ jobs: REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} run: | - make -C custombuild build-image TARGET=authgearx TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + make -C custombuild build-image TARGET=authgearx BUILD_MULTI_PLATFORMS=true TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 with: @@ -93,13 +95,15 @@ jobs: REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} run: | - make -C custombuild build-image TARGET=authgearx TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + make -C custombuild build-image TARGET=authgearx BUILD_MULTI_PLATFORMS=true TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" portal-image-custom: runs-on: ubuntu-24.04 if: ${{ github.repository == 'authgear/authgear-server' }} steps: - uses: actions/checkout@v4 + - name: Install qemu for multi arch build + run: docker run --privileged --rm tonistiigi/binfmt --install all # https://aran.dev/posts/github-actions-go-private-modules/ - name: Set up SSH key env: @@ -135,7 +139,7 @@ jobs: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} run: | - make -C custombuild build-image TARGET=portalx TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + make -C custombuild build-image TARGET=portalx BUILD_MULTI_PLATFORMS=true TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 with: @@ -152,4 +156,4 @@ jobs: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} run: | - make -C custombuild build-image TARGET=portalx TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + make -C custombuild build-image TARGET=portalx BUILD_MULTI_PLATFORMS=true TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" diff --git a/Makefile b/Makefile index 24511fa775..e885e089a2 100644 --- a/Makefile +++ b/Makefile @@ -175,6 +175,9 @@ endif endif # endif TAG_IMAGE ifeq ($(PUSH_IMAGE),true) $(eval BUILD_OPTS += --push) +endif +ifeq ($(BUILD_MULTI_PLATFORMS),true) + $(eval BUILD_OPTS += --platform linux/amd64,linux/arm64) endif @# Add --pull so that we are using the latest base image. docker buildx build --pull \ diff --git a/custombuild/Makefile b/custombuild/Makefile index 5e8fca6592..8563755b89 100644 --- a/custombuild/Makefile +++ b/custombuild/Makefile @@ -59,6 +59,9 @@ endif endif # endif TAG_IMAGE ifeq ($(PUSH_IMAGE),true) $(eval BUILD_OPTS += --push) +endif +ifeq ($(BUILD_MULTI_PLATFORMS),true) + $(eval BUILD_OPTS += --platform linux/amd64,linux/arm64) endif @# Add --pull so that we are using the latest base image. @# The build context is the parent directory From 65eeb92d4dfed05727158dc544dffe747bd72372 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Fri, 13 Dec 2024 12:21:56 +0800 Subject: [PATCH 04/40] Setup docker container builder --- .github/workflows/ci.yaml | 12 ++++++++++++ .github/workflows/custom-build.yaml | 14 ++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a7dd8d40db..5d31341d2a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,6 +127,12 @@ jobs: - uses: actions/checkout@v4 - name: Install qemu for multi arch build run: docker run --privileged --rm tonistiigi/binfmt --install all + - name: Setup container builder + run: | + docker buildx create \ + --name container-builder \ + --driver docker-container \ + --bootstrap --use - name: docker login if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} env: @@ -228,6 +234,12 @@ jobs: - uses: actions/checkout@v4 - name: Install qemu for multi arch build run: docker run --privileged --rm tonistiigi/binfmt --install all + - name: Setup container builder + run: | + docker buildx create \ + --name container-builder \ + --driver docker-container \ + --bootstrap --use - name: docker login if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} env: diff --git a/.github/workflows/custom-build.yaml b/.github/workflows/custom-build.yaml index 503085f17c..cf11dc7427 100644 --- a/.github/workflows/custom-build.yaml +++ b/.github/workflows/custom-build.yaml @@ -44,6 +44,13 @@ jobs: - uses: actions/checkout@v4 - name: Install qemu for multi arch build run: docker run --privileged --rm tonistiigi/binfmt --install all + - name: Setup container builder + run: | + docker buildx create \ + --name container-builder \ + --driver docker-container \ + --bootstrap --use + # https://aran.dev/posts/github-actions-go-private-modules/ - name: Set up SSH key env: @@ -104,6 +111,13 @@ jobs: - uses: actions/checkout@v4 - name: Install qemu for multi arch build run: docker run --privileged --rm tonistiigi/binfmt --install all + - name: Setup container builder + run: | + docker buildx create \ + --name container-builder \ + --driver docker-container \ + --bootstrap --use + # https://aran.dev/posts/github-actions-go-private-modules/ - name: Set up SSH key env: From 62e19c49173f9b1117c092bfd66ae30c8d9027e2 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Fri, 13 Dec 2024 13:06:54 +0800 Subject: [PATCH 05/40] Always use build platform to build js assets Because the produced js should be the same no matter in which platform --- cmd/authgear/Dockerfile | 2 +- cmd/portal/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/authgear/Dockerfile b/cmd/authgear/Dockerfile index 0cc86ad817..fd0093662a 100644 --- a/cmd/authgear/Dockerfile +++ b/cmd/authgear/Dockerfile @@ -21,7 +21,7 @@ RUN make build BIN_NAME=authgear TARGET=authgear GIT_HASH=$GIT_HASH # RUN readelf -d ./authgear | grep 'There is no dynamic section in this file' # Stage 2: Build the static files -FROM node:20.9.0-bookworm as stage2 +FROM --platform=$BUILDPLATFORM node:20.9.0-bookworm as stage2 ARG GIT_HASH WORKDIR /usr/src/app COPY ./scripts/npm/package.json ./scripts/npm/package-lock.json ./scripts/npm/ diff --git a/cmd/portal/Dockerfile b/cmd/portal/Dockerfile index 4a9720ee14..d1a792c689 100644 --- a/cmd/portal/Dockerfile +++ b/cmd/portal/Dockerfile @@ -21,7 +21,7 @@ RUN make build BIN_NAME=authgear-portal TARGET=portal GIT_HASH=$GIT_HASH # RUN readelf -d ./authgear | grep 'There is no dynamic section in this file' # Stage 2: Build the static files -FROM node:20.9.0-bookworm as stage2 +FROM --platform=$BUILDPLATFORM node:20.9.0-bookworm as stage2 ARG GIT_HASH WORKDIR /usr/src/app COPY ./scripts/npm/package.json ./scripts/npm/package-lock.json ./scripts/npm/ @@ -36,7 +36,7 @@ COPY . . RUN make authui GIT_HASH=$GIT_HASH # Stage 3: Build the portal static files -FROM node:20.9.0-bookworm as stage3 +FROM --platform=$BUILDPLATFORM node:20.9.0-bookworm as stage3 ARG GIT_HASH # If the working directory is /src, Parcel will have some problem with it. WORKDIR /usr/src/app From 1d38d58fd4e1bf325145684d6416d5712170681a Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Fri, 13 Dec 2024 14:48:34 +0800 Subject: [PATCH 06/40] Only build amd64 image in prs to reduce build time --- .github/workflows/ci.yaml | 6 ++++-- .github/workflows/custom-build.yaml | 9 ++++----- Makefile | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5d31341d2a..0e6dcc41fb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -123,6 +123,7 @@ jobs: TARGET: portal IMAGE_NAME: authgear-portal PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + BUILD_PLATFORMS: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}" steps: - uses: actions/checkout@v4 - name: Install qemu for multi arch build @@ -140,7 +141,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} run: | printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make build-image BUILD_MULTI_PLATFORMS=true PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME + - run: make build-image BUILD_PLATFORMS=$BUILD_PLATFORMS PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} run: | @@ -230,6 +231,7 @@ jobs: TARGET: authgear IMAGE_NAME: authgear-server PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + BUILD_PLATFORMS: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}" steps: - uses: actions/checkout@v4 - name: Install qemu for multi arch build @@ -247,7 +249,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} run: | printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make build-image BUILD_MULTI_PLATFORMS=true PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME + - run: make build-image BUILD_PLATFORMS=$BUILD_PLATFORMS PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} run: | diff --git a/.github/workflows/custom-build.yaml b/.github/workflows/custom-build.yaml index cf11dc7427..252b594227 100644 --- a/.github/workflows/custom-build.yaml +++ b/.github/workflows/custom-build.yaml @@ -85,7 +85,7 @@ jobs: REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} run: | - make -C custombuild build-image TARGET=authgearx BUILD_MULTI_PLATFORMS=true TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + make -C custombuild build-image TARGET=authgearx BUILD_PLATFORMS=linux/amd64,linux/arm64 TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 with: @@ -102,7 +102,7 @@ jobs: REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} run: | - make -C custombuild build-image TARGET=authgearx BUILD_MULTI_PLATFORMS=true TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + make -C custombuild build-image TARGET=authgearx BUILD_PLATFORMS=linux/amd64,linux/arm64 TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" portal-image-custom: runs-on: ubuntu-24.04 @@ -129,7 +129,6 @@ jobs: printf "$AUTHGEAR_PRIVATE_DEPLOY_KEY" | base64 --decode | ssh-add - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - run: make -C custombuild build-image TARGET=portalx IMAGE_NAME=authgear-portal-custom - name: Clean up SSH key if: ${{ always() }} run: | @@ -153,7 +152,7 @@ jobs: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} run: | - make -C custombuild build-image TARGET=portalx BUILD_MULTI_PLATFORMS=true TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + make -C custombuild build-image TARGET=portalx BUILD_PLATFORMS=linux/amd64,linux/arm64 TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 with: @@ -170,4 +169,4 @@ jobs: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} run: | - make -C custombuild build-image TARGET=portalx BUILD_MULTI_PLATFORMS=true TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + make -C custombuild build-image TARGET=portalx BUILD_PLATFORMS=linux/amd64,linux/arm64 TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" diff --git a/Makefile b/Makefile index e885e089a2..d0f2a8c82e 100644 --- a/Makefile +++ b/Makefile @@ -176,8 +176,8 @@ endif # endif TAG_IMAGE ifeq ($(PUSH_IMAGE),true) $(eval BUILD_OPTS += --push) endif -ifeq ($(BUILD_MULTI_PLATFORMS),true) - $(eval BUILD_OPTS += --platform linux/amd64,linux/arm64) +ifneq ($(BUILD_PLATFORMS),) + $(eval BUILD_OPTS += --platform $(BUILD_PLATFORMS)) endif @# Add --pull so that we are using the latest base image. docker buildx build --pull \ From 16c776d57c49fc45e73d91a6bac69431195af1ff Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Fri, 13 Dec 2024 14:49:17 +0800 Subject: [PATCH 07/40] Fix ssh key is cleaned up too early --- .github/workflows/custom-build.yaml | 30 +++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/custom-build.yaml b/.github/workflows/custom-build.yaml index 252b594227..f3882366b2 100644 --- a/.github/workflows/custom-build.yaml +++ b/.github/workflows/custom-build.yaml @@ -62,13 +62,6 @@ jobs: printf "$AUTHGEAR_PRIVATE_DEPLOY_KEY" | base64 --decode | ssh-add - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - name: Clean up SSH key - if: ${{ always() }} - run: | - ssh-add -D - ssh-agent -k - echo "SSH_AUTH_SOCK=" >> "$GITHUB_ENV" - echo "SSH_AGENT_PID=" >> "$GITHUB_ENV" - uses: oursky/google-github-actions-auth@v2 with: @@ -104,6 +97,14 @@ jobs: run: | make -C custombuild build-image TARGET=authgearx BUILD_PLATFORMS=linux/amd64,linux/arm64 TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + - name: Clean up SSH key + if: ${{ always() }} + run: | + ssh-add -D + ssh-agent -k + echo "SSH_AUTH_SOCK=" >> "$GITHUB_ENV" + echo "SSH_AGENT_PID=" >> "$GITHUB_ENV" + portal-image-custom: runs-on: ubuntu-24.04 if: ${{ github.repository == 'authgear/authgear-server' }} @@ -129,13 +130,6 @@ jobs: printf "$AUTHGEAR_PRIVATE_DEPLOY_KEY" | base64 --decode | ssh-add - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - name: Clean up SSH key - if: ${{ always() }} - run: | - ssh-add -D - ssh-agent -k - echo "SSH_AUTH_SOCK=" >> "$GITHUB_ENV" - echo "SSH_AGENT_PID=" >> "$GITHUB_ENV" - uses: oursky/google-github-actions-auth@v2 with: @@ -170,3 +164,11 @@ jobs: REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} run: | make -C custombuild build-image TARGET=portalx BUILD_PLATFORMS=linux/amd64,linux/arm64 TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + + - name: Clean up SSH key + if: ${{ always() }} + run: | + ssh-add -D + ssh-agent -k + echo "SSH_AUTH_SOCK=" >> "$GITHUB_ENV" + echo "SSH_AGENT_PID=" >> "$GITHUB_ENV" From cbe4c968d43dff3ff9f4caade65e83b44ed98c1f Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Fri, 13 Dec 2024 15:46:19 +0800 Subject: [PATCH 08/40] Support BUILD_PLATFORMS in build-image of custombuild --- custombuild/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/custombuild/Makefile b/custombuild/Makefile index 8563755b89..6548cadb77 100644 --- a/custombuild/Makefile +++ b/custombuild/Makefile @@ -60,8 +60,8 @@ endif # endif TAG_IMAGE ifeq ($(PUSH_IMAGE),true) $(eval BUILD_OPTS += --push) endif -ifeq ($(BUILD_MULTI_PLATFORMS),true) - $(eval BUILD_OPTS += --platform linux/amd64,linux/arm64) +ifneq ($(BUILD_PLATFORMS),) + $(eval BUILD_OPTS += --platform $(BUILD_PLATFORMS)) endif @# Add --pull so that we are using the latest base image. @# The build context is the parent directory From 16181dd30f643cc1e06ae402e8e1b75b762a3ce0 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Fri, 13 Dec 2024 16:01:40 +0800 Subject: [PATCH 09/40] Extract common scripts to common.mk --- .github/workflows/ci.yaml | 4 +- Makefile | 77 +++------------------------------------ common.mk | 73 +++++++++++++++++++++++++++++++++++++ custombuild/Makefile | 74 ++----------------------------------- 4 files changed, 84 insertions(+), 144 deletions(-) create mode 100644 common.mk diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0e6dcc41fb..43ac8cb7ed 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -121,7 +121,7 @@ jobs: needs: portal-test env: TARGET: portal - IMAGE_NAME: authgear-portal + IMAGE_NAME: quay.io/theauthgear/authgear-portal PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" BUILD_PLATFORMS: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}" steps: @@ -229,7 +229,7 @@ jobs: needs: ["authgear-test", "authui-test"] env: TARGET: authgear - IMAGE_NAME: authgear-server + IMAGE_NAME: quay.io/theauthgear/authgear-server PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" BUILD_PLATFORMS: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}" steps: diff --git a/Makefile b/Makefile index d0f2a8c82e..c592049477 100644 --- a/Makefile +++ b/Makefile @@ -1,46 +1,8 @@ -# The use of variables -# -# We use simply expanded variables in this Makefile. -# -# This means -# 1. You use ::= instead of = because = defines a recursively expanded variable. -# See https://www.gnu.org/software/make/manual/html_node/Simple-Assignment.html -# 2. You use ::= instead of := because ::= is a POSIX standard. -# See https://www.gnu.org/software/make/manual/html_node/Simple-Assignment.html -# 3. You do not use ?= because it is shorthand to define a recursively expanded variable. -# See https://www.gnu.org/software/make/manual/html_node/Conditional-Assignment.html -# You should use the long form documented in the above link instead. -# 4. When you override a variable in the command line, as documented in https://www.gnu.org/software/make/manual/html_node/Overriding.html -# you specify the variable with ::= instead of = or := -# If you fail to do so, the variable becomes recursively expanded variable accidentally. -# -# GIT_NAME could be empty. -ifeq ($(origin GIT_NAME), undefined) - GIT_NAME ::= $(shell git describe --exact-match 2>/dev/null) -endif -ifeq ($(origin GIT_HASH), undefined) - GIT_HASH ::= git-$(shell git rev-parse --short=12 HEAD) -endif -ifeq ($(origin LDFLAGS), undefined) - LDFLAGS ::= "-X github.com/authgear/authgear-server/pkg/version.Version=${GIT_HASH}" -endif - -# osusergo: https://godoc.org/github.com/golang/go/src/os/user -# netgo: https://golang.org/doc/go1.5#net -# static_build: https://github.com/golang/go/issues/26492#issuecomment-635563222 -# The binary is static on Linux only. It is not static on macOS. -# timetzdata: https://golang.org/doc/go1.15#time/tzdata -GO_BUILD_TAGS ::= osusergo netgo static_build timetzdata -GO_RUN_TAGS ::= - - -.PHONY: start -start: - go run -tags "$(GO_RUN_TAGS)" -ldflags ${LDFLAGS} ./cmd/authgear start - -.PHONY: start-portal -start-portal: - go run -tags "$(GO_RUN_TAGS)" -ldflags ${LDFLAGS} ./cmd/portal start +CMD_AUTHGEAR ::= authgear +CMD_PORTAL ::= portal +BUILD_CTX ::= . + +include ./common.mk .PHONY: authgearonce-start authgearonce-start: GO_RUN_TAGS += authgearonce @@ -128,10 +90,6 @@ fmt: govulncheck: govulncheck -show traces,version,verbose ./... -.PHONY: build -build: - go build -o $(BIN_NAME) -tags "$(GO_BUILD_TAGS)" -ldflags ${LDFLAGS} ./cmd/$(TARGET) - .PHONY: binary binary: GO_BUILD_TAGS += authgearlite binary: @@ -160,31 +118,6 @@ check-tidy: make -C authui check-tidy make -C portal check-tidy -.PHONY: build-image -build-image: - $(eval DOCKER_IMAGE ::= quay.io/theauthgear/$(IMAGE_NAME)) - $(eval BUILD_OPTS ::= --tag $(DOCKER_IMAGE)) -ifeq (${TAG_IMAGE},true) # if TAG_IMAGE - $(eval BUILD_OPTS += --tag $(DOCKER_IMAGE):latest) -ifneq (${GIT_HASH},) - $(eval BUILD_OPTS += --tag $(DOCKER_IMAGE):$(GIT_HASH)) -endif -ifneq (${GIT_NAME},) - $(eval BUILD_OPTS += --tag $(DOCKER_IMAGE):$(GIT_NAME)) -endif -endif # endif TAG_IMAGE -ifeq ($(PUSH_IMAGE),true) - $(eval BUILD_OPTS += --push) -endif -ifneq ($(BUILD_PLATFORMS),) - $(eval BUILD_OPTS += --platform $(BUILD_PLATFORMS)) -endif - @# Add --pull so that we are using the latest base image. - docker buildx build --pull \ - --file ./cmd/$(TARGET)/Dockerfile \ - $(BUILD_OPTS) \ - --build-arg GIT_HASH=$(GIT_HASH) . - .PHONY: html-email html-email: # Generate `.mjml` templates from `.mjml.gotemplate` files diff --git a/common.mk b/common.mk new file mode 100644 index 0000000000..e0a896e253 --- /dev/null +++ b/common.mk @@ -0,0 +1,73 @@ +# The use of variables +# +# We use simply expanded variables in this Makefile. +# +# This means +# 1. You use ::= instead of = because = defines a recursively expanded variable. +# See https://www.gnu.org/software/make/manual/html_node/Simple-Assignment.html +# 2. You use ::= instead of := because ::= is a POSIX standard. +# See https://www.gnu.org/software/make/manual/html_node/Simple-Assignment.html +# 3. You do not use ?= because it is shorthand to define a recursively expanded variable. +# See https://www.gnu.org/software/make/manual/html_node/Conditional-Assignment.html +# You should use the long form documented in the above link instead. +# 4. When you override a variable in the command line, as documented in https://www.gnu.org/software/make/manual/html_node/Overriding.html +# you specify the variable with ::= instead of = or := +# If you fail to do so, the variable becomes recursively expanded variable accidentally. +# +# GIT_NAME could be empty. +ifeq ($(origin GIT_NAME), undefined) + GIT_NAME ::= $(shell git describe --exact-match 2>/dev/null) +endif +ifeq ($(origin GIT_HASH), undefined) + GIT_HASH ::= git-$(shell git rev-parse --short=12 HEAD) +endif +ifeq ($(origin LDFLAGS), undefined) + LDFLAGS ::= "-X github.com/authgear/authgear-server/pkg/version.Version=${GIT_HASH}" +endif + + +# osusergo: https://godoc.org/github.com/golang/go/src/os/user +# netgo: https://golang.org/doc/go1.5#net +# static_build: https://github.com/golang/go/issues/26492#issuecomment-635563222 +# The binary is static on Linux only. It is not static on macOS. +# timetzdata: https://golang.org/doc/go1.15#time/tzdata +GO_BUILD_TAGS ::= osusergo netgo static_build timetzdata +GO_RUN_TAGS ::= + + +.PHONY: start +start: + go run -tags "$(GO_RUN_TAGS)" -ldflags ${LDFLAGS} ./cmd/${CMD_AUTHGEAR} start + +.PHONY: start-portal +start-portal: + go run -tags "$(GO_RUN_TAGS)" -ldflags ${LDFLAGS} ./cmd/${CMD_PORTAL} start + +.PHONY: build +build: + go build -o $(BIN_NAME) -tags "$(GO_BUILD_TAGS)" -ldflags ${LDFLAGS} ./cmd/$(TARGET) + +.PHONY: build-image +build-image: + $(eval BUILD_OPTS ::= --tag $(IMAGE_NAME)) +ifeq (${TAG_IMAGE},true) # if TAG_IMAGE + $(eval BUILD_OPTS += --tag $(IMAGE_NAME):latest) +ifneq (${GIT_HASH},) + $(eval BUILD_OPTS += --tag $(IMAGE_NAME):$(GIT_HASH)) +endif +ifneq (${GIT_NAME},) + $(eval BUILD_OPTS += --tag $(IMAGE_NAME):$(GIT_NAME)) +endif +endif # endif TAG_IMAGE +ifeq ($(PUSH_IMAGE),true) + $(eval BUILD_OPTS += --push) +endif +ifneq ($(BUILD_PLATFORMS),) + $(eval BUILD_OPTS += --platform $(BUILD_PLATFORMS)) +endif + @# Add --pull so that we are using the latest base image. + @# The build context is the parent directory + docker build --pull --ssh=default \ + --file ./cmd/$(TARGET)/Dockerfile \ + $(BUILD_OPTS) \ + --build-arg GIT_HASH=$(GIT_HASH) ${BUILD_CTX} diff --git a/custombuild/Makefile b/custombuild/Makefile index 6548cadb77..1f4fb59183 100644 --- a/custombuild/Makefile +++ b/custombuild/Makefile @@ -1,71 +1,5 @@ -# The use of variables -# -# We use simply expanded variables in this Makefile. -# -# This means -# 1. You use ::= instead of = because = defines a recursively expanded variable. -# See https://www.gnu.org/software/make/manual/html_node/Simple-Assignment.html -# 2. You use ::= instead of := because ::= is a POSIX standard. -# See https://www.gnu.org/software/make/manual/html_node/Simple-Assignment.html -# 3. You do not use ?= because it is shorthand to define a recursively expanded variable. -# See https://www.gnu.org/software/make/manual/html_node/Conditional-Assignment.html -# You should use the long form documented in the above link instead. -# 4. When you override a variable in the command line, as documented in https://www.gnu.org/software/make/manual/html_node/Overriding.html -# you specify the variable with ::= instead of = or := -# If you fail to do so, the variable becomes recursively expanded variable accidentally. -# -# GIT_NAME could be empty. -ifeq ($(origin GIT_NAME), undefined) - GIT_NAME ::= $(shell git describe --exact-match 2>/dev/null) -endif -ifeq ($(origin GIT_HASH), undefined) - GIT_HASH ::= git-$(shell git rev-parse --short=12 HEAD) -endif -ifeq ($(origin LDFLAGS), undefined) - LDFLAGS ::= "-X github.com/authgear/authgear-server/pkg/version.Version=${GIT_HASH}" -endif +CMD_AUTHGEAR ::= authgearx +CMD_PORTAL ::= portalx +BUILD_CTX ::= ../ -# osusergo: https://godoc.org/github.com/golang/go/src/os/user -# netgo: https://golang.org/doc/go1.5#net -# static_build: https://github.com/golang/go/issues/26492#issuecomment-635563222 -# The binary is static on Linux only. It is not static on macOS. -# timetzdata: https://golang.org/doc/go1.15#time/tzdata -GO_BUILD_TAGS ::= osusergo netgo static_build timetzdata -GO_RUN_TAGS ::= - -.PHONY: start -start: - go run -tags "$(GO_RUN_TAGS)" -ldflags ${LDFLAGS} ./cmd/authgearx start - -.PHONY: start-portal -start-portal: - go run -tags "$(GO_RUN_TAGS)" -ldflags ${LDFLAGS} ./cmd/portalx start - -.PHONY: build -build: - go build -o $(BIN_NAME) -tags "$(GO_BUILD_TAGS)" -ldflags ${LDFLAGS} ./cmd/$(TARGET) - -.PHONY: build-image -build-image: - $(eval BUILD_OPTS ::= --tag $(IMAGE_NAME)) -ifeq (${TAG_IMAGE},true) # if TAG_IMAGE - $(eval BUILD_OPTS += --tag $(IMAGE_NAME):latest) -ifneq (${GIT_HASH},) - $(eval BUILD_OPTS += --tag $(IMAGE_NAME):$(GIT_HASH)) -endif -ifneq (${GIT_NAME},) - $(eval BUILD_OPTS += --tag $(IMAGE_NAME):$(GIT_NAME)) -endif -endif # endif TAG_IMAGE -ifeq ($(PUSH_IMAGE),true) - $(eval BUILD_OPTS += --push) -endif -ifneq ($(BUILD_PLATFORMS),) - $(eval BUILD_OPTS += --platform $(BUILD_PLATFORMS)) -endif - @# Add --pull so that we are using the latest base image. - @# The build context is the parent directory - docker build --pull --ssh=default \ - --file ./cmd/$(TARGET)/Dockerfile \ - $(BUILD_OPTS) \ - --build-arg GIT_HASH=$(GIT_HASH) ../ +include ../common.mk From a3a365ebd292b56fd5b496ee06019cada2ec67e6 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Fri, 13 Dec 2024 16:27:20 +0800 Subject: [PATCH 10/40] Extract go-mod-outdated to another file --- Makefile | 8 +------- custombuild/Makefile | 1 + e2e/Makefile | 2 ++ k6/Makefile | 2 ++ scripts/make/go-mod-outdated.mk | 6 ++++++ 5 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 scripts/make/go-mod-outdated.mk diff --git a/Makefile b/Makefile index c592049477..7130103533 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ CMD_PORTAL ::= portal BUILD_CTX ::= . include ./common.mk +include ./scripts/make/go-mod-outdated.mk .PHONY: authgearonce-start authgearonce-start: GO_RUN_TAGS += authgearonce @@ -29,13 +30,6 @@ vendor: $(MAKE) authui $(MAKE) portal -.PHONY: go-mod-outdated -go-mod-outdated: - # https://stackoverflow.com/questions/55866604/whats-the-go-mod-equivalent-of-npm-outdated - # Since go 1.21, this command will exit 2 when one of the dependencies require a go version newer than us. - # This implies we have to use the latest verion of Go whenever possible. - go list -u -m -f '{{if .Update}}{{if not .Indirect}}{{.}}{{end}}{{end}}' all - .PHONY: ensure-important-modules-up-to-date ensure-important-modules-up-to-date: # If grep matches something, it exits 0, otherwise it exits 1. diff --git a/custombuild/Makefile b/custombuild/Makefile index 1f4fb59183..5d480fb7a5 100644 --- a/custombuild/Makefile +++ b/custombuild/Makefile @@ -3,3 +3,4 @@ CMD_PORTAL ::= portalx BUILD_CTX ::= ../ include ../common.mk +include ../scripts/make/go-mod-outdated.mk diff --git a/e2e/Makefile b/e2e/Makefile index cfe520e937..21c2849b78 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -1,3 +1,5 @@ +include ../scripts/make/go-mod-outdated.mk + .PHONY: ci ci: @# Check if any `focus: true` is present in the test files diff --git a/k6/Makefile b/k6/Makefile index 7ada571185..f4e2519d7d 100644 --- a/k6/Makefile +++ b/k6/Makefile @@ -1,3 +1,5 @@ +include ../scripts/make/go-mod-outdated.mk + #export K6_HTTP_DEBUG=true # We cannot use K6_DURATION and K6_VUS because # using them will make k6 to ignore scenarios. diff --git a/scripts/make/go-mod-outdated.mk b/scripts/make/go-mod-outdated.mk new file mode 100644 index 0000000000..94cbf676b0 --- /dev/null +++ b/scripts/make/go-mod-outdated.mk @@ -0,0 +1,6 @@ +.PHONY: go-mod-outdated +go-mod-outdated: + # https://stackoverflow.com/questions/55866604/whats-the-go-mod-equivalent-of-npm-outdated + # Since go 1.21, this command will exit 2 when one of the dependencies require a go version newer than us. + # This implies we have to use the latest verion of Go whenever possible. + go list -u -m -f '{{if .Update}}{{if not .Indirect}}{{.}}{{end}}{{end}}' all From 7fa2269e544eeb29d3639bd28b20715d6acceeb7 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Fri, 13 Dec 2024 16:29:04 +0800 Subject: [PATCH 11/40] Extract govulncheck to another file --- Makefile | 1 + custombuild/Makefile | 1 + e2e/Makefile | 1 + k6/Makefile | 1 + scripts/make/govulncheck.mk | 3 +++ 5 files changed, 7 insertions(+) create mode 100644 scripts/make/govulncheck.mk diff --git a/Makefile b/Makefile index 7130103533..9a96c900d1 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ BUILD_CTX ::= . include ./common.mk include ./scripts/make/go-mod-outdated.mk +include ./scripts/make/govulncheck.mk .PHONY: authgearonce-start authgearonce-start: GO_RUN_TAGS += authgearonce diff --git a/custombuild/Makefile b/custombuild/Makefile index 5d480fb7a5..d498461315 100644 --- a/custombuild/Makefile +++ b/custombuild/Makefile @@ -4,3 +4,4 @@ BUILD_CTX ::= ../ include ../common.mk include ../scripts/make/go-mod-outdated.mk +include ../scripts/make/govulncheck.mk diff --git a/e2e/Makefile b/e2e/Makefile index 21c2849b78..ec0e055346 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -1,4 +1,5 @@ include ../scripts/make/go-mod-outdated.mk +include ../scripts/make/govulncheck.mk .PHONY: ci ci: diff --git a/k6/Makefile b/k6/Makefile index f4e2519d7d..ff8193d643 100644 --- a/k6/Makefile +++ b/k6/Makefile @@ -1,4 +1,5 @@ include ../scripts/make/go-mod-outdated.mk +include ../scripts/make/govulncheck.mk #export K6_HTTP_DEBUG=true # We cannot use K6_DURATION and K6_VUS because diff --git a/scripts/make/govulncheck.mk b/scripts/make/govulncheck.mk new file mode 100644 index 0000000000..53fdb95d68 --- /dev/null +++ b/scripts/make/govulncheck.mk @@ -0,0 +1,3 @@ +.PHONY: govulncheck +govulncheck: + govulncheck -show traces,version,verbose ./... From 67e74ea31c015d39fa4913279a763d3d8149069b Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Fri, 13 Dec 2024 20:04:00 +0800 Subject: [PATCH 12/40] Build different arch image parallelly --- .github/workflows/ci.yaml | 61 +++++++++++++++++++++++++++++++++++++-- common.mk | 45 +++++++++++++++++++++-------- 2 files changed, 91 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 43ac8cb7ed..011fd85dd3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -223,7 +223,7 @@ jobs: safe_image_name=$(echo -n "$image" | tr '/:' '_') docker save "$image" -o ~/.cache/images/"$safe_image_name".tar done - authgear-image: + authgear-image-amd64: if: ${{ github.repository != 'oursky/authgear-server' }} runs-on: ubuntu-24.04 needs: ["authgear-test", "authui-test"] @@ -231,7 +231,7 @@ jobs: TARGET: authgear IMAGE_NAME: quay.io/theauthgear/authgear-server PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - BUILD_PLATFORMS: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}" + BUILD_ARCH: amd64 steps: - uses: actions/checkout@v4 - name: Install qemu for multi arch build @@ -249,7 +249,62 @@ jobs: DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} run: | printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make build-image BUILD_PLATFORMS=$BUILD_PLATFORMS PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME + - run: make build-image BUILD_ARCH=$BUILD_ARCH PUSH_IMAGE=$PUSH_IMAGE TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME + - name: docker logout + if: ${{ always() }} + run: | + docker logout quay.io + authgear-image-arm64: + if: ${{ github.repository != 'oursky/authgear-server' }} + runs-on: ubuntu-24.04 + needs: ["authgear-test", "authui-test"] + env: + TARGET: authgear + IMAGE_NAME: quay.io/theauthgear/authgear-server + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + BUILD_ARCH: arm64 + steps: + - uses: actions/checkout@v4 + - name: Install qemu for multi arch build + run: docker run --privileged --rm tonistiigi/binfmt --install all + - name: Setup container builder + run: | + docker buildx create \ + --name container-builder \ + --driver docker-container \ + --bootstrap --use + - name: docker login + if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} + env: + DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} + run: | + printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io + - run: make build-image BUILD_ARCH=$BUILD_ARCH PUSH_IMAGE=$PUSH_IMAGE TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME + - name: docker logout + if: ${{ always() }} + run: | + docker logout quay.io + + authgear-image: + if: ${{ github.repository != 'oursky/authgear-server' }} + runs-on: ubuntu-24.04 + needs: ["authgear-image-amd64", "authgear-image-arm64"] + env: + TARGET: authgear + IMAGE_NAME: quay.io/theauthgear/authgear-server + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + SOURCE_ARCHS: arm64,amd64 + steps: + - uses: actions/checkout@v4 + - name: docker login + if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} + env: + DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} + run: | + printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io + - run: make tag-image SOURCE_ARCHS=$SOURCE_ARCHS IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} run: | diff --git a/common.mk b/common.mk index e0a896e253..e76f275143 100644 --- a/common.mk +++ b/common.mk @@ -47,27 +47,48 @@ start-portal: build: go build -o $(BIN_NAME) -tags "$(GO_BUILD_TAGS)" -ldflags ${LDFLAGS} ./cmd/$(TARGET) + + .PHONY: build-image build-image: - $(eval BUILD_OPTS ::= --tag $(IMAGE_NAME)) -ifeq (${TAG_IMAGE},true) # if TAG_IMAGE - $(eval BUILD_OPTS += --tag $(IMAGE_NAME):latest) -ifneq (${GIT_HASH},) - $(eval BUILD_OPTS += --tag $(IMAGE_NAME):$(GIT_HASH)) + $(eval IMAGE_TAG_BASE ::= $(IMAGE_NAME):$(GIT_HASH)) + $(eval BUILD_OPTS ::= ) +ifeq ($(BUILD_ARCH),amd64) + $(eval BUILD_OPTS += --platform linux/$(BUILD_ARCH) ) + $(eval BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-amd64 ) +else ifeq ($(BUILD_ARCH),arm64) + $(eval BUILD_OPTS += --platform linux/$(BUILD_ARCH) ) + $(eval BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-arm64 ) +else + $(eval BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-$(BUILD_ARCH)-unknown ) endif -ifneq (${GIT_NAME},) - $(eval BUILD_OPTS += --tag $(IMAGE_NAME):$(GIT_NAME)) -endif -endif # endif TAG_IMAGE ifeq ($(PUSH_IMAGE),true) $(eval BUILD_OPTS += --push) endif -ifneq ($(BUILD_PLATFORMS),) - $(eval BUILD_OPTS += --platform $(BUILD_PLATFORMS)) +ifeq ($(EXTRA_BUILD_OPTS),true) + $(eval BUILD_OPTS += $(EXTRA_BUILD_OPTS)) endif @# Add --pull so that we are using the latest base image. @# The build context is the parent directory - docker build --pull --ssh=default \ + docker build --pull \ --file ./cmd/$(TARGET)/Dockerfile \ $(BUILD_OPTS) \ --build-arg GIT_HASH=$(GIT_HASH) ${BUILD_CTX} + +.PHONY: tag-image +tag-image: + $(eval IMAGE_SOURCES ::= ) + $(eval TAGS ::= --tag $(IMAGE_NAME):latest ) + $(eval TAGS += --tag $(IMAGE_NAME):$(GIT_HASH)) +ifneq (${GIT_NAME},) + $(eval TAGS += --tag $(IMAGE_NAME):$(GIT_NAME)) +endif +ifneq ($(findstring amd64,$(SOURCE_ARCHS)),) + $(eval IMAGE_SOURCES += $(IMAGE_NAME):$(GIT_HASH)-amd64 ) +endif +ifneq ($(findstring arm64,$(SOURCE_ARCHS)),) + $(eval IMAGE_SOURCES += $(IMAGE_NAME):$(GIT_HASH)-arm64 ) +endif + docker buildx imagetools create \ + $(TAGS) \ + $(IMAGE_SOURCES) From 7fa51ed7ed900d9bdda552889d3936ffa47241f6 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Fri, 13 Dec 2024 21:26:58 +0800 Subject: [PATCH 13/40] Build different arch portal image parallelly --- .github/workflows/ci.yaml | 64 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 011fd85dd3..5b64c84eb3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -115,7 +115,7 @@ jobs: working-directory: ./portal if: ${{ !cancelled() }} - portal-image: + portal-image-amd64: if: ${{ github.repository != 'oursky/authgear-server' }} runs-on: ubuntu-24.04 needs: portal-test @@ -123,7 +123,7 @@ jobs: TARGET: portal IMAGE_NAME: quay.io/theauthgear/authgear-portal PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - BUILD_PLATFORMS: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}" + BUILD_ARCH: amd64 steps: - uses: actions/checkout@v4 - name: Install qemu for multi arch build @@ -141,12 +141,70 @@ jobs: DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} run: | printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make build-image BUILD_PLATFORMS=$BUILD_PLATFORMS PUSH_IMAGE=$PUSH_IMAGE TAG_IMAGE=true TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME + - run: make build-image BUILD_ARCH=$BUILD_ARCH PUSH_IMAGE=$PUSH_IMAGE TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} run: | docker logout quay.io + portal-image-arm64: + if: ${{ github.repository != 'oursky/authgear-server' }} + runs-on: ubuntu-24.04 + needs: portal-test + env: + TARGET: portal + IMAGE_NAME: quay.io/theauthgear/authgear-portal + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + BUILD_ARCH: arm64 + steps: + - uses: actions/checkout@v4 + - name: Install qemu for multi arch build + run: docker run --privileged --rm tonistiigi/binfmt --install all + - name: Setup container builder + run: | + docker buildx create \ + --name container-builder \ + --driver docker-container \ + --bootstrap --use + - name: docker login + if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} + env: + DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} + run: | + printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io + - run: make build-image BUILD_ARCH=$BUILD_ARCH PUSH_IMAGE=$PUSH_IMAGE TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME + - name: docker logout + if: ${{ always() }} + run: | + docker logout quay.io + + portal-image: + if: ${{ github.repository != 'oursky/authgear-server' }} + runs-on: ubuntu-24.04 + needs: ["portal-image-amd64", "portal-image-arm64"] + env: + TARGET: authgear + IMAGE_NAME: quay.io/theauthgear/authgear-portal + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + SOURCE_ARCHS: arm64,amd64 + steps: + - uses: actions/checkout@v4 + - name: docker login + if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} + env: + DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} + run: | + printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io + - run: make tag-image SOURCE_ARCHS=$SOURCE_ARCHS IMAGE_NAME=$IMAGE_NAME + - name: docker logout + if: ${{ always() }} + run: | + docker logout quay.io + + + authgear-test: if: ${{ github.repository != 'oursky/authgear-server' }} runs-on: ubuntu-24.04 From 5bf0931d7c4d0caaa9f339294cd5c1f972273061 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Fri, 13 Dec 2024 21:31:41 +0800 Subject: [PATCH 14/40] Update custom build workflows --- .github/workflows/custom-build.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/custom-build.yaml b/.github/workflows/custom-build.yaml index f3882366b2..3fd19fc10d 100644 --- a/.github/workflows/custom-build.yaml +++ b/.github/workflows/custom-build.yaml @@ -78,7 +78,8 @@ jobs: REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} run: | - make -C custombuild build-image TARGET=authgearx BUILD_PLATFORMS=linux/amd64,linux/arm64 TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 with: @@ -95,7 +96,8 @@ jobs: REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} run: | - make -C custombuild build-image TARGET=authgearx BUILD_PLATFORMS=linux/amd64,linux/arm64 TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" - name: Clean up SSH key if: ${{ always() }} @@ -146,7 +148,8 @@ jobs: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} run: | - make -C custombuild build-image TARGET=portalx BUILD_PLATFORMS=linux/amd64,linux/arm64 TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 with: @@ -163,7 +166,8 @@ jobs: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} run: | - make -C custombuild build-image TARGET=portalx BUILD_PLATFORMS=linux/amd64,linux/arm64 TAG_IMAGE=true PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" - name: Clean up SSH key if: ${{ always() }} From 00fab940d9fe3446afe011893ba14ad1edd7391a Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 11:35:33 +0800 Subject: [PATCH 15/40] Do not build arm64 image in pr and do not tag image --- .github/workflows/ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5b64c84eb3..5b83deac76 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -148,7 +148,7 @@ jobs: docker logout quay.io portal-image-arm64: - if: ${{ github.repository != 'oursky/authgear-server' }} + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} runs-on: ubuntu-24.04 needs: portal-test env: @@ -180,7 +180,7 @@ jobs: docker logout quay.io portal-image: - if: ${{ github.repository != 'oursky/authgear-server' }} + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} runs-on: ubuntu-24.04 needs: ["portal-image-amd64", "portal-image-arm64"] env: @@ -313,7 +313,7 @@ jobs: run: | docker logout quay.io authgear-image-arm64: - if: ${{ github.repository != 'oursky/authgear-server' }} + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} runs-on: ubuntu-24.04 needs: ["authgear-test", "authui-test"] env: @@ -345,7 +345,7 @@ jobs: docker logout quay.io authgear-image: - if: ${{ github.repository != 'oursky/authgear-server' }} + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} runs-on: ubuntu-24.04 needs: ["authgear-image-amd64", "authgear-image-arm64"] env: From 177d6f1f732b28aaa5468538a4ce2186bf42c107 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 14:43:37 +0800 Subject: [PATCH 16/40] Use composite action to build image --- .github/actions/build-image/action.yaml | 52 ++++++++++ .github/workflows/ci.yaml | 128 +++++++----------------- 2 files changed, 88 insertions(+), 92 deletions(-) create mode 100644 .github/actions/build-image/action.yaml diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml new file mode 100644 index 0000000000..d2e6f0fb1b --- /dev/null +++ b/.github/actions/build-image/action.yaml @@ -0,0 +1,52 @@ +name: Build image +inputs: + target: + required: true + image_name: + required: true + push_image: + required: true + build_arch: + required: true + docker_registry: + required: false + docker_username: + required: false + docker_password: + required: false +runs: + using: "composite" + steps: + - name: Install qemu for multi arch build + shell: bash + run: docker run --privileged --rm tonistiigi/binfmt --install all + - name: Setup container builder + shell: bash + run: | + docker buildx create \ + --name container-builder \ + --driver docker-container \ + --bootstrap --use + - name: docker login + if: ${{ inputs.push_image == 'true' }} + env: + DOCKER_USERNAME: ${{ inputs.docker_username }} + DOCKER_PASSWORD: ${{ inputs.docker_password }} + DOCKER_REGISTRY: ${{ inputs.docker_registry }} + shell: bash + run: | + printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" $DOCKER_REGISTRY + - run: make build-image BUILD_ARCH=$BUILD_ARCH PUSH_IMAGE=$PUSH_IMAGE TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME + shell: bash + env: + TARGET: ${{ inputs.target }} + IMAGE_NAME: ${{ inputs.image_name }} + PUSH_IMAGE: ${{ inputs.push_image }} + BUILD_ARCH: ${{ inputs.build_arch }} + - name: docker logout + if: ${{ always() }} + env: + DOCKER_REGISTRY: ${{ inputs.docker_registry }} + shell: bash + run: | + docker logout $DOCKER_REGISTRY diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5b83deac76..f9dd021deb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -120,64 +120,36 @@ jobs: runs-on: ubuntu-24.04 needs: portal-test env: - TARGET: portal - IMAGE_NAME: quay.io/theauthgear/authgear-portal PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - BUILD_ARCH: amd64 steps: - uses: actions/checkout@v4 - - name: Install qemu for multi arch build - run: docker run --privileged --rm tonistiigi/binfmt --install all - - name: Setup container builder - run: | - docker buildx create \ - --name container-builder \ - --driver docker-container \ - --bootstrap --use - - name: docker login - if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} - env: - DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} - run: | - printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make build-image BUILD_ARCH=$BUILD_ARCH PUSH_IMAGE=$PUSH_IMAGE TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - - name: docker logout - if: ${{ always() }} - run: | - docker logout quay.io + - uses: ./.github/actions/build-image + with: + target: portal + image_name: quay.io/theauthgear/authgear-portal + push_image: "${{ env.PUSH_IMAGE }}" + build_arch: amd64 + docker_registry: quay.io + docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" + docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" portal-image-arm64: if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} runs-on: ubuntu-24.04 needs: portal-test env: - TARGET: portal - IMAGE_NAME: quay.io/theauthgear/authgear-portal PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - BUILD_ARCH: arm64 steps: - uses: actions/checkout@v4 - - name: Install qemu for multi arch build - run: docker run --privileged --rm tonistiigi/binfmt --install all - - name: Setup container builder - run: | - docker buildx create \ - --name container-builder \ - --driver docker-container \ - --bootstrap --use - - name: docker login - if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} - env: - DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} - run: | - printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make build-image BUILD_ARCH=$BUILD_ARCH PUSH_IMAGE=$PUSH_IMAGE TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - - name: docker logout - if: ${{ always() }} - run: | - docker logout quay.io + - uses: ./.github/actions/build-image + with: + target: portal + image_name: quay.io/theauthgear/authgear-portal + push_image: "${{ env.PUSH_IMAGE }}" + build_arch: arm64 + docker_registry: quay.io + docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" + docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" portal-image: if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} @@ -286,63 +258,35 @@ jobs: runs-on: ubuntu-24.04 needs: ["authgear-test", "authui-test"] env: - TARGET: authgear - IMAGE_NAME: quay.io/theauthgear/authgear-server PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - BUILD_ARCH: amd64 steps: - uses: actions/checkout@v4 - - name: Install qemu for multi arch build - run: docker run --privileged --rm tonistiigi/binfmt --install all - - name: Setup container builder - run: | - docker buildx create \ - --name container-builder \ - --driver docker-container \ - --bootstrap --use - - name: docker login - if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} - env: - DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} - run: | - printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make build-image BUILD_ARCH=$BUILD_ARCH PUSH_IMAGE=$PUSH_IMAGE TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - - name: docker logout - if: ${{ always() }} - run: | - docker logout quay.io + - uses: ./.github/actions/build-image + with: + target: authgear + image_name: quay.io/theauthgear/authgear-server + push_image: "${{ env.PUSH_IMAGE }}" + build_arch: amd64 + docker_registry: quay.io + docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" + docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" authgear-image-arm64: if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} runs-on: ubuntu-24.04 needs: ["authgear-test", "authui-test"] env: - TARGET: authgear - IMAGE_NAME: quay.io/theauthgear/authgear-server PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - BUILD_ARCH: arm64 steps: - uses: actions/checkout@v4 - - name: Install qemu for multi arch build - run: docker run --privileged --rm tonistiigi/binfmt --install all - - name: Setup container builder - run: | - docker buildx create \ - --name container-builder \ - --driver docker-container \ - --bootstrap --use - - name: docker login - if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} - env: - DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} - run: | - printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make build-image BUILD_ARCH=$BUILD_ARCH PUSH_IMAGE=$PUSH_IMAGE TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME - - name: docker logout - if: ${{ always() }} - run: | - docker logout quay.io + - uses: ./.github/actions/build-image + with: + target: authgear + image_name: quay.io/theauthgear/authgear-server + push_image: "${{ env.PUSH_IMAGE }}" + build_arch: arm64 + docker_registry: quay.io + docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" + docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" authgear-image: if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} From a45af4ee7a9b50eca5a8d2dbc2ff223461b3d960 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 15:18:15 +0800 Subject: [PATCH 17/40] Fix custom build missing ssh forwarding --- .github/workflows/custom-build.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/custom-build.yaml b/.github/workflows/custom-build.yaml index 3fd19fc10d..f3f46779e1 100644 --- a/.github/workflows/custom-build.yaml +++ b/.github/workflows/custom-build.yaml @@ -78,7 +78,7 @@ jobs: REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} run: | - make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" + make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" EXTRA_BUILD_OPTS="--ssh=default" make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 @@ -96,7 +96,7 @@ jobs: REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} run: | - make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" + make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" EXTRA_BUILD_OPTS="--ssh=default" make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" - name: Clean up SSH key @@ -148,7 +148,7 @@ jobs: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} run: | - make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" + make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" EXTRA_BUILD_OPTS="--ssh=default" make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 @@ -166,7 +166,7 @@ jobs: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} run: | - make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" + make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" EXTRA_BUILD_OPTS="--ssh=default" make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" - name: Clean up SSH key From 0b63af2ade3b925ebf5dcb6c22288c9a98bc275b Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 15:27:33 +0800 Subject: [PATCH 18/40] Fix custom build scripts --- .github/workflows/custom-build.yaml | 8 ++++---- .github/workflows/oursky.yaml | 27 +++++++++++++-------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/workflows/custom-build.yaml b/.github/workflows/custom-build.yaml index f3f46779e1..73cd5c33c9 100644 --- a/.github/workflows/custom-build.yaml +++ b/.github/workflows/custom-build.yaml @@ -79,7 +79,7 @@ jobs: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} run: | make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 with: @@ -97,7 +97,7 @@ jobs: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} run: | make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" - name: Clean up SSH key if: ${{ always() }} @@ -149,7 +149,7 @@ jobs: REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} run: | make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" - uses: oursky/google-github-actions-auth@v2 with: @@ -167,7 +167,7 @@ jobs: REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} run: | make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" - name: Clean up SSH key if: ${{ always() }} diff --git a/.github/workflows/oursky.yaml b/.github/workflows/oursky.yaml index ea0b997fda..de60fdc898 100644 --- a/.github/workflows/oursky.yaml +++ b/.github/workflows/oursky.yaml @@ -25,7 +25,12 @@ jobs: printf "$AUTHGEAR_PRIVATE_DEPLOY_KEY" | base64 --decode | ssh-add - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - run: make -C custombuild build-image TARGET=authgearx IMAGE_NAME=authgear-server-custom + - name: Build and Push + env: + REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} + run: | + make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" EXTRA_BUILD_OPTS="--ssh=default" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-server" - name: Clean up SSH key if: ${{ always() }} run: | @@ -33,12 +38,6 @@ jobs: ssh-agent -k echo "SSH_AUTH_SOCK=" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=" >> "$GITHUB_ENV" - - name: Push - env: - REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} - run: | - make -C custombuild tag-image IMAGE_NAME=authgear-server-custom REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-server" - make -C custombuild push-image REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-server" portal-image-custom: if: ${{ github.repository == 'oursky/authgear-server' }} @@ -56,7 +55,12 @@ jobs: printf "$AUTHGEAR_PRIVATE_DEPLOY_KEY" | base64 --decode | ssh-add - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - run: make -C custombuild build-image TARGET=portalx IMAGE_NAME=authgear-portal-custom + - name: Build and Push + env: + REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} + run: | + make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" EXTRA_BUILD_OPTS="--ssh=default" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-portal" - name: Clean up SSH key if: ${{ always() }} run: | @@ -64,9 +68,4 @@ jobs: ssh-agent -k echo "SSH_AUTH_SOCK=" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=" >> "$GITHUB_ENV" - - name: Push - env: - REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} - run: | - make -C custombuild tag-image IMAGE_NAME=authgear-portal-custom REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-portal" - make -C custombuild push-image REMOTE_IMAGE_NAME="$REPO_PREFIX/authgear-portal" + From 78a96c95fe4a6523156e0c36100e55ab489886e4 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 15:45:11 +0800 Subject: [PATCH 19/40] Reduce code duplication in custom build by composite action --- .../actions/build-custom-image/action.yaml | 42 +++++++++ .github/workflows/custom-build.yaml | 92 ++++++------------- 2 files changed, 70 insertions(+), 64 deletions(-) create mode 100644 .github/actions/build-custom-image/action.yaml diff --git a/.github/actions/build-custom-image/action.yaml b/.github/actions/build-custom-image/action.yaml new file mode 100644 index 0000000000..a767d53a60 --- /dev/null +++ b/.github/actions/build-custom-image/action.yaml @@ -0,0 +1,42 @@ +name: Build custom image +inputs: + target: + required: true + image_name: + required: true + gcp_project_id: + required: true + gcp_workload_identity_provider: + required: true + docker_repo: + required: false +runs: + using: "composite" + steps: + - uses: oursky/google-github-actions-auth@v2 + with: + project_id: ${{ inputs.gcp_project_id }} + workload_identity_provider: ${{ inputs.gcp_workload_identity_provider }} + - uses: oursky/google-github-actions-setup-gcloud@v2 + - name: Configure docker + env: + REPO: ${{ inputs.docker_repo }} + shell: bash + run: | + gcloud auth configure-docker "$REPO" + - name: Build and Push + env: + TARGET: ${{ inputs.target }} + IMAGE_NAME: ${{ inputs.image_name }} + REPO: ${{ inputs.docker_repo }} + shell: bash + run: | + make -C custombuild build-image TARGET=$TARGET BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME=$IMAGE_NAME EXTRA_BUILD_OPTS="--ssh=default" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME" + - name: docker logout + if: ${{ always() }} + env: + REPO: ${{ inputs.docker_repo }} + shell: bash + run: | + docker logout "$REPO" diff --git a/.github/workflows/custom-build.yaml b/.github/workflows/custom-build.yaml index 73cd5c33c9..b86976acbc 100644 --- a/.github/workflows/custom-build.yaml +++ b/.github/workflows/custom-build.yaml @@ -63,41 +63,23 @@ jobs: echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - uses: oursky/google-github-actions-auth@v2 + - name: Build and push to HK + uses: ./.github/actions/build-custom-image with: - project_id: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_PROJECT_ID_HK }} - workload_identity_provider: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_WORKLOAD_IDENTITY_PROVIDER_HK }} - - uses: oursky/google-github-actions-setup-gcloud@v2 - - name: Configure docker - env: - REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} - run: | - gcloud auth configure-docker "$REPO" - - name: Push to HK - env: - REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} - REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} - run: | - make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + target: authgearx + image_name: "${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }}/authgear-server" + gcp_project_id: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_PROJECT_ID_HK }} + gcp_workload_identity_provider: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_WORKLOAD_IDENTITY_PROVIDER_HK }} + docker_repo: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} - - uses: oursky/google-github-actions-auth@v2 + - name: Build and push to US + uses: ./.github/actions/build-custom-image with: - project_id: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_PROJECT_ID_US }} - workload_identity_provider: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_WORKLOAD_IDENTITY_PROVIDER_US }} - - uses: oursky/google-github-actions-setup-gcloud@v2 - - name: Configure docker - env: - REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} - run: | - gcloud auth configure-docker "$REPO" - - name: Push to US - env: - REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} - REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} - run: | - make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-server" || docker logout "$REPO" + target: authgearx + image_name: "${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }}/authgear-server" + gcp_project_id: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_PROJECT_ID_US }} + gcp_workload_identity_provider: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_WORKLOAD_IDENTITY_PROVIDER_US }} + docker_repo: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} - name: Clean up SSH key if: ${{ always() }} @@ -133,41 +115,23 @@ jobs: echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - uses: oursky/google-github-actions-auth@v2 + - name: Build and push to HK + uses: ./.github/actions/build-custom-image with: - project_id: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_PROJECT_ID_HK }} - workload_identity_provider: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_WORKLOAD_IDENTITY_PROVIDER_HK }} - - uses: oursky/google-github-actions-setup-gcloud@v2 - - name: Configure docker - env: - REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} - run: | - gcloud auth configure-docker "$REPO" - - name: Push to HK - env: - REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }} - REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} - run: | - make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + target: portalx + image_name: "${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_HK }}/authgear-portal" + gcp_project_id: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_PROJECT_ID_HK }} + gcp_workload_identity_provider: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_WORKLOAD_IDENTITY_PROVIDER_HK }} + docker_repo: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_HK }} - - uses: oursky/google-github-actions-auth@v2 + - name: Build and push to US + uses: ./.github/actions/build-custom-image with: - project_id: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_PROJECT_ID_US }} - workload_identity_provider: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_WORKLOAD_IDENTITY_PROVIDER_US }} - - uses: oursky/google-github-actions-setup-gcloud@v2 - - name: Configure docker - env: - REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} - run: | - gcloud auth configure-docker "$REPO" - - name: Push to US - env: - REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }} - REPO: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} - run: | - make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-portal" || docker logout "$REPO" + target: portalx + image_name: "${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX_US }}/authgear-portal" + gcp_project_id: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_PROJECT_ID_US }} + gcp_workload_identity_provider: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_GOOGLE_WORKLOAD_IDENTITY_PROVIDER_US }} + docker_repo: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_US }} - name: Clean up SSH key if: ${{ always() }} From a8399c6a8b5c19e88a88372c699512a077957e5b Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 15:56:55 +0800 Subject: [PATCH 20/40] Fix EXTRA_BUILD_OPTS is not added to build options --- common.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.mk b/common.mk index e76f275143..e23bdeefdd 100644 --- a/common.mk +++ b/common.mk @@ -65,7 +65,7 @@ endif ifeq ($(PUSH_IMAGE),true) $(eval BUILD_OPTS += --push) endif -ifeq ($(EXTRA_BUILD_OPTS),true) +ifneq ($(EXTRA_BUILD_OPTS),) $(eval BUILD_OPTS += $(EXTRA_BUILD_OPTS)) endif @# Add --pull so that we are using the latest base image. From 27e29527436d417b2822ec30b1688e98e205eae2 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 15:57:33 +0800 Subject: [PATCH 21/40] Remove duplicated govulncheck --- Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Makefile b/Makefile index 9a96c900d1..6f17cc8ff6 100644 --- a/Makefile +++ b/Makefile @@ -81,10 +81,6 @@ fmt: find ./devtools ./pkg ./cmd ./e2e -name '*.go' -not -name 'wire_gen.go' -not -name '*_mock_test.go' | sort | xargs goimports -w -format-only -local github.com/authgear/authgear-server $(MAKE) sort-translations -.PHONY: govulncheck -govulncheck: - govulncheck -show traces,version,verbose ./... - .PHONY: binary binary: GO_BUILD_TAGS += authgearlite binary: From 24297f25e3961960f95790fd67d9ac04b4935ed8 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 16:05:35 +0800 Subject: [PATCH 22/40] Remove extra quote in script --- .github/actions/build-custom-image/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-custom-image/action.yaml b/.github/actions/build-custom-image/action.yaml index a767d53a60..bde7c0f034 100644 --- a/.github/actions/build-custom-image/action.yaml +++ b/.github/actions/build-custom-image/action.yaml @@ -32,7 +32,7 @@ runs: shell: bash run: | make -C custombuild build-image TARGET=$TARGET BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME=$IMAGE_NAME EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME" + make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} env: From dc7cb1a778a986a38ba47b2dcc1f37bfd2d2718b Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 16:33:10 +0800 Subject: [PATCH 23/40] Reorganize workflows --- .github/workflows/ci-branchs.yaml | 21 ++ .github/workflows/ci-prs.yaml | 15 ++ .github/workflows/ci.yaml | 335 ----------------------------- .github/workflows/run-builds.yaml | 121 +++++++++++ .github/workflows/run-checks.yaml | 185 ++++++++++++++++ .github/workflows/run-release.yaml | 26 +++ 6 files changed, 368 insertions(+), 335 deletions(-) create mode 100644 .github/workflows/ci-branchs.yaml create mode 100644 .github/workflows/ci-prs.yaml delete mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/run-builds.yaml create mode 100644 .github/workflows/run-checks.yaml create mode 100644 .github/workflows/run-release.yaml diff --git a/.github/workflows/ci-branchs.yaml b/.github/workflows/ci-branchs.yaml new file mode 100644 index 0000000000..dfff0ec10a --- /dev/null +++ b/.github/workflows/ci-branchs.yaml @@ -0,0 +1,21 @@ +name: CI - Branches + +on: + push: + branches: + - '*' + - '!gh-pages' + tags: + - '*' + +jobs: + checks: + uses: ./.github/workflows/run-checks.yaml + builds: + needs: checks + uses: ./.github/workflows/run-builds.yaml + secrets: inherit + release: + needs: builds + uses: ./.github/workflows/run-release.yaml + secrets: inherit diff --git a/.github/workflows/ci-prs.yaml b/.github/workflows/ci-prs.yaml new file mode 100644 index 0000000000..3a413e56aa --- /dev/null +++ b/.github/workflows/ci-prs.yaml @@ -0,0 +1,15 @@ +name: CI - Pull Requests + +on: + pull_request: + branches: + - '*' + - '!gh-pages' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + checks: + uses: ./.github/workflows/run-checks.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index f9dd021deb..0000000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,335 +0,0 @@ -name: CI - -on: - push: - branches: - - '*' - - '!gh-pages' - tags: - - '*' - pull_request: - branches: - - '*' - - '!gh-pages' - -jobs: - authui-test: - if: ${{ github.repository != 'oursky/authgear-server' }} - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version-file: "./.tool-versions" - - run: npm ci - working-directory: ./authui - - name: Run npm audit - env: - AUDIT_LEVEL: ${{ vars.NPM_AUDIT_LEVEL }} - run: | - if [ -z "${AUDIT_LEVEL}" ]; then - npm audit - else - npm audit --audit-level="${AUDIT_LEVEL}" - fi - working-directory: ./authui - if: ${{ !cancelled() }} - - run: npm run typecheck - working-directory: ./authui - if: ${{ !cancelled() }} - - run: npm run eslint - working-directory: ./authui - if: ${{ !cancelled() }} - - run: npm run stylelint - working-directory: ./authui - if: ${{ !cancelled() }} - - run: npm run prettier - working-directory: ./authui - if: ${{ !cancelled() }} - - run: npm run test - working-directory: ./authui - if: ${{ !cancelled() }} - - run: make check-tidy - working-directory: ./authui - if: ${{ !cancelled() }} - - run: npm run build - working-directory: ./authui - if: ${{ !cancelled() }} - - run: npm run ensure-reproducible-build - working-directory: ./authui - if: ${{ !cancelled() }} - - run: npm run browserslist-coverage-lint - working-directory: ./authui - if: ${{ !cancelled() }} - - portal-test: - if: ${{ github.repository != 'oursky/authgear-server' }} - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/install-native-deps - - uses: actions/setup-node@v4 - with: - node-version-file: "./.tool-versions" - - run: npm ci - working-directory: ./portal - - name: Run npm audit - env: - AUDIT_LEVEL: ${{ vars.NPM_AUDIT_LEVEL }} - run: | - if [ -z "${AUDIT_LEVEL}" ]; then - npm audit - else - npm audit --audit-level="${AUDIT_LEVEL}" - fi - working-directory: ./portal - if: ${{ !cancelled() }} - - run: npm run typecheck - working-directory: ./portal - if: ${{ !cancelled() }} - - run: npm run eslint - working-directory: ./portal - if: ${{ !cancelled() }} - - run: npm run stylelint - working-directory: ./portal - if: ${{ !cancelled() }} - - run: npm run prettier - working-directory: ./portal - if: ${{ !cancelled() }} - - run: npm run test - working-directory: ./portal - if: ${{ !cancelled() }} - - run: npm run gentype - working-directory: ./portal - if: ${{ !cancelled() }} - - run: make check-tidy - working-directory: ./portal - if: ${{ !cancelled() }} - - run: npm run build - working-directory: ./portal - if: ${{ !cancelled() }} - - run: npm run ensure-reproducible-build - working-directory: ./portal - if: ${{ !cancelled() }} - - run: npm run browserslist-coverage-lint - working-directory: ./portal - if: ${{ !cancelled() }} - - portal-image-amd64: - if: ${{ github.repository != 'oursky/authgear-server' }} - runs-on: ubuntu-24.04 - needs: portal-test - env: - PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/build-image - with: - target: portal - image_name: quay.io/theauthgear/authgear-portal - push_image: "${{ env.PUSH_IMAGE }}" - build_arch: amd64 - docker_registry: quay.io - docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" - docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" - - portal-image-arm64: - if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} - runs-on: ubuntu-24.04 - needs: portal-test - env: - PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/build-image - with: - target: portal - image_name: quay.io/theauthgear/authgear-portal - push_image: "${{ env.PUSH_IMAGE }}" - build_arch: arm64 - docker_registry: quay.io - docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" - docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" - - portal-image: - if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} - runs-on: ubuntu-24.04 - needs: ["portal-image-amd64", "portal-image-arm64"] - env: - TARGET: authgear - IMAGE_NAME: quay.io/theauthgear/authgear-portal - PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - SOURCE_ARCHS: arm64,amd64 - steps: - - uses: actions/checkout@v4 - - name: docker login - if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} - env: - DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} - run: | - printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make tag-image SOURCE_ARCHS=$SOURCE_ARCHS IMAGE_NAME=$IMAGE_NAME - - name: docker logout - if: ${{ always() }} - run: | - docker logout quay.io - - - - authgear-test: - if: ${{ github.repository != 'oursky/authgear-server' }} - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/install-native-deps - - uses: actions/setup-go@v4 - with: - go-version-file: "./go.mod" - - uses: actions/setup-node@v4 - with: - node-version-file: "./.tool-versions" - - run: make vendor - - run: make check-dockerignore - if: ${{ !cancelled() }} - - run: make govulncheck - if: ${{ !cancelled() }} - # We cannot enable this until we move to mjml@5 - # https://github.com/mjmlio/mjml/issues/2589 - #- run: npm audit - # working-directory: ./scripts/npm - # if: ${{ !cancelled() }} - - run: make ensure-important-modules-up-to-date - if: ${{ !cancelled() }} - - run: make lint - if: ${{ !cancelled() }} - - run: make test - if: ${{ !cancelled() }} - - run: make fmt - if: ${{ !cancelled() }} - - run: make check-tidy - if: ${{ !cancelled() }} - - run: make -C packagetracker check-if-new-version-available - if: ${{ !cancelled() }} - - authgear-e2e: - if: ${{ github.repository != 'oursky/authgear-server' }} - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/install-native-deps - - - uses: actions/setup-go@v4 - with: - go-version-file: "./go.mod" - - run: make vendor - - # Cache docker images - - run: mkdir -p ~/.cache/images - - id: cache-docker-images - uses: actions/cache@v3 - with: - path: ~/.cache/images - key: ${{ runner.os }}-e2e-docker-images - continue-on-error: true - env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 - - name: Load docker image cache - if: ${{ steps.cache-docker-images.outputs.cache-hit == 'true' }} - run: | - set -x - for image in $(grep 'image: ' ./e2e/docker-compose.yaml | awk '{ print $2 }'); do - safe_image_name=$(echo -n "$image" | tr '/:' '_') - docker load -i ~/.cache/images/"$safe_image_name".tar || true - done - - run: make -C e2e ci - if: ${{ !cancelled() }} - env: - COMPOSE_INTERACTIVE_NO_CLI: 1 - - name: Save docker image cache - run: | - set -x - for image in $(grep 'image: ' ./e2e/docker-compose.yaml | awk '{ print $2 }'); do - safe_image_name=$(echo -n "$image" | tr '/:' '_') - docker save "$image" -o ~/.cache/images/"$safe_image_name".tar - done - authgear-image-amd64: - if: ${{ github.repository != 'oursky/authgear-server' }} - runs-on: ubuntu-24.04 - needs: ["authgear-test", "authui-test"] - env: - PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/build-image - with: - target: authgear - image_name: quay.io/theauthgear/authgear-server - push_image: "${{ env.PUSH_IMAGE }}" - build_arch: amd64 - docker_registry: quay.io - docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" - docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" - authgear-image-arm64: - if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} - runs-on: ubuntu-24.04 - needs: ["authgear-test", "authui-test"] - env: - PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/build-image - with: - target: authgear - image_name: quay.io/theauthgear/authgear-server - push_image: "${{ env.PUSH_IMAGE }}" - build_arch: arm64 - docker_registry: quay.io - docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" - docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" - - authgear-image: - if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} - runs-on: ubuntu-24.04 - needs: ["authgear-image-amd64", "authgear-image-arm64"] - env: - TARGET: authgear - IMAGE_NAME: quay.io/theauthgear/authgear-server - PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - SOURCE_ARCHS: arm64,amd64 - steps: - - uses: actions/checkout@v4 - - name: docker login - if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} - env: - DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} - run: | - printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make tag-image SOURCE_ARCHS=$SOURCE_ARCHS IMAGE_NAME=$IMAGE_NAME - - name: docker logout - if: ${{ always() }} - run: | - docker logout quay.io - - release: - if: ${{ github.repository != 'oursky/authgear-server' }} - runs-on: ubuntu-24.04 - needs: ["portal-test", "authgear-test"] - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/install-native-deps - - uses: actions/setup-go@v4 - with: - go-version-file: "./go.mod" - - uses: actions/setup-node@v4 - with: - node-version-file: "./.tool-versions" - - run: make vendor - - run: make binary - - uses: oursky/action-gh-release@v2 - if: ${{ github.repository == 'authgear/authgear-server' && startsWith(github.ref, 'refs/tags/') && !startsWith(github.ref, 'refs/tags/staging-') }} - with: - draft: true - files: | - ./dist/* diff --git a/.github/workflows/run-builds.yaml b/.github/workflows/run-builds.yaml new file mode 100644 index 0000000000..6b5f0ea9af --- /dev/null +++ b/.github/workflows/run-builds.yaml @@ -0,0 +1,121 @@ +name: Do builds + +on: + workflow_call: + +jobs: + authgear-image-amd64: + if: ${{ github.repository != 'oursky/authgear-server' }} + runs-on: ubuntu-24.04 + env: + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-image + with: + target: authgear + image_name: quay.io/theauthgear/authgear-server + push_image: "${{ env.PUSH_IMAGE }}" + build_arch: amd64 + docker_registry: quay.io + docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" + docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" + authgear-image-arm64: + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} + runs-on: ubuntu-24.04 + env: + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-image + with: + target: authgear + image_name: quay.io/theauthgear/authgear-server + push_image: "${{ env.PUSH_IMAGE }}" + build_arch: arm64 + docker_registry: quay.io + docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" + docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" + + authgear-image: + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} + runs-on: ubuntu-24.04 + needs: ["authgear-image-amd64", "authgear-image-arm64"] + env: + TARGET: authgear + IMAGE_NAME: quay.io/theauthgear/authgear-server + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + SOURCE_ARCHS: arm64,amd64 + steps: + - uses: actions/checkout@v4 + - name: docker login + if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} + env: + DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} + run: | + printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io + - run: make tag-image SOURCE_ARCHS=$SOURCE_ARCHS IMAGE_NAME=$IMAGE_NAME + - name: docker logout + if: ${{ always() }} + run: | + docker logout quay.io + + + portal-image-amd64: + if: ${{ github.repository != 'oursky/authgear-server' }} + runs-on: ubuntu-24.04 + env: + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-image + with: + target: portal + image_name: quay.io/theauthgear/authgear-portal + push_image: "${{ env.PUSH_IMAGE }}" + build_arch: amd64 + docker_registry: quay.io + docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" + docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" + + portal-image-arm64: + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} + runs-on: ubuntu-24.04 + env: + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-image + with: + target: portal + image_name: quay.io/theauthgear/authgear-portal + push_image: "${{ env.PUSH_IMAGE }}" + build_arch: arm64 + docker_registry: quay.io + docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" + docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" + + portal-image: + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} + runs-on: ubuntu-24.04 + needs: ["portal-image-amd64", "portal-image-arm64"] + env: + TARGET: authgear + IMAGE_NAME: quay.io/theauthgear/authgear-portal + PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" + SOURCE_ARCHS: arm64,amd64 + steps: + - uses: actions/checkout@v4 + - name: docker login + if: ${{ github.repository == 'authgear/authgear-server' && github.event_name == 'push' }} + env: + DOCKER_USERNAME: ${{ secrets.QUAY_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} + run: | + printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io + - run: make tag-image SOURCE_ARCHS=$SOURCE_ARCHS IMAGE_NAME=$IMAGE_NAME + - name: docker logout + if: ${{ always() }} + run: | + docker logout quay.io diff --git a/.github/workflows/run-checks.yaml b/.github/workflows/run-checks.yaml new file mode 100644 index 0000000000..9ea5f68dba --- /dev/null +++ b/.github/workflows/run-checks.yaml @@ -0,0 +1,185 @@ +name: Run checks + +on: + workflow_call: + +jobs: + authui-test: + if: ${{ github.repository != 'oursky/authgear-server' }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: "./.tool-versions" + - run: npm ci + working-directory: ./authui + - name: Run npm audit + env: + AUDIT_LEVEL: ${{ vars.NPM_AUDIT_LEVEL }} + run: | + if [ -z "${AUDIT_LEVEL}" ]; then + npm audit + else + npm audit --audit-level="${AUDIT_LEVEL}" + fi + working-directory: ./authui + if: ${{ !cancelled() }} + - run: npm run typecheck + working-directory: ./authui + if: ${{ !cancelled() }} + - run: npm run eslint + working-directory: ./authui + if: ${{ !cancelled() }} + - run: npm run stylelint + working-directory: ./authui + if: ${{ !cancelled() }} + - run: npm run prettier + working-directory: ./authui + if: ${{ !cancelled() }} + - run: npm run test + working-directory: ./authui + if: ${{ !cancelled() }} + - run: make check-tidy + working-directory: ./authui + if: ${{ !cancelled() }} + - run: npm run build + working-directory: ./authui + if: ${{ !cancelled() }} + - run: npm run ensure-reproducible-build + working-directory: ./authui + if: ${{ !cancelled() }} + - run: npm run browserslist-coverage-lint + working-directory: ./authui + if: ${{ !cancelled() }} + + portal-test: + if: ${{ github.repository != 'oursky/authgear-server' }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/install-native-deps + - uses: actions/setup-node@v4 + with: + node-version-file: "./.tool-versions" + - run: npm ci + working-directory: ./portal + - name: Run npm audit + env: + AUDIT_LEVEL: ${{ vars.NPM_AUDIT_LEVEL }} + run: | + if [ -z "${AUDIT_LEVEL}" ]; then + npm audit + else + npm audit --audit-level="${AUDIT_LEVEL}" + fi + working-directory: ./portal + if: ${{ !cancelled() }} + - run: npm run typecheck + working-directory: ./portal + if: ${{ !cancelled() }} + - run: npm run eslint + working-directory: ./portal + if: ${{ !cancelled() }} + - run: npm run stylelint + working-directory: ./portal + if: ${{ !cancelled() }} + - run: npm run prettier + working-directory: ./portal + if: ${{ !cancelled() }} + - run: npm run test + working-directory: ./portal + if: ${{ !cancelled() }} + - run: npm run gentype + working-directory: ./portal + if: ${{ !cancelled() }} + - run: make check-tidy + working-directory: ./portal + if: ${{ !cancelled() }} + - run: npm run build + working-directory: ./portal + if: ${{ !cancelled() }} + - run: npm run ensure-reproducible-build + working-directory: ./portal + if: ${{ !cancelled() }} + - run: npm run browserslist-coverage-lint + working-directory: ./portal + if: ${{ !cancelled() }} + + + authgear-test: + if: ${{ github.repository != 'oursky/authgear-server' }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/install-native-deps + - uses: actions/setup-go@v4 + with: + go-version-file: "./go.mod" + - uses: actions/setup-node@v4 + with: + node-version-file: "./.tool-versions" + - run: make vendor + - run: make check-dockerignore + if: ${{ !cancelled() }} + - run: make govulncheck + if: ${{ !cancelled() }} + # We cannot enable this until we move to mjml@5 + # https://github.com/mjmlio/mjml/issues/2589 + #- run: npm audit + # working-directory: ./scripts/npm + # if: ${{ !cancelled() }} + - run: make ensure-important-modules-up-to-date + if: ${{ !cancelled() }} + - run: make lint + if: ${{ !cancelled() }} + - run: make test + if: ${{ !cancelled() }} + - run: make fmt + if: ${{ !cancelled() }} + - run: make check-tidy + if: ${{ !cancelled() }} + - run: make -C packagetracker check-if-new-version-available + if: ${{ !cancelled() }} + + authgear-e2e: + if: ${{ github.repository != 'oursky/authgear-server' }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/install-native-deps + + - uses: actions/setup-go@v4 + with: + go-version-file: "./go.mod" + - run: make vendor + + # Cache docker images + - run: mkdir -p ~/.cache/images + - id: cache-docker-images + uses: actions/cache@v3 + with: + path: ~/.cache/images + key: ${{ runner.os }}-e2e-docker-images + continue-on-error: true + env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 + - name: Load docker image cache + if: ${{ steps.cache-docker-images.outputs.cache-hit == 'true' }} + run: | + set -x + for image in $(grep 'image: ' ./e2e/docker-compose.yaml | awk '{ print $2 }'); do + safe_image_name=$(echo -n "$image" | tr '/:' '_') + docker load -i ~/.cache/images/"$safe_image_name".tar || true + done + - run: make -C e2e ci + if: ${{ !cancelled() }} + env: + COMPOSE_INTERACTIVE_NO_CLI: 1 + - name: Save docker image cache + run: | + set -x + for image in $(grep 'image: ' ./e2e/docker-compose.yaml | awk '{ print $2 }'); do + safe_image_name=$(echo -n "$image" | tr '/:' '_') + docker save "$image" -o ~/.cache/images/"$safe_image_name".tar + done diff --git a/.github/workflows/run-release.yaml b/.github/workflows/run-release.yaml new file mode 100644 index 0000000000..c744af182c --- /dev/null +++ b/.github/workflows/run-release.yaml @@ -0,0 +1,26 @@ +name: Run checks + +on: + workflow_call: + +jobs: + release: + if: ${{ github.repository != 'oursky/authgear-server' }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/install-native-deps + - uses: actions/setup-go@v4 + with: + go-version-file: "./go.mod" + - uses: actions/setup-node@v4 + with: + node-version-file: "./.tool-versions" + - run: make vendor + - run: make binary + - uses: oursky/action-gh-release@v2 + if: ${{ github.repository == 'authgear/authgear-server' && startsWith(github.ref, 'refs/tags/') && !startsWith(github.ref, 'refs/tags/staging-') }} + with: + draft: true + files: | + ./dist/* From af42e74fd057476af12490be1eed8956abf960ab Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 16:41:06 +0800 Subject: [PATCH 24/40] Build amd64 image in prs --- .github/workflows/ci-prs.yaml | 5 +++++ .github/workflows/run-builds.yaml | 13 +++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-prs.yaml b/.github/workflows/ci-prs.yaml index 3a413e56aa..72f69c51c2 100644 --- a/.github/workflows/ci-prs.yaml +++ b/.github/workflows/ci-prs.yaml @@ -13,3 +13,8 @@ concurrency: jobs: checks: uses: ./.github/workflows/run-checks.yaml + builds: + needs: checks + uses: ./.github/workflows/run-builds.yaml + with: + amd64-build-only: true diff --git a/.github/workflows/run-builds.yaml b/.github/workflows/run-builds.yaml index 6b5f0ea9af..ca3924c2e1 100644 --- a/.github/workflows/run-builds.yaml +++ b/.github/workflows/run-builds.yaml @@ -2,6 +2,11 @@ name: Do builds on: workflow_call: + inputs: + amd64-build-only: + required: false + default: false + type: boolean jobs: authgear-image-amd64: @@ -21,7 +26,7 @@ jobs: docker_username: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_USERNAME || '' }}" docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" authgear-image-arm64: - if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push' && !inputs.amd64-build-only) }} runs-on: ubuntu-24.04 env: PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" @@ -38,7 +43,7 @@ jobs: docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" authgear-image: - if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push' && !inputs.amd64-build-only) }} runs-on: ubuntu-24.04 needs: ["authgear-image-amd64", "authgear-image-arm64"] env: @@ -80,7 +85,7 @@ jobs: docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" portal-image-arm64: - if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push' && !inputs.amd64-build-only) }} runs-on: ubuntu-24.04 env: PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" @@ -97,7 +102,7 @@ jobs: docker_password: "${{ env.PUSH_IMAGE == 'true' && secrets.QUAY_ROBOT_TOKEN || '' }}" portal-image: - if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') }} + if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push' && !inputs.amd64-build-only) }} runs-on: ubuntu-24.04 needs: ["portal-image-amd64", "portal-image-arm64"] env: From 3331fed2e192e2470d86a027eb9a866bf2fab16a Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 17:11:08 +0800 Subject: [PATCH 25/40] Fix workflow names --- .github/workflows/run-builds.yaml | 2 +- .github/workflows/run-release.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-builds.yaml b/.github/workflows/run-builds.yaml index ca3924c2e1..ef30a266f4 100644 --- a/.github/workflows/run-builds.yaml +++ b/.github/workflows/run-builds.yaml @@ -1,4 +1,4 @@ -name: Do builds +name: Run builds on: workflow_call: diff --git a/.github/workflows/run-release.yaml b/.github/workflows/run-release.yaml index c744af182c..dd35c0928a 100644 --- a/.github/workflows/run-release.yaml +++ b/.github/workflows/run-release.yaml @@ -1,4 +1,4 @@ -name: Run checks +name: Run releases on: workflow_call: From d390f69da467684fcf302c1d5770765ab2512b73 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Mon, 16 Dec 2024 17:17:47 +0800 Subject: [PATCH 26/40] Fix a typo in filename --- .github/workflows/{ci-branchs.yaml => ci-branches.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci-branchs.yaml => ci-branches.yaml} (100%) diff --git a/.github/workflows/ci-branchs.yaml b/.github/workflows/ci-branches.yaml similarity index 100% rename from .github/workflows/ci-branchs.yaml rename to .github/workflows/ci-branches.yaml From 35f62cb83fb485f40ba6585da49fd1c1c9d7fa51 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Tue, 17 Dec 2024 15:52:14 +0800 Subject: [PATCH 27/40] Write Makefile in idiomatic way --- common.mk | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/common.mk b/common.mk index e23bdeefdd..482b2ba90e 100644 --- a/common.mk +++ b/common.mk @@ -51,23 +51,24 @@ build: .PHONY: build-image build-image: - $(eval IMAGE_TAG_BASE ::= $(IMAGE_NAME):$(GIT_HASH)) - $(eval BUILD_OPTS ::= ) +IMAGE_TAG_BASE ::= $(IMAGE_NAME):$(GIT_HASH) +BUILD_OPTS ::= ifeq ($(BUILD_ARCH),amd64) - $(eval BUILD_OPTS += --platform linux/$(BUILD_ARCH) ) - $(eval BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-amd64 ) +BUILD_OPTS += --platform linux/$(BUILD_ARCH) +BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-amd64 else ifeq ($(BUILD_ARCH),arm64) - $(eval BUILD_OPTS += --platform linux/$(BUILD_ARCH) ) - $(eval BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-arm64 ) +BUILD_OPTS += --platform linux/$(BUILD_ARCH) +BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-arm64 else - $(eval BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-$(BUILD_ARCH)-unknown ) +BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-$(BUILD_ARCH)-unknown endif ifeq ($(PUSH_IMAGE),true) - $(eval BUILD_OPTS += --push) +BUILD_OPTS += --push endif ifneq ($(EXTRA_BUILD_OPTS),) - $(eval BUILD_OPTS += $(EXTRA_BUILD_OPTS)) +BUILD_OPTS += $(EXTRA_BUILD_OPTS) endif +build-image: @# Add --pull so that we are using the latest base image. @# The build context is the parent directory docker build --pull \ @@ -77,18 +78,19 @@ endif .PHONY: tag-image tag-image: - $(eval IMAGE_SOURCES ::= ) - $(eval TAGS ::= --tag $(IMAGE_NAME):latest ) - $(eval TAGS += --tag $(IMAGE_NAME):$(GIT_HASH)) +IMAGE_SOURCES ::= +TAGS ::= --tag $(IMAGE_NAME):latest +TAGS += --tag $(IMAGE_NAME):$(GIT_HASH) ifneq (${GIT_NAME},) - $(eval TAGS += --tag $(IMAGE_NAME):$(GIT_NAME)) +TAGS += --tag $(IMAGE_NAME):$(GIT_NAME) endif ifneq ($(findstring amd64,$(SOURCE_ARCHS)),) - $(eval IMAGE_SOURCES += $(IMAGE_NAME):$(GIT_HASH)-amd64 ) +IMAGE_SOURCES += $(IMAGE_NAME):$(GIT_HASH)-amd64 endif ifneq ($(findstring arm64,$(SOURCE_ARCHS)),) - $(eval IMAGE_SOURCES += $(IMAGE_NAME):$(GIT_HASH)-arm64 ) +IMAGE_SOURCES += $(IMAGE_NAME):$(GIT_HASH)-arm64 endif +tag-image: docker buildx imagetools create \ $(TAGS) \ $(IMAGE_SOURCES) From f500227abaf51b93213958ff25596aebb14d002c Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Tue, 17 Dec 2024 18:45:28 +0800 Subject: [PATCH 28/40] Push arm64 and amd64 image layer by digest, and create manifest by the digests --- .github/actions/build-image/action.yaml | 14 +++++++++++++- .github/workflows/run-builds.yaml | 24 ++++++++++++++++++------ common.mk | 20 +++++++------------- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml index d2e6f0fb1b..0e577bb6ce 100644 --- a/.github/actions/build-image/action.yaml +++ b/.github/actions/build-image/action.yaml @@ -14,6 +14,9 @@ inputs: required: false docker_password: required: false +outputs: + image_digest: + value: ${{ steps.build_image.outputs.image_digest }} runs: using: "composite" steps: @@ -36,7 +39,16 @@ runs: shell: bash run: | printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" $DOCKER_REGISTRY - - run: make build-image BUILD_ARCH=$BUILD_ARCH PUSH_IMAGE=$PUSH_IMAGE TARGET=$TARGET IMAGE_NAME=$IMAGE_NAME + - id: build_image + run: | + make build-image \ + BUILD_ARCH=$BUILD_ARCH \ + OUTPUT=type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true \ + TARGET=$TARGET \ + IMAGE_NAME=$IMAGE_NAME \ + METADATA_FILE=metadata.json + DIGEST="$(cat metadata.json | jq '.["containerimage.digest"]' -r)" + echo "image_digest=$DIGEST" >> "$GITHUB_OUTPUT" shell: bash env: TARGET: ${{ inputs.target }} diff --git a/.github/workflows/run-builds.yaml b/.github/workflows/run-builds.yaml index ef30a266f4..f24f671466 100644 --- a/.github/workflows/run-builds.yaml +++ b/.github/workflows/run-builds.yaml @@ -12,11 +12,14 @@ jobs: authgear-image-amd64: if: ${{ github.repository != 'oursky/authgear-server' }} runs-on: ubuntu-24.04 + outputs: + image_digest: ${{ steps.build_image.outputs.image_digest }} env: PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/build-image + - id: build_image + uses: ./.github/actions/build-image with: target: authgear image_name: quay.io/theauthgear/authgear-server @@ -28,11 +31,14 @@ jobs: authgear-image-arm64: if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push' && !inputs.amd64-build-only) }} runs-on: ubuntu-24.04 + outputs: + image_digest: ${{ steps.build_image.outputs.image_digest }} env: PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/build-image + - id: build_image + uses: ./.github/actions/build-image with: target: authgear image_name: quay.io/theauthgear/authgear-server @@ -50,7 +56,8 @@ jobs: TARGET: authgear IMAGE_NAME: quay.io/theauthgear/authgear-server PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - SOURCE_ARCHS: arm64,amd64 + AMD64_DIGEST: "${{needs.authgear-image-amd64.outputs.image_digest}}" + ARM64_DIGEST: "${{needs.authgear-image-arm64.outputs.image_digest}}" steps: - uses: actions/checkout@v4 - name: docker login @@ -60,7 +67,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} run: | printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make tag-image SOURCE_ARCHS=$SOURCE_ARCHS IMAGE_NAME=$IMAGE_NAME + - run: make tag-image SOURCE_DIGESTS="$AMD64_DIGEST $ARM64_DIGEST" IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} run: | @@ -69,6 +76,8 @@ jobs: portal-image-amd64: if: ${{ github.repository != 'oursky/authgear-server' }} + outputs: + image_digest: ${{ steps.build_image.outputs.image_digest }} runs-on: ubuntu-24.04 env: PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" @@ -86,6 +95,8 @@ jobs: portal-image-arm64: if: ${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push' && !inputs.amd64-build-only) }} + outputs: + image_digest: ${{ steps.build_image.outputs.image_digest }} runs-on: ubuntu-24.04 env: PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" @@ -109,7 +120,8 @@ jobs: TARGET: authgear IMAGE_NAME: quay.io/theauthgear/authgear-portal PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" - SOURCE_ARCHS: arm64,amd64 + AMD64_DIGEST: "${{needs.portal-image-amd64.outputs.image_digest}}" + ARM64_DIGEST: "${{needs.portal-image-arm64.outputs.image_digest}}" steps: - uses: actions/checkout@v4 - name: docker login @@ -119,7 +131,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.QUAY_ROBOT_TOKEN }} run: | printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" quay.io - - run: make tag-image SOURCE_ARCHS=$SOURCE_ARCHS IMAGE_NAME=$IMAGE_NAME + - run: make tag-image SOURCE_DIGESTS="$AMD64_DIGEST $ARM64_DIGEST" IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} run: | diff --git a/common.mk b/common.mk index 482b2ba90e..415bc65e74 100644 --- a/common.mk +++ b/common.mk @@ -55,19 +55,18 @@ IMAGE_TAG_BASE ::= $(IMAGE_NAME):$(GIT_HASH) BUILD_OPTS ::= ifeq ($(BUILD_ARCH),amd64) BUILD_OPTS += --platform linux/$(BUILD_ARCH) -BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-amd64 else ifeq ($(BUILD_ARCH),arm64) BUILD_OPTS += --platform linux/$(BUILD_ARCH) -BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-arm64 -else -BUILD_OPTS += --tag $(IMAGE_TAG_BASE)-$(BUILD_ARCH)-unknown endif -ifeq ($(PUSH_IMAGE),true) -BUILD_OPTS += --push +ifneq ($(OUTPUT),) +BUILD_OPTS += --output=$(OUTPUT) endif ifneq ($(EXTRA_BUILD_OPTS),) BUILD_OPTS += $(EXTRA_BUILD_OPTS) endif +ifneq ($(METADATA_FILE),) +BUILD_OPTS += --metadata-file $(METADATA_FILE) +endif build-image: @# Add --pull so that we are using the latest base image. @# The build context is the parent directory @@ -84,13 +83,8 @@ TAGS += --tag $(IMAGE_NAME):$(GIT_HASH) ifneq (${GIT_NAME},) TAGS += --tag $(IMAGE_NAME):$(GIT_NAME) endif -ifneq ($(findstring amd64,$(SOURCE_ARCHS)),) -IMAGE_SOURCES += $(IMAGE_NAME):$(GIT_HASH)-amd64 -endif -ifneq ($(findstring arm64,$(SOURCE_ARCHS)),) -IMAGE_SOURCES += $(IMAGE_NAME):$(GIT_HASH)-arm64 -endif +IMAGE_SOURCES := $(foreach digest,$(SOURCE_DIGESTS),${IMAGE_NAME}@${digest} ) tag-image: docker buildx imagetools create \ $(TAGS) \ - $(IMAGE_SOURCES) + ${IMAGE_SOURCES} From dcd9abf904b48d7dce565d9054f4fe7c41612d63 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Tue, 17 Dec 2024 18:50:55 +0800 Subject: [PATCH 29/40] Use custom builder --- .github/actions/build-image/action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml index 0e577bb6ce..bc537e6a55 100644 --- a/.github/actions/build-image/action.yaml +++ b/.github/actions/build-image/action.yaml @@ -46,7 +46,8 @@ runs: OUTPUT=type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true \ TARGET=$TARGET \ IMAGE_NAME=$IMAGE_NAME \ - METADATA_FILE=metadata.json + METADATA_FILE=metadata.json \ + EXTRA_BUILD_OPTS="--builder=container-builder" DIGEST="$(cat metadata.json | jq '.["containerimage.digest"]' -r)" echo "image_digest=$DIGEST" >> "$GITHUB_OUTPUT" shell: bash From d8c5b175018ddffd18d1226a0125b24585f0bbd3 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Tue, 17 Dec 2024 19:35:44 +0800 Subject: [PATCH 30/40] Do not always push image --- .github/actions/build-image/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml index bc537e6a55..97bbdad9e2 100644 --- a/.github/actions/build-image/action.yaml +++ b/.github/actions/build-image/action.yaml @@ -43,7 +43,7 @@ runs: run: | make build-image \ BUILD_ARCH=$BUILD_ARCH \ - OUTPUT=type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true \ + OUTPUT=$OUTPUT \ TARGET=$TARGET \ IMAGE_NAME=$IMAGE_NAME \ METADATA_FILE=metadata.json \ @@ -54,7 +54,7 @@ runs: env: TARGET: ${{ inputs.target }} IMAGE_NAME: ${{ inputs.image_name }} - PUSH_IMAGE: ${{ inputs.push_image }} + OUTPUT: ${{ (inputs.push_image == 'true') && 'type=image,name=$$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true' || ''}} BUILD_ARCH: ${{ inputs.build_arch }} - name: docker logout if: ${{ always() }} From b6804fd6ca0f06b1bc8e684e1c412aa3d01ec770 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Wed, 18 Dec 2024 13:25:10 +0800 Subject: [PATCH 31/40] Fix portal build step missing id --- .github/workflows/run-builds.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-builds.yaml b/.github/workflows/run-builds.yaml index f24f671466..7f537854a2 100644 --- a/.github/workflows/run-builds.yaml +++ b/.github/workflows/run-builds.yaml @@ -83,7 +83,8 @@ jobs: PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/build-image + - id: build_image + uses: ./.github/actions/build-image with: target: portal image_name: quay.io/theauthgear/authgear-portal @@ -102,7 +103,8 @@ jobs: PUSH_IMAGE: "${{ (github.repository == 'authgear/authgear-server' && github.event_name == 'push') && 'true' || 'false' }}" steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/build-image + - id: build_image + uses: ./.github/actions/build-image with: target: portal image_name: quay.io/theauthgear/authgear-portal From ce2625a24c4d140cb9ace3883c301b19ad40d0e4 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Wed, 18 Dec 2024 14:58:20 +0800 Subject: [PATCH 32/40] Use digest in custom build scripts --- .../actions/build-custom-image/action.yaml | 14 ++++++-- .github/workflows/oursky.yaml | 36 ++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/.github/actions/build-custom-image/action.yaml b/.github/actions/build-custom-image/action.yaml index bde7c0f034..0dc6263c91 100644 --- a/.github/actions/build-custom-image/action.yaml +++ b/.github/actions/build-custom-image/action.yaml @@ -10,6 +10,8 @@ inputs: required: true docker_repo: required: false + builder: + required: true runs: using: "composite" steps: @@ -29,10 +31,18 @@ runs: TARGET: ${{ inputs.target }} IMAGE_NAME: ${{ inputs.image_name }} REPO: ${{ inputs.docker_repo }} + BUILDER: ${{ inputs.builder }} shell: bash run: | - make -C custombuild build-image TARGET=$TARGET BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME=$IMAGE_NAME EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME=$IMAGE_NAME + make -C custombuild build-image \ + TARGET=$TARGET \ + BUILD_ARCH=amd64 \ + OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ + IMAGE_NAME=$IMAGE_NAME \ + METADATA_FILE=metadata.json \ + EXTRA_BUILD_OPTS="--ssh=default --builder=container-builder" + DIGEST="$(cat metadata.json | jq '.["containerimage.digest"]' -r)" + make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} env: diff --git a/.github/workflows/oursky.yaml b/.github/workflows/oursky.yaml index de60fdc898..231512820c 100644 --- a/.github/workflows/oursky.yaml +++ b/.github/workflows/oursky.yaml @@ -25,12 +25,26 @@ jobs: printf "$AUTHGEAR_PRIVATE_DEPLOY_KEY" | base64 --decode | ssh-add - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" + - name: Setup container builder + run: | + docker buildx create \ + --name container-builder \ + --driver docker-container \ + --bootstrap --use - name: Build and Push env: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} run: | - make -C custombuild build-image TARGET=authgearx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-server" EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-server" + IMAGE_NAME="$REPO_PREFIX/authgear-server" + make -C custombuild build-image \ + TARGET=authgearx \ + BUILD_ARCH=amd64 \ + OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ + IMAGE_NAME=$IMAGE_NAME \ + METADATA_FILE=metadata.json \ + EXTRA_BUILD_OPTS="--ssh=default --builder=container-builder" + DIGEST="$(cat metadata.json | jq '.["containerimage.digest"]' -r)" + make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME - name: Clean up SSH key if: ${{ always() }} run: | @@ -55,12 +69,26 @@ jobs: printf "$AUTHGEAR_PRIVATE_DEPLOY_KEY" | base64 --decode | ssh-add - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" + - name: Setup container builder + run: | + docker buildx create \ + --name container-builder \ + --driver docker-container \ + --bootstrap --use - name: Build and Push env: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} run: | - make -C custombuild build-image TARGET=portalx BUILD_ARCH=amd64 PUSH_IMAGE=true IMAGE_NAME="$REPO_PREFIX/authgear-portal" EXTRA_BUILD_OPTS="--ssh=default" - make -C custombuild tag-image SOURCE_ARCHS=amd64 IMAGE_NAME="$REPO_PREFIX/authgear-portal" + IMAGE_NAME="$REPO_PREFIX/authgear-portal" + make -C custombuild build-image \ + TARGET=portalx \ + BUILD_ARCH=amd64 \ + OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ + IMAGE_NAME=$IMAGE_NAME \ + METADATA_FILE=metadata.json \ + EXTRA_BUILD_OPTS="--ssh=default --builder=container-builder" + DIGEST="$(cat metadata.json | jq '.["containerimage.digest"]' -r)" + make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME - name: Clean up SSH key if: ${{ always() }} run: | From 21b2bb3e30ecd3328880fb708fe3b2ee57ca0103 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Wed, 18 Dec 2024 15:22:41 +0800 Subject: [PATCH 33/40] Print metadata json for debug --- .github/actions/build-custom-image/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/build-custom-image/action.yaml b/.github/actions/build-custom-image/action.yaml index 0dc6263c91..656ecb0da9 100644 --- a/.github/actions/build-custom-image/action.yaml +++ b/.github/actions/build-custom-image/action.yaml @@ -41,6 +41,7 @@ runs: IMAGE_NAME=$IMAGE_NAME \ METADATA_FILE=metadata.json \ EXTRA_BUILD_OPTS="--ssh=default --builder=container-builder" + cat metadata.json DIGEST="$(cat metadata.json | jq '.["containerimage.digest"]' -r)" make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME - name: docker logout From 0dd8d2168d51d2b8f5fd886ededb24b6bc3eea16 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Wed, 18 Dec 2024 15:52:28 +0800 Subject: [PATCH 34/40] Use a unique manifest name --- .github/actions/build-custom-image/action.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/build-custom-image/action.yaml b/.github/actions/build-custom-image/action.yaml index 656ecb0da9..c2e4a53ebd 100644 --- a/.github/actions/build-custom-image/action.yaml +++ b/.github/actions/build-custom-image/action.yaml @@ -34,15 +34,16 @@ runs: BUILDER: ${{ inputs.builder }} shell: bash run: | + MANIFEST_NAME="./$(uuidgen).json" make -C custombuild build-image \ TARGET=$TARGET \ BUILD_ARCH=amd64 \ OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ IMAGE_NAME=$IMAGE_NAME \ - METADATA_FILE=metadata.json \ + METADATA_FILE=$MANIFEST_NAME \ EXTRA_BUILD_OPTS="--ssh=default --builder=container-builder" - cat metadata.json - DIGEST="$(cat metadata.json | jq '.["containerimage.digest"]' -r)" + cat $MANIFEST_NAME + DIGEST="$(cat $MANIFEST_NAME | jq '.["containerimage.digest"]' -r)" make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} From 9a6a37f97ede2bcc30f5a12f94852e5d00f46a57 Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Wed, 18 Dec 2024 16:15:14 +0800 Subject: [PATCH 35/40] Fix manifest path in custombuild --- .github/actions/build-custom-image/action.yaml | 4 ++-- .github/workflows/oursky.yaml | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/actions/build-custom-image/action.yaml b/.github/actions/build-custom-image/action.yaml index c2e4a53ebd..a32458dc73 100644 --- a/.github/actions/build-custom-image/action.yaml +++ b/.github/actions/build-custom-image/action.yaml @@ -42,8 +42,8 @@ runs: IMAGE_NAME=$IMAGE_NAME \ METADATA_FILE=$MANIFEST_NAME \ EXTRA_BUILD_OPTS="--ssh=default --builder=container-builder" - cat $MANIFEST_NAME - DIGEST="$(cat $MANIFEST_NAME | jq '.["containerimage.digest"]' -r)" + cat custombuild/$MANIFEST_NAME + DIGEST="$(cat custombuild/$MANIFEST_NAME | jq '.["containerimage.digest"]' -r)" make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} diff --git a/.github/workflows/oursky.yaml b/.github/workflows/oursky.yaml index 231512820c..e68594dbe0 100644 --- a/.github/workflows/oursky.yaml +++ b/.github/workflows/oursky.yaml @@ -35,15 +35,17 @@ jobs: env: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} run: | + MANIFEST_NAME="./$(uuidgen).json" IMAGE_NAME="$REPO_PREFIX/authgear-server" make -C custombuild build-image \ TARGET=authgearx \ BUILD_ARCH=amd64 \ OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ IMAGE_NAME=$IMAGE_NAME \ - METADATA_FILE=metadata.json \ + METADATA_FILE=$MANIFEST_NAME \ EXTRA_BUILD_OPTS="--ssh=default --builder=container-builder" - DIGEST="$(cat metadata.json | jq '.["containerimage.digest"]' -r)" + cat custombuild/$MANIFEST_NAME + DIGEST="$(cat custombuild/$MANIFEST_NAME | jq '.["containerimage.digest"]' -r)" make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME - name: Clean up SSH key if: ${{ always() }} @@ -79,15 +81,17 @@ jobs: env: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} run: | + MANIFEST_NAME="./$(uuidgen).json" IMAGE_NAME="$REPO_PREFIX/authgear-portal" make -C custombuild build-image \ TARGET=portalx \ BUILD_ARCH=amd64 \ OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ IMAGE_NAME=$IMAGE_NAME \ - METADATA_FILE=metadata.json \ + METADATA_FILE=$MANIFEST_NAME \ EXTRA_BUILD_OPTS="--ssh=default --builder=container-builder" - DIGEST="$(cat metadata.json | jq '.["containerimage.digest"]' -r)" + cat custombuild/$MANIFEST_NAME + DIGEST="$(cat custombuild/$MANIFEST_NAME | jq '.["containerimage.digest"]' -r)" make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME - name: Clean up SSH key if: ${{ always() }} From 8fbad4763d41764d4376e6dcd5febbe9ff0ec6df Mon Sep 17 00:00:00 2001 From: Tung Wu Date: Wed, 18 Dec 2024 16:16:36 +0800 Subject: [PATCH 36/40] Turn off provenance in image build --- common.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/common.mk b/common.mk index 415bc65e74..eebe541fe8 100644 --- a/common.mk +++ b/common.mk @@ -71,6 +71,7 @@ build-image: @# Add --pull so that we are using the latest base image. @# The build context is the parent directory docker build --pull \ + --provenance=false \ --file ./cmd/$(TARGET)/Dockerfile \ $(BUILD_OPTS) \ --build-arg GIT_HASH=$(GIT_HASH) ${BUILD_CTX} From f3f07aab4d736fed312635880ddd693393f86f41 Mon Sep 17 00:00:00 2001 From: Louis Chan Date: Thu, 19 Dec 2024 15:43:17 +0800 Subject: [PATCH 37/40] Use BUILDX_BUILDER instead of --builder --- .../actions/build-custom-image/action.yaml | 2 +- .github/actions/build-image/action.yaml | 11 ++--------- .../actions/docker-buildx-create/action.yaml | 15 +++++++++++++++ .github/workflows/custom-build.yaml | 14 ++------------ .github/workflows/oursky.yaml | 19 ++++--------------- 5 files changed, 24 insertions(+), 37 deletions(-) create mode 100644 .github/actions/docker-buildx-create/action.yaml diff --git a/.github/actions/build-custom-image/action.yaml b/.github/actions/build-custom-image/action.yaml index a32458dc73..e0b4a71902 100644 --- a/.github/actions/build-custom-image/action.yaml +++ b/.github/actions/build-custom-image/action.yaml @@ -41,7 +41,7 @@ runs: OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ IMAGE_NAME=$IMAGE_NAME \ METADATA_FILE=$MANIFEST_NAME \ - EXTRA_BUILD_OPTS="--ssh=default --builder=container-builder" + EXTRA_BUILD_OPTS="--ssh=default" cat custombuild/$MANIFEST_NAME DIGEST="$(cat custombuild/$MANIFEST_NAME | jq '.["containerimage.digest"]' -r)" make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml index 97bbdad9e2..d024abdd2b 100644 --- a/.github/actions/build-image/action.yaml +++ b/.github/actions/build-image/action.yaml @@ -23,13 +23,7 @@ runs: - name: Install qemu for multi arch build shell: bash run: docker run --privileged --rm tonistiigi/binfmt --install all - - name: Setup container builder - shell: bash - run: | - docker buildx create \ - --name container-builder \ - --driver docker-container \ - --bootstrap --use + - uses: ./.github/actions/docker-buildx-create - name: docker login if: ${{ inputs.push_image == 'true' }} env: @@ -46,8 +40,7 @@ runs: OUTPUT=$OUTPUT \ TARGET=$TARGET \ IMAGE_NAME=$IMAGE_NAME \ - METADATA_FILE=metadata.json \ - EXTRA_BUILD_OPTS="--builder=container-builder" + METADATA_FILE=metadata.json DIGEST="$(cat metadata.json | jq '.["containerimage.digest"]' -r)" echo "image_digest=$DIGEST" >> "$GITHUB_OUTPUT" shell: bash diff --git a/.github/actions/docker-buildx-create/action.yaml b/.github/actions/docker-buildx-create/action.yaml new file mode 100644 index 0000000000..0fcd20c876 --- /dev/null +++ b/.github/actions/docker-buildx-create/action.yaml @@ -0,0 +1,15 @@ +name: "Create a builder and set BUILDX_BUILDER" +description: "Create a builder and set BUILDX_BUILDER" +runs: + using: "composite" + steps: + - name: "docker buildx create" + shell: bash + env: + BUILDER_NAME: container-builder + run: | + docker buildx create \ + --name "$BUILDER_NAME" \ + --driver docker-container \ + --bootstrap --use + echo "BUILDX_BUILDER=$BUILDER_NAME" >> "$GITHUB_ENV" diff --git a/.github/workflows/custom-build.yaml b/.github/workflows/custom-build.yaml index b86976acbc..3919a0f871 100644 --- a/.github/workflows/custom-build.yaml +++ b/.github/workflows/custom-build.yaml @@ -44,12 +44,7 @@ jobs: - uses: actions/checkout@v4 - name: Install qemu for multi arch build run: docker run --privileged --rm tonistiigi/binfmt --install all - - name: Setup container builder - run: | - docker buildx create \ - --name container-builder \ - --driver docker-container \ - --bootstrap --use + - uses: ./.github/actions/docker-buildx-create # https://aran.dev/posts/github-actions-go-private-modules/ - name: Set up SSH key @@ -96,12 +91,7 @@ jobs: - uses: actions/checkout@v4 - name: Install qemu for multi arch build run: docker run --privileged --rm tonistiigi/binfmt --install all - - name: Setup container builder - run: | - docker buildx create \ - --name container-builder \ - --driver docker-container \ - --bootstrap --use + - uses: ./.github/actions/docker-buildx-create # https://aran.dev/posts/github-actions-go-private-modules/ - name: Set up SSH key diff --git a/.github/workflows/oursky.yaml b/.github/workflows/oursky.yaml index e68594dbe0..90cab08edf 100644 --- a/.github/workflows/oursky.yaml +++ b/.github/workflows/oursky.yaml @@ -25,12 +25,7 @@ jobs: printf "$AUTHGEAR_PRIVATE_DEPLOY_KEY" | base64 --decode | ssh-add - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - name: Setup container builder - run: | - docker buildx create \ - --name container-builder \ - --driver docker-container \ - --bootstrap --use + - uses: ./.github/actions/docker-buildx-create - name: Build and Push env: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} @@ -43,7 +38,7 @@ jobs: OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ IMAGE_NAME=$IMAGE_NAME \ METADATA_FILE=$MANIFEST_NAME \ - EXTRA_BUILD_OPTS="--ssh=default --builder=container-builder" + EXTRA_BUILD_OPTS="--ssh=default" cat custombuild/$MANIFEST_NAME DIGEST="$(cat custombuild/$MANIFEST_NAME | jq '.["containerimage.digest"]' -r)" make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME @@ -71,12 +66,7 @@ jobs: printf "$AUTHGEAR_PRIVATE_DEPLOY_KEY" | base64 --decode | ssh-add - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - name: Setup container builder - run: | - docker buildx create \ - --name container-builder \ - --driver docker-container \ - --bootstrap --use + - uses: ./.github/actions/docker-buildx-create - name: Build and Push env: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} @@ -89,7 +79,7 @@ jobs: OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ IMAGE_NAME=$IMAGE_NAME \ METADATA_FILE=$MANIFEST_NAME \ - EXTRA_BUILD_OPTS="--ssh=default --builder=container-builder" + EXTRA_BUILD_OPTS="--ssh=default" cat custombuild/$MANIFEST_NAME DIGEST="$(cat custombuild/$MANIFEST_NAME | jq '.["containerimage.digest"]' -r)" make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME @@ -100,4 +90,3 @@ jobs: ssh-agent -k echo "SSH_AUTH_SOCK=" >> "$GITHUB_ENV" echo "SSH_AGENT_PID=" >> "$GITHUB_ENV" - From 6487fda635eed7cbf513676d8a82339808f581cd Mon Sep 17 00:00:00 2001 From: Louis Chan Date: Thu, 19 Dec 2024 15:48:20 +0800 Subject: [PATCH 38/40] Remove unused input `builder` from action `build-custom-image` --- .github/actions/build-custom-image/action.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/actions/build-custom-image/action.yaml b/.github/actions/build-custom-image/action.yaml index e0b4a71902..f0f63d83c2 100644 --- a/.github/actions/build-custom-image/action.yaml +++ b/.github/actions/build-custom-image/action.yaml @@ -10,8 +10,6 @@ inputs: required: true docker_repo: required: false - builder: - required: true runs: using: "composite" steps: @@ -31,7 +29,6 @@ runs: TARGET: ${{ inputs.target }} IMAGE_NAME: ${{ inputs.image_name }} REPO: ${{ inputs.docker_repo }} - BUILDER: ${{ inputs.builder }} shell: bash run: | MANIFEST_NAME="./$(uuidgen).json" From 58e96baeb5eb2ab7847085d33b3dc102c940b655 Mon Sep 17 00:00:00 2001 From: Louis Chan Date: Thu, 19 Dec 2024 16:03:00 +0800 Subject: [PATCH 39/40] Write METADATA_FILE to temp dir --- .github/actions/build-custom-image/action.yaml | 9 +++++---- .github/actions/build-image/action.yaml | 7 +++++-- .github/workflows/oursky.yaml | 18 ++++++++++-------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/actions/build-custom-image/action.yaml b/.github/actions/build-custom-image/action.yaml index f0f63d83c2..4d82fa4a60 100644 --- a/.github/actions/build-custom-image/action.yaml +++ b/.github/actions/build-custom-image/action.yaml @@ -31,16 +31,17 @@ runs: REPO: ${{ inputs.docker_repo }} shell: bash run: | - MANIFEST_NAME="./$(uuidgen).json" + METADATA_FILE="$(mktemp).json" + echo "METADATA_FILE=$METADATA_FILE" make -C custombuild build-image \ TARGET=$TARGET \ BUILD_ARCH=amd64 \ OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ IMAGE_NAME=$IMAGE_NAME \ - METADATA_FILE=$MANIFEST_NAME \ + METADATA_FILE="$METADATA_FILE" \ EXTRA_BUILD_OPTS="--ssh=default" - cat custombuild/$MANIFEST_NAME - DIGEST="$(cat custombuild/$MANIFEST_NAME | jq '.["containerimage.digest"]' -r)" + (set -x && cat "$METADATA_FILE") + DIGEST="$(jq < "$METADATA_FILE" '.["containerimage.digest"]' -r)" make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME - name: docker logout if: ${{ always() }} diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml index d024abdd2b..e39aa3cf48 100644 --- a/.github/actions/build-image/action.yaml +++ b/.github/actions/build-image/action.yaml @@ -35,13 +35,16 @@ runs: printf "$DOCKER_PASSWORD" | docker login --password-stdin --username "$DOCKER_USERNAME" $DOCKER_REGISTRY - id: build_image run: | + METADATA_FILE="$(mktemp).json" + echo "METADATA_FILE=$METADATA_FILE" make build-image \ BUILD_ARCH=$BUILD_ARCH \ OUTPUT=$OUTPUT \ TARGET=$TARGET \ IMAGE_NAME=$IMAGE_NAME \ - METADATA_FILE=metadata.json - DIGEST="$(cat metadata.json | jq '.["containerimage.digest"]' -r)" + METADATA_FILE="$METADATA_FILE" + (set -x && cat "$METADATA_FILE") + DIGEST="$(jq < "$METADATA_FILE" '.["containerimage.digest"]' -r)" echo "image_digest=$DIGEST" >> "$GITHUB_OUTPUT" shell: bash env: diff --git a/.github/workflows/oursky.yaml b/.github/workflows/oursky.yaml index 90cab08edf..cde1be46f3 100644 --- a/.github/workflows/oursky.yaml +++ b/.github/workflows/oursky.yaml @@ -30,17 +30,18 @@ jobs: env: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} run: | - MANIFEST_NAME="./$(uuidgen).json" + METADATA_FILE="$(mktemp).json" + echo "METADATA_FILE=$METADATA_FILE" IMAGE_NAME="$REPO_PREFIX/authgear-server" make -C custombuild build-image \ TARGET=authgearx \ BUILD_ARCH=amd64 \ OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ IMAGE_NAME=$IMAGE_NAME \ - METADATA_FILE=$MANIFEST_NAME \ + METADATA_FILE="$METADATA_FILE" \ EXTRA_BUILD_OPTS="--ssh=default" - cat custombuild/$MANIFEST_NAME - DIGEST="$(cat custombuild/$MANIFEST_NAME | jq '.["containerimage.digest"]' -r)" + (set -x && cat "$METADATA_FILE") + DIGEST="$(jq < "$METADATA_FILE" '.["containerimage.digest"]' -r)" make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME - name: Clean up SSH key if: ${{ always() }} @@ -71,17 +72,18 @@ jobs: env: REPO_PREFIX: ${{ secrets.AUTHGEAR_CUSTOM_BUILD_REPO_PREFIX }} run: | - MANIFEST_NAME="./$(uuidgen).json" + METADATA_FILE="$(mktemp).json" + echo "METADATA_FILE=$METADATA_FILE" IMAGE_NAME="$REPO_PREFIX/authgear-portal" make -C custombuild build-image \ TARGET=portalx \ BUILD_ARCH=amd64 \ OUTPUT="type=image,name=$IMAGE_NAME,push-by-digest=true,name-canonical=true,push=true" \ IMAGE_NAME=$IMAGE_NAME \ - METADATA_FILE=$MANIFEST_NAME \ + METADATA_FILE="$METADATA_FILE" \ EXTRA_BUILD_OPTS="--ssh=default" - cat custombuild/$MANIFEST_NAME - DIGEST="$(cat custombuild/$MANIFEST_NAME | jq '.["containerimage.digest"]' -r)" + (set -x && cat "$METADATA_FILE") + DIGEST="$(jq < "$METADATA_FILE" '.["containerimage.digest"]' -r)" make -C custombuild tag-image SOURCE_DIGESTS="$DIGEST" IMAGE_NAME=$IMAGE_NAME - name: Clean up SSH key if: ${{ always() }} From 0e9442008979af9a41911d6334fd3c7ebec6733c Mon Sep 17 00:00:00 2001 From: Louis Chan Date: Thu, 19 Dec 2024 17:50:53 +0800 Subject: [PATCH 40/40] Document --provenance=false --- common.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common.mk b/common.mk index eebe541fe8..684b532c3f 100644 --- a/common.mk +++ b/common.mk @@ -70,6 +70,8 @@ endif build-image: @# Add --pull so that we are using the latest base image. @# The build context is the parent directory + @# --provenance=false because we have no idea to figure out how to deal with the unknown manifest yet. + @# See https://github.com/authgear/authgear-server/pull/4943#discussion_r1891263998 docker build --pull \ --provenance=false \ --file ./cmd/$(TARGET)/Dockerfile \