From 2d926d47c575e7d5efdc86b1880253aa2c451fe0 Mon Sep 17 00:00:00 2001 From: Kevin DeShawn <126115026+KevDevSha@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:24:53 -0500 Subject: [PATCH 1/6] add ghcr --- .github/workflows/docker.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index e4e6f83551..b23701c7b8 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -12,7 +12,7 @@ on: workflow_dispatch: {} jobs: docker-build: - runs-on: mosaic-8wide + runs-on: mosaic-4wide if: github.repository_owner == 'mosaicml' strategy: matrix: @@ -47,6 +47,12 @@ jobs: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: Login to GHCR + uses: docker/login-action@v2 + with: + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_PASSWORD }} + - name: Calculate Docker Image Variables run: | set -euxo pipefail @@ -60,24 +66,29 @@ jobs: if [ "${{ github.event_name }}" == "pull_request" ]; then echo "Triggered by pull_request event." STAGING_REPO="mosaicml/ci-staging" + GHCR_STAGING_REPO="ghcr.io/databricks-mosaic/ci-staging" IMAGE_TAG="${STAGING_REPO}:${{matrix.name}}-${GIT_SHA}" + GHCR_IMAGE_TAG="${GHCR_STAGING_REPO}:${{matrix.name}}-${GIT_SHA}" IMAGE_CACHE="${STAGING_REPO}:${{matrix.name}}-buildcache" else # Triggered by push or workflow_dispatch event echo "Triggered by ${{ github.event_name }} event, releasing to prod" PROD_REPO="mosaicml/llm-foundry" IMAGE_TAG="${PROD_REPO}:${{matrix.name}}-${GIT_SHA},${PROD_REPO}:${{matrix.name}}-latest" + GHCR_PROD_REPO="ghcr.io/databricks-mosaic/llm-foundry" + GHCR_IMAGE_TAG="${GHCR_PROD_REPO}:${{matrix.name}}-${GIT_SHA},${GHCR_PROD_REPO}:${{matrix.name}}-latest" IMAGE_CACHE="${PROD_REPO}:${{matrix.name}}-buildcache" fi echo "IMAGE_TAG=${IMAGE_TAG}" >> ${GITHUB_ENV} echo "IMAGE_CACHE=${IMAGE_CACHE}" >> ${GITHUB_ENV} + echo "GHCR_IMAGE_TAG=${GHCR_IMAGE_TAG}" >> ${GITHUB_ENV} - name: Build and Push the Docker Image uses: docker/build-push-action@v3 with: context: . - tags: ${{ env.IMAGE_TAG }} + tags: ${{ env.IMAGE_TAG }},${{ env.GHCR_IMAGE_TAG}} push: true cache-from: type=registry,ref=${{ env.IMAGE_CACHE }} cache-to: type=registry,ref=${{ env.IMAGE_CACHE }},mode=max From 01e452ebed8087ae416946cc6ca8b7b6238b6d47 Mon Sep 17 00:00:00 2001 From: Kevin DeShawn <126115026+KevDevSha@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:26:02 -0500 Subject: [PATCH 2/6] remove maximize build space --- .github/workflows/docker.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index b23701c7b8..32abb1fe0b 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -24,13 +24,6 @@ jobs: base_image: mosaicml/pytorch:2.3.1_cu121-python3.11-ubuntu20.04-aws dep_groups: "[gpu]" steps: - - name: Maximize Build Space on Worker - uses: easimon/maximize-build-space@v4 - with: - overprovision-lvm: true - remove-dotnet: true - remove-android: true - remove-haskell: true - name: Checkout uses: actions/checkout@v3 From 166dbb133fbed598ce911c23bb7e5cca45cc6d69 Mon Sep 17 00:00:00 2001 From: Kevin DeShawn <126115026+KevDevSha@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:36:37 -0500 Subject: [PATCH 3/6] Update docker.yaml --- .github/workflows/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 32abb1fe0b..4144a593e0 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -44,7 +44,7 @@ jobs: uses: docker/login-action@v2 with: username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_PASSWORD }} + password: ${{ secrets.GHCR_TOKEN }} - name: Calculate Docker Image Variables run: | From eaf5a72b1887b48ad074ee0356d9e3045eea6c4f Mon Sep 17 00:00:00 2001 From: Kevin DeShawn <126115026+KevDevSha@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:41:13 -0500 Subject: [PATCH 4/6] Update docker.yaml --- .github/workflows/docker.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 4144a593e0..823ac0b004 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -12,7 +12,7 @@ on: workflow_dispatch: {} jobs: docker-build: - runs-on: mosaic-4wide + runs-on: mosaic-8wide if: github.repository_owner == 'mosaicml' strategy: matrix: @@ -45,6 +45,7 @@ jobs: with: username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} + registry: ghcr.io - name: Calculate Docker Image Variables run: | From 386955d883e2c187da84cda450a3a60d88b7d2ba Mon Sep 17 00:00:00 2001 From: Kevin DeShawn <126115026+KevDevSha@users.noreply.github.com> Date: Tue, 30 Jul 2024 17:10:37 -0500 Subject: [PATCH 5/6] concat --- .github/workflows/docker.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 823ac0b004..e97aa6b94f 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -61,16 +61,16 @@ jobs: echo "Triggered by pull_request event." STAGING_REPO="mosaicml/ci-staging" GHCR_STAGING_REPO="ghcr.io/databricks-mosaic/ci-staging" - IMAGE_TAG="${STAGING_REPO}:${{matrix.name}}-${GIT_SHA}" GHCR_IMAGE_TAG="${GHCR_STAGING_REPO}:${{matrix.name}}-${GIT_SHA}" + IMAGE_TAG="${STAGING_REPO}:${{matrix.name}}-${GIT_SHA},${GHCR_IMAGE_TAG}" IMAGE_CACHE="${STAGING_REPO}:${{matrix.name}}-buildcache" else # Triggered by push or workflow_dispatch event echo "Triggered by ${{ github.event_name }} event, releasing to prod" PROD_REPO="mosaicml/llm-foundry" - IMAGE_TAG="${PROD_REPO}:${{matrix.name}}-${GIT_SHA},${PROD_REPO}:${{matrix.name}}-latest" GHCR_PROD_REPO="ghcr.io/databricks-mosaic/llm-foundry" GHCR_IMAGE_TAG="${GHCR_PROD_REPO}:${{matrix.name}}-${GIT_SHA},${GHCR_PROD_REPO}:${{matrix.name}}-latest" + IMAGE_TAG="${PROD_REPO}:${{matrix.name}}-${GIT_SHA},${PROD_REPO}:${{matrix.name}}-latest,${GHCR_IMAGE_TAG}" IMAGE_CACHE="${PROD_REPO}:${{matrix.name}}-buildcache" fi @@ -82,7 +82,7 @@ jobs: uses: docker/build-push-action@v3 with: context: . - tags: ${{ env.IMAGE_TAG }},${{ env.GHCR_IMAGE_TAG}} + tags: ${{ env.IMAGE_TAG }} push: true cache-from: type=registry,ref=${{ env.IMAGE_CACHE }} cache-to: type=registry,ref=${{ env.IMAGE_CACHE }},mode=max From b4a48e3676ec8d87bde54640116b2d68558eb065 Mon Sep 17 00:00:00 2001 From: Kevin DeShawn <126115026+KevDevSha@users.noreply.github.com> Date: Tue, 30 Jul 2024 17:19:47 -0500 Subject: [PATCH 6/6] Update docker.yaml --- .github/workflows/docker.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index e97aa6b94f..17bb976a5d 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -76,7 +76,6 @@ jobs: echo "IMAGE_TAG=${IMAGE_TAG}" >> ${GITHUB_ENV} echo "IMAGE_CACHE=${IMAGE_CACHE}" >> ${GITHUB_ENV} - echo "GHCR_IMAGE_TAG=${GHCR_IMAGE_TAG}" >> ${GITHUB_ENV} - name: Build and Push the Docker Image uses: docker/build-push-action@v3