prod-image-build.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Build PROD images
on:  # yamllint disable-line rule:truthy
  workflow_call:
    inputs:
      runs-on-as-json-public:
        description: "The array of labels (in json form) determining default runner used for the build."
        required: true
        type: string
      build-type:
        description: >
          Name of the 'type' of the build - usually 'Regular' but other types are used to test image
          variations.
        required: true
        type: string
      do-build:
        description: >
          Whether to actually do the build (true/false). If set to false, the build is done
          already in pull-request-target workflow, so we skip it here.
        required: false
        default: "true"
        type: string
      upload-package-artifact:
        description: >
          Whether to upload package artifacts (true/false). If false, the job will rely on artifacts prepared
          by the main prod-image build job.
        required: true
        type: string
      target-commit-sha:
        description: "The commit SHA to checkout for the build"
        required: false
        default: ""
        type: string
      pull-request-target:
        description: "Whether we are running this from pull-request-target workflow (true/false)"
        required: false
        default: "false"
        type: string
      is-committer-build:
        description: "Whether the build is executed by committer (true/false)"
        required: false
        default: "false"
        type: string
      push-image:
        description: "Whether to push image to the registry (true/false)"
        required: true
        type: string
      debian-version:
        description: "Base Debian distribution to use for the build (bookworm)"
        type: string
        default: "bookworm"
      install-mysql-client-type:
        description: "MySQL client type to use during build (mariadb/mysql)"
        type: string
        default: "mariadb"
      use-uv:
        description: "Whether to use uv to build the image (true/false)"
        required: true
        type: string
      image-tag:
        description: "Tag to set for the image"
        required: true
        type: string
      python-versions:
        description: "JSON-formatted array of Python versions to build images from"
        required: true
        type: string
      default-python-version:
        description: "Which version of python should be used by default"
        required: true
        type: string
      platform:
        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
        required: true
        type: string
      branch:
        description: "Branch used to run the CI jobs in (main/v2_*_test)."
        required: true
        type: string
      constraints-branch:
        description: "Branch used to construct constraints URL from."
        required: true
        type: string
      build-provider-packages:
        description: "Whether to build provider packages (true/false). If false providers are from PyPI"
        required: true
        type: string
      upgrade-to-newer-dependencies:
        description: "Whether to attempt to upgrade image to newer dependencies (true/false)"
        required: true
        type: string
      chicken-egg-providers:
        description: "Space-separated list of providers that should be installed from context files"
        required: true
        type: string
      docker-cache:
        description: "Docker cache specification to build the image (registry, local, disabled)."
        required: true
        type: string
      disable-airflow-repo-cache:
        description: "Disable airflow repo cache read from main."
        required: true
        type: string
jobs:

  build-prod-packages:
    name: "${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} Airflow and provider packages"
    timeout-minutes: 10
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
    env:
      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
      VERSION_SUFFIX_FOR_PYPI: ${{ inputs.branch == 'main' && 'dev0' || '' }}
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
        if: inputs.do-build == 'true' && inputs.upload-package-artifact == 'true'
      - name: "Checkout target branch"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Checkout target commit"
        uses: ./.github/actions/checkout_target_commit
        with:
          target-commit-sha: ${{ inputs.target-commit-sha }}
          pull-request-target: ${{ inputs.pull-request-target }}
          is-committer-build: ${{ inputs.is-committer-build }}
        if: inputs.do-build == 'true' && inputs.upload-package-artifact == 'true'
      - name: "Cleanup docker"
        run: ./scripts/ci/cleanup_docker.sh
        if: inputs.do-build == 'true' && inputs.upload-package-artifact == 'true'
      - uses: actions/setup-python@v5
        with:
          python-version: "${{ inputs.default-python-version }}"
        if: inputs.do-build == 'true' && inputs.upload-package-artifact == 'true'
      - name: "Cleanup dist and context file"
        shell: bash
        run: rm -fv ./dist/* ./docker-context-files/*
        if: inputs.do-build == 'true' && inputs.upload-package-artifact == 'true'
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
        if: inputs.do-build == 'true' && inputs.upload-package-artifact == 'true'
      - name: "Prepare providers packages"
        shell: bash
        run: >
          breeze release-management prepare-provider-packages
          --package-list-file ./prod_image_installed_providers.txt
          --package-format wheel
        if: >
          inputs.do-build == 'true' &&
          inputs.upload-package-artifact == 'true' &&
          inputs.build-provider-packages == 'true'
      - name: "Prepare chicken-eggs provider packages"
        shell: bash
        run: >
          breeze release-management prepare-provider-packages
          --package-format wheel ${{ inputs.chicken-egg-providers }}
        if: >
          inputs.do-build  == 'true' &&
          inputs.upload-package-artifact == 'true' &&
          inputs.chicken-egg-providers != ''
      - name: "Prepare airflow package"
        shell: bash
        run: >
          breeze release-management prepare-airflow-package --package-format wheel
        if: inputs.do-build  == 'true' && inputs.upload-package-artifact == 'true'
      - name: "Prepare task-sdk package"
        shell: bash
        run: >
          breeze release-management prepare-task-sdk-package --package-format wheel
        if: inputs.do-build  == 'true' && inputs.upload-package-artifact == 'true'
      - name: "Upload prepared packages as artifacts"
        uses: actions/upload-artifact@v4
        with:
          name: prod-packages
          path: ./dist
          retention-days: 7
          if-no-files-found: error
        if: inputs.do-build  == 'true' && inputs.upload-package-artifact == 'true'

  build-prod-images:
    strategy:
      fail-fast: false
      matrix:
        # yamllint disable-line rule:line-length
        python-version: ${{ inputs.do-build == 'true' && fromJSON(inputs.python-versions) || fromJSON('[""]') }}
    timeout-minutes: 80
    name: "\
${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \
PROD ${{ inputs.build-type }} image\
${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\
${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
    needs:
      - build-prod-packages
    env:
      BACKEND: sqlite
      DEFAULT_BRANCH: ${{ inputs.branch }}
      DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
      VERSION_SUFFIX_FOR_PYPI: ${{ inputs.branch == 'main' && 'dev0' || '' }}
      INCLUDE_NOT_READY_PROVIDERS: "true"
      # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
      # Airflow one is going to be used
      CONSTRAINTS_GITHUB_REPOSITORY: >-
        ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
        secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
      # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one
      GITHUB_REPOSITORY: ${{ github.repository }}
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GITHUB_USERNAME: ${{ github.actor }}
      USE_UV: ${{ inputs.use-uv }}
      VERBOSE: "true"
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
        if: inputs.do-build == 'true'
      - name: "Checkout target branch"
        uses: actions/checkout@v4
        with:
          persist-credentials: false
      - name: "Checkout target commit"
        uses: ./.github/actions/checkout_target_commit
        with:
          target-commit-sha: ${{ inputs.target-commit-sha }}
          pull-request-target: ${{ inputs.pull-request-target }}
          is-committer-build: ${{ inputs.is-committer-build }}
        if: inputs.do-build == 'true'
      - name: "Cleanup docker"
        run: ./scripts/ci/cleanup_docker.sh
        if: inputs.do-build == 'true'
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
        if: inputs.do-build == 'true'
      - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
        shell: bash
        run: |
          pip install rich>=12.4.4 pyyaml
          python scripts/ci/pre_commit/update_providers_dependencies.py
        if: inputs.do-build == 'true' && inputs.upgrade-to-newer-dependencies != 'false'
      - name: "Cleanup dist and context file"
        shell: bash
        run: rm -fv ./dist/* ./docker-context-files/*
        if: inputs.do-build == 'true'
      - name: "Download packages prepared as artifacts"
        uses: actions/download-artifact@v4
        with:
          name: prod-packages
          path: ./docker-context-files
        if: inputs.do-build == 'true'
      - name: "Download constraints"
        uses: actions/download-artifact@v4
        with:
          name: constraints
          path: ./docker-context-files
        if: inputs.do-build == 'true'
      - name: Login to ghcr.io
        shell: bash
        run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
        if: inputs.do-build == 'true'
      - name: "Build PROD images w/ source providers ${{ matrix.python-version }}:${{ inputs.image-tag }}"
        shell: bash
        run: >
          breeze prod-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}"
          --commit-sha "${{ github.sha }}"
          --install-packages-from-context --airflow-constraints-mode constraints-source-providers
          --use-constraints-for-context-packages --python "${{ matrix.python-version }}"
        env:
          PUSH: ${{ inputs.push-image }}
          DOCKER_CACHE: ${{ inputs.docker-cache }}
          DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }}
          DEBIAN_VERSION: ${{ inputs.debian-version }}
          INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
          INCLUDE_NOT_READY_PROVIDERS: "true"
        if: inputs.do-build == 'true' && inputs.build-provider-packages == 'true'
      - name: "Build PROD images with PyPi providers ${{ matrix.python-version }}:${{ inputs.image-tag }}"
        shell: bash
        run: >
          breeze prod-image build --builder airflow_cache --tag-as-latest
          --image-tag "${{ inputs.image-tag }}" --commit-sha "${{ github.sha }}"
          --install-packages-from-context --airflow-constraints-mode constraints
          --use-constraints-for-context-packages --python "${{ matrix.python-version }}"
        env:
          PUSH: ${{ inputs.push-image }}
          DOCKER_CACHE: ${{ inputs.docker-cache }}
          DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }}
          DEBIAN_VERSION: ${{ inputs.debian-version }}
          INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
          INCLUDE_NOT_READY_PROVIDERS: "true"
        if: inputs.do-build == 'true' && inputs.build-provider-packages != 'true'
      - name: Verify PROD image ${{ matrix.python-version }}:${{ inputs.image-tag }}
        run: >
          breeze prod-image verify --image-tag "${{ inputs.image-tag }}"
          --python "${{ matrix.python-version }}"
        if: inputs.do-build == 'true'