airflow-plugin.yml

name: Airflow Plugin
on:
  push:
    branches:
      - master
    paths:
      - ".github/workflows/airflow-plugin.yml"
      - "metadata-ingestion-modules/airflow-plugin/**"
      - "metadata-ingestion/**"
      - "metadata-models/**"
  pull_request:
    branches:
      - "**"
    paths:
      - ".github/workflows/airflow-plugin.yml"
      - "metadata-ingestion-modules/airflow-plugin/**"
      - "metadata-ingestion/**"
      - "metadata-models/**"
  release:
    types: [published]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  airflow-plugin:
    runs-on: ubuntu-latest
    env:
      SPARK_VERSION: 3.0.3
      DATAHUB_TELEMETRY_ENABLED: false
    strategy:
      matrix:
        include:
          # Note: this should be kept in sync with tox.ini.
          - python-version: "3.8"
            extra_pip_requirements: "apache-airflow~=2.3.4"
            extra_pip_extras: test-airflow23
          - python-version: "3.10"
            extra_pip_requirements: "apache-airflow~=2.4.3"
            extra_pip_extras: test-airflow24
          - python-version: "3.10"
            extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt"
          - python-version: "3.10"
            extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
          - python-version: "3.10"
            extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
          - python-version: "3.11"
            extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
          - python-version: "3.11"
            extra_pip_requirements: "apache-airflow~=2.10.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt"
      fail-fast: false
    steps:
      - name: Set up JDK 17
        uses: actions/setup-java@v4
        with:
          distribution: "zulu"
          java-version: 17
      - uses: gradle/actions/setup-gradle@v3
      - uses: acryldata/sane-checkout-action@v3
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
      - name: Install dependencies
        run: ./metadata-ingestion/scripts/install_deps.sh
      - name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }})
        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
      - name: pip freeze show list installed
        if: always()
        run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze
      - uses: actions/upload-artifact@v3
        if: ${{ always() && matrix.python-version == '3.10' && matrix.extra_pip_requirements == 'apache-airflow>=2.7.0' }}
        with:
          name: Test Results (Airflow Plugin ${{ matrix.python-version}})
          path: |
            **/build/reports/tests/test/**
            **/build/test-results/test/**
            **/junit.*.xml
            !**/binary/**
      - name: Upload coverage to Codecov
        if: always()
        uses: codecov/codecov-action@v5
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          directory: ./build/coverage-reports/
          fail_ci_if_error: false
          flags: airflow,airflow-${{ matrix.extra_pip_extras }}
          name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
          verbose: true

  event-file:
    runs-on: ubuntu-latest
    steps:
      - name: Upload
        uses: actions/upload-artifact@v3
        with:
          name: Event File
          path: ${{ github.event_path }}