forked from datahub-project/datahub
-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ci: separate airflow build and test (datahub-project#8688)
Co-authored-by: Harshal Sheth <[email protected]>
- Loading branch information
1 parent
1282e5b
commit e867dbc
Showing
52 changed files
with
2,037 additions
and
1,874 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| name: Airflow Plugin | ||
| on: | ||
| push: | ||
| branches: | ||
| - master | ||
| paths: | ||
| - ".github/workflows/airflow-plugin.yml" | ||
| - "metadata-ingestion-modules/airflow-plugin/**" | ||
| - "metadata-ingestion/**" | ||
| - "metadata-models/**" | ||
| pull_request: | ||
| branches: | ||
| - master | ||
| paths: | ||
| - ".github/**" | ||
| - "metadata-ingestion-modules/airflow-plugin/**" | ||
| - "metadata-ingestion/**" | ||
| - "metadata-models/**" | ||
| release: | ||
| types: [published] | ||
|
|
||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | ||
| cancel-in-progress: true | ||
|
|
||
| jobs: | ||
| airflow-plugin: | ||
| runs-on: ubuntu-latest | ||
| env: | ||
| SPARK_VERSION: 3.0.3 | ||
| DATAHUB_TELEMETRY_ENABLED: false | ||
| strategy: | ||
| matrix: | ||
| include: | ||
| - python-version: "3.7" | ||
| extraPythonRequirement: "apache-airflow~=2.1.0" | ||
| - python-version: "3.7" | ||
| extraPythonRequirement: "apache-airflow~=2.2.0" | ||
| - python-version: "3.10" | ||
| extraPythonRequirement: "apache-airflow~=2.4.0" | ||
| - python-version: "3.10" | ||
| extraPythonRequirement: "apache-airflow~=2.6.0" | ||
| - python-version: "3.10" | ||
| extraPythonRequirement: "apache-airflow>2.6.0" | ||
| fail-fast: false | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ matrix.python-version }} | ||
| cache: "pip" | ||
| - name: Install dependencies | ||
| run: ./metadata-ingestion/scripts/install_deps.sh | ||
| - name: Install airflow package and test (extras ${{ matrix.extraPythonRequirement }}) | ||
| run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:airflow-plugin:lint :metadata-ingestion-modules:airflow-plugin:testQuick | ||
| - name: pip freeze show list installed | ||
| if: always() | ||
| run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && pip freeze | ||
| - uses: actions/upload-artifact@v3 | ||
| if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'apache-airflow>2.6.0' }} | ||
| with: | ||
| name: Test Results (Airflow Plugin ${{ matrix.python-version}}) | ||
| path: | | ||
| **/build/reports/tests/test/** | ||
| **/build/test-results/test/** | ||
| **/junit.*.xml | ||
| - name: Upload coverage to Codecov | ||
| if: always() | ||
| uses: codecov/codecov-action@v3 | ||
| with: | ||
| token: ${{ secrets.CODECOV_TOKEN }} | ||
| directory: . | ||
| fail_ci_if_error: false | ||
| flags: airflow-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }} | ||
| name: pytest-airflow | ||
| verbose: true | ||
|
|
||
| event-file: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Upload | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: Event File | ||
| path: ${{ github.event_path }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
12 changes: 12 additions & 0 deletions
12
metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_airflow_compat.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| # This module must be imported before any Airflow imports in any of our files. | ||
| # The AIRFLOW_PATCHED just helps avoid flake8 errors. | ||
|
|
||
| from datahub.utilities._markupsafe_compat import MARKUPSAFE_PATCHED | ||
|
|
||
| assert MARKUPSAFE_PATCHED | ||
|
|
||
| AIRFLOW_PATCHED = True | ||
|
|
||
| __all__ = [ | ||
| "AIRFLOW_PATCHED", | ||
| ] |
29 changes: 29 additions & 0 deletions
29
metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_airflow_shims.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| from airflow.models.baseoperator import BaseOperator | ||
|
|
||
| from datahub_airflow_plugin._airflow_compat import AIRFLOW_PATCHED | ||
|
|
||
| try: | ||
| from airflow.models.mappedoperator import MappedOperator | ||
| from airflow.models.operator import Operator | ||
| from airflow.operators.empty import EmptyOperator | ||
| except ModuleNotFoundError: | ||
| # Operator isn't a real class, but rather a type alias defined | ||
| # as the union of BaseOperator and MappedOperator. | ||
| # Since older versions of Airflow don't have MappedOperator, we can just use BaseOperator. | ||
| Operator = BaseOperator # type: ignore | ||
| MappedOperator = None # type: ignore | ||
| from airflow.operators.dummy import DummyOperator as EmptyOperator # type: ignore | ||
|
|
||
| try: | ||
| from airflow.sensors.external_task import ExternalTaskSensor | ||
| except ImportError: | ||
| from airflow.sensors.external_task_sensor import ExternalTaskSensor # type: ignore | ||
|
|
||
| assert AIRFLOW_PATCHED | ||
|
|
||
| __all__ = [ | ||
| "Operator", | ||
| "MappedOperator", | ||
| "EmptyOperator", | ||
| "ExternalTaskSensor", | ||
| ] |
Oops, something went wrong.