Adapter Integration Tests #59
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # **what?** | |
| # Runs integration tests. | |
| # **why?** | |
| # Ensure code runs as expected. | |
| # **when?** | |
| # This will run for all PRs, when code is pushed to a release | |
| # branch, and when manually triggered. | |
| name: Adapter Integration Tests | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| - "*.latest" | |
| - "releases/*" | |
| pull_request_target: | |
| workflow_dispatch: | |
| inputs: | |
| dbt-core-branch: | |
| description: "branch of dbt-core to use in dev-requirements.txt" | |
| required: false | |
| type: string | |
| # explicitly turn off permissions for `GITHUB_TOKEN` | |
| permissions: read-all | |
| # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request_target') && github.event.pull_request.head.ref || github.sha }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| # generate test metadata about what files changed and the testing matrix to use | |
| test-metadata: | |
| # run if not a PR from a forked repository or has a label to mark as safe to test | |
| if: >- | |
| github.event_name != 'pull_request_target' || | |
| github.event.pull_request.head.repo.full_name == github.repository || | |
| contains(github.event.pull_request.labels.*.name, 'ok to test') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.generate-matrix.outputs.result }} | |
| steps: | |
| - name: Check out the repository (non-PR) | |
| if: github.event_name != 'pull_request_target' | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Check out the repository (PR) | |
| if: github.event_name == 'pull_request_target' | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Check if relevant files changed | |
| if: github.event_name == 'pull_request_target' | |
| # https://github.com/marketplace/actions/paths-changes-filter | |
| # For each filter, it sets output variable named by the filter to the text: | |
| # 'true' - if any of changed files matches any of filter rules | |
| # 'false' - if none of changed files matches any of filter rules | |
| # also, returns: | |
| # `changes` - JSON array with names of all filters matching any of the changed files | |
| uses: dorny/paths-filter@v2 | |
| id: get-changes | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| filters: | | |
| spark: | |
| - 'dbt/**' | |
| - 'tests/**' | |
| - 'dev-requirements.txt' | |
| test: | |
| name: ${{ matrix.test }} / python ${{ matrix.python-version }} / ubuntu-latest | |
| # run if not a PR from a forked repository or has a label to mark as safe to test | |
| # also checks that the matrix generated is not empty | |
| if: >- | |
| ( | |
| github.event_name != 'pull_request_target' || | |
| github.event.pull_request.head.repo.full_name == github.repository || | |
| contains(github.event.pull_request.labels.*.name, 'ok to test') | |
| ) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.8" | |
| test: | |
| - "spark-thrift" | |
| - "spark-session" | |
| - "spark-databricks-odbc-sql-endpoint" | |
| - "spark-databricks-odbc-cluster" | |
| - "spark-databricks-http" | |
| env: | |
| TOXENV: integration-${{ matrix.test }} | |
| PYTEST_ADDOPTS: "-v --color=yes --csv integration_results.csv" | |
| DBT_INVOCATION_ENV: github-actions | |
| DD_CIVISIBILITY_AGENTLESS_ENABLED: true | |
| DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} | |
| DD_SITE: datadoghq.com | |
| DD_ENV: ci | |
| DD_SERVICE: ${{ github.event.repository.name }} | |
| DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }} | |
| DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} | |
| DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} | |
| DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} | |
| DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USERNAME }} | |
| DBT_TEST_USER_1: "[email protected]" | |
| DBT_TEST_USER_2: "[email protected]" | |
| DBT_TEST_USER_3: "[email protected]" | |
| ODBC_DRIVER: "Simba" | |
| steps: | |
| - name: Check out the repository | |
| if: github.event_name != 'pull_request_target' | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| # explicitly checkout the branch for the PR, | |
| # this is necessary for the `pull_request` event | |
| - name: Check out the repository (PR) | |
| if: github.event_name == 'pull_request_target' | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install python dependencies | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip install tox | |
| python -m pip --version | |
| tox --version | |
| - name: Update dev_requirements.txt | |
| if: inputs.dbt-core-branch != '' | |
| run: | | |
| pip install bumpversion | |
| ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} | |
| - uses: isbang/[email protected] | |
| if: ${{ matrix.test == 'spark-thrift'}} | |
| with: | |
| compose-file: "./docker-compose.yml" | |
| - name: Configure ODBC | |
| if: ${{ matrix.test == 'spark-databricks-odbc-sql-endpoint' || matrix.test == 'spark-databricks-odbc-cluster' }} | |
| run: | | |
| ./.github/scripts/configure_odbc.sh | |
| - name: Run tox for ${{ matrix.test }} | |
| run: tox -- --ddtrace |