Add microbatch incremental strategy #1126
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: dbt-trino tests | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - "*.*.latest" | |
| paths-ignore: | |
| - "**/*.md" | |
| pull_request: | |
| branches: | |
| - master | |
| - "*.*.latest" | |
| paths-ignore: | |
| - "**/*.md" | |
| jobs: | |
| checks: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: "Checkout the source code" | |
| uses: actions/checkout@v4 | |
| - name: "Install Python" | |
| uses: actions/setup-python@v5 | |
| - name: "Install dev requirements" | |
| run: pip install -r dev_requirements.txt | |
| - name: "Run pre-commit checks" | |
| run: pre-commit run --all-files | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| engine: | |
| - "trino" | |
| - "starburst" | |
| - "starburst_galaxy" | |
| python: | |
| - "3.9" | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| isStarburstBranch: | |
| - ${{ (github.event_name == 'pull_request' && contains(github.event.pull_request.head.repo.full_name, 'starburstdata')) || github.event_name != 'pull_request' }} | |
| exclude: | |
| - engine: "starburst_galaxy" | |
| python: "3.12" | |
| isStarburstBranch: false | |
| - engine: "starburst_galaxy" | |
| python: "3.11" | |
| - engine: "starburst_galaxy" | |
| python: "3.10" | |
| - engine: "starburst_galaxy" | |
| python: "3.9" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Run dbt-trino tests against ${{ matrix.engine }} on python ${{ matrix.python }} | |
| env: | |
| DBT_TESTS_STARBURST_GALAXY_HOST: ${{ secrets.DBT_TESTS_STARBURST_GALAXY_HOST }} | |
| DBT_TESTS_STARBURST_GALAXY_USER: ${{ secrets.DBT_TESTS_STARBURST_GALAXY_USER }} | |
| DBT_TESTS_STARBURST_GALAXY_PASSWORD: ${{ secrets.DBT_TESTS_STARBURST_GALAXY_PASSWORD }} | |
| run: | | |
| if [[ ${{ matrix.engine }} == "trino" || ${{ matrix.engine }} == "starburst" ]]; then | |
| make dbt-${{ matrix.engine }}-tests | |
| elif [[ ${{ matrix.engine }} == "starburst_galaxy" ]]; then | |
| python -m pip install -e . -r dev_requirements.txt | |
| python -m pytest tests/functional --profile starburst_galaxy | |
| fi | |
| - name: Remove container on failure | |
| if: failure() | |
| run: ./docker/remove_${{ matrix.engine }}.bash || true |