integration.yml

# **what?**
# Runs integration tests.

# **why?**
# Ensure code runs as expected.

# **when?**
# This will run for all PRs, when code is pushed to a release
# branch, and when manually triggered.

name: Integration tests

on:
  push:
    branches:
      - "main"
      - "*.latest"
      - "releases/*"
  pull_request:
  workflow_dispatch:
    inputs:
      dbt-core-branch:
        description: "branch of dbt-core to use in dev-requirements.txt"
        required: false
        type: string

# explicitly turn off permissions for `GITHUB_TOKEN`
permissions: read-all

# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  # generate test metadata about what files changed and the testing matrix to use
  test-metadata:
    # run if not a PR from a forked repository or has a label to mark as safe to test
    if: >-
      github.event_name != 'pull_request' ||
      github.event.pull_request.head.repo.full_name == github.repository ||
      contains(github.event.pull_request.labels.*.name, 'ok to test')
    runs-on: ubuntu-latest

    outputs:
      matrix: ${{ steps.generate-matrix.outputs.result }}
      run-python-tests: ${{ steps.filter.outputs.bigquery-python }}

    steps:
      - name: Check out the repository (non-PR)
        if: github.event_name != 'pull_request'
        uses: actions/checkout@v3
        with:
          persist-credentials: false

      - name: Check out the repository (PR)
        if: github.event_name == 'pull_request'
        uses: actions/checkout@v3
        with:
          persist-credentials: false
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Check if relevant files changed
        if: github.event_name == 'pull_request'
        # https://github.com/marketplace/actions/paths-changes-filter
        # For each filter, it sets output variable named by the filter to the text:
        #  'true' - if any of changed files matches any of filter rules
        #  'false' - if none of changed files matches any of filter rules
        # also, returns:
        #  `changes` - JSON array with names of all filters matching any of the changed files
        uses: dorny/paths-filter@v2
        id: get-changes
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          filters: |
            spark:
              - 'dbt/**'
              - 'tests/**'
              - 'dev-requirements.txt'

  local-tests:
    name: test spark local against python ${{ matrix.python-version }}

    # run if not a PR from a forked repository or has a label to mark as safe to test
    # also checks that the matrix generated is not empty
    if: >-
      (
        github.event_name != 'pull_request' ||
        github.event.pull_request.head.repo.full_name == github.repository ||
        contains(github.event.pull_request.labels.*.name, 'ok to test')
      )

    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.8"
        test:
          - "spark-thrift"
          - "spark-session"

    env:
      PYTEST_ADDOPTS: "-v --color=yes --csv test_results.csv"
      DBT_INVOCATION_ENV: github-actions
      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
      DD_SITE: datadoghq.com
      DD_ENV: ci
      DD_SERVICE: ${{ github.event.repository.name }}

    steps:
      - name: Check out the repository
        if: github.event_name != 'pull_request'
        uses: actions/checkout@v3
        with:
          persist-credentials: false

      # explicity checkout the branch for the PR,
      # this is necessary for the `pull_request` event
      - name: Check out the repository (PR)
        if: github.event_name == 'pull_request'
        uses: actions/checkout@v3
        with:
          persist-credentials: false
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install python dependencies
        run: |
          python -m pip install --user --upgrade pip
          python -m pip install tox
          python -m pip --version
          tox --version

      - name: Update dev_requirements.txt
        if: inputs.dbt-core-branch != ''
        run: |
          pip install bumpversion
          ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}

      - uses: isbang/compose-action@v1.5.1
        if: ${{ matrix.test == 'spark-thrift'}}
        with:
          compose-file: "./docker-compose.yml"

      - name: Run tox for Spark ${{ matrix.test }}
        run: tox -e integration-${{ matrix.test }}

  databricks-tests:
    name: test spark databricks against python ${{ matrix.python-version }}
    # run if not a PR from a forked repository or has a label to mark as safe to test
    # also checks that the matrix generated is not empty
    if: >-
      (
        github.event_name != 'pull_request' ||
        github.event.pull_request.head.repo.full_name == github.repository ||
        contains(github.event.pull_request.labels.*.name, 'ok to test')
      )

    runs-on: ubuntu-latest
    container:
      image: "fishtownanalytics/test-container:10"
      options: --user root
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.8"
        test:
          - "databricks-odbc-sql-endpoint"
          - "databricks-odbc-cluster"
          - "spark-databricks-http"

    env:
      PYTEST_ADDOPTS: "-v --color=yes --csv test_results.csv"
      DBT_INVOCATION_ENV: github-actions
      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
      DD_SITE: datadoghq.com
      DD_ENV: ci
      DD_SERVICE: ${{ github.event.repository.name }}
      DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }}
      DBT_DATABRICKS_HOSTNAME: ${{ secrets.DBT_DATABRICKS_HOST }}
      DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }}
      DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }}
      DBT_DATABRICS_USER: ${{ secrets.DBT_DATABRICKS_USER }}

    steps:
      - name: Check out the repository
        if: github.event_name != 'pull_request'
        uses: actions/checkout@v3
        with:
          persist-credentials: false

      # explicity checkout the branch for the PR,
      # this is necessary for the `pull_request` event
      - name: Check out the repository (PR)
        if: github.event_name == 'pull_request'
        uses: actions/checkout@v3
        with:
          persist-credentials: false
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Install python dependencies
        run: |
          python -m pip install --user --upgrade pip
          python -m pip install tox
          python -m pip --version
          tox --version

      - name: Update dev_requirements.txt
        if: inputs.dbt-core-branch != ''
        run: |
          pip install bumpversion
          ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}

      - name: Configure ODBC
        if: ${{ matrix.test != 'spark-databricks-http' }}
        shell: bash
        run: |
          apt-get update && apt-get install -y --no-install-recommends \
          g++ \
          unixodbc-dev \
          unzip
          
          apt-get install  -y curl
          rm -rf /tmp && mkdir /tmp
          
          curl -OL "https://databricks.com/wp-content/uploads/drivers-2020/SimbaSparkODBC-2.6.16.1019-Debian-64bit.zip" \

          unzip SimbaSparkODBC-2.6.16.1019-Debian-64bit.zip -d /tmp/ \
          && dpkg -i /tmp/SimbaSparkODBC-*/*.deb \
          && echo "[Simba]\nDriver = /opt/simba/spark/lib/64/libsparkodbc_sb64.so" >> /etc/odbcinst.ini \
          && rm -rf /tmp

      - name: Run tox for Spark ${{ matrix.test }}
        run: tox -e integration-${{ matrix.test }}