integration.yml

# **what?**
# Runs integration tests.

# **why?**
# Ensure code runs as expected.

# **when?**
# This will run for all PRs, when code is pushed to a release
# branch, and when manually triggered.

name: Adapter Integration Tests

run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') &&  inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}"

on:
  push:
    branches:
      - "main"
      - "*.latest"

  pull_request_target:
    paths-ignore:
      - ".changes/**"
      - ".gitignore"
      - "**.md"

  workflow_dispatch:
    inputs:
      name:
        description: "Name to associate with run (example: 'dbt-adapters-242')"
        required: false
        type: string
        default: "Adapter Integration Tests"
      adapter_branch:
        description: "The branch of this adapter repository to use"
        type: string
        required: false
        default: "main"
      dbt_adapters_branch:
        description: "The branch of dbt-adapters to use"
        type: string
        required: false
        default: "main"
      dbt_core_branch:
        description: "The branch of dbt-core to use"
        type: string
        required: false
        default: "main"
      dbt_common_branch:
        description: "The branch of dbt-common to use"
        type: string
        required: false
        default: "main"

# explicitly turn off permissions for `GITHUB_TOKEN`
permissions: read-all

# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request_target') && github.event.pull_request.head.ref || github.sha }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:

  test:
    name: ${{ matrix.test }}
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        test:
          - "apache_spark"
          - "spark_session"
          - "databricks_sql_endpoint"
          - "databricks_cluster"
          - "databricks_http_cluster"

    env:
      DBT_INVOCATION_ENV: github-actions
      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
      DD_SITE: datadoghq.com
      DD_ENV: ci
      DD_SERVICE: ${{ github.event.repository.name }}
      DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }}
      DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }}
      DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }}
      DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }}
      DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USERNAME }}
      DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com"
      DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com"
      DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com"

    steps:
      - name: Check out the repository (push)
        if: github.event_name == 'push'
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Check out the repository (workflow_dispatch)
        if: github.event_name == 'workflow_dispatch'
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          ref: ${{ inputs.adapter_branch }}

      # explicitly checkout the branch for the PR,
      # this is necessary for the `pull_request` event
      - name: Check out the repository (PR)
        if: github.event_name == 'pull_request_target'
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          ref: ${{ github.event.pull_request.head.sha }}

      # the python version used here is not what is used in the tests themselves
      - name: Set up Python for dagger
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Update Adapters and Core branches (update dev_requirements.txt)
        if: github.event_name == 'workflow_dispatch'
        run: |
          ./.github/scripts/update_dev_dependency_branches.sh \
            ${{ inputs.dbt_adapters_branch }} \
            ${{ inputs.dbt_core_branch }} \
            ${{ inputs.dbt_common_branch }}
          cat dev-requirements.txt

      - name: Install python dependencies
        run: |
          python -m pip install --user --upgrade pip
          python -m pip --version
          python -m pip install -r dagger/requirements.txt

      - name: Run tests for ${{ matrix.test }}
        run: python dagger/run_dbt_spark_tests.py --profile ${{ matrix.test }}