test-repeater.yml

# **what?**
# This workflow will test all test(s) at the input path given number of times to determine if it's flaky or not.  You can test with any supported OS/Python combination.
# This is batched in 10 to allow more test iterations faster.

# **why?**
# Testing if a test is flaky and if a previously flaky test has been fixed.  This allows easy testing on supported python versions and OS combinations.

# **when?**
# This is triggered manually from dbt-core.

name: Flaky Tester

on:
  workflow_dispatch:
    inputs:
      branch:
        description: 'Branch to check out'
        type: string
        required: true
        default: 'main'
      test_path:
        description: 'Path to single test to run (ex: tests/functional/retry/test_retry.py::TestRetry::test_fail_fast)'
        type: string
        required: true
        default: 'tests/functional/...'
      python_version:
        description: 'Version of Python to Test Against'
        type: choice
        options:
          - '3.9'
          - '3.10'
          - '3.11'
      os:
        description: 'OS to run test in'
        type: choice
        options:
          - 'ubuntu-latest'
          - 'macos-12'
          - 'windows-latest'
      num_runs_per_batch:
        description: 'Max number of times to run the test per batch.  We always run 10 batches.'
        type: number
        required: true
        default: '50'

permissions: read-all

defaults:
  run:
    shell: bash

jobs:
  debug:
    runs-on: ubuntu-latest
    steps:
      - name: "[DEBUG] Output Inputs"
        run: |
          echo "Branch: ${{ inputs.branch }}"
          echo "test_path: ${{ inputs.test_path }}"
          echo "python_version: ${{ inputs.python_version }}"
          echo "os: ${{ inputs.os }}"
          echo "num_runs_per_batch: ${{ inputs.num_runs_per_batch }}"

  pytest:
    runs-on: ${{ inputs.os }}
    strategy:
      # run all batches, even if one fails.  This informs how flaky the test may be.
      fail-fast: false
      # using a matrix to speed up the jobs since the matrix will run in parallel when runners are available
      matrix:
        batch: ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]
    env:
      PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
      DBT_TEST_USER_1: dbt_test_user_1
      DBT_TEST_USER_2: dbt_test_user_2
      DBT_TEST_USER_3: dbt_test_user_3
      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
      DD_SITE: datadoghq.com
      DD_ENV: ci
      DD_SERVICE: ${{ github.event.repository.name }}

    steps:
      - name: "Checkout code"
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.branch }}

      - name: "Setup Python"
        uses: actions/setup-python@v5
        with:
          python-version: "${{ inputs.python_version }}"

      - name: "Setup Dev Environment"
        run: make dev

      - name: "Set up postgres (linux)"
        if: inputs.os == 'ubuntu-latest'
        run: make setup-db

        # mac and windows don't use make due to limitations with docker with those runners in GitHub
      - name: "Set up postgres (macos)"
        if: inputs.os == 'macos-12'
        uses: ./.github/actions/setup-postgres-macos

      - name: "Set up postgres (windows)"
        if: inputs.os == 'windows-latest'
        uses: ./.github/actions/setup-postgres-windows

      - name: "Test Command"
        id: command
        run: |
          test_command="python -m pytest ${{ inputs.test_path }}"
          echo "test_command=$test_command" >> $GITHUB_OUTPUT

      - name: "Run test ${{ inputs.num_runs_per_batch }} times"
        id: pytest
        run: |
          set +e
          for ((i=1; i<=${{ inputs.num_runs_per_batch }}; i++))
          do
            echo "Running pytest iteration $i..."
            python -m pytest --ddtrace ${{ inputs.test_path }}
            exit_code=$?

            if [[ $exit_code -eq 0 ]]; then
              success=$((success + 1))
              echo "Iteration $i: Success"
            else
              failure=$((failure + 1))
              echo "Iteration $i: Failure"
            fi

            echo
            echo "==========================="
            echo "Successful runs: $success"
            echo "Failed runs: $failure"
            echo "==========================="
            echo
          done

          echo "failure=$failure" >> $GITHUB_OUTPUT

      - name: "Success and Failure Summary: ${{ inputs.os }}/Python ${{ inputs.python_version }}"
        run: |
          echo "Batch: ${{ matrix.batch }}"
          echo "Successful runs: ${{ steps.pytest.outputs.success }}"
          echo "Failed runs: ${{ steps.pytest.outputs.failure }}"

      - name: "Error for Failures"
        if: ${{ steps.pytest.outputs.failure }}
        run: |
            echo "Batch ${{ matrix.batch }} failed ${{ steps.pytest.outputs.failure }} of ${{ inputs.num_runs_per_batch }} tests"
            exit 1