Skip to content

Commit

Permalink
Mark flaky tests (#813) (#826)
Browse files Browse the repository at this point in the history
backport #813

(cherry picked from commit b57890e)
  • Loading branch information
mikealfare authored May 14, 2024
1 parent 4b72580 commit f32c1ed
Show file tree
Hide file tree
Showing 11 changed files with 149 additions and 166 deletions.
87 changes: 0 additions & 87 deletions .github/scripts/integration-test-matrix.js

This file was deleted.

170 changes: 99 additions & 71 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,88 +53,129 @@ defaults:
shell: bash

jobs:
# generate test metadata about what files changed and the testing matrix to use
test-metadata:
test:
name: redshift / python ${{ matrix.python-version }} / ${{ matrix.os }}

# run if not a PR from a forked repository or has a label to mark as safe to test
if: >-
github.event_name != 'pull_request_target' ||
github.event.pull_request.head.repo.full_name == github.repository ||
contains(github.event.pull_request.labels.*.name, 'ok to test')
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}

outputs:
matrix: ${{ steps.generate-matrix.outputs.result }}
strategy:
fail-fast: false
max-parallel: 3
matrix:
os: [ubuntu-22.04, macos12, windows-2022]
python-version: ["3.8"]
include:
- os: ubuntu-22.04
python-version: "3.9"
- os: ubuntu-22.04
python-version: "3.10"
- os: ubuntu-22.04
python-version: "3.11"

env:
TOXENV: integration-redshift
PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
DBT_INVOCATION_ENV: github-actions
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
DD_SITE: datadoghq.com
DD_ENV: ci
DD_SERVICE: ${{ github.event.repository.name }}

steps:
- name: Check out the repository (non-PR)
- name: Check out the repository
if: github.event_name != 'pull_request_target'
uses: actions/checkout@v3
with:
persist-credentials: false

# explicity checkout the branch for the PR,
# this is necessary for the `pull_request_target` event
- name: Check out the repository (PR)
if: github.event_name == 'pull_request_target'
uses: actions/checkout@v3
with:
persist-credentials: false
ref: ${{ github.event.pull_request.head.sha }}

- name: Check if relevant files changed
if: github.event_name == 'pull_request_target'
# https://github.com/marketplace/actions/paths-changes-filter
# For each filter, it sets output variable named by the filter to the text:
# 'true' - if any of changed files matches any of filter rules
# 'false' - if none of changed files matches any of filter rules
# also, returns:
# `changes` - JSON array with names of all filters matching any of the changed files
uses: dorny/paths-filter@v2
id: get-changes
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
token: ${{ secrets.GITHUB_TOKEN }}
filters: |
redshift:
- 'dbt/**'
- 'tests/**'
- 'dev-requirements.txt'
- name: Generate integration test matrix
id: generate-matrix
uses: actions/github-script@v6
python-version: ${{ matrix.python-version }}

- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install tox
python -m pip --version
tox --version
- name: Update dev_requirements.txt
if: inputs.dbt-core-branch != ''
run: |
pip install bumpversion
./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
- name: Run tox (redshift)
env:
CHANGES: ${{ steps.get-changes.outputs.changes }}
REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
DBT_TEST_USER_1: dbt_test_user_1
DBT_TEST_USER_2: dbt_test_user_2
DBT_TEST_USER_3: dbt_test_user_3
run: tox -- -m "not flaky" --ddtrace

- uses: actions/upload-artifact@v3
if: always()
with:
script: |
const script = require('./.github/scripts/integration-test-matrix.js')
const matrix = script({ context })
console.log(matrix)
return matrix
test:
name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }}
name: logs
path: ./logs

- name: Get current date
if: always()
id: date
run: |
echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
- uses: actions/upload-artifact@v3
if: always()
with:
name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_redshift-${{ steps.date.outputs.date }}.csv
path: integration_results.csv

test-flaky:
name: redshift / python ${{ matrix.python-version }} / ubuntu-22.04 - flaky

# run this after the norm integration tests to avoid collisions
needs: test

# run if not a PR from a forked repository or has a label to mark as safe to test
# also checks that the matrix generated is not empty
if: >-
needs.test-metadata.outputs.matrix &&
fromJSON( needs.test-metadata.outputs.matrix ).include[0] &&
(
github.event_name != 'pull_request_target' ||
github.event.pull_request.head.repo.full_name == github.repository ||
contains(github.event.pull_request.labels.*.name, 'ok to test')
)
runs-on: ${{ matrix.os }}

needs: test-metadata
github.event_name != 'pull_request_target' ||
github.event.pull_request.head.repo.full_name == github.repository ||
contains(github.event.pull_request.labels.*.name, 'ok to test')
runs-on: ubuntu-22.04

strategy:
fail-fast: false
max-parallel: 3
matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }}
max-parallel: 1
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]

env:
TOXENV: integration-${{ matrix.adapter }}
PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
TOXENV: integration-redshift
PYTEST_ADDOPTS: "-v --color=yes -n1 --csv integration_results.csv"
DBT_INVOCATION_ENV: github-actions
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
DD_INSTRUMENTATION_TELEMETRY_ENABLED: false
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
DD_SITE: datadoghq.com
DD_ENV: ci
Expand Down Expand Up @@ -175,38 +216,25 @@ jobs:
./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
- name: Run tox (redshift)
if: matrix.adapter == 'redshift'
env:
REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }}
REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }}
REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }}
REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }}
REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }}
REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}
DBT_TEST_USER_1: dbt_test_user_1
DBT_TEST_USER_2: dbt_test_user_2
DBT_TEST_USER_3: dbt_test_user_3
run: tox -- --ddtrace

- uses: actions/upload-artifact@v3
if: always()
with:
name: logs
path: ./logs

- name: Get current date
if: always()
id: date
run: |
echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
- uses: actions/upload-artifact@v3
if: always()
with:
name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv
path: integration_results.csv
run: tox -- -m flaky --ddtrace

require-label-comment:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04

needs: test

Expand All @@ -229,7 +257,7 @@ jobs:
check_for_duplicate_msg: true

post-failure:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
needs: test
if: ${{ failure() }}

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-12, windows-latest]
os: [ubuntu-22.04, macos-12, windows-2022]
python-version: ['3.8', '3.9', '3.10', '3.11']

steps:
Expand Down
2 changes: 2 additions & 0 deletions pytest.ini
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,5 @@ env_files =
testpaths =
tests/unit
tests/functional
markers =
flaky: marks tests as flaky so they run one at a time (de-select with '-m "not flaky"')
11 changes: 11 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,14 @@ def dbt_profile_target():
"pass": os.getenv("REDSHIFT_TEST_PASS"),
"dbname": os.getenv("REDSHIFT_TEST_DBNAME"),
}


def pytest_sessionfinish(session, exitstatus):
"""
Configures pytest to treat a scenario with no tests as passing
pytest returns a code 5 when it collects no tests in an effort to warn when tests are expected but not collected
We don't want this when running tox because some combinations of markers and test segments return nothing
"""
if exitstatus == 5:
session.exitstatus = 0
2 changes: 2 additions & 0 deletions tests/functional/adapter/catalog_tests/test_get_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ def setup(self, project):


class TestGetCatalogByRelations(BaseGetCatalog):
@pytest.mark.flaky
def test_get_one_catalog_by_relations(self, project, adapter):
project.adapter._capabilities[Capability.SchemaMetadataByRelations] = CapabilitySupport(
support=Support.Full
Expand All @@ -37,6 +38,7 @@ def test_get_one_catalog_by_relations(self, project, adapter):


class TestGetCatalogBySchemas(BaseGetCatalog):
@pytest.mark.flaky
def test_get_one_catalog_by_schemas(self, project, adapter):
project.adapter._capabilities[Capability.SchemaMetadataByRelations] = CapabilitySupport(
support=Support.NotImplemented
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def docs(self, project):
run_dbt(["run"])
yield run_dbt(["docs", "generate"])

@pytest.mark.flaky
@pytest.mark.parametrize(
"node_name,relation_type",
[
Expand Down
Loading

0 comments on commit f32c1ed

Please sign in to comment.