Mark flaky tests (#813)

backport #813

(cherry picked from commit b57890e)

.github/workflows/integration.yml

@@ -53,88 +53,129 @@ defaults:
     shell: bash
 
 jobs:
-  # generate test metadata about what files changed and the testing matrix to use
-  test-metadata:
+  test:
+    name: redshift / python ${{ matrix.python-version }} / ${{ matrix.os }}
+
     # run if not a PR from a forked repository or has a label to mark as safe to test
     if: >-
       github.event_name != 'pull_request_target' ||
       github.event.pull_request.head.repo.full_name == github.repository ||
       contains(github.event.pull_request.labels.*.name, 'ok to test')
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
 
-    outputs:
-      matrix: ${{ steps.generate-matrix.outputs.result }}
+    strategy:
+      fail-fast: false
+      max-parallel: 3
+      matrix:
+        os: [ubuntu-22.04, macos12, windows-2022]
+        python-version: ["3.8"]
+        include:
+          - os: ubuntu-22.04
+            python-version: "3.9"
+          - os: ubuntu-22.04
+            python-version: "3.10"
+          - os: ubuntu-22.04
+            python-version: "3.11"
+
+    env:
+      TOXENV: integration-redshift
+      PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
+      DBT_INVOCATION_ENV: github-actions
+      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+      DD_SITE: datadoghq.com
+      DD_ENV: ci
+      DD_SERVICE: ${{ github.event.repository.name }}
 
     steps:
-      - name: Check out the repository (non-PR)
+      - name: Check out the repository
         if: github.event_name != 'pull_request_target'
         uses: actions/checkout@v3
         with:
           persist-credentials: false
 
+      # explicity checkout the branch for the PR,
+      # this is necessary for the `pull_request_target` event
       - name: Check out the repository (PR)
         if: github.event_name == 'pull_request_target'
         uses: actions/checkout@v3
         with:
           persist-credentials: false
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Check if relevant files changed
-        if: github.event_name == 'pull_request_target'
-        # https://github.com/marketplace/actions/paths-changes-filter
-        # For each filter, it sets output variable named by the filter to the text:
-        #  'true' - if any of changed files matches any of filter rules
-        #  'false' - if none of changed files matches any of filter rules
-        # also, returns:
-        #  `changes` - JSON array with names of all filters matching any of the changed files
-        uses: dorny/paths-filter@v2
-        id: get-changes
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          filters: |
-            redshift:
-              - 'dbt/**'
-              - 'tests/**'
-              - 'dev-requirements.txt'
-
-      - name: Generate integration test matrix
-        id: generate-matrix
-        uses: actions/github-script@v6
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install  --user --upgrade pip
+          python -m pip install tox
+          python -m pip --version
+          tox --version
+
+      - name: Update dev_requirements.txt
+        if: inputs.dbt-core-branch != ''
+        run: |
+          pip install bumpversion
+          ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
+
+      - name: Run tox (redshift)
         env:
-          CHANGES: ${{ steps.get-changes.outputs.changes }}
+          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
+          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
+          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
+          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
+          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
+          DBT_TEST_USER_1: dbt_test_user_1
+          DBT_TEST_USER_2: dbt_test_user_2
+          DBT_TEST_USER_3: dbt_test_user_3
+        run: tox -- -m "not flaky" --ddtrace
+
+      - uses: actions/upload-artifact@v3
+        if: always()
         with:
-          script: |
-            const script = require('./.github/scripts/integration-test-matrix.js')
-            const matrix = script({ context })
-            console.log(matrix)
-            return matrix
-  test:
-    name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }}
+          name: logs
+          path: ./logs
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: |
+          echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_redshift-${{ steps.date.outputs.date }}.csv
+          path: integration_results.csv
+
+  test-flaky:
+    name: redshift / python ${{ matrix.python-version }} / ubuntu-22.04 - flaky
+
+    # run this after the norm integration tests to avoid collisions
+    needs: test
 
     # run if not a PR from a forked repository or has a label to mark as safe to test
-    # also checks that the matrix generated is not empty
     if: >-
-      needs.test-metadata.outputs.matrix &&
-      fromJSON( needs.test-metadata.outputs.matrix ).include[0] &&
-      (
-        github.event_name != 'pull_request_target' ||
-        github.event.pull_request.head.repo.full_name == github.repository ||
-        contains(github.event.pull_request.labels.*.name, 'ok to test')
-      )
-    runs-on: ${{ matrix.os }}
-
-    needs: test-metadata
+      github.event_name != 'pull_request_target' ||
+      github.event.pull_request.head.repo.full_name == github.repository ||
+      contains(github.event.pull_request.labels.*.name, 'ok to test')
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
-      max-parallel: 3
-      matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }}
+      max-parallel: 1
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     env:
-      TOXENV: integration-${{ matrix.adapter }}
-      PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
+      TOXENV: integration-redshift
+      PYTEST_ADDOPTS: "-v --color=yes -n1 --csv integration_results.csv"
       DBT_INVOCATION_ENV: github-actions
       DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+      DD_INSTRUMENTATION_TELEMETRY_ENABLED: false
       DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
       DD_SITE: datadoghq.com
       DD_ENV: ci
@@ -175,38 +216,25 @@ jobs:
           ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
 
       - name: Run tox (redshift)
-        if: matrix.adapter == 'redshift'
         env:
           REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
           REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
           REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
           REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
           REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
+          REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }}
+          REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }}
+          REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }}
+          REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }}
+          REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }}
+          REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}
           DBT_TEST_USER_1: dbt_test_user_1
           DBT_TEST_USER_2: dbt_test_user_2
           DBT_TEST_USER_3: dbt_test_user_3
-        run: tox -- --ddtrace
-
-      - uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: logs
-          path: ./logs
-
-      - name: Get current date
-        if: always()
-        id: date
-        run: |
-          echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
-
-      - uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv
-          path: integration_results.csv
+        run: tox -- -m flaky --ddtrace
 
   require-label-comment:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     needs: test
 
@@ -229,7 +257,7 @@ jobs:
           check_for_duplicate_msg: true
 
   post-failure:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: test
     if: ${{ failure() }}
 

.github/workflows/main.yml

@@ -173,7 +173,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-12, windows-latest]
+        os: [ubuntu-22.04, macos-12, windows-2022]
         python-version: ['3.8', '3.9', '3.10', '3.11']
 
     steps:

pytest.ini

@@ -7,3 +7,5 @@ env_files =
 testpaths =
     tests/unit
     tests/functional
+markers =
+    flaky: marks tests as flaky so they run one at a time (de-select with '-m "not flaky"')

tests/conftest.py

@@ -20,3 +20,14 @@ def dbt_profile_target():
         "pass": os.getenv("REDSHIFT_TEST_PASS"),
         "dbname": os.getenv("REDSHIFT_TEST_DBNAME"),
     }
+
+
+def pytest_sessionfinish(session, exitstatus):
+    """
+    Configures pytest to treat a scenario with no tests as passing
+
+    pytest returns a code 5 when it collects no tests in an effort to warn when tests are expected but not collected
+    We don't want this when running tox because some combinations of markers and test segments return nothing
+    """
+    if exitstatus == 5:
+        session.exitstatus = 0

tests/functional/adapter/catalog_tests/test_get_catalog.py

@@ -25,6 +25,7 @@ def setup(self, project):
 
 
 class TestGetCatalogByRelations(BaseGetCatalog):
+    @pytest.mark.flaky
     def test_get_one_catalog_by_relations(self, project, adapter):
         project.adapter._capabilities[Capability.SchemaMetadataByRelations] = CapabilitySupport(
             support=Support.Full
@@ -37,6 +38,7 @@ def test_get_one_catalog_by_relations(self, project, adapter):
 
 
 class TestGetCatalogBySchemas(BaseGetCatalog):
+    @pytest.mark.flaky
     def test_get_one_catalog_by_schemas(self, project, adapter):
         project.adapter._capabilities[Capability.SchemaMetadataByRelations] = CapabilitySupport(
             support=Support.NotImplemented

tests/functional/adapter/catalog_tests/test_relation_types.py

@@ -24,6 +24,7 @@ def docs(self, project):
         run_dbt(["run"])
         yield run_dbt(["docs", "generate"])
 
+    @pytest.mark.flaky
     @pytest.mark.parametrize(
         "node_name,relation_type",
         [