From 6a0fe314b29de86430ec2a453ed12da029aafb5a Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Mon, 13 May 2024 16:20:15 +0100 Subject: [PATCH 1/3] Fix #9907: Add retry to tox to reduce flaky tests due to network failures --- .github/workflows/main.yml | 16 ++++++++++++---- .../structured-logging-schema-check.yml | 6 +++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5590f0bf2e9..c55b97b4e3f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -95,8 +95,12 @@ jobs: python -m pip install tox tox --version - - name: Run tox - run: tox + - name: Run unit tests + uses: nick-fields/retry@v3 + with: + timeout_minutes: 5 + max_attempts: 3 + command: tox -e unit - name: Get current date if: always() @@ -201,8 +205,12 @@ jobs: python -m pip install tox tox --version - - name: Run tests - run: tox -- --ddtrace + - name: Run integration tests + uses: nick-fields/retry@v3 + with: + timeout_minutes: 10 + max_attempts: 3 + command: tox -- --ddtrace env: PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }} diff --git a/.github/workflows/structured-logging-schema-check.yml b/.github/workflows/structured-logging-schema-check.yml index 3c4b2eb0406..9e19acecbaa 100644 --- a/.github/workflows/structured-logging-schema-check.yml +++ b/.github/workflows/structured-logging-schema-check.yml @@ -94,7 +94,11 @@ jobs: # integration tests generate a ton of logs in different files. the next step will find them all. # we actually care if these pass, because the normal test run doesn't usually include many json log outputs - name: Run integration tests - run: tox -e integration -- -nauto + uses: nick-fields/retry@v3 + with: + timeout_minutes: 10 + max_attempts: 3 + command: tox -e integration -- -nauto env: PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }} From 16f35eaf437e9ac14fa5a56f6de4e1cace84bafe Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Mon, 13 May 2024 16:53:54 +0100 Subject: [PATCH 2/3] set integration timeout to 30 mins --- .github/workflows/main.yml | 2 +- .github/workflows/structured-logging-schema-check.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c55b97b4e3f..e42e15d51d2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -208,7 +208,7 @@ jobs: - name: Run integration tests uses: nick-fields/retry@v3 with: - timeout_minutes: 10 + timeout_minutes: 30 max_attempts: 3 command: tox -- --ddtrace env: diff --git a/.github/workflows/structured-logging-schema-check.yml b/.github/workflows/structured-logging-schema-check.yml index 9e19acecbaa..4934bffcaeb 100644 --- a/.github/workflows/structured-logging-schema-check.yml +++ b/.github/workflows/structured-logging-schema-check.yml @@ -96,7 +96,7 @@ jobs: - name: Run integration tests uses: nick-fields/retry@v3 with: - timeout_minutes: 10 + timeout_minutes: 30 max_attempts: 3 command: tox -e integration -- -nauto env: From 668413d67b1c6b970e6ac076348556a7e0474554 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Mon, 13 May 2024 19:36:30 +0100 Subject: [PATCH 3/3] Make timeout consistent --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e42e15d51d2..b596bf7293f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,7 +98,7 @@ jobs: - name: Run unit tests uses: nick-fields/retry@v3 with: - timeout_minutes: 5 + timeout_minutes: 10 max_attempts: 3 command: tox -e unit