[forge] Add junit support

Add junit support for forge This will allow us to onboard forge onto trunk flaky test detection Also make suite choice more clean Test Plan: running on PR
aptos-labs · Oct 22, 2024 · 9a3f392 · 9a3f392
1 parent fbf09ce
commit 9a3f392
Show file tree

Hide file tree

Showing 14 changed files with 798 additions and 529 deletions.
diff --git a/.github/workflows/docker-build-test.yaml b/.github/workflows/docker-build-test.yaml
@@ -277,6 +277,7 @@ jobs:
       # by this GHA. If there is a Forge namespace collision, Forge will pre-empt the existing test running in the namespace.
       FORGE_NAMESPACE: forge-e2e-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
       SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
+      SEND_RESULTS_TO_TRUNK: true
 
   # This job determines the last released docker image tag, which is used by forge compat test.
   fetch-last-released-docker-image-tag:
@@ -356,6 +357,7 @@ jobs:
       COMMENT_HEADER: forge-compat
       FORGE_NAMESPACE: forge-compat-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
       SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
+      SEND_RESULTS_TO_TRUNK: true
 
   # Run forge framework upgradability test. This is a PR required job.
   forge-framework-upgrade-test:
@@ -385,6 +387,7 @@ jobs:
       COMMENT_HEADER: forge-framework-upgrade
       FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
       SKIP_JOB: ${{ !contains(github.event.pull_request.labels.*.name, 'CICD:run-framework-upgrade-test') && (needs.test-target-determinator.outputs.run_framework_upgrade_test == 'false') }}
+      SEND_RESULTS_TO_TRUNK: true
 
   forge-consensus-only-perf-test:
     needs:

diff --git a/.github/workflows/forge-stable.yaml b/.github/workflows/forge-stable.yaml
@@ -136,6 +136,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 7200 # Run for 2 hours
       FORGE_TEST_SUITE: framework_upgrade
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   run-forge-realistic-env-load-sweep:
     if: ${{ github.event_name != 'pull_request' && always() }}
@@ -148,6 +149,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 1800 # Run for 30 minutes (6 tests, each for 300 seconds)
       FORGE_TEST_SUITE: realistic_env_load_sweep
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   run-forge-realistic-env-workload-sweep:
     if: ${{ github.event_name != 'pull_request' && always() }}
@@ -160,6 +162,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 2000 # Run for 33 minutes (5 tests, each for 400 seconds)
       FORGE_TEST_SUITE: realistic_env_workload_sweep
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   run-forge-realistic-env-graceful-overload:
     if: ${{ github.event_name != 'pull_request' && always() }}
@@ -172,6 +175,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 1200 # Run for 20 minutes
       FORGE_TEST_SUITE: realistic_env_graceful_overload
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   run-forge-realistic-env-graceful-workload-sweep:
     if: ${{ github.event_name != 'pull_request' && always() }}
@@ -184,6 +188,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 2100 # Run for 5 minutes per test, 7 tests.
       FORGE_TEST_SUITE: realistic_env_graceful_workload_sweep
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   run-forge-realistic-env-fairness-workload-sweep:
     if: ${{ github.event_name != 'pull_request' && always() }}
@@ -196,6 +201,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 900 # Run for 5 minutes per test, 3 tests.
       FORGE_TEST_SUITE: realistic_env_fairness_workload_sweep
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   run-forge-realistic-network-tuned-for-throughput:
     if: ${{ github.event_name != 'pull_request' && always() }}
@@ -209,6 +215,7 @@ jobs:
       FORGE_TEST_SUITE: realistic_network_tuned_for_throughput
       FORGE_ENABLE_PERFORMANCE: true
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   ### Forge Correctness/Componenet/Stress tests
 
@@ -223,6 +230,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 2400 # Run for 40 minutes
       FORGE_TEST_SUITE: consensus_stress_test
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   run-forge-workload-mix-test:
     if: ${{ github.event_name != 'pull_request' && always() }}
@@ -235,6 +243,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
       FORGE_TEST_SUITE: workload_mix
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   run-forge-single-vfn-perf:
     if: ${{ github.event_name != 'pull_request' && always() }}
@@ -247,6 +256,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 480 # Run for 8 minutes
       FORGE_TEST_SUITE: single_vfn_perf
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   run-forge-fullnode-reboot-stress-test:
     if: ${{ github.event_name != 'pull_request' && always() }}
@@ -259,6 +269,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 1800 # Run for 30 minutes
       FORGE_TEST_SUITE: fullnode_reboot_stress_test
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   ### Compatibility Forge tests
 
@@ -275,6 +286,7 @@ jobs:
       IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG_FOR_COMPAT_TEST }}
       GIT_SHA: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }} # this is the git ref to checkout
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
   ### Changing working quorum Forge tests
 
@@ -289,6 +301,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 1200 # Run for 20 minutes
       FORGE_TEST_SUITE: changing_working_quorum_test
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
       FORGE_ENABLE_FAILPOINTS: true
 
   run-forge-changing-working-quorum-test-high-load:
@@ -302,6 +315,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
       FORGE_TEST_SUITE: changing_working_quorum_test_high_load
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
       FORGE_ENABLE_FAILPOINTS: true
 
   # Measures PFN latencies with a constant TPS (with a realistic environment)
@@ -316,6 +330,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
       FORGE_TEST_SUITE: pfn_const_tps_with_realistic_env
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
 
 
   # longest test for last, to get useful signal from short tests first
@@ -331,3 +346,4 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: 7200 # Run for 2 hours
       FORGE_TEST_SUITE: realistic_env_max_load_large
       POST_TO_SLACK: true
+      SEND_RESULTS_TO_TRUNK: true
diff --git a/.github/workflows/workflow-run-forge.yaml b/.github/workflows/workflow-run-forge.yaml
@@ -87,6 +87,10 @@ on:
         required: false
         type: string
         description: The deployer profile used to spin up and configure forge infrastructure
+      SEND_RESULTS_TO_TRUNK:
+        required: false
+        type: boolean
+        description: Send forge results to trunk.io
 
 env:
   AWS_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
@@ -118,6 +122,7 @@ env:
   VERBOSE: true
   FORGE_NUM_VALIDATORS: ${{ inputs.FORGE_NUM_VALIDATORS }}
   FORGE_NUM_VALIDATOR_FULLNODES: ${{ inputs.FORGE_NUM_VALIDATOR_FULLNODES }}
+  FORGE_JUNIT_XML_PATH: ${{ inputs.SEND_RESULTS_TO_TRUNK ? '/tmp/test.xml' : '' }}
 
 # TODO: should we migrate this to a composite action, so that we can skip it
 # at the call site, and don't need to wrap each step in an if statement?
@@ -228,3 +233,14 @@ jobs:
       # Print out whether the job was skipped.
       - run: echo "Skipping forge test!"
         if: ${{ inputs.SKIP_JOB }}
+
+    - name: Upload results
+      # Run this step even if the test step ahead fails
+      if: "!cancelled() && ${{ inputs.SEND_RESULTS_TO_TRUNK }}"
+      uses: trunk-io/analytics-uploader@main
+      with:
+        # Configured in the nextest.toml file
+        junit-paths: ${{ env.FORGE_JUNIT_XML_PATH }}
+        org-slug: aptoslabs
+        token: ${{ secrets.TRUNK_API_TOKEN }}
+      continue-on-error: true