Skip to content

Continuous Forge Tests - Stable #33

Continuous Forge Tests - Stable

Continuous Forge Tests - Stable #33

Workflow file for this run

# Continuously run stable forge tests against the latest main branch.
name: Continuous Forge Tests - Stable
permissions:
issues: write
pull-requests: write
contents: read
id-token: write
actions: write #required for workflow cancellation via check-aptos-core
concurrency:
group: forge-stable-${{ github.ref_name }}
cancel-in-progress: true
on:
# Allow triggering manually
workflow_dispatch:
inputs:
IMAGE_TAG:
required: false
type: string
description: The docker image tag to test. This may be a git SHA1, or a tag like "<branch>_<git SHA1>". If not specified, Forge will find the latest build based on the git history (starting from GIT_SHA input)
GIT_SHA:
required: false
type: string
description: The git SHA1 to checkout. This affects the Forge test runner that is used. If not specified, the latest main will be used
# NOTE: to support testing different branches on different schedules, you need to specify the cron schedule in the 'determine-test-branch' step as well below
# Reference: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
schedule:
- cron: "0 22 * * 0,2,4" # The main branch cadence. This runs every Sun,Tues,Thurs
pull_request:
paths:
- ".github/workflows/forge-stable.yaml"
- "testsuite/find_latest_image.py"
env:
AWS_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
IMAGE_TAG: ${{ inputs.IMAGE_TAG }} # this is only used for workflow_dispatch, otherwise defaults to empty
AWS_REGION: us-west-2
jobs:
# This job determines the image tag and branch to test, and passes them to the other jobs
# NOTE: this may be better as a separate workflow as the logic is quite complex but generalizable
determine-test-metadata:
runs-on: ubuntu-latest
outputs:
IMAGE_TAG: ${{ steps.get-docker-image-tag.outputs.IMAGE_TAG }}
BRANCH: ${{ steps.determine-test-branch.outputs.BRANCH }}
BRANCH_HASH: ${{ steps.hash-branch.outputs.BRANCH_HASH }}
steps:
- uses: actions/checkout@v3
- name: Determine branch based on cadence
id: determine-test-branch
# NOTE: the schedule cron MUST match the one in the 'on.schedule.cron' section above
run: |
if [[ "${{ github.event_name }}" == "schedule" ]]; then
if [[ "${{ github.event.schedule }}" == "0 22 * * 0,2,4" ]]; then
echo "Branch: main"
echo "BRANCH=main" >> $GITHUB_OUTPUT
else
echo "Unknown schedule: ${{ github.event.schedule }}"
exit 1
fi
elif [[ "${{ github.event_name }}" == "push" ]]; then
echo "Branch: ${{ github.ref_name }}"
echo "BRANCH=${{ github.ref_name }}" >> $GITHUB_OUTPUT
else
echo "Using GIT_SHA"
# on workflow_dispatch, this will simply use the inputs.GIT_SHA given (or the default)
# on pull_request, this will default to null and the following "checkout" step will use the PR's base branch
echo "BRANCH=${{ inputs.GIT_SHA }}" >> $GITHUB_OUTPUT
fi
# Use the branch hash instead of the full branch name to stay under kubernetes namespace length limit
- name: Hash the branch
id: hash-branch
run: |
# If BRANCH is empty, default to "main"
if [ -z "${{ steps.determine-test-branch.outputs.BRANCH }}" ]; then
BRANCH="main"
else
BRANCH="${{ steps.determine-test-branch.outputs.BRANCH }}"
fi
# Hashing the branch name
echo "BRANCH_HASH=$(echo -n "$BRANCH" | sha256sum | cut -c1-10)" >> $GITHUB_OUTPUT
- uses: aptos-labs/aptos-core/.github/actions/check-aptos-core@main
with:
cancel-workflow: ${{ github.event_name == 'schedule' }} # Cancel the workflow if it is scheduled on a fork
# actions/get-latest-docker-image-tag requires docker utilities and having authenticated to internal docker image registries
- uses: aptos-labs/aptos-core/.github/actions/docker-setup@main
id: docker-setup
with:
GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
EXPORT_GCP_PROJECT_VARIABLES: "false"
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DOCKER_ARTIFACT_REPO: ${{ secrets.AWS_DOCKER_ARTIFACT_REPO }}
GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
- uses: aptos-labs/aptos-core/.github/actions/get-latest-docker-image-tag@main
id: get-docker-image-tag
with:
branch: ${{ steps.determine-test-branch.outputs.BRANCH }}
variants: "failpoints performance"
- name: Write summary
run: |
IMAGE_TAG=${{ steps.get-docker-image-tag.outputs.IMAGE_TAG }}
BRANCH=${{ steps.determine-test-branch.outputs.BRANCH }}
if [ -n "${BRANCH}" ]; then
echo "BRANCH: [${BRANCH}](https://github.com/${{ github.repository }}/tree/${BRANCH})" >> $GITHUB_STEP_SUMMARY
fi
echo "IMAGE_TAG: [${IMAGE_TAG}](https://github.com/${{ github.repository }}/commit/${IMAGE_TAG})" >> $GITHUB_STEP_SUMMARY
### Real-world-network tests.
# Run forge framework upgradability test. This is a PR required job.
run-forge-framework-upgrade-test:
if: ${{ github.event_name != 'pull_request' }}
needs:
- determine-test-metadata
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: d1bf834728a0cf166d993f4728dfca54f3086fb0 #aptos-node-v1.18.0
FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 7200 # Run for 2 hours
FORGE_TEST_SUITE: framework_upgrade
POST_TO_SLACK: true
run-forge-realistic-env-load-sweep:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-framework-upgrade-test] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-load-sweep-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 1800 # Run for 30 minutes (6 tests, each for 300 seconds)
FORGE_TEST_SUITE: realistic_env_load_sweep
POST_TO_SLACK: true
run-forge-realistic-env-workload-sweep:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-realistic-env-load-sweep] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-workload-sweep-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 2000 # Run for 33 minutes (5 tests, each for 400 seconds)
FORGE_TEST_SUITE: realistic_env_workload_sweep
POST_TO_SLACK: true
run-forge-realistic-env-graceful-overload:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-realistic-env-workload-sweep] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-graceful-overload-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 1200 # Run for 20 minutes
FORGE_TEST_SUITE: realistic_env_graceful_overload
POST_TO_SLACK: true
run-forge-realistic-env-graceful-workload-sweep:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-realistic-env-graceful-overload] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-graceful-workload-sweep-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 2100 # Run for 5 minutes per test, 7 tests.
FORGE_TEST_SUITE: realistic_env_graceful_workload_sweep
POST_TO_SLACK: true
run-forge-realistic-env-fairness-workload-sweep:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-realistic-env-graceful-workload-sweep] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-fairness-workload-sweep-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 900 # Run for 5 minutes per test, 3 tests.
FORGE_TEST_SUITE: realistic_env_fairness_workload_sweep
POST_TO_SLACK: true
run-forge-realistic-network-tuned-for-throughput:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [ determine-test-metadata, run-forge-realistic-env-fairness-workload-sweep ] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-network-tuned-for-throughput-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
FORGE_TEST_SUITE: realistic_network_tuned_for_throughput
FORGE_ENABLE_PERFORMANCE: true
POST_TO_SLACK: true
### Forge Correctness/Componenet/Stress tests
run-forge-consensus-stress-test:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-realistic-network-tuned-for-throughput] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-consensus-stress-test-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 2400 # Run for 40 minutes
FORGE_TEST_SUITE: consensus_stress_test
POST_TO_SLACK: true
run-forge-workload-mix-test:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-consensus-stress-test] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-workload-mix-test-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
FORGE_TEST_SUITE: workload_mix
POST_TO_SLACK: true
run-forge-single-vfn-perf:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-workload-mix-test] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-continuous-e2e-single-vfn-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 480 # Run for 8 minutes
FORGE_TEST_SUITE: single_vfn_perf
POST_TO_SLACK: true
run-forge-fullnode-reboot-stress-test:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-single-vfn-perf] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-fullnode-reboot-stress-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 1800 # Run for 30 minutes
FORGE_TEST_SUITE: fullnode_reboot_stress_test
POST_TO_SLACK: true
### Compatibility Forge tests
run-forge-compat:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-fullnode-reboot-stress-test] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
FORGE_NAMESPACE: forge-compat-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 300 # Run for 5 minutes
# This will upgrade from testnet branch to the latest main
FORGE_TEST_SUITE: compat
IMAGE_TAG: d1bf834728a0cf166d993f4728dfca54f3086fb0 #aptos-node-v1.18.0
GIT_SHA: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }} # this is the git ref to checkout
POST_TO_SLACK: true
### Changing working quorum Forge tests
run-forge-changing-working-quorum-test:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-compat] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-changing-working-quorum-test-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 1200 # Run for 20 minutes
FORGE_TEST_SUITE: changing_working_quorum_test
POST_TO_SLACK: true
FORGE_ENABLE_FAILPOINTS: true
run-forge-changing-working-quorum-test-high-load:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-changing-working-quorum-test] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-changing-working-quorum-test-high-load-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
FORGE_TEST_SUITE: changing_working_quorum_test_high_load
POST_TO_SLACK: true
FORGE_ENABLE_FAILPOINTS: true
# Measures PFN latencies with a constant TPS (with a realistic environment)
run-forge-pfn-const-tps-realistic-env:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-changing-working-quorum-test-high-load] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-pfn-const-tps-with-realistic-env-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
FORGE_TEST_SUITE: pfn_const_tps_with_realistic_env
POST_TO_SLACK: true
# longest test for last, to get useful signal from short tests first
run-forge-realistic-env-max-load-long:
if: ${{ github.event_name != 'pull_request' && always() }}
needs: [determine-test-metadata, run-forge-pfn-const-tps-realistic-env] # Only run after the previous job completes
uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
secrets: inherit
with:
IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
FORGE_NAMESPACE: forge-realistic-env-max-load-long-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
FORGE_RUNNER_DURATION_SECS: 7200 # Run for 2 hours
FORGE_TEST_SUITE: realistic_env_max_load_large
POST_TO_SLACK: true