Merge branch 'aptos-labs:main' into sentio-dev

.github/actions/run-faucet-tests/action.yaml

@@ -3,7 +3,10 @@ description: |
   Runs the tests for the Aptos faucet against a local testnet built from a particular release branch
 inputs:
   NETWORK:
-    description: "The network branch for running the local testnet: devnet or testnet."
+    description: "The network branch for running the local testnet: devnet, testnet, or custom for a specific tag."
+    required: true
+  IMAGE_TAG:
+    description: "The docker image tag to use for the local testnet if NETWORK=custom."
     required: true
   GCP_DOCKER_ARTIFACT_REPO:
     description: "The GCP Docker artifact repository."
@@ -35,7 +38,7 @@ runs:
     # testnet, moving the mint key where the tests expect it to be, and running the
     # integration tests.
     - name: Run integration tests
-      run: poetry run python main.py --base-network ${{ inputs.NETWORK }} --external-test-dir ${{ runner.temp }}/testnet --image-repo-with-project ${{ inputs.GCP_DOCKER_ARTIFACT_REPO }}
+      run: poetry run python main.py --base-network ${{ inputs.NETWORK }} --tag ${{ inputs.IMAGE_TAG }} --external-test-dir ${{ runner.temp }}/testnet --image-repo-with-project ${{ inputs.GCP_DOCKER_ARTIFACT_REPO }}
       working-directory: crates/aptos-faucet/integration-tests
       shell: bash
 

.github/actions/rust-setup/action.yaml

@@ -9,10 +9,7 @@ runs:
     - run: sudo apt-get update && sudo apt-get install build-essential ca-certificates clang curl git libpq-dev libssl-dev pkg-config lsof lld --no-install-recommends --assume-yes
       shell: bash
 
-    - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
-      with:
-        override: true
-        components: rustfmt, clippy
+    - uses: dsherret/rust-toolchain-file@v1
 
     # rust-cache action will cache ~/.cargo and ./target
     # https://github.com/Swatinem/rust-cache#cache-details
@@ -32,3 +29,8 @@ runs:
       run: |
         git config --global credential.helper store
         echo "${{ inputs.GIT_CREDENTIALS }}" > ~/.git-credentials
+
+    # Display the rust toolchain version being installed
+    - name: Setup rust toolchain
+      shell: bash
+      run: rustup show

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    reviewers:
+      - "aptos-labs/security"
+      - "aptos-labs/prod-eng"
+    labels:
+      - "CICD:run-e2e-tests"
+      - "dependencies"
+      - "rust"

.github/workflows/adhoc-forge.yaml

@@ -2,14 +2,18 @@ name: "Ad-hoc Forge Run"
 on:
   workflow_dispatch:
     inputs:
-      IMAGE_TAG:
+      GIT_SHA:
         required: true
         type: string
-        description: The docker image tag to test. If not specified, falls back on GIT_SHA, and then to the latest commits on the current branch
+        description: The git SHA1 to checkout and test
+      IMAGE_TAG:
+        required: false
+        type: string
+        description: The docker image tag to test. If not specified, falls back on GIT_SHA
       FORGE_IMAGE_TAG:
-        required: true
+        required: false
         type: string
-        description: The docker image tag to use for forge runner. If not specified, falls back on GIT_SHA, and then to the latest commits on the current branch
+        description: The docker image tag to use for forge runner. If not specified, falls back on GIT_SHA
       FORGE_RUNNER_DURATION_SECS:
         required: false
         type: string
@@ -25,9 +29,6 @@ on:
         type: string
         description: The Forge k8s cluster to be used for test
 
-env:
-  GIT_SHA: ${{ github.sha }}
-
 permissions:
   contents: read
   id-token: write #required for GCP Workload Identity federation which we use to login into Google Artifact Registry
@@ -40,14 +41,14 @@ jobs:
     steps:
       - name: collect metadata
         run: |
-          echo "GIT_SHA: ${{ env.GIT_SHA }}"
+          echo "GIT_SHA: ${{ inputs.GIT_SHA }}"
           echo "IMAGE_TAG: ${{ inputs.IMAGE_TAG }}"
           echo "FORGE_IMAGE_TAG: ${{ inputs.FORGE_IMAGE_TAG }}"
           echo "FORGE_RUNNER_DURATION_SECS: ${{ inputs.FORGE_RUNNER_DURATION_SECS }}"
           echo "FORGE_TEST_SUITE: ${{ inputs.FORGE_TEST_SUITE }}"
           echo "FORGE_CLUSTER_NAME: ${{ inputs.FORGE_CLUSTER_NAME }}"
     outputs:
-      gitSha: ${{ env.GIT_SHA }}
+      gitSha: ${{ inputs.GIT_SHA }}
       imageTag: ${{ inputs.IMAGE_TAG }}
       forgeImageTag: ${{ inputs.FORGE_IMAGE_TAG }}
       forgeRunnerDurationSecs: ${{ inputs.FORGE_RUNNER_DURATION_SECS }}

.github/workflows/cargo-metadata-upload.yaml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
+      - uses: dsherret/rust-toolchain-file@v1
       - id: auth
         uses: "google-github-actions/auth@v1"
         with:

.github/workflows/docker-build-test.yaml

@@ -229,6 +229,22 @@ jobs:
       GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
       SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
 
+  faucet-tests-main:
+    needs: [permission-check, rust-images, determine-docker-build-metadata, file_change_determinator] # runs with the default release docker build variant "rust-images"
+    if: |
+      (
+        github.event_name == 'push' ||
+        github.event_name == 'workflow_dispatch' ||
+        contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') ||
+        github.event.pull_request.auto_merge != null) ||
+        contains(github.event.pull_request.body, '#e2e'
+      )
+    uses: ./.github/workflows/faucet-tests-main.yaml
+    secrets: inherit
+    with:
+      GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
+      SKIP_JOB: ${{ needs.file_change_determinator.outputs.only_docs_changed == 'true' }}
+
   indexer-grpc-e2e-tests:
     needs: [permission-check, rust-images, determine-docker-build-metadata] # runs with the default release docker build variant "rust-images"
     if: |

.github/workflows/faucet-tests-main.yaml

@@ -0,0 +1,74 @@
+## IMPORTANT NOTE TO EDITORS OF THIS FILE ##
+
+## If you are trying to change how this CI works, you MUST go read the important
+## note at the top of docker-build-test.yaml. In short, to test this, you must temporarily
+## change docker-build-test to use the pull_request trigger instead of pull_request_target.
+
+## Make sure to add the CICD:CICD:build-images and CICD:run-e2e-tests labels to test
+## this within an in-review PR.
+
+## If the above approach is too slow (since you have to wait for the rust images
+## to build), you can cut the iteration time dramatically by changing the envs
+##   - Replace env.IMAGE_TAG for a known image tag
+##   - env.GIT_SHA will resolve to that of your PR branch
+
+# These tests ensure that changes to the node don't break compatibility with the faucet.
+#
+# For tests that prevent changes to the faucet breaking compatibility with the production
+# networks, see faucet-tests-prod.
+#
+# This test uses the node image built from the previous step, hence the workflow_call
+# trigger.
+
+name: "Faucet Integration Tests: Main"
+on:
+  # This is called from within the docker-build-test.yaml workflow since we depend
+  # on the images having been built before this workflow runs.
+  workflow_call:
+    inputs:
+      GIT_SHA:
+        required: true
+        type: string
+        description: Use this to override the git SHA1, branch name (e.g. devnet) or tag to release the SDK from
+      SKIP_JOB:
+        required: false
+        default: false
+        type: boolean
+        description: Set to true to skip this job. Useful for PRs that don't require this workflow.
+
+env:
+  # This is the docker image tag that will be used for the SDK release.
+  # It is also used to pull the docker images for the CI.
+  IMAGE_TAG: ${{ inputs.GIT_SHA || 'devnet' }} # default to "devnet" tag when not running on workflow_call
+  GIT_SHA: ${{ inputs.GIT_SHA || github.event.pull_request.head.sha || github.sha }} # default to PR branch sha when not running on workflow_call
+
+jobs:
+  # These tests ensure that the faucet works with a node built from main. If we only
+  # upgrade devnet if this job is green and we always update the faucet image alongside
+  # the image for the faucet private fullnode, the faucet and fullnode should always
+  # be compatible in production.
+  run-tests-main:
+    if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests')
+    runs-on: high-perf-docker
+    steps:
+      - uses: actions/checkout@v3
+        if: ${{ !inputs.SKIP_JOB }}
+        with:
+          ref: ${{ env.GIT_SHA }}
+
+      - uses: aptos-labs/aptos-core/.github/actions/docker-setup@main
+        if: ${{ !inputs.SKIP_JOB }}
+        with:
+          GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DOCKER_ARTIFACT_REPO: ${{ secrets.AWS_DOCKER_ARTIFACT_REPO }}
+          GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
+
+      - uses: ./.github/actions/run-faucet-tests
+        if: ${{ !inputs.SKIP_JOB }}
+        with:
+          NETWORK: custom
+          IMAGE_TAG: ${{ env.IMAGE_TAG }}
+          GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}

.github/workflows/faucet-tests.yaml → .github/workflows/faucet-tests-prod.yaml

@@ -1,4 +1,13 @@
-name: "Faucet Integration Tests"
+# These tests ensure that changes to the faucet itself or its code dependencies don't
+# cause the faucet to stop working with the production (devnet and testnet) networks.
+#
+# For tests that prevent the node from breaking compatibility with the faucet, see
+# faucet-tests-main.
+#
+# These tests use node images built from devnet / testnet so we don't need to use
+# workflow_call to wait for the image build to finish.
+
+name: "Faucet Integration Tests: Prod"
 on:
   pull_request_target:
     types: [labeled, opened, synchronize, reopened, auto_merge_enabled]
@@ -24,6 +33,7 @@ jobs:
           required-permission: write
           comment-not-permitted: Sorry, you don't have permission to trigger this workflow.
 
+  # These tests ensure that the faucet works with the nodes running on devnet.
   run-tests-devnet:
     if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests')
     needs: [permission-check]
@@ -43,6 +53,7 @@ jobs:
           NETWORK: devnet
           GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
 
+  # These tests ensure that the faucet works with the nodes running on testnet.
   run-tests-testnet:
     if: contains(github.event.pull_request.labels.*.name, 'CICD:non-required-tests')
     needs: [permission-check]

.github/workflows/forge-pfn.yaml

@@ -23,10 +23,6 @@ on:
   pull_request:
     paths:
       - ".github/workflows/forge-pfn.yaml"
-      - "testsuite/find_latest_image.py"
-  push:
-    branches:
-      - aptos-release-v* # The aptos release branches
 
 env:
   AWS_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}

.github/workflows/forge-state-sync.yaml

@@ -23,10 +23,6 @@ on:
   pull_request:
     paths:
       - ".github/workflows/forge-state-sync.yaml"
-      - "testsuite/find_latest_image.py"
-  push:
-    branches:
-      - aptos-release-v* # The aptos release branches
 
 env:
   AWS_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}

.github/workflows/replay-verify.yaml

@@ -69,7 +69,7 @@ jobs:
     if: |
       github.event_name == 'schedule' ||
       github.event_name == 'push' ||
-      github.event_name == 'workflow_dispatch' && (inputs.CHAIN_NAME == 'testnet' || inputs.CHAIN_NAME == 'all' )
+      github.event_name == 'workflow_dispatch' && (inputs.CHAIN_NAME == 'mainnet' || inputs.CHAIN_NAME == 'all' )
     needs: determine-test-metadata
     uses: aptos-labs/aptos-core/.github/workflows/workflow-run-replay-verify.yaml@main
     secrets: inherit
@@ -83,7 +83,7 @@ jobs:
       BACKUP_CONFIG_TEMPLATE_PATH: terraform/helm/fullnode/files/backup/s3-public.yaml
       # workflow config
       RUNS_ON: "high-perf-docker-with-local-ssd"
-      TIMEOUT_MINUTES: 240
+      TIMEOUT_MINUTES: 300
 
   test-replay:
     if: ${{ github.event_name == 'pull_request' }}

.gitignore

@@ -76,6 +76,10 @@ aptos-move/move-examples/scripts/minter/build/Minter/*
 # Docker incremental build temporary files and directories
 target-out-docker
 **/Dockerfile.tmp
+# Indexer grpc temporary certificates.
+docker/compose/indexer-grpc/data-service-grpc-server.crt
+docker/compose/indexer-grpc/data-service-grpc-server.key
+
 
 # Doc generation output
 *.md.old
@@ -90,8 +94,9 @@ target-out-docker
 .*\#
 \#*\#
 
-# Aptos CLI files
+# Aptos CLI / local testnet files
 .aptos
+**/*.rdb
 
 # VSCode settings
 .vscode/