ccip-load-tests.yml

name: CCIP Load Test
on: 
  push:
    branches:
      - ccip-develop
    tags:
      - '*'
  workflow_dispatch:
    inputs:
      base64_test_input: # base64 encoded toml for test input
        description: 'Base64 encoded toml test input'
        required: false
      test_secrets_override_key:
        description: 'Key to run tests with custom test secrets'
        required: false
        type: string        
  pull_request: # DEBUG: Checking run conditions

# Only run 1 of this workflow at a time per PR
concurrency:
  group: load-ccip-tests-chainlink-${{ github.ref }}
  cancel-in-progress: true

env:
  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
  CHAINLINK_VERSION: ${{ github.sha}}
  INPUT_CHAINLINK_TEST_VERSION: ${{ github.sha}}
  ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }}
  INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
  AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws
  MOD_CACHE_VERSION: 1
  E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
  E2E_TEST_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
  E2E_TEST_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }}
  E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
  E2E_TEST_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }}
  # Default private key test secret loaded from Github Secret as only security team has access to it.
  # this key secrets.QA_SHARED_803C_KEY has a story behind it. To know more, see CCIP-2875 and SECHD-16575 tickets.
  E2E_TEST_ETHEREUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_ARBITRUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_BASE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_WEMIX_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_AVALANCHE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_ZKSYNC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_MODE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_METIS_ANDROMEDA_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_OPTIMISM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_KROMA_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_GNOSIS_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_POLYGON_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}
  E2E_TEST_BSC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }}

jobs:
  build-chainlink:
    environment: integration
    permissions:
      id-token: write
      contents: read
    name: Build Chainlink Image
    runs-on: ubuntu20.04-16cores-64GB
    steps:
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
      - name: Check if image exists
        id: check-image
        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
        with:
          repository: chainlink
          tag: ${{ env.CHAINLINK_VERSION }}
          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
      - name: Build Image
        if: steps.check-image.outputs.exists == 'false'
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
        env:
          GH_TOKEN: ${{ github.token }}
        with:
          cl_repo: smartcontractkit/chainlink-ccip
          cl_ref: ${{ env.CHAINLINK_VERSION }}
          push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ env.CHAINLINK_VERSION }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
        with:
          id: ccip-load-test-build-chainlink-image
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Build Chainlink Image
        continue-on-error: true

  build-test-image:
    environment: integration
    permissions:
      id-token: write
      contents: read
    name: Build Test Image
    runs-on: ubuntu20.04-16cores-64GB
    steps:
      - name: Collect Metrics
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
        with:
          id: ccip-load-test-build-test-image
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Build Test Image
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
      - name: Build Test Image
        uses: ./.github/actions/build-test-image
        with:
          tag: ${{ env.INPUT_CHAINLINK_TEST_VERSION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}

  ccip-load-test:
    environment: integration
    needs: [ build-chainlink, build-test-image ]
    if: ${{ always() && !contains(needs.*.result, 'failure') }}
    permissions:
      issues: read
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    env:
      CHAINLINK_ENV_USER: ${{ github.actor }}
      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
      SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
      TEST_LOG_LEVEL: info
      REF_NAME: ${{ github.head_ref || github.ref_name }}
    strategy:
      fail-fast: false
      matrix:
        type:
          - name: stable-load
            run: ^TestLoadCCIPStableRPS$
            os: ubuntu-latest
          - name: load-with-arm-curse-uncurse
            run: ^TestLoadCCIPStableRPSAfterARMCurseAndUncurse$
            config_path: ./integration-tests/ccip-tests/testconfig/tomls/load-with-arm-curse-uncurse.toml
            os: ubuntu-latest
    runs-on: ${{ matrix.type.os }}
    name: CCIP ${{ matrix.type.name }}
    steps:
      - name: Collect Metrics
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
        with:
          id: ccip-load-test-${{ matrix.type.name }}
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: CCIP ${{ matrix.type.name }}
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          ref: ${{ env.REF_NAME }}
      - name: Sets env vars
        id: set_override_config
        shell: bash
        run: |
          # if the matrix.type.config_path is set, use it as the override config
          if [ -n "${{ matrix.type.config_path }}" ]; then
              BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ${{ matrix.type.config_path }})
              echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
              echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
          fi
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH)
            echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
            echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT
          fi
      - name: step summary
        shell: bash
        run: |
          echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
          echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY
          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
          echo "\`${{ env.INPUT_CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY
      - name: Prepare Base64 TOML override for CCIP secrets
        uses: ./.github/actions/setup-create-base64-config-ccip
        id: setup_create_base64_config_ccip
        with:
          runId: ${{ github.run_id }}
          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
          chainlinkVersion: ${{ github.sha }}
          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
      - name: Run Tests
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
        env:
          TEST_SUITE: load
          TEST_ARGS: -test.timeout 900h
          DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
          RR_MEM: 8Gi
          RR_CPU: 4
          TEST_TRIGGERED_BY: ccip-load-test-ci-${{ matrix.type.name }}
          BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}
          TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }}          
          E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests"
        with:
          test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ${{ matrix.type.run }} ./load 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          # Load default test secrets
          test_secrets_defaults_base64: ${{ secrets.CCIP_DEFAULT_TEST_SECRETS }}
          # Override default test secrets with custom test secrets if provided
          test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }}
          token: ${{ secrets.GITHUB_TOKEN }}
          go_mod_path: ./integration-tests/go.mod
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
          triggered_by: ${{ env.TEST_TRIGGERED_BY }}
          publish_check_name: ${{ matrix.type.name }}
          artifacts_location: ./integration-tests/load/logs/payload_ccip.json
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
          cache_restore_only: "true"
          should_cleanup: "true"

  # Reporting Jobs
  start-slack-thread:
    name: Start Slack Thread
    if: ${{ failure() && needs.ccip-load-test.result != 'skipped' && needs.ccip-load-test.result != 'cancelled' }}
    environment: integration
    outputs:
      thread_ts: ${{ steps.slack.outputs.thread_ts }}
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    runs-on: ubuntu-latest
    needs: [ccip-load-test]
    steps:
      - name: Debug Result
        run: echo ${{ join(needs.*.result, ',') }}
      - name: Main Slack Notification
        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
        id: slack
        with:
          channel-id: "#ccip-testing"
          payload: |
            {
              "attachments": [
                {
                  "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}",
                  "blocks": [
                    {
                      "type": "header",
                      "text": {
                        "type": "plain_text",
                        "text": "CCIP load tests results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}",
                        "emoji": true
                      }
                    },
                    {
                      "type": "divider"
                    },
                    {
                      "type": "section",
                      "text": {
                        "type": "mrkdwn",
                        "text": "<${{ github.server_url }}/${{ github.repository }}/${{contains(github.ref_name, 'release') && 'releases/tag' || 'tree'}}/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>"
                      }
                    }
                  ]
                }
              ]
            }
        env:
          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

  post-test-results-to-slack:
    name: Post Test Results
    if: ${{ failure() && needs.start-slack-thread.result != 'skipped' && needs.start-slack-thread.result != 'cancelled' }}
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    runs-on: ubuntu-latest
    needs: start-slack-thread
    steps:
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Post Test Results
        uses: ./.github/actions/notify-slack-jobs-result
        with:
          github_token: ${{ github.token }}
          github_repository: ${{ github.repository }}
          workflow_run_id: ${{ github.run_id }}
          github_job_name_regex: ^CCIP (.*)$
          message_title: CCIP Jobs
          slack_channel_id: "#ccip-testing"
          slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }}
          slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }}

  # End Reporting Jobs