Improvements in test configurability (#227) #151
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CCIP Load Test | |
| on: | |
| push: | |
| branches: | |
| - ccip-develop | |
| workflow_dispatch: | |
| inputs: | |
| existing-deployment: | |
| description: 'Run load tests on already deployed contracts and jobs' | |
| required: false | |
| default: false | |
| type: boolean | |
| networks: | |
| description: Network pairs to run tests on (comma separated list) | |
| type: choice | |
| default: "SIMULATED_1,SIMULATED_2" | |
| options: | |
| - "SIMULATED_1,SIMULATED_2" | |
| - "OPTIMISM_GOERLI,SEPOLIA,GOERLI,AVALANCHE_FUJI,ARBITRUM_GOERLI,MUMBAI" | |
| numberOfChains: | |
| description: Number of networks to run tests on | |
| type: choice | |
| default: "2" | |
| options: | |
| - "2" | |
| - "3" | |
| - "4" | |
| - "5" | |
| - "6" | |
| chainlinkImage: | |
| description: Container image location for the Chainlink nodes | |
| required: false | |
| default: "" | |
| chainlinkVersion: | |
| description: Container image version for the Chainlink nodes | |
| required: false | |
| default: "" | |
| chainlinkTestVersion: | |
| description: Container image version for the Chainlink tests | |
| required: false | |
| default: "" | |
| testDuration: | |
| description: Duration of the test (time string) | |
| required: false | |
| default: 15m | |
| send_request_rate: | |
| description: Number of ccip-send requests to trigger per unit time duration | |
| required: true | |
| default: 2 | |
| type: number | |
| send_request_rate_duration: | |
| description: Time duration unit for ccip-send request rate | |
| required: false | |
| default: 1s | |
| slackMemberID: | |
| description: Slack Member ID (Not your @) | |
| required: true | |
| default: U01A2B2C3D4 | |
| type: string | |
| # Only run 1 of this workflow at a time per PR | |
| concurrency: | |
| group: load-ccip-tests-chainlink-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| INPUT_CHAINLINK_TEST_VERSION: ${{ inputs.chainlinkTestVersion || github.sha}} | |
| INPUT_CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }} | |
| INPUT_CHAINLINK_VERSION: ${{ inputs.chainlinkVersion || github.sha}} | |
| ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ inputs.chainlinkTestVersion || github.sha }} | |
| INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com | |
| jobs: | |
| build-chainlink: | |
| environment: integration | |
| if: ${{ inputs.chainlinkImage == '' || inputs.chainlinkVersion == '' }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| name: Build Chainlink Image | |
| runs-on: ubuntu20.04-16cores-64GB | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 | |
| - name: Check if image exists | |
| id: check-image | |
| uses: smartcontractkit/chainlink-github-actions/docker/image-exists@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1 | |
| with: | |
| repository: chainlink | |
| tag: ${{ env.INPUT_CHAINLINK_VERSION }} | |
| AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| - name: Build Image | |
| if: steps.check-image.outputs.exists == 'false' | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| with: | |
| cl_repo: smartcontractkit/chainlink-ccip | |
| cl_ref: ${{ env.INPUT_CHAINLINK_VERSION }} | |
| push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ env.INPUT_CHAINLINK_VERSION }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| - name: Collect Metrics | |
| if: always() | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@8163dcea2f01a0a8fec84b284406ff7af1d2e1c0 | |
| with: | |
| basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} | |
| this-job-name: Build Chainlink Image | |
| continue-on-error: true | |
| build-test-image: | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| name: Build Test Image | |
| runs-on: ubuntu20.04-16cores-64GB | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@8163dcea2f01a0a8fec84b284406ff7af1d2e1c0 | |
| with: | |
| basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} | |
| this-job-name: Build Test Image | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 | |
| - name: Build Test Image | |
| uses: ./.github/actions/build-test-image | |
| with: | |
| tag: ${{ env.INPUT_CHAINLINK_TEST_VERSION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| ccip_load_test: | |
| name: CCIP Load Test | |
| environment: integration | |
| runs-on: ubuntu-latest | |
| needs: [ build-chainlink, build-test-image ] | |
| if: ${{ always() && !contains(needs.*.result, 'failure') }} | |
| permissions: | |
| issues: read | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| env: | |
| CHAINLINK_COMMIT_SHA: ${{ inputs.chainlinkVersion }} | |
| CHAINLINK_ENV_USER: ${{ github.actor }} | |
| SLACK_USER: ${{ inputs.slackMemberID }} | |
| SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} | |
| SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} | |
| SELECTED_NETWORKS: SIMULATED,${{ inputs.networks }} | |
| EVM_KEYS: ${{ secrets.QA_EVM_KEYS }} # replace it with input parameter once debug log is updated to not print the key | |
| CCIP_TEST_DURATION: ${{ inputs.testDuration }} | |
| CCIP_PHASE_VALIDATION_TIMEOUT: ${{ inputs.phase_time_out }} | |
| CCIP_LOAD_TEST_RATE: ${{ inputs.send_request_rate }} | |
| CCIP_LOAD_TEST_RATEUNIT: ${{ inputs.send_request_rate_duration }} | |
| CCIP_MSG_TYPE: WithoutToken | |
| CCIP_CHAINLINK_NODE_FUNDING: 20 | |
| CCIP_TESTS_ON_EXISTING_DEPLOYMENT: ${{ inputs.existing-deployment }} | |
| CCIP_NO_OF_NETWORKS: ${{ inputs.numberOfChains }} | |
| CCIP_KEEP_ENV_TTL: 5h | |
| CCIP_REUSE_CONTRACTS: True | |
| TEST_LOG_LEVEL: info | |
| REF_NAME: ${{ github.head_ref || github.ref_name }} | |
| ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests | |
| GOERLI_URLS: ${{ secrets.QA_GOERLI_URLS }} | |
| TEST_GOERLI_URLS: ${{ secrets.QA_GOERLI_URLS }} | |
| GOERLI_HTTP_URLS: ${{ secrets.QA_GOERLI_HTTP_URLS }} | |
| TEST_GOERLI_HTTP_URLS: ${{ secrets.QA_GOERLI_HTTP_URLS }} | |
| OPTIMISM_GOERLI_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_URLS }} | |
| TEST_OPTIMISM_GOERLI_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_URLS }} | |
| OPTIMISM_GOERLI_HTTP_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_HTTP_URLS }} | |
| TEST_OPTIMISM_GOERLI_HTTP_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_HTTP_URLS }} | |
| SEPOLIA_URLS: ${{ secrets.QA_SEPOLIA_URLS }} | |
| TEST_SEPOLIA_URLS: ${{ secrets.QA_SEPOLIA_URLS }} | |
| SEPOLIA_HTTP_URLS: ${{ secrets.QA_SEPOLIA_HTTP_URLS }} | |
| TEST_SEPOLIA_HTTP_URLS: ${{ secrets.QA_SEPOLIA_HTTP_URLS }} | |
| AVALANCHE_FUJI_URLS: ${{ secrets.QA_AVALANCHE_FUJI_URLS }} | |
| TEST_AVALANCHE_FUJI_URLS: ${{ secrets.QA_AVALANCHE_FUJI_URLS }} | |
| AVALANCHE_FUJI_HTTP_URLS: ${{ secrets.QA_AVALANCHE_FUJI_HTTP_URLS }} | |
| TEST_AVALANCHE_FUJI_HTTP_URLS: ${{ secrets.QA_AVALANCHE_FUJI_HTTP_URLS }} | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@8163dcea2f01a0a8fec84b284406ff7af1d2e1c0 | |
| with: | |
| basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} | |
| this-job-name: ${{ inputs.networks }} CCIP Load Test | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 | |
| with: | |
| ref: ${{ env.REF_NAME }} | |
| - name: Sets env vars | |
| shell: bash | |
| run: | | |
| if [[ "${{ github.event_name }}" != "workflow_dispatch" ]]; then | |
| echo "SELECTED_NETWORKS=SIMULATED,SIMULATED_1,SIMULATED_2" >> $GITHUB_ENV | |
| echo "CCIP_NO_OF_NETWORKS=2" >> $GITHUB_ENV | |
| fi | |
| if [[ "${{ env.INPUT_CHAINLINK_IMAGE }}" == "" || "${{ env.INPUT_CHAINLINK_VERSION }}" == "" ]]; then | |
| echo "INPUT_CHAINLINK_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink" >> $GITHUB_ENV | |
| echo "INPUT_CHAINLINK_VERSION=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| if [[ "${{ env.INPUT_CHAINLINK_TEST_VERSION }}" == "" ]]; then | |
| echo "INPUT_CHAINLINK_TEST_VERSION=${{ github.sha }}" >> $GITHUB_ENV | |
| fi | |
| if [[ "${{ env.CCIP_NO_OF_NETWORKS }}" != "2" ]]; then | |
| echo "RR_MEM=8Gi" >> $GITHUB_ENV | |
| echo "RR_CPU=4" >> $GITHUB_ENV | |
| fi | |
| - name: step summary | |
| shell: bash | |
| env: | |
| EXISTING_DEPLOYMENT: ${{ inputs.existing-deployment }} | |
| run: | | |
| echo "ENV_JOB_IMAGE=${{ env.ENV_JOB_IMAGE_BASE }}:${{ env.INPUT_CHAINLINK_TEST_VERSION }}" >> $GITHUB_ENV | |
| echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY | |
| echo "\`${{ env.INPUT_CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY | |
| echo "\`${{ env.INPUT_CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY | |
| - name: Run Tests | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ad22fbd6f4d108b82aaf49b527bcf40f32babea8 # v2.2.1 | |
| env: | |
| TEST_SUITE: load | |
| TEST_ARGS: -test.timeout 900h | |
| DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable | |
| TEST_LOKI_TOKEN: ${{ secrets.LOKI_TOKEN }} | |
| TEST_LOKI_URL: ${{ secrets.LOKI_URL }} | |
| with: | |
| test_command_to_run: make test_need_operator_assets && cd ./integration-tests/ccip-tests && go test -v -timeout 900h -count=1 -json -run ^TestLoadCCIPStableRPS$ ./load 2>&1 | tee /tmp/gotest.log | gotestfmt | |
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
| cl_repo: ${{ env.INPUT_CHAINLINK_IMAGE }} | |
| cl_image_tag: ${{ env.INPUT_CHAINLINK_VERSION }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| go_mod_path: ./integration-tests/go.mod | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
| artifacts_location: ./integration-tests/load/logs/payload_ccip.json |