From 513e9010afb717cb62480fe9596f0d3e01ec02ca Mon Sep 17 00:00:00 2001 From: Anindita Ghosh <88458927+AnieeG@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:23:17 -0700 Subject: [PATCH] handle blank TTL (#234) --- .gitignore | 3 +- integration-tests/ccip-tests/Makefile | 43 +-- integration-tests/ccip-tests/README.md | 294 +++--------------- integration-tests/ccip-tests/ccip-example.env | 138 -------- .../ccip-tests/testconfig/override/main.go | 6 +- .../testconfig/override/override.toml | 161 +++++++--- .../ccip-tests/testconfig/secrets.env | 103 ++++++ .../ccip-tests/testconfig/tomls/default.toml | 9 + .../ccip-tests/testsetups/ccip.go | 42 +-- 9 files changed, 323 insertions(+), 476 deletions(-) delete mode 100644 integration-tests/ccip-tests/ccip-example.env create mode 100644 integration-tests/ccip-tests/testconfig/secrets.env diff --git a/.gitignore b/.gitignore index 2273b11d57..9eb3d7c35a 100644 --- a/.gitignore +++ b/.gitignore @@ -60,8 +60,7 @@ core/web/assets # Integration Tests integration-tests/**/logs/ integration-tests/**/tmp_* -integration-tests/**/testconfig/**/override.toml -integration-tests/**/testconfig/**/**.env +integration-tests/**/testconfig/override/**.env tests-*.xml *.test tmp-manifest-*.yaml diff --git a/integration-tests/ccip-tests/Makefile b/integration-tests/ccip-tests/Makefile index 7cf396868c..5af55f6c3a 100644 --- a/integration-tests/ccip-tests/Makefile +++ b/integration-tests/ccip-tests/Makefile @@ -1,50 +1,53 @@ ## To Override the default config, -# example usage: make override_config overrideconfig="" override_toml=../config/config.toml env=../.env +# example usage: make override_config overridestring="" override_toml=../config/config.toml env=../.env .PHONY: override_config override_config: cd ./testconfig/override && \ - go run . --overridewith=$(overrideconfig) --path=$(override_toml) --output=$(env) && \ + go run . --overridewith=$(overridestring) --path=$(override_toml) --output=$(env) && \ cd ../.. + # example usage: make test_load_ccip_simulated_k8 image=chainlink-ccip tag=latest testimage=chainlink-ccip-tests:latest testname=TestLoadCCIPStableRequestTriggeringWithNetworkChaos -.PHONY: test_load_ccip_simulated_k8 -test_load_ccip_simulated_k8:override_config +.PHONY: test_load_ccip +include test-config.env +test_load_ccip: source ./testconfig/override/$(env) && \ CHAINLINK_IMAGE=$(image) \ CHAINLINK_VERSION=$(tag) \ ENV_JOB_IMAGE=$(testimage) \ + TEST_SUITE=load \ + TEST_ARGS="-test.timeout 900h" \ + DETACH_RUNNER=true \ go test -timeout 24h -count=1 -v -run ^$(testname)$$ ./load -# example usage: make test_load_ccip_simulated_k8 image=chainlink-ccip tag=latest testimage=chainlink-ccip-tests:latest -.PHONY: test_load_ccip_testnet_k8 +# example usage: make test_smoke_ccip image=chainlink-ccip tag=latest testimage=chainlink-ccip-tests:latest testname=TestSmokeCCIPForBidirectionalLane overridestring="" override_toml=../config/config.toml env=test-config.env +.PHONY: test_smoke_ccip include test-config.env -overrideconfig := "CCIP.Env.Networks = ['SEPOLIA', 'OPTIMISM_GOERLI', 'ARBITRUM_GOERLI', 'AVALANCHE_FUJI', 'BSC_TESTNET', 'MUMBAI', 'BASE_GOERLI]" -override overrideconfig += "CCIP.Groups.load.NodeFunding = 10" -test_load_ccip_testnet_k8: override_config +test_smoke_ccip: override_config source ./testconfig/override/$(env) && \ CHAINLINK_IMAGE=$(image) \ CHAINLINK_VERSION=$(tag) \ ENV_JOB_IMAGE=$(testimage) \ - go test -timeout 24h -count=1 -v -run ^TestLoadCCIPStableRPS$$ ./load - -# example usage: make test_smoke_ccip_simulated_local image=chainlink-ccip tag=latest testname=TestSmokeCCIPForBidirectionalLane override_toml=../config/config.toml env=test-config.env -.PHONY: test_smoke_ccip_simulated_local -include test-config.env -test_smoke_ccip_simulated_local: override_config - source ./testconfig/override/$(env) && \ - CHAINLINK_IMAGE=$(image) \ - CHAINLINK_VERSION=$(tag) \ - ENV_JOB_IMAGE="" \ + TEST_SUITE=smoke \ + TEST_ARGS="-test.timeout 900h" \ + DETACH_RUNNER=true \ go test -timeout 24h -count=1 -v -run ^$(testname)$$ ./smoke # example usage: make test_smoke_ccip_simulated_local_besu image=chainlink-ccip tag=latest override_toml=../config/config.toml env=test-config.env .PHONY: test_smoke_ccip_simulated_local_besu include test-config.env -overrideconfig := "CCIP.Env.Networks = ['SIMULATED_BESU_NONDEV_1', 'SIMULATED_BESU_NONDEV_2']" +overridestring := "CCIP.Env.Networks = ['SIMULATED_BESU_NONDEV_1', 'SIMULATED_BESU_NONDEV_2']" test_smoke_ccip_simulated_local_besu: override_config source ./testconfig/override/$(env) && \ CHAINLINK_IMAGE=$(image) \ CHAINLINK_VERSION=$(tag) \ ENV_JOB_IMAGE="" \ go test -timeout 24h -count=1 -v -run ^TestSmokeCCIPForBidirectionalLane$$ ./smoke + +# image: the name for the chainlink image being built, example: image=chainlink +# tag: the tag for the chainlink image being built, example: tag=latest +# example usage: make build_ccip_image image=chainlink-ccip tag=latest +.PHONY: build_ccip_image +build_ccip_image: + docker build -f ../../core/chainlink.Dockerfile --build-arg COMMIT_SHA=$(git rev-parse HEAD) --build-arg CHAINLINK_USER=chainlink -t $(image):$(tag) ../../ diff --git a/integration-tests/ccip-tests/README.md b/integration-tests/ccip-tests/README.md index ea252313d2..f0a194a11c 100644 --- a/integration-tests/ccip-tests/README.md +++ b/integration-tests/ccip-tests/README.md @@ -1,271 +1,71 @@ -# Integration Tests +# CCIP Tests -Here lives the integration tests for chainlink, utilizing our [chainlink-testing-framework](https://github.com/smartcontractkit/chainlink-testing-framework). +Here lives the integration tests for ccip, utilizing our [chainlink-testing-framework](https://github.com/smartcontractkit/chainlink-testing-framework) and [integration-tests](https://github.com/smartcontractkit/ccip/integration-tests) -## Full Setup +## Running the tests -Prerequisites to run the tests from your local machine. Best for debugging and developing new tests, or checking changes to the framework. This can be a bit complex however, so if you just want to run the tests, see the [Just Run](#just-run) section. +### Setting up test inputs : -
- Details +In order to run the tests the first step is to set up the test inputs. There are two kinds of inputs - +1. Generic test input - set via TOML - If no specific input is set; the tests will run with default inputs mentioned in [default.toml](./testconfig/tomls/default.toml) +2. Secrets - set via env variables. Please refer to [secrets.toml](./testconfig/secrets.env) for the list of env variables that need to be set. -### Install Dependencies +If you want to override the default inputs, you need to set an env var `BASE64_TEST_CONFIG_OVERRIDE` containing the base64 encoded TOML file content. +For example, if you want to override the `Networks` input in test and want to run your test on `avalanche testnet` and `arbitrum goerli` network, you can create a TOML file with the following content: +```toml +[CCIP] -Run the below command to install all dependencies. - -```sh -make install_qa_tools -``` - -Or you can choose to do it manually. - -
- Install Go - - [Install](https://go.dev/doc/install) -
- -
- Install NodeJS - - [Install](https://nodejs.org/en/download/) -
- -
- Install Helm Charts - - [Install Helm](https://helm.sh/docs/intro/install/#through-package-managers) if you don't already have it. Then add necessary charts with the below commands. - - ```sh - helm repo add chainlink-qa https://raw.githubusercontent.com/smartcontractkit/qa-charts/gh-pages/ - helm repo add bitnami https://charts.bitnami.com/bitnami - helm repo update - ``` - -
- -## Connect to a Kubernetes Cluster - -Integration tests require a connection to an actively running kubernetes cluster. [Minikube](https://minikube.sigs.k8s.io/docs/start/) -can work fine for some tests, but in order to run more rigorous tests, or to run with any parallelism, you'll need to either -increase minikube's resources significantly, or get a more substantial cluster. -This is necessary to deploy ephemeral testing environments, which include external adapters, chainlink nodes and their DBs, -as well as some simulated blockchains, all depending on the types of tests and networks being used. - - -### Setup Kubernetes Cluster using k3d - -[k3d](https://k3d.io/) is a lightweight wrapper to run k3s (a lightweight kubernetes distribution) in docker. It's a great way to run a local kubernetes cluster for testing. -To create a new cluster you can run: - -```sh -k3d cluster create test-k8s --registry-create test-k8s-registry:0.0.0.0:5000 -``` - -This will create a cluster with a local registry running on port 5000. You can then use the registry to push images to and pull images from. - -To build and push chainlink image to the registry you can run: - -```sh -make build_push_docker_image -```` - -To stop the cluster you can run: - -```sh -k3d cluster stop test-k8s -``` - -To start an existing cluster you can run: - -```sh -k3d cluster start test-k8s -``` - -## Configure Environment - -See the [example.env](./example.env) file and use it as a template for your own `.env` file. This allows you to configure general settings like what name to associate with your tests, and which Chainlink version to use when running them. - -You can also specify `EVM_KEYS` and `EVM_URLS` for running on live chains, or use specific identifiers as shown in the [example.env](./example.env) file. - -Other `EVM_*` variables are retrieved when running with the `@general` tag, and is helpful for doing quick sanity checks on new chains or when tweaking variables. - -**The tests will not automatically load your .env file. Remember to run `source .env` for changes to take effect.** -## How to Run - -Most of the time, you'll want to run tests on a simulated chain, for the purposes of speed and cost. - -### Smoke - -Run all smoke tests with the below command. Will use your `SELECTED_NETWORKS` env var for which network to run on. - -```sh -make test_smoke # Run all smoke tests on the chosen SELECTED_NETWORKS -SELECTED_NETWORKS="GOERLI" make test_smoke # Run all smoke tests on GOERLI network -make test_smoke_simulated # Run all smoke tests on a simulated network -``` - -Run all smoke tests in parallel, only using simulated blockchains. *Note: As of now, you can only run tests in parallel on simulated chains, not on live ones. Running on parallel tests on live chains will give errors* - -```sh -make test_smoke_simulated args="-test.parallel=" +[CCIP.Env] +Networks = ['AVALANCHE_FUJI', 'ARBITRUM_GOERLI'] ``` - -You can also run specific tests and debug tests in vscode by setting up your .vscode/settings.json with this information. Just replace all the "" with your information before running a test. - -```json -{ - "makefile.extensionOutputFolder": "./.vscode", - "go.testEnvVars": { - "LOG_LEVEL": "debug", - "SELECTED_NETWORKS": "SIMULATED,SIMULATED_1,SIMULATED_2", - "CHAINLINK_IMAGE":".dkr.ecr.us-west-2.amazonaws.com/chainlink", - "CHAINLINK_VERSION":"develop", - "CHAINLINK_ENV_USER":"", - "TEST_LOG_LEVEL":"debug", - "AWS_ACCESS_KEY_ID":"", - "AWS_SECRET_ACCESS_KEY":"", - "AWS_SESSION_TOKEN":"" - }, - "go.testTimeout": "900s" -} -``` - -You can also run your tests inside of kubernetes instead of from locally to reduce local resource usage and the number of ports that get forwarded to the cluster. This is not recommended for normal developement since building and pushing the image can be time heavy depending on your internet upload speeds. To do this you will want to either pull down an already built chainlink-tests image or build one yourself. To build and push one yourself you can run: - -```sh -make build_test_image tag= base_tag=latest suite="smoke soak chaos reorg migration performance" push=true +and then encode it using `base64` command and set the env var `BASE64_TEST_CONFIG_OVERRIDE` with the encoded content. +```bash +export BASE64_TEST_CONFIG_OVERRIDE=$(base64 -w 0 < path-to-toml-file) ``` -Once that is done building you can add this to your go.testEnvVars in .vscode/settings.json with the correct account number and tag filled out. - -```json - "TEST_SUITE": "smoke", - "TEST_ARGS": "-test.timeout 30m", - "ENV_JOB_IMAGE":".dkr.ecr.us-west-2.amazonaws.com/chainlink-env-tests:", +Alternatively, you can also use the make command to invoke a go script to do the same. +```bash +## if overridestring is set, override_toml is ignored +make override_config overridestring="" override_toml="" env="<.env file with BASE64_TEST_CONFIG_OVERRIDE value>" ``` -Once that is done you can run/debug your test using the vscode test view just like normal. - -### Soak - -Currently we have 2 soak tests, both can be triggered using make commands. - -```sh -make test_soak_ocr -make test_soak_keeper +In order to set the secrets, you need to set the env vars mentioned in [secrets.toml](./testconfig/secrets.env) file and source the file. +```bash +source ./testconfig/secrets.env ``` -Soak tests will pull all their network information from the env vars that you can set in the `.env` file. *Reminder to run `source .env` for changes to take effect.* - -To configure specific parameters of how the soak tests run (e.g. test length, number of contracts), adjust the values in your `.env` file, you can use `example.env` as reference - - -#### Running with custom image -On each PR navigate to the `integration-tests` job, here you will find the images for both chainlink-tests and core. In your env file you need to replace: - -`ENV_JOB_IMAGE="image-location/chainlink-tests:"` - -`CHAINLINK_IMAGE="public.ecr.aws/chainlink/chainlink"` - -`export CHAINLINK_VERSION=""` - -After all the env vars are exported, run the tests. This will kick off a remote runner that will be in charge of running the tests. Locally the test should pass quickly and a namespace will be displayed in the output e.g -`INF Creating new namespace Namespace=soak-ocr-goerli-testnet-957b2` - -#### Logs and monitoring -- Pod logs: `kubectl logs -n soak-ocr-goerli-testnet-957b2 -c node -f chainlink-0-1` -- Remote runner logs: `kubectl logs -n soak-ocr-goerli-testnet-957b2 -f remote-runner-cs2as` -- Navigate to Grafana chainlink testing insights for all logs - -### Performance - -Currently, all performance tests are only run on simulated blockchains. - -```sh -make test_perf +Please note that the secrets.env should not be checked in to the repo and should be kept locally. +You can run the command to ignore the changes to the file. +```bash +git update-index --skip-worktree ./testconfig/secrets.env ``` -## CCIP Tests - -### Configure Environment for CCIP - -> CCIP tests follow some additional set up as it needs multi-chain set-up to run the tests. -> To run any kind of CCIP tests you need to provide the some mandatory and optional env variables. -> See the [ccip-example.env](ccip/ccip-example.env) file and use it as a template for your own .env file. -> The tests will not automatically load your .env file. Remember to run source .env for changes to take effect. - +### Triggering the tests +There are two ways to run the tests: +1. Using local docker containers +2. Using remote kubernetes cluster +### Using local docker containers -### How to Run CCIP Tests - ->After following the instructions [Install Dependencies](#setup) , [K8 Setup](#connect-to-a-kubernetes-cluster) ->and [Configure Environment](#configure-environment-for-ccip) , you can run the tests using the following commands. - -#### Smoke tests -Mostly you would run tests on simulated networks for cost and speed- -```sh -make test_smoke_ccip_simulated -``` -On specific networks (set up according to envionment variables) - -```sh -make test_smoke_ccip +In order to run the tests locally, you need to have docker installed and running on your machine. +You can use a specific chainlink image and tag (if you already have one) for the tests. Otherwise, you can build the image using the following command: +```bash +make build_ccip_image image=chainlink-ccip tag=latest-dev # please choose the image and tag name as per your choice ``` -#### Load tests -On simulated networks - -```sh -make test_load_ccip_simulated -``` -On specific networks (set up according to envionment variables) - -```sh -make test_load_ccip -``` +Currently, for local run the tests creates two new private geth networks and runs the tests on them. Running tests on testnet and mainnet is not supported yet for local run. +Please refer to [Using remote kubernetes cluster](#using-remote-kubernetes-cluster) section for running the tests on live networks like testnet and mainnet. -#### Soak tests -Soak tests are just load tests run with long duration and lesser rate of request triggering. -```sh -make test_soak_ccip -``` - -#### Chaos tests -Chaos tests can only be run in simulated networks to inject specific chaos events in custom environments. -You need to trigger podchaos tests and network chaos tests separately. +You can use the following command to run the tests locally with your specific chainlink image. -```sh -make test_chaos_ccip_pods_raw +#### Smoke Tests +```bash +# if overridestring is set, override_toml is ignored +# mark the testimage as empty for running the tests in local docker containers +make test_smoke_ccip image=chainlink-ccip tag=latest-dev testimage="" testname=TestSmokeCCIPForBidirectionalLane overridestring="" override_toml="" env="<.env file with BASE64_TEST_CONFIG_OVERRIDE value>" ``` +Currently other types of tests like load and chaos can only be run using remote kubernetes cluster. -```sh -make test_chaos_ccip_network_raw -``` -#### Soak/Load/Smoke tests on Existing Deployments -> Tests can be run on existing deployments by setting the env var CCIP_TESTS_ON_EXISTING_DEPLOYMENT to true. -> 1. Create a branch out of the code ref using which the contracts were deployed. (This is to ensure that the gethwrappers are compatible with the deployed contracts) -> 2. Update [contracts.json](ccip/contracts/laneconfig/contracts.json) file with the contract addresses of the deployment under test. -> 3. Set the env var CCIP_TESTS_ON_EXISTING_DEPLOYMENT to true in the .env file. -> 4. Set up other env vars denoting the network and other load config parameters in .env file as mentioned in [Configure Environment](#configure-environment-for-ccip) section. -> 5. Run the tests in same branch using the following command - -> ```sh -> source -> make test_soak_ccip # Or the corresponding make _test command for load or smoke tests -> ``` - -## Common Issues - -- When upgrading to a new version, it's possible the helm charts have changed. There are a myriad of errors that can result from this, so it's best to just try running `helm repo update` when encountering an error you're unsure of. -- Docker failing to pull image, make sure you are referencing the correct ECR repo in AWS since develop images are not pushed to the public one. - - If tests hang for some time this is usually the case, so make sure to check the logs each time tests are failing to start - -
- - -## Just Run - -If you're making changes to chainlink code, or just want to run some tests without a complex setup, follow the below steps. +### Using remote kubernetes cluster -1. [Install Go](https://go.dev/doc/install) -2. [Install GitHub CLI](https://cli.github.com/) -3. Authenticate with GitHub CLI: `gh auth login` -4. `make run` -5. Follow the setup wizard and watch your tests run in the GitHub Action. \ No newline at end of file +These tests remain bound to a Kubernetes run environment, and require more complex setup and running instructions. We endeavor to make these easier to run and configure, but for the time being please seek a member of the QA/Test Tooling team if you want to run these. \ No newline at end of file diff --git a/integration-tests/ccip-tests/ccip-example.env b/integration-tests/ccip-tests/ccip-example.env deleted file mode 100644 index 5ad27719fc..0000000000 --- a/integration-tests/ccip-tests/ccip-example.env +++ /dev/null @@ -1,138 +0,0 @@ -# An example template that you can use for your own .env file for integration test settings -# `source ./integration-tests/.env` - -########## Mandatory Env Vars ########## -export CHAINLINK_IMAGE="***.dkr.ecr.***.amazonaws.com/chainlink-ccip" # link to pull chainlink image from which contains ccip codebase -export CHAINLINK_VERSION="0.4.0-beta.0+core2.1.0" # Version of the chainlink-ccip image to pull - -########## Network Settings ########## - -# Select a pre-defined network(s). Networks are pre-defined under `./integration-tests/known_networks.go` -# The network provided in index 0(SIMULATED in this example) -# is overlooked by ccip tests. The tests are run with networks provided -# from index 1 onwards (AVALANCHE_FUJI,SEPOLIA in this example) -# You need provide at least two networks (from index 1) to run the ccip tests -export SELECTED_NETWORKS="SIMULATED,AVALANCHE_FUJI,SEPOLIA" - -# The following env vars are used to run the tests on non-simulated networks. -# Ignore the following when the tests are run using simulated networks -# export SELECTED_NETWORKS="SIMULATED,SIMULATED_1,SIMULATED_2" -# General private values that will be retrieved when running on non-simulated networks -export EVM_KEYS="private,funding,keys" # Comma-separated list of private keys to use when running on live networks - -# Specific private values for urls and keys when running on specified chains -# Goerli -export GOERLI_URLS="" -export GOERLI_HTTP_URLS="" -export GOERLI_KEYS="goerli,funding,keys" # optional if you want to use EVM_KEYS for the goerli network - -# Sepolia -export SEPOLIA_URLS="" -export SEPOLIA_HTTP_URLS="" -export SEPOLIA_KEYS="sepolia,funding,keys" # optional if you want to use EVM_KEYS for the sepolia network - -# Arbitrum Goerli -export ARBITRUM_GOERLI_URLS="" -export ARBITRUM_GOERLI_HTTP_URLS="" -export ARBITRUM_GOERLI_KEYS="arbitrum,funding,keys" # optional if you want to use EVM_KEYS for the arbitrum network - -# Optimism Goerli -export OPTIMISM_GOERLI_URLS="" -export OPTIMISM_GOERLI_HTTP_URLS="" -export OPTIMISM_GOERLI_KEYS="optimism,funding,keys" # optional if you want to use EVM_KEYS for the Optimism network - -# Avalanche Fuji -export AVALANCHE_FUJI_URLS="wss://avalanche.io/ws/" -export AVALANCHE_FUJI_HTTP_URLS="http://avalanche.io/ws/" -export AVALANCHE_FUJI_KEYS="avalanche,funding,keys" # optional if you want to use EVM_KEYS for the Avalanche Fuji network - -# Polygon Testnet -export MUMBAI_URLS="" -export MUMBAI_HTTP_URLS="" -export MUMBAI_KEYS="mumbai,funding,keys" # optional if you want to use EVM_KEYS for the Polygon Testnet network - -# Ethereum Mainnet -export ETHEREUM_MAINNET_URLS="" -export ETHEREUM_MAINNET_HTTP_URLS="" -export ETHEREUM_MAINNET_KEYS="ethereum,funding,keys" # optional if you want to use EVM_KEYS for the Ethereum Mainnet network - -export AVALANCHE_MAINNET_URLS="" -export AVALANCHE_MAINNET_HTTP_URLS="" -export AVALANCHE_MAINNET_KEYS="avalanche,funding,keys" # optional if you want to use EVM_KEYS for the Avalanche Mainnet network - -export ARBITRUM_MAINNET_URLS="" -export ARBITRUM_MAINNET_HTTP_URLS="" -export ARBITRUM_MAINNET_KEYS="arbitrum,funding,keys" # optional if you want to use EVM_KEYS for the Arbitrum Mainnet network - -export POLYGON_MAINNET_URLS="" -export POLYGON_MAINNET_HTTP_URLS="" -export POLYGON_MAINNET_KEYS="polygon,funding,keys" # optional if you want to use EVM_KEYS for the Polygon Mainnet network - -export OPTIMISM_MAINNET_URLS="" -export OPTIMISM_MAINNET_HTTP_URLS="" -export OPTIMISM_MAINNET_KEYS="optimism,funding,keys" # optional if you want to use EVM_KEYS for the Optimism Mainnet network - -########## Optional CCIP Test Settings ########## - -# Number of networks to run the ccip tests on. Default value is 2 unless specified -# If you want to run the ccip tests on more than 2 networks, you need to provide the env name accordingly in SELECTED_NETWORKS -export CCIP_NO_OF_NETWORKS=2 - -# Whether to run the ccip tests on an existing deployment(staging,prod cluster). Default value is false unless specified -# If existing deployment is used, test will not deploy/configure the contracts or jobs. In this case -# the tests will assume that deployment has already been completed and will ensure the ccip-send and receive is working -# with the provided contracts under `./integration-tests/contracts/ccip/laneconfig/contracts.json` -export CCIP_TESTS_ON_EXISTING_DEPLOYMENT=false - -# Even if the CCIP_TESTS_ON_EXISTING_DEPLOYMENT is false, test will try to reuse contracts from `./integration-tests/contracts/ccip/laneconfig/contracts.json` -# if CCIP_REUSE_CONTRACTS is true. If you want tests to deploy new contracts, set CCIP_REUSE_CONTRACTS to false -export CCIP_REUSE_CONTRACTS=true # Whether to reuse the contracts deployed in the previous run. Default value is true unless specified - -# Duration to wait for the each phase validation(SendRequested, Commit, Execution) to time-out. -# Default value is 5m for smoke and 30m for load unless specified -export CCIP_PHASE_VALIDATION_TIMEOUT=5m - -# Amount of native currency to fund the chainlink node with for each network. Default value is 1 for smoke and 20 unless specified -export CCIP_CHAINLINK_NODE_FUNDING=1 - -### Optional Load Test Settings ### -# Msg type to use for the load test. Default value is WithToken unless specified. -# Values to choose from WithToken,WithoutToken -export CCIP_MSG_TYPE=WithToken - -# Duration to run the load test for. Default value is 10m unless specified -export CCIP_TEST_DURATION=10m - -# Rate unit to use for the load test. The ccip-send requests will be triggerred at a rate of duration. -# Default value is 1s unless specified -export CCIP_LOAD_TEST_RATEUNIT=1s - -# Rate to use for the load test. Default value is 2 unless specified -# The ccip-send requests will be triggerred at a rate of per -export CCIP_LOAD_TEST_RATE=2 - -# Chaos interval to use for the load test. Default value is 1m unless specified -# The chaos interval is the duration to wait before subsequent chaos condition is applied in the test environment. -# This is mainly used for load tests with chaos. -export CCIP_LOAD_TEST_CHAOS_INTERVAL=1m - -# Keep environment alive for the specified duration after the test is completed. Default value is 20m unless specified -export CCIP_KEEP_ENV_TTL=20m - -# if you want to send loki logs to a loki instance for populating dashboards from load test result, please provide the following values -export TEST_LOKI_TOKEN="" -export TEST_LOKI_URL="" - -##### This is applicable for load tests. If you want to be notified on slack when the load tests are completed, please provide the following values #### -#### Slack Notification Settings #### -export SLACK_API_KEY="xoxb-example-key" # API key used to report soak test results to slack -export SLACK_CHANNEL="C000000000" # Channel ID for the slack bot to post test results -export SLACK_USER="U000000000" # User ID of the person running the soak tests to properly notify them - -#### Some Additional Settings #### -# Whether to keep the environment alive after the tests are completed. Default value is Never unless specified -export KEEP_ENVIRONMENTS="Never" # Always | OnFail | Never -# Name of the person running the tests (change to your own). This will be used in environment tags. -export CHAINLINK_ENV_USER="Satoshi-Nakamoto" -# Log level for the tests. Default value is info unless specified -export TEST_LOG_LEVEL="info" # info | debug | trace \ No newline at end of file diff --git a/integration-tests/ccip-tests/testconfig/override/main.go b/integration-tests/ccip-tests/testconfig/override/main.go index d6cc153126..65e147a927 100644 --- a/integration-tests/ccip-tests/testconfig/override/main.go +++ b/integration-tests/ccip-tests/testconfig/override/main.go @@ -46,7 +46,11 @@ func main() { os.Exit(1) } defer envFile.Close() - _, err = envFile.WriteString("export BASE64_TEST_CONFIG_OVERRIDE=" + encoded) + envVars := fmt.Sprintf(` +export BASE64_TEST_CONFIG_OVERRIDE=%s +export TEST_BASE64_TEST_CONFIG_OVERRIDE=%s +`, encoded, encoded) + _, err = envFile.WriteString(envVars) if err != nil { log.Println("unable to write to .env file - ", err) os.Exit(1) diff --git a/integration-tests/ccip-tests/testconfig/override/override.toml b/integration-tests/ccip-tests/testconfig/override/override.toml index 9665af0040..6f3a25ddad 100644 --- a/integration-tests/ccip-tests/testconfig/override/override.toml +++ b/integration-tests/ccip-tests/testconfig/override/override.toml @@ -1,53 +1,116 @@ [CCIP] -[CCIP.Env] -[CCIP.Env.Chainlink.Common] -NodeConfig = """ -[Feature] -LogPoller = true -CCIP = true - -[Log] -Level = 'debug' -JSONConsole = true - -[Log.File] -MaxSize = '0b' - -[WebServer] -AllowOrigins = '*' -HTTPPort = 6688 -SecureCookies = false -HTTPWriteTimeout = '1m' - -[WebServer.RateLimit] -Authenticated = 2000 -Unauthenticated = 1000 -[WebServer.TLS] -HTTPSPort = 0 - -[Database] -MaxIdleConns = 10 -MaxOpenConns = 20 -MigrateOnStartup = true - -[OCR2] -Enabled = true -DefaultTransactionQueueDepth = 0 - -[OCR] -Enabled = false -DefaultTransactionQueueDepth = 0 +[CCIP.Env] +Networks = ['AVALANCHE_FUJI', 'OPTIMISM_GOERLI'] -[P2P.V1] -Enabled = false -ListenPort = 0 +[CCIP.Deployments] +Data = """ +{ + "lane_configs": { + "Arbitrum Goerli": { + "is_mock_arm": true, + "fee_token": "0xd14838A68E8AFBAdE5efb411d5871ea0011AFd28", + "bridge_tokens": [ + "0xd14838A68E8AFBAdE5efb411d5871ea0011AFd28", + "0xfd064A18f3BF249cf1f87FC203E90D8f650f2d63" + ], + "bridge_tokens_pools": [ + "0xC274570C956B840b71eDe9771e2d3B7Ad1A8C164", + "0xA2A7E89A684784dd73e80bdCF2F2368Ef7C7A690" + ], + "arm": "0xa763a65f7697be1d9a890c2ad006386ad0a16fcb", + "router": "0x2798d664bad9dd2ec6588447715664857b7a8743", + "price_registry": "0xaa0090ab7944648c7d3a950f714b6acd3338774b", + "wrapped_native": "0x32d5D5978905d9c6c2D4C417F0E06Fe768a4FB5a", + "src_contracts": { + "Avalanche Fuji": { + "on_ramp": "0x5922E1b06FaB45BB31a4AB8b8922a9b9763A2795", + "deployed_at": 48392911 + } + }, + "dest_contracts": { + "Avalanche Fuji": { + "off_ramp": "0x1D3e13D896031Aed6C047A96C9879a05DA5F5913", + "commit_store": "0x701ccF1c98Bd3284649Dc6cf08c69410d90c8590", + "receiver_dapp": "0x8Ed27Ab67aea9BC50628fba25ab52D23ae8d0eae" + } + } + }, + "Avalanche Fuji": { + "is_mock_arm": true, + "fee_token": "0x0b9d5D9136855f6FEc3c0993feE6E9CE8a297846", + "bridge_tokens": [ + "0x2Ca7afAC86D0b28d9f7e512b969cEaD9E3048303", + "0x4e51e0a8a5e31Acb0a0B5EB9E5301a21DeBFAbbe", + "0x14f7b60b3234aCF3321198fE6d9792dE130a502c", + "0x5425890298aed601595a70AB815c96711a31Bc65", + "0x0b9d5D9136855f6FEc3c0993feE6E9CE8a297846" + ], + "bridge_tokens_pools": [ + "0xd8D45Dd3278c62214160Db39AfbC709a1a758764", + "0x00D857FA8603968A41D8f61684A565021aA3D4A4", + "0x45227357466953b84338Dc9Df4841a37BEE43a23", + "0x2dec3577e37336cd57C40E207C74A3189aCc839d", + "0x30d1594322EbBA3b6Bf78239F72a857A39c897dD" + ], + "arm": "0xD4A51dC0F5C680A8A18eA4Ec3A2f25C6db9424B7", + "router": "0xFAf82271A26e85A12f447Dc535a7e516b0657401", + "price_registry": "0xdbeA1a10AC6a2B729bF128aE9281Ed420dbE7113", + "wrapped_native": "0xd00ae08403B9bbb9124bB305C09058E32C39A48c", + "src_contracts": { + "Optimism Goerli": { + "on_ramp": "0xa27054c76f64335f5a583a797aecf35d5ce3da79", + "deployed_at": 27066483 + } + }, + "dest_contracts": { + "Optimism Goerli": { + "off_ramp": "0xc7e336c9db98dce3c3bc99ce87498ee646b7e37c", + "commit_store": "0x52a927ed15fced4bf0875b26c2add12ded868910", + "receiver_dapp": "0x8Ed27Ab67aea9BC50628fba25ab52D23ae8d0eae" + } + } + }, + "Optimism Goerli": { + "is_mock_arm": true, + "fee_token": "0xdc2CC710e42857672E7907CF474a69B63B93089f", + "bridge_tokens": [ + "0xdc2CC710e42857672E7907CF474a69B63B93089f", + "0xe05606174bac4A6364B31bd0eCA4bf4dD368f8C6" + ], + "bridge_tokens_pools": [ + "0x7d3baccb582f1F40DD366f47bb70d495bFA71100", + "0xb4A7854bA68FfB9bfDd84b80691b36a3bC28A4a7" + ], + "arm": "0xC1b9bcdC6bE9f4D6224672F18fC46ff330F487e1", + "router": "0xa9ec8d47df64003367a1fc3ae2a0f28858438370", + "price_registry": "0x89143a4AC2aB3472b41687FDd7A58efFbA3461DA", + "wrapped_native": "0x4200000000000000000000000000000000000006", + "src_contracts": { + "Avalanche Fuji": { + "on_ramp": "0xf6c909ee2a3a24665abe30fc2e00d22279c5ab4f", + "deployed_at": 16360032 + } + }, + "dest_contracts": { + "Avalanche Fuji": { + "off_ramp": "0x1c5caecdbcc88c02d11fc47737e214aa22f1aadd", + "commit_store": "0xbebc239a4ede352f97cf61ddb1aa46a1bf0d0de4", + "receiver_dapp": "0x8Ed27Ab67aea9BC50628fba25ab52D23ae8d0eae" + } + } + } + } +} +""" -[P2P] -[P2P.V2] -Enabled = true -ListenAddresses = ['0.0.0.0:6690'] -AnnounceAddresses = ['0.0.0.0:6690'] -DeltaDial = '500ms' -DeltaReconcile = '5s' -""" \ No newline at end of file +[CCIP.Groups] +[CCIP.Groups.smoke] +MsgType = 'WithToken' +PhaseTimeout = '50m' +LocalCluster = false +ExistingDeployment = true +ReuseContracts = true +NoOfTokensPerChain = 2 +NoOfTokensInMsg = 2 +AmountPerToken = 1 \ No newline at end of file diff --git a/integration-tests/ccip-tests/testconfig/secrets.env b/integration-tests/ccip-tests/testconfig/secrets.env new file mode 100644 index 0000000000..4949706424 --- /dev/null +++ b/integration-tests/ccip-tests/testconfig/secrets.env @@ -0,0 +1,103 @@ +# An example template that you can use for your own .env file for integration test settings +# `source ./secrets.env` + +## these are only secrets, please do not commit this file to git +## please use the following command to ignore this file from git +## `git update-index --skip-worktree secrets.env` + +## if you want to run the test on a k8 pod, the following test image will be used to run the test +export ENV_JOB_IMAGE="***.dkr.ecr.***.amazonaws.com/chainlink-ccip-tests:latest" # link to pull image from which contains ccip codebase + +########## Chainlink Secrets ########## +export CHAINLINK_IMAGE="***.dkr.ecr.***.amazonaws.com/chainlink-ccip" # link to pull chainlink image from which contains ccip codebase +export CHAINLINK_VERSION="****" # Version of the chainlink-ccip image to pull + +########## Network Secrets ########## + +# General private values that will be retrieved when running on non-simulated networks +export EVM_KEYS="private keys for funding wallet to run test" + +# Specific private values for urls and keys when running on specified chains +# Goerli +export GOERLI_URLS="" +export GOERLI_HTTP_URLS="" +export GOERLI_KEYS="" # optional if you want to use EVM_KEYS for the goerli network + +# Sepolia +export SEPOLIA_URLS="" +export SEPOLIA_HTTP_URLS="" +export SEPOLIA_KEYS="" # optional if you want to use EVM_KEYS for the sepolia network + +# Arbitrum Goerli +export ARBITRUM_GOERLI_URLS="" +export ARBITRUM_GOERLI_HTTP_URLS="" +export ARBITRUM_GOERLI_KEYS="" # optional if you want to use EVM_KEYS for the arbitrum network + +# Optimism Goerli +export OPTIMISM_GOERLI_URLS="" +export OPTIMISM_GOERLI_HTTP_URLS="" +export OPTIMISM_GOERLI_KEYS="" # optional if you want to use EVM_KEYS for the Optimism network + +# Avalanche Fuji +export AVALANCHE_FUJI_URLS="wss://avalanche.io/ws/" +export AVALANCHE_FUJI_HTTP_URLS="http://avalanche.io/ws/" +export AVALANCHE_FUJI_KEYS="" # optional if you want to use EVM_KEYS for the Avalanche Fuji network + +# Polygon Testnet +export MUMBAI_URLS="" +export MUMBAI_HTTP_URLS="" +export MUMBAI_KEYS="" # optional if you want to use EVM_KEYS for the Polygon Testnet network + +# Base Goerli +export BASE_GOERLI_URLS="" +export BASE_GOERLI_HTTP_URLS="" +export BASE_GOERLI_KEYS="" # optional if you want to use EVM_KEYS for the base goerli network + +# Binance Testnet +export BSC_TESTNET_URLS="" +export BSC_TESTNET_HTTP_URLS="" +export BSC_TESTNET_KEYS="" # optional if you want to use EVM_KEYS for the binance testnet network + + +# Ethereum Mainnet +export ETHEREUM_MAINNET_URLS="" +export ETHEREUM_MAINNET_HTTP_URLS="" +export ETHEREUM_MAINNET_KEYS="" # optional if you want to use EVM_KEYS for the Ethereum Mainnet network + +# Avalanche Mainnet +export AVALANCHE_MAINNET_URLS="" +export AVALANCHE_MAINNET_HTTP_URLS="" +export AVALANCHE_MAINNET_KEYS="" # optional if you want to use EVM_KEYS for the Avalanche Mainnet network + +# Arbitrum Mainnet +export ARBITRUM_MAINNET_URLS="" +export ARBITRUM_MAINNET_HTTP_URLS="" +export ARBITRUM_MAINNET_KEYS="" # optional if you want to use EVM_KEYS for the Arbitrum Mainnet network + +# Polygon Mainnet +export POLYGON_MAINNET_URLS="" +export POLYGON_MAINNET_HTTP_URLS="" +export POLYGON_MAINNET_KEYS="" # optional if you want to use EVM_KEYS for the Polygon Mainnet network + +# Optimism Mainnet +export OPTIMISM_MAINNET_URLS="" +export OPTIMISM_MAINNET_HTTP_URLS="" +export OPTIMISM_MAINNET_KEYS="" # optional if you want to use EVM_KEYS for the Optimism Mainnet network + +# Binance Mainnet +export BSC_MAINNET_URLS="" +export BSC_MAINNET_HTTP_URLS="" +export BSC_MAINNET_KEYS="" # optional if you want to use EVM_KEYS for the Binance Mainnet network + +# Base Mainnet +export BASE_MAINNET_URLS="" +export BASE_MAINNET_HTTP_URLS="" +export BASE_MAINNET_KEYS="" # optional if you want to use EVM_KEYS for the base mainnet network + + +# if you want to send loki logs to a loki instance for populating dashboards from load test result, please provide the following values +export TEST_LOKI_TENANT_ID="" +export TEST_LOKI_URL="" + +export LOKI_TENANT_ID="" +export LOKI_URL="" diff --git a/integration-tests/ccip-tests/testconfig/tomls/default.toml b/integration-tests/ccip-tests/testconfig/tomls/default.toml index 2753225a64..82fa803182 100644 --- a/integration-tests/ccip-tests/testconfig/tomls/default.toml +++ b/integration-tests/ccip-tests/testconfig/tomls/default.toml @@ -1,7 +1,16 @@ +# this file contains the deafult configuration for the test +# all secrets must be stored in .env file and sourced before running the test [CCIP] +# all variables to set up the test environment [CCIP.Env] +# networks between which lanes will be set up and the messages will be sent +# if more than 2 networks are specified, then lanes will be set up between all possible pairs of networks +# default value is ['SIMULATED_1', 'SIMULATED_2'] which means that test will create two private geth networks from scratch and set up lanes between them Networks = ['SIMULATED_1', 'SIMULATED_2'] +# these values will be used to set up chainlink DON +# along with these values, the secrets needs to be specified as part of .env variables +# [CCIP.Env.Chainlink] NoOfNodes = 6 NodeMemory = '4Gi' diff --git a/integration-tests/ccip-tests/testsetups/ccip.go b/integration-tests/ccip-tests/testsetups/ccip.go index f8e04687ce..0e571260df 100644 --- a/integration-tests/ccip-tests/testsetups/ccip.go +++ b/integration-tests/ccip-tests/testsetups/ccip.go @@ -16,6 +16,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog" chainselectors "github.com/smartcontractkit/chain-selectors" + "github.com/smartcontractkit/chainlink-env/config" "github.com/smartcontractkit/chainlink-env/environment" "github.com/smartcontractkit/chainlink-testing-framework/blockchain" "github.com/smartcontractkit/chainlink-testing-framework/networks" @@ -646,6 +647,13 @@ func CCIPDefaultTestSetUp( configureCLNode := !pointer.GetBool(inputs.TestGroupInput.ExistingDeployment) var deployCL func() error var local *test_env.CLClusterTestEnv + envConfig := &environment.Config{ + NamespacePrefix: envName, + Test: t, + } + if inputs.EnvInput.TTL != nil { + envConfig.TTL = inputs.EnvInput.TTL.Duration() + } if configureCLNode { if pointer.GetBool(inputs.TestGroupInput.LocalCluster) { local, deployCL = DeployLocalCluster(t, inputs) @@ -654,13 +662,7 @@ func CCIPDefaultTestSetUp( } } else { // deploy the env if configureCLNode is true - k8Env = DeployEnvironments( - t, - &environment.Config{ - TTL: inputs.EnvInput.TTL.Duration(), - NamespacePrefix: envName, - Test: t, - }, inputs) + k8Env = DeployEnvironments(t, envConfig, inputs) ccipEnv = &actions.CCIPTestEnv{K8Env: k8Env} } @@ -671,17 +673,14 @@ func CCIPDefaultTestSetUp( } } else { // if configureCLNode is false, use a placeholder env to create remote runner - k8Env = environment.New( - &environment.Config{ - TTL: inputs.EnvInput.TTL.Duration(), - NamespacePrefix: envName, - Test: t, - }) - err = k8Env.Run() - require.NoErrorf(t, err, "error creating environment remote runner") - setUpArgs.Env = &actions.CCIPTestEnv{K8Env: k8Env} - if k8Env.WillUseRemoteRunner() { - return setUpArgs + if value, set := os.LookupEnv(config.EnvVarJobImage); set && value != "" { + k8Env = environment.New(envConfig) + err = k8Env.Run() + require.NoErrorf(t, err, "error creating environment remote runner") + setUpArgs.Env = &actions.CCIPTestEnv{K8Env: k8Env} + if k8Env.WillUseRemoteRunner() { + return setUpArgs + } } } @@ -699,7 +698,12 @@ func CCIPDefaultTestSetUp( if _, ok := chainByChainID[n.ChainID]; ok { continue } - ec, err := blockchain.NewEVMClient(n, k8Env, lggr) + var ec blockchain.EVMClient + if k8Env == nil { + ec, err = blockchain.ConnectEVMClient(n, lggr) + } else { + ec, err = blockchain.NewEVMClient(n, k8Env, lggr) + } require.NoError(t, err, "Connecting to blockchain nodes shouldn't fail") chains = append(chains, ec) chainByChainID[n.ChainID] = ec