Skip to content

Commit

Permalink
CCIP-2694 : lane specific load Frequency (#1149)
Browse files Browse the repository at this point in the history 
## Motivation
The load frequency is set as a global frequency for all lanes under
test. We need to specify destination specific load frequency so that
frequency for slow-destination network lanes can be set differently than
frequency of fast-destination network lanes.

## Solution
  • Loading branch information
@AnieeG
AnieeG authored Jul 9, 2024
1 parent 5f54e6d commit 93ce593
Show file tree
Hide file tree
Showing 8 changed files with 170 additions and 74 deletions.
6 changes: 1 addition & 5 deletions integration-tests/ccip-tests/load/ccip_loadgen.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,10 +248,7 @@ func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.Response {
// if the token address is 0x0 it will use Native as fee token and the fee amount should be mentioned in bind.TransactOpts's value
fee, err := sourceCCIP.Common.Router.GetFee(destChainSelector, msg)
if err != nil {
res.Error = fmt.Sprintf("reqNo %d err %s - while getting fee from router - msg Data %x FeeToken %s TokenAmounts %+v ExtraArgs %x Receiver %x",
msgSerialNo, err.Error(),
msg.Data, msg.FeeToken, msg.TokenAmounts, msg.ExtraArgs, msg.Receiver)

res.Error = fmt.Sprintf("reqNo %d err %s - while getting fee from router", msgSerialNo, err.Error())
res.Failed = true
return res
}
Expand All @@ -271,7 +268,6 @@ func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.Response {
}

txConfirmationTime := time.Now().UTC()
// wait for the tx to be mined, timeout is set to 10 minutes
lggr.Info().Str("tx", sendTx.Hash().Hex()).Msg("waiting for tx to be mined")
lggr = lggr.With().Str("Msg Tx", sendTx.Hash().String()).Logger()

Expand Down
59 changes: 42 additions & 17 deletions integration-tests/ccip-tests/load/helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"math"
"math/big"
"strings"
"sync"
"testing"
"time"
Expand All @@ -16,6 +17,8 @@ import (
"go.uber.org/atomic"
"golang.org/x/sync/errgroup"

"github.com/smartcontractkit/chainlink-common/pkg/config"

"github.com/smartcontractkit/chainlink-testing-framework/k8s/chaos"
"github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext"

Expand All @@ -32,11 +35,30 @@ type ChaosConfig struct {
WaitBetweenChaos time.Duration
}

// WaspSchedule calculates the load schedule based on the provided request per unit time and duration
// if multiple step durations are provided, it will calculate the schedule based on the step duration and
// corresponding request per unit time by matching the index of the request per unit time and step duration slice
func WaspSchedule(rps []int64, duration *config.Duration, steps []*config.Duration) []*wasp.Segment {
var segments []*wasp.Segment
var segmentDuration time.Duration

if len(rps) > 1 {
for i, req := range rps {
duration := steps[i].Duration()
segmentDuration += duration
segments = append(segments, wasp.Plain(req, duration)...)
}
totalDuration := duration.Duration()
repeatTimes := totalDuration.Seconds() / segmentDuration.Seconds()
return wasp.CombineAndRepeat(int(math.Round(repeatTimes)), segments)
}
return wasp.Plain(rps[0], duration.Duration())
}

type LoadArgs struct {
t *testing.T
Ctx context.Context
lggr *zerolog.Logger
schedules []*wasp.Segment
RunnerWg *errgroup.Group // to wait on individual load generators run
LoadStarterWg *sync.WaitGroup // waits for all the runners to start
TestCfg *testsetups.CCIPTestConfig
Expand Down Expand Up @@ -82,23 +104,27 @@ func (l *LoadArgs) Setup() {
l.SetReportParams()
}

func (l *LoadArgs) setSchedule() {
var segments []*wasp.Segment
var segmentDuration time.Duration
func (l *LoadArgs) scheduleForDest(destNetworkName string) []*wasp.Segment {
require.Greater(l.t, len(l.TestCfg.TestGroupInput.LoadProfile.RequestPerUnitTime), 0, "RequestPerUnitTime must be set")

if len(l.TestCfg.TestGroupInput.LoadProfile.RequestPerUnitTime) > 1 {
for i, req := range l.TestCfg.TestGroupInput.LoadProfile.RequestPerUnitTime {
duration := l.TestCfg.TestGroupInput.LoadProfile.StepDuration[i].Duration()
segmentDuration += duration
segments = append(segments, wasp.Plain(req, duration)...)
// try to locate if there is a frequency provided for the destination network
// to locate the frequency, we check if the destination network name contains the network name in the frequency map
// if found, use that frequency for the destination network
// otherwise, use the default frequency
if l.TestCfg.TestGroupInput.LoadProfile.FrequencyByDestination != nil {
for networkName, freq := range l.TestCfg.TestGroupInput.LoadProfile.FrequencyByDestination {
if strings.Contains(destNetworkName, networkName) {
return WaspSchedule(
freq.RequestPerUnitTime,
l.TestCfg.TestGroupInput.LoadProfile.TestDuration,
freq.StepDuration)
}
}
totalDuration := l.TestCfg.TestGroupInput.LoadProfile.TestDuration.Duration()
repeatTimes := totalDuration.Seconds() / segmentDuration.Seconds()
l.schedules = wasp.CombineAndRepeat(int(math.Round(repeatTimes)), segments)
} else {
l.schedules = wasp.Plain(l.TestCfg.TestGroupInput.LoadProfile.RequestPerUnitTime[0], l.TestCfg.TestGroupInput.LoadProfile.TestDuration.Duration())
}

return WaspSchedule(
l.TestCfg.TestGroupInput.LoadProfile.RequestPerUnitTime,
l.TestCfg.TestGroupInput.LoadProfile.TestDuration,
l.TestCfg.TestGroupInput.LoadProfile.StepDuration)
}

func (l *LoadArgs) SanityCheck() {
Expand Down Expand Up @@ -233,7 +259,6 @@ func (l *LoadArgs) ValidateCurseFollowedByUncurse() {
}

func (l *LoadArgs) TriggerLoadByLane() {
l.setSchedule()
l.TestSetupArgs.Reporter.SetDuration(l.TestCfg.TestGroupInput.LoadProfile.TestDuration.Duration())

// start load for a lane
Expand Down Expand Up @@ -269,7 +294,7 @@ func (l *LoadArgs) TriggerLoadByLane() {
waspCfg := &wasp.Config{
T: l.TestCfg.Test,
GenName: fmt.Sprintf("lane %s-> %s", lane.SourceNetworkName, lane.DestNetworkName),
Schedule: l.schedules,
Schedule: l.scheduleForDest(lane.DestNetworkName),
LoadType: wasp.RPS,
RateLimitUnitDuration: l.TestCfg.TestGroupInput.LoadProfile.TimeUnit.Duration(),
CallResultBufLen: 10, // we keep the last 10 call results for each generator, as the detailed report is generated at the end of the test
Expand Down
15 changes: 11 additions & 4 deletions integration-tests/ccip-tests/testconfig/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,7 +528,12 @@ Specifies the number of routers to be set up for each network.

### CCIP.Groups.[testgroup].MaxNoOfLanes
Specifies the maximum number of lanes to be set up between networks. If this values is not set, the test will set up lanes between all possible pairs of networks mentioned in `selected_networks` in [CCIP.Env.Networks](#ccipenvnetworksselectednetworks).
For example, if `selected_networks = ['SIMULATED_1', 'SIMULATED_2', 'SIMULATED_3']`, and `MaxNoOfLanes` is set to 2, it denotes that the test will select the first 2 lanes between all possible pairs `SIMULATED_1`, `SIMULATED_2`, and `SIMULATED_3` for the test run.
For example, if `selected_networks = ['SIMULATED_1', 'SIMULATED_2', 'SIMULATED_3']`, and `MaxNoOfLanes` is set to 2, it denotes that the test will randomly select the 2 lanes between all possible pairs `SIMULATED_1`, `SIMULATED_2`, and `SIMULATED_3` for the test run.

### CCIP.Groups.[testgroup].DenselyConnectedNetworkChainIds
This is applicable only if [MaxNoOfLanes](#ccipgroupstestgroupmaxnooflanes) is specified.
Specifies the chain ids for networks to be densely connected. If this is provided the test will include all possible pairs of networks mentioned in `DenselyConnectedNetworkChainIds`.
The rest of the networks will be connected randomly based on the value of `MaxNoOfLanes`.

### CCIP.Groups.[testgroup].ChaosDuration
Specifies the duration for which the chaos experiment is to be run. This is only valid if the test type is 'chaos'.
Expand Down Expand Up @@ -595,14 +600,16 @@ This is only valid if the tests are run on remote runners in k8s. If set to true
### CCIP.Groups.[testgroup].LoadProfile
Specifies the load profile for the test. Only valid if the testgroup is 'load'.

### CCIP.Groups.[testgroup].LoadProfile.LoadFrequency.[DestNetworkName]

#### CCIP.Groups.[testgroup].LoadProfile.RequestPerUnitTime
Specifies the number of requests to be sent per unit time.
Specifies the number of requests to be sent per unit time. This is applicable to all networks if [LoadFrequency](#ccipgroupstestgrouploadprofileloadfrequencydestnetworkname) is not specified for a destination network.

#### CCIP.Groups.[testgroup].LoadProfile.TimeUnit
Specifies the unit of time for the load profile.
Specifies the unit of time for the load profile. This is applicable to all networks if [LoadFrequency](#ccipgroupstestgrouploadprofileloadfrequencydestnetworkname) is not specified for a destination network.

#### CCIP.Groups.[testgroup].LoadProfile.StepDuration
Specifies the duration for each step in the load profile.
Specifies the duration for each step in the load profile. This is applicable to all networks if [LoadFrequency](#ccipgroupstestgrouploadprofileloadfrequencydestnetworkname) is not specified for a destination network.

#### CCIP.Groups.[testgroup].LoadProfile.TestDuration
Specifies the total duration for the load test.
Expand Down
84 changes: 46 additions & 38 deletions integration-tests/ccip-tests/testconfig/ccip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,19 +204,26 @@ func (m *MsgProfile) Validate() error {
return nil
}

type LoadFrequency struct {
RequestPerUnitTime []int64 `toml:",omitempty"`
TimeUnit *config.Duration `toml:",omitempty"`
StepDuration []*config.Duration `toml:",omitempty"`
}

type LoadProfile struct {
MsgProfile *MsgProfile `toml:",omitempty"`
RequestPerUnitTime []int64 `toml:",omitempty"`
TimeUnit *config.Duration `toml:",omitempty"`
StepDuration []*config.Duration `toml:",omitempty"`
TestDuration *config.Duration `toml:",omitempty"`
NetworkChaosDelay *config.Duration `toml:",omitempty"`
WaitBetweenChaosDuringLoad *config.Duration `toml:",omitempty"`
SkipRequestIfAnotherRequestTriggeredWithin *config.Duration `toml:",omitempty"`
OptimizeSpace *bool `toml:",omitempty"`
FailOnFirstErrorInLoad *bool `toml:",omitempty"`
SendMaxDataInEveryMsgCount *int64 `toml:",omitempty"`
TestRunName string `toml:",omitempty"`
MsgProfile *MsgProfile `toml:",omitempty"`
FrequencyByDestination map[string]*LoadFrequency `toml:",omitempty"`
RequestPerUnitTime []int64 `toml:",omitempty"`
TimeUnit *config.Duration `toml:",omitempty"`
StepDuration []*config.Duration `toml:",omitempty"`
TestDuration *config.Duration `toml:",omitempty"`
NetworkChaosDelay *config.Duration `toml:",omitempty"`
WaitBetweenChaosDuringLoad *config.Duration `toml:",omitempty"`
SkipRequestIfAnotherRequestTriggeredWithin *config.Duration `toml:",omitempty"`
OptimizeSpace *bool `toml:",omitempty"`
FailOnFirstErrorInLoad *bool `toml:",omitempty"`
SendMaxDataInEveryMsgCount *int64 `toml:",omitempty"`
TestRunName string `toml:",omitempty"`
}

func (l *LoadProfile) Validate() error {
Expand Down Expand Up @@ -246,32 +253,33 @@ func (l *LoadProfile) SetTestRunName(name string) {

// CCIPTestGroupConfig defines configuration input to change how a particular CCIP test group should run
type CCIPTestGroupConfig struct {
Type string `toml:",omitempty"`
KeepEnvAlive *bool `toml:",omitempty"`
BiDirectionalLane *bool `toml:",omitempty"`
CommitAndExecuteOnSameDON *bool `toml:",omitempty"`
NoOfCommitNodes int `toml:",omitempty"`
MsgDetails *MsgDetails `toml:",omitempty"`
TokenConfig *TokenConfig `toml:",omitempty"`
MulticallInOneTx *bool `toml:",omitempty"`
NoOfSendsInMulticall int `toml:",omitempty"`
PhaseTimeout *config.Duration `toml:",omitempty"`
LocalCluster *bool `toml:",omitempty"`
ExistingDeployment *bool `toml:",omitempty"`
ReuseContracts *bool `toml:",omitempty"`
NodeFunding float64 `toml:",omitempty"`
NetworkPairs []string `toml:",omitempty"`
NoOfNetworks int `toml:",omitempty"`
NoOfRoutersPerPair int `toml:",omitempty"`
MaxNoOfLanes int `toml:",omitempty"`
ChaosDuration *config.Duration `toml:",omitempty"`
USDCMockDeployment *bool `toml:",omitempty"`
CommitOCRParams *contracts.OffChainAggregatorV2Config `toml:",omitempty"`
ExecOCRParams *contracts.OffChainAggregatorV2Config `toml:",omitempty"`
OffRampConfig *OffRampConfig `toml:",omitempty"`
CommitInflightExpiry *config.Duration `toml:",omitempty"`
StoreLaneConfig *bool `toml:",omitempty"`
LoadProfile *LoadProfile `toml:",omitempty"`
Type string `toml:",omitempty"`
KeepEnvAlive *bool `toml:",omitempty"`
BiDirectionalLane *bool `toml:",omitempty"`
CommitAndExecuteOnSameDON *bool `toml:",omitempty"`
NoOfCommitNodes int `toml:",omitempty"`
MsgDetails *MsgDetails `toml:",omitempty"`
TokenConfig *TokenConfig `toml:",omitempty"`
MulticallInOneTx *bool `toml:",omitempty"`
NoOfSendsInMulticall int `toml:",omitempty"`
PhaseTimeout *config.Duration `toml:",omitempty"`
LocalCluster *bool `toml:",omitempty"`
ExistingDeployment *bool `toml:",omitempty"`
ReuseContracts *bool `toml:",omitempty"`
NodeFunding float64 `toml:",omitempty"`
NetworkPairs []string `toml:",omitempty"`
DenselyConnectedNetworkChainIds []string `toml:",omitempty"`
NoOfNetworks int `toml:",omitempty"`
NoOfRoutersPerPair int `toml:",omitempty"`
MaxNoOfLanes int `toml:",omitempty"`
ChaosDuration *config.Duration `toml:",omitempty"`
USDCMockDeployment *bool `toml:",omitempty"`
CommitOCRParams *contracts.OffChainAggregatorV2Config `toml:",omitempty"`
ExecOCRParams *contracts.OffChainAggregatorV2Config `toml:",omitempty"`
OffRampConfig *OffRampConfig `toml:",omitempty"`
CommitInflightExpiry *config.Duration `toml:",omitempty"`
StoreLaneConfig *bool `toml:",omitempty"`
LoadProfile *LoadProfile `toml:",omitempty"`
}

func (c *CCIPTestGroupConfig) Validate() error {
Expand Down
3 changes: 2 additions & 1 deletion integration-tests/ccip-tests/testconfig/tomls/ccip1.4-stress-2024-06-27/baseline.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
## throughput 1msg / 5s
## 20% Token, 60% DataWithToken, 15% Regular size msgs, 5% Large msgs
##
## make test_load_ccip testimage=amazonaws.com/chainlink-ccip-tests:ccip-develop \
## make test_load_ccip testimage=<aws account number>.dkr.ecr.<aws region>.amazonaws.com/chainlink-ccip-tests:ccip-develop \
## testname=TestLoadCCIPStableRequestTriggeringWithNetworkChaos \
## override_toml=./testconfig/tomls/baseline.toml \
## secret_toml=./testconfig/tomls/secrets.toml
Expand Down Expand Up @@ -58,6 +58,7 @@ evm_finality_depth = 200

[CCIP.Env.Network.AnvilConfigs.PRIVATE-CHAIN-1]
block_time = 1

#
[CCIP.Env.Network.AnvilConfigs.PRIVATE-CHAIN-2]
block_time = 1
Expand Down
26 changes: 20 additions & 6 deletions integration-tests/ccip-tests/testconfig/tomls/ccip1.4-stress-2024-06-27/tier-a.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
## throughput 1msg / 5s
## 20% Token, 60% DataWithToken, 15% Regular size msgs, 5% Large msgs
##
## make test_load_ccip testimage=795953128386.dkr.ecr.us-west-2.amazonaws.com/chainlink-ccip-tests:ccip-develop \
## make test_load_ccip testimage=<aws account number>.dkr.ecr.<aws region>.amazonaws.com/chainlink-ccip-tests:ccip-develop \
## testname=TestLoadCCIPStableRequestTriggeringWithNetworkChaos \
## override_toml=./testconfig/tomls/baseline.toml \
## override_toml=./testconfig/tomls/ccip1.4-stress-2024-06-27/tier-a.toml \
## secret_toml=./testconfig/tomls/secrets.toml

[CCIP]
Expand Down Expand Up @@ -89,7 +89,8 @@ evm_default_gas_limit = 6000000
evm_finality_depth = 1

[CCIP.Env.Network.AnvilConfigs.PRIVATE-CHAIN-1]
block_time = 1
block_time = 2

#
[CCIP.Env.Network.AnvilConfigs.SLOW-CHAIN-1]
block_time = 12
Expand Down Expand Up @@ -172,6 +173,7 @@ DeltaReconcile = '5s'

[CCIP.Groups]
[CCIP.Groups.load]
DenselyConnectedNetworkChainIds = ['90000001', '90000002', '1337']
KeepEnvAlive = true
NoOfCommitNodes = 16
PhaseTimeout = '40m'
Expand All @@ -187,15 +189,27 @@ BatchGasLimit = 11000000
TimeoutForPriceUpdate = '15m'
NoOfTokensPerChain = 60
NoOfTokensWithDynamicPrice = 15
DynamicPriceUpdateInterval ='15s'
DynamicPriceUpdateInterval ='5m'
CCIPOwnerTokens = true

[CCIP.Groups.load.LoadProfile]
TestDuration = '4h'
TimeUnit = '5s'
RequestPerUnitTime = [1]
OptimizeSpace = true
NetworkChaosDelay = '100ms'
TimeUnit = '5s'
RequestPerUnitTime = [1]

[CCIP.Groups.load.LoadProfile.LoadFrequency.slow-chain-1]
TimeUnit = '10s'
RequestPerUnitTime = [1]

[CCIP.Groups.load.LoadProfile.LoadFrequency.slow-chain-2]
TimeUnit = '10s'
RequestPerUnitTime = [1]

[CCIP.Groups.load.LoadProfile.LoadFrequency.slow-chain-3]
TimeUnit = '10s'
RequestPerUnitTime = [1]

# to represent 20%, 60%, 15%, 5% of the total messages
[CCIP.Groups.load.LoadProfile.MsgProfile]
Expand Down
Loading

0 comments on commit 93ce593

Please sign in to comment.