diff --git a/integration-tests/ccip-tests/actions/ccip_helpers.go b/integration-tests/ccip-tests/actions/ccip_helpers.go index b7bd69bb38..e81949bc47 100644 --- a/integration-tests/ccip-tests/actions/ccip_helpers.go +++ b/integration-tests/ccip-tests/actions/ccip_helpers.go @@ -1704,7 +1704,7 @@ func (lane *CCIPLane) Multicall(noOfRequests int, msgType string, multiSendAddr } } } - stat := testreporters.NewCCIPRequestStats(int64(lane.NumberOfReq + i)) + stat := testreporters.NewCCIPRequestStats(int64(lane.NumberOfReq+i), lane.SourceNetworkName, lane.DestNetworkName) txstats = append(txstats, testreporters.TransactionStats{ Fee: fee.String(), NoOfTokensSent: len(msg.TokenAmounts), @@ -1756,7 +1756,7 @@ func (lane *CCIPLane) Multicall(noOfRequests int, msgType string, multiSendAddr func (lane *CCIPLane) SendRequests(noOfRequests int, msgType string, gasLimit *big.Int) error { for i := 1; i <= noOfRequests; i++ { msg := fmt.Sprintf("msg %d", i) - stat := testreporters.NewCCIPRequestStats(int64(lane.NumberOfReq + i)) + stat := testreporters.NewCCIPRequestStats(int64(lane.NumberOfReq+i), lane.SourceNetworkName, lane.DestNetworkName) txHash, txConfirmationDur, fee, err := lane.Source.SendRequest( lane.Dest.ReceiverDapp.EthAddress, msgType, msg, gasLimit, diff --git a/integration-tests/ccip-tests/contracts/multicall.go b/integration-tests/ccip-tests/contracts/multicall.go index 3f55656805..8d75cbd923 100644 --- a/integration-tests/ccip-tests/contracts/multicall.go +++ b/integration-tests/ccip-tests/contracts/multicall.go @@ -47,6 +47,20 @@ type CCIPMsgData struct { Fee *big.Int } +func TransferTokenCallData(to common.Address, amount *big.Int) ([]byte, error) { + erc20ABI, err := abi.JSON(strings.NewReader(erc20.ERC20ABI)) + if err != nil { + return nil, err + } + transferToken := erc20ABI.Methods["transfer"] + inputs, err := transferToken.Inputs.Pack(to, amount) + if err != nil { + return nil, err + } + inputs = append(transferToken.ID[:], inputs...) + return inputs, nil +} + // ApproveTokenCallData returns the call data for approving a token with approve function of erc20 contract func ApproveTokenCallData(to common.Address, amount *big.Int) ([]byte, error) { erc20ABI, err := abi.JSON(strings.NewReader(erc20.ERC20ABI)) @@ -90,7 +104,7 @@ func WaitForSuccessfulTxMined(evmClient blockchain.EVMClient, tx *types.Transact if receipt.Status != types.ReceiptStatusSuccessful { return fmt.Errorf("tx failed %s", tx.Hash().Hex()) } - log.Info().Str("tx", tx.Hash().Hex()).Msg("tx mined successfully") + log.Info().Str("tx", tx.Hash().Hex()).Str("Network", evmClient.GetNetworkName()).Msg("tx mined successfully") return nil } @@ -122,6 +136,9 @@ func MultiCallCCIP( allValue := big.NewInt(0) // create call data for each msg for _, msg := range msgData { + if msg.Msg.FeeToken != (common.Address{}) { + return nil, fmt.Errorf("fee token should be %s for native as fee", common.HexToAddress("0x0").Hex()) + } // approve bridge token for _, tokenAndAmount := range msg.Msg.TokenAmounts { inputs, err := ApproveTokenCallData(msg.RouterAddr, tokenAndAmount.Amount) @@ -139,21 +156,28 @@ func MultiCallCCIP( callData = append(callData, data) allValue.Add(allValue, msg.Fee) } + opts, err := evmClient.TransactionOpts(evmClient.GetDefaultWallet()) if err != nil { return nil, err } // the value of transactionOpts is the sum of the value of all msg, which is the total fee of all ccip-sends opts.Value = allValue + // call aggregate3Value to group all msg call data and send them in a single transaction tx, err := boundContract.Transact(opts, "aggregate3Value", callData) if err != nil { return nil, err } + err = evmClient.MarkTxAsSentOnL2(tx) + if err != nil { + return nil, err + } err = WaitForSuccessfulTxMined(evmClient, tx) if err != nil { - return nil, errors.Wrapf(err, "multicall failed for ccip-send; router %s", contractAddress.Hex()) + return nil, errors.Wrapf(err, "multicall failed for ccip-send; multicall %s", contractAddress.Hex()) } + return tx, nil } // if with feetoken, use aggregate3 to send msg without value var callData []Call @@ -213,3 +237,45 @@ func MultiCallCCIP( } return tx, nil } + +func TransferTokens( + evmClient blockchain.EVMClient, + contractAddress common.Address, + tokens []*ERC20Token, +) error { + multiCallABI, err := abi.JSON(strings.NewReader(MultiCallABI)) + if err != nil { + return err + } + var callData []Call + boundContract := bind.NewBoundContract(contractAddress, multiCallABI, evmClient.Backend(), evmClient.Backend(), evmClient.Backend()) + for _, token := range tokens { + var inputs []byte + balance, err := token.BalanceOf(nil, contractAddress.Hex()) + if err != nil { + return err + } + inputs, err = TransferTokenCallData(common.HexToAddress(evmClient.GetDefaultWallet().Address()), balance) + if err != nil { + return err + } + data := Call{Target: token.ContractAddress, AllowFailure: false, CallData: inputs} + callData = append(callData, data) + } + + opts, err := evmClient.TransactionOpts(evmClient.GetDefaultWallet()) + if err != nil { + return err + } + + // call aggregate3 to group all msg call data and send them in a single transaction + tx, err := boundContract.Transact(opts, "aggregate3", callData) + if err != nil { + return err + } + err = WaitForSuccessfulTxMined(evmClient, tx) + if err != nil { + return errors.Wrapf(err, "token transfer failed for token; router %s", contractAddress.Hex()) + } + return nil +} diff --git a/integration-tests/ccip-tests/load/ccip_loadgen.go b/integration-tests/ccip-tests/load/ccip_loadgen.go index 3175f87b86..40b862b1a9 100644 --- a/integration-tests/ccip-tests/load/ccip_loadgen.go +++ b/integration-tests/ccip-tests/load/ccip_loadgen.go @@ -1,12 +1,11 @@ package load import ( - "bytes" "context" + crypto_rand "crypto/rand" "encoding/base64" "fmt" "math/big" - "math/rand" "strconv" "testing" "time" @@ -14,6 +13,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/rs/zerolog" chain_selectors "github.com/smartcontractkit/chain-selectors" "github.com/smartcontractkit/wasp" "github.com/stretchr/testify/require" @@ -47,7 +47,7 @@ func NewCCIPLoad(t *testing.T, lane *actions.CCIPLane, timeout time.Duration, no CurrentMsgSerialNo: atomic.NewInt64(1), CallTimeOut: timeout, NoOfReq: noOfReq, - SendMaxDataIntermittently: false, + SendMaxDataIntermittently: true, } } @@ -87,7 +87,19 @@ func (c *CCIPE2ELoad) BeforeAllCall(msgType string) { require.NoError(c.t, err, "failed to fetch dynamic config") c.MaxDataBytes = dCfg.MaxDataBytes } - + // if the msg is sent via multicall, transfer the token transfer amount to multicall contract + if sourceCCIP.Common.MulticallEnabled && sourceCCIP.Common.MulticallContract != (common.Address{}) { + for i, amount := range sourceCCIP.TransferAmount { + token := sourceCCIP.Common.BridgeTokens[i] + amountToApprove := new(big.Int).Mul(amount, big.NewInt(c.NoOfReq)) + bal, err := token.BalanceOf(context.Background(), sourceCCIP.Common.MulticallContract.Hex()) + require.NoError(c.t, err, "Failed to get token balance") + if bal.Cmp(amountToApprove) < 0 { + err := token.Transfer(sourceCCIP.Common.MulticallContract.Hex(), amountToApprove) + require.NoError(c.t, err, "Failed to approve token transfer amount") + } + } + } // wait for any pending txs before moving on err = sourceCCIP.Common.ChainClient.WaitForEvents() require.NoError(c.t, err, "Failed to wait for events") @@ -100,46 +112,40 @@ func (c *CCIPE2ELoad) BeforeAllCall(msgType string) { destCCIP.Common.ChainClient.ParallelTransactions(false) } -func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.CallResult { - res := &wasp.CallResult{} - sourceCCIP := c.Lane.Source +func (c *CCIPE2ELoad) CCIPMsg() (router.ClientEVM2AnyMessage, *testreporters.RequestStat) { msgSerialNo := c.CurrentMsgSerialNo.Load() c.CurrentMsgSerialNo.Inc() - lggr := c.Lane.Logger.With().Int("msg Number", int(msgSerialNo)).Logger() - stats := testreporters.NewCCIPRequestStats(msgSerialNo) - defer c.Lane.Reports.UpdatePhaseStatsForReq(stats) + stats := testreporters.NewCCIPRequestStats(msgSerialNo, c.Lane.SourceNetworkName, c.Lane.DestNetworkName) // form the message for transfer msgStr := fmt.Sprintf("new message with Id %d", msgSerialNo) - if c.SendMaxDataIntermittently { - lggr.Info().Msg("sending max data intermittently") - // every 10th message will have extra data with almost MaxDataBytes - if msgSerialNo%10 == 0 { + // every 100th message will have extra data with almost MaxDataBytes + if msgSerialNo%100 == 0 { length := c.MaxDataBytes - 1 b := make([]byte, c.MaxDataBytes-1) - _, err := rand.Read(b) - if err != nil { - res.Error = err.Error() - res.Failed = true - return res + _, err := crypto_rand.Read(b) + if err == nil { + randomString := base64.URLEncoding.EncodeToString(b) + msgStr = randomString[:length] } - randomString := base64.URLEncoding.EncodeToString(b) - msgStr = randomString[:length] } } msg := c.msg - // if msg contains more than 2 tokens, selectively choose random 2 tokens - if len(msg.TokenAmounts) > 2 { - // randomize the order of elements in the slice - rand.Shuffle(len(msg.TokenAmounts), func(i, j int) { - msg.TokenAmounts[i], msg.TokenAmounts[j] = msg.TokenAmounts[j], msg.TokenAmounts[i] - }) - // select first 2 tokens - msg.TokenAmounts = msg.TokenAmounts[:2] - } msg.Data = []byte(msgStr) + return msg, stats +} + +func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.CallResult { + res := &wasp.CallResult{} + sourceCCIP := c.Lane.Source + + msg, stats := c.CCIPMsg() + msgSerialNo := stats.ReqNo + lggr := c.Lane.Logger.With().Int64("msg Number", stats.ReqNo).Logger() + + defer c.Lane.Reports.UpdatePhaseStatsForReq(stats) feeToken := sourceCCIP.Common.FeeToken.EthAddress // initiate the transfer lggr.Debug().Str("triggeredAt", time.Now().GoString()).Msg("triggering transfer") @@ -167,7 +173,12 @@ func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.CallResult { } else { sendTx, err = sourceCCIP.Common.Router.CCIPSend(destChainSelector, msg, fee) } - + err = sourceCCIP.Common.ChainClient.MarkTxAsSentOnL2(sendTx) + if err != nil { + res.Error = err.Error() + res.Failed = true + return res + } if err != nil { stats.UpdateState(lggr, 0, testreporters.TX, time.Since(startTime), testreporters.Failure) res.Error = fmt.Sprintf("ccip-send tx error %+v for msg ID %d", err, msgSerialNo) @@ -196,26 +207,24 @@ func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.CallResult { NoOfTokensSent: len(msg.TokenAmounts), MessageBytesLength: len(msg.Data), }) - // wait for - // - CCIPSendRequested Event log to be generated, - msgLogs, sourceLogTime, err := c.Lane.Source.AssertEventCCIPSendRequested(lggr, sendTx.Hash().Hex(), c.CallTimeOut, txConfirmationTime, []*testreporters.RequestStat{stats}) - - if err != nil || msgLogs == nil || len(msgLogs) == 0 { + err = c.Validate(lggr, sendTx, txConfirmationTime, []*testreporters.RequestStat{stats}) + if err != nil { res.Error = err.Error() - res.Data = stats.StatusByPhase res.Failed = true + res.Data = stats.StatusByPhase return res } - msgLog := msgLogs[0] - sentMsg := msgLog.Message - seqNum := sentMsg.SequenceNumber - lggr = lggr.With().Str("msgId ", fmt.Sprintf("0x%x", sentMsg.MessageId[:])).Logger() + res.Data = stats.StatusByPhase + return res +} - if bytes.Compare(sentMsg.Data, []byte(msgStr)) != 0 { - res.Error = fmt.Sprintf("the message byte didnot match expected %s received %s msg ID %d", msgStr, string(sentMsg.Data), msgSerialNo) - res.Data = stats.StatusByPhase - res.Failed = true - return res +func (c *CCIPE2ELoad) Validate(lggr zerolog.Logger, sendTx *types.Transaction, txConfirmationTime time.Time, stats []*testreporters.RequestStat) error { + // wait for + // - CCIPSendRequested Event log to be generated, + msgLogs, sourceLogTime, err := c.Lane.Source.AssertEventCCIPSendRequested(lggr, sendTx.Hash().Hex(), c.CallTimeOut, txConfirmationTime, stats) + + if err != nil || msgLogs == nil || len(msgLogs) == 0 { + return err } lstFinalizedBlock := c.LastFinalizedTxBlock.Load() @@ -223,61 +232,61 @@ func (c *CCIPE2ELoad) Call(_ *wasp.Generator) *wasp.CallResult { // if the finality tag is enabled and the last finalized block is greater than the block number of the message // consider the message finalized if c.Lane.Source.Common.ChainClient.GetNetworkConfig().FinalityDepth == 0 && - lstFinalizedBlock != 0 && lstFinalizedBlock > msgLog.Raw.BlockNumber { + lstFinalizedBlock != 0 && lstFinalizedBlock > msgLogs[0].Raw.BlockNumber { sourceLogFinalizedAt = c.LastFinalizedTimestamp.Load() - stats.UpdateState(lggr, seqNum, testreporters.SourceLogFinalized, - sourceLogFinalizedAt.Sub(sourceLogTime), testreporters.Success, - testreporters.TransactionStats{ - TxHash: msgLog.Raw.TxHash.String(), - FinalizedByBlock: strconv.FormatUint(lstFinalizedBlock, 10), - FinalizedAt: sourceLogFinalizedAt.String(), - }) + for _, stat := range stats { + stat.UpdateState(lggr, stat.SeqNum, testreporters.SourceLogFinalized, + sourceLogFinalizedAt.Sub(sourceLogTime), testreporters.Success, + testreporters.TransactionStats{ + TxHash: msgLogs[0].Raw.TxHash.String(), + FinalizedByBlock: strconv.FormatUint(lstFinalizedBlock, 10), + FinalizedAt: sourceLogFinalizedAt.String(), + }) + } } else { var finalizingBlock uint64 sourceLogFinalizedAt, finalizingBlock, err = c.Lane.Source.AssertSendRequestedLogFinalized( - lggr, sendTx.Hash(), sourceLogTime, []*testreporters.RequestStat{stats}) + lggr, sendTx.Hash(), sourceLogTime, stats) if err != nil { - res.Error = err.Error() - res.Data = stats.StatusByPhase - res.Failed = true - return res + return err } c.LastFinalizedTxBlock.Store(finalizingBlock) c.LastFinalizedTimestamp.Store(sourceLogFinalizedAt) } - // wait for - // - CommitStore to increase the seq number, - err = c.Lane.Dest.AssertSeqNumberExecuted(lggr, seqNum, c.CallTimeOut, sourceLogFinalizedAt, stats) - if err != nil { - res.Error = err.Error() - res.Data = stats.StatusByPhase - res.Failed = true - return res - } - // wait for ReportAccepted event - commitReport, reportAcceptedAt, err := c.Lane.Dest.AssertEventReportAccepted(lggr, seqNum, c.CallTimeOut, sourceLogFinalizedAt, stats) - if err != nil || commitReport == nil { - res.Error = err.Error() - res.Data = stats.StatusByPhase - res.Failed = true - return res - } - blessedAt, err := c.Lane.Dest.AssertReportBlessed(lggr, seqNum, c.CallTimeOut, *commitReport, reportAcceptedAt, stats) - if err != nil { - res.Error = err.Error() - res.Data = stats.StatusByPhase - res.Failed = true - return res - } - _, err = c.Lane.Dest.AssertEventExecutionStateChanged(lggr, seqNum, c.CallTimeOut, blessedAt, stats, testhelpers.ExecutionStateSuccess) - if err != nil { - res.Error = err.Error() - res.Data = stats.StatusByPhase - res.Failed = true - return res + for _, msgLog := range msgLogs { + seqNum := msgLog.Message.SequenceNumber + var reqStat *testreporters.RequestStat + lggr = lggr.With().Str("msgId ", fmt.Sprintf("0x%x", msgLog.Message.MessageId[:])).Logger() + for _, stat := range stats { + if stat.SeqNum == seqNum { + reqStat = stat + break + } + } + if reqStat == nil { + return fmt.Errorf("could not find request stat for seq number %d", seqNum) + } + // wait for + // - CommitStore to increase the seq number, + err = c.Lane.Dest.AssertSeqNumberExecuted(lggr, seqNum, c.CallTimeOut, sourceLogFinalizedAt, reqStat) + if err != nil { + return err + } + // wait for ReportAccepted event + commitReport, reportAcceptedAt, err := c.Lane.Dest.AssertEventReportAccepted(lggr, seqNum, c.CallTimeOut, sourceLogFinalizedAt, reqStat) + if err != nil || commitReport == nil { + return err + } + blessedAt, err := c.Lane.Dest.AssertReportBlessed(lggr, seqNum, c.CallTimeOut, *commitReport, reportAcceptedAt, reqStat) + if err != nil { + return err + } + _, err = c.Lane.Dest.AssertEventExecutionStateChanged(lggr, seqNum, c.CallTimeOut, blessedAt, reqStat, testhelpers.ExecutionStateSuccess) + if err != nil { + return err + } } - res.Data = stats.StatusByPhase - return res + return nil } diff --git a/integration-tests/ccip-tests/load/ccip_multicall_loadgen.go b/integration-tests/ccip-tests/load/ccip_multicall_loadgen.go new file mode 100644 index 0000000000..c46deaad63 --- /dev/null +++ b/integration-tests/ccip-tests/load/ccip_multicall_loadgen.go @@ -0,0 +1,274 @@ +package load + +import ( + "context" + "fmt" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/prometheus/common/model" + "github.com/rs/zerolog" + chain_selectors "github.com/smartcontractkit/chain-selectors" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + "github.com/smartcontractkit/chainlink-testing-framework/logging" + "github.com/smartcontractkit/wasp" + "golang.org/x/sync/errgroup" + + "github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/actions" + "github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/contracts" + "github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/testreporters" + "github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/testsetups" +) + +// CCIPMultiCallLoadGenerator represents a load generator for the CCIP lanes originating from same network +// The purpose of this load generator is to group ccip-send calls for the CCIP lanes originating from same network +// This is to avoid the scenario of hitting rpc rate limit for the same network if the load generator is sending +// too many ccip-send calls to the same network hitting the rpc rate limit +type CCIPMultiCallLoadGenerator struct { + t *testing.T + logger zerolog.Logger + client blockchain.EVMClient + E2ELoads map[string]*CCIPE2ELoad + MultiCall string + NoOfRequestsPerUnitTime int64 + labels model.LabelSet + loki *wasp.LokiClient + responses chan map[string]MultiCallReturnValues + Done chan struct{} +} + +type MultiCallReturnValues struct { + Msgs []contracts.CCIPMsgData + Stats []*testreporters.RequestStat +} + +func NewMultiCallLoadGenerator(testCfg *testsetups.CCIPTestConfig, lanes []*actions.CCIPLane, noOfRequestsPerUnitTime int64, labels map[string]string) (*CCIPMultiCallLoadGenerator, error) { + // check if all lanes are from same network + source := lanes[0].SourceChain.GetChainID() + multiCall := lanes[0].SrcNetworkLaneCfg.Multicall + if multiCall == "" { + return nil, fmt.Errorf("multicall address cannot be empty") + } + for i := 1; i < len(lanes); i++ { + if source.String() != lanes[i].SourceChain.GetChainID().String() { + return nil, fmt.Errorf("all lanes should be from same network; expected %s, got %s", source, lanes[i].SourceChain.GetChainID()) + } + if lanes[i].SrcNetworkLaneCfg.Multicall != multiCall { + return nil, fmt.Errorf("multicall address should be same for all lanes") + } + } + client := lanes[0].SourceChain + lggr := logging.GetTestLogger(testCfg.Test).With().Str("Source Network", client.GetNetworkName()).Logger() + ls := wasp.LabelsMapToModel(labels) + if err := ls.Validate(); err != nil { + return nil, err + } + loki, err := wasp.NewLokiClient(wasp.NewEnvLokiConfig()) + if err != nil { + return nil, err + } + m := &CCIPMultiCallLoadGenerator{ + t: testCfg.Test, + client: client, + MultiCall: multiCall, + logger: lggr, + NoOfRequestsPerUnitTime: noOfRequestsPerUnitTime, + E2ELoads: make(map[string]*CCIPE2ELoad), + labels: ls, + loki: loki, + responses: make(chan map[string]MultiCallReturnValues), + Done: make(chan struct{}), + } + for _, lane := range lanes { + ccipLoad := NewCCIPLoad(testCfg.Test, lane, testCfg.TestGroupInput.PhaseTimeout.Duration(), 100000) + ccipLoad.BeforeAllCall(testCfg.TestGroupInput.MsgType) + m.E2ELoads[fmt.Sprintf("%s-%s", lane.SourceNetworkName, lane.DestNetworkName)] = ccipLoad + } + + m.StartLokiStream() + return m, nil +} + +func (m *CCIPMultiCallLoadGenerator) Stop() error { + m.Done <- struct{}{} + tokenMap := make(map[string]struct{}) + var tokens []*contracts.ERC20Token + for _, e2eLoad := range m.E2ELoads { + for i := range e2eLoad.Lane.Source.TransferAmount { + if _, ok := tokenMap[e2eLoad.Lane.Source.Common.BridgeTokens[i].Address()]; !ok { + tokens = append(tokens, e2eLoad.Lane.Source.Common.BridgeTokens[i]) + } + } + } + if len(tokens) > 0 { + return contracts.TransferTokens(m.client, common.HexToAddress(m.MultiCall), tokens) + } + return nil +} + +func (m *CCIPMultiCallLoadGenerator) StartLokiStream() { + go func() { + for { + select { + case <-m.Done: + m.logger.Info().Msg("stopping loki client from multi call load generator") + m.loki.Stop() + return + case rValues := <-m.responses: + m.HandleLokiLogs(rValues) + } + } + }() +} + +func (m *CCIPMultiCallLoadGenerator) HandleLokiLogs(rValues map[string]MultiCallReturnValues) { + for dest, rValue := range rValues { + labels := m.labels.Merge(model.LabelSet{ + "dest_chain": model.LabelValue(dest), + "test_data_type": "responses", + "go_test_name": model.LabelValue(m.t.Name()), + }) + for _, stat := range rValue.Stats { + err := m.loki.HandleStruct(labels, time.Now().UTC(), stat.StatusByPhase) + if err != nil { + m.logger.Error().Err(err).Msg("error while handling loki logs") + } + } + } +} + +func (m *CCIPMultiCallLoadGenerator) Call(_ *wasp.Generator) *wasp.CallResult { + res := &wasp.CallResult{} + msgs, returnValuesByDest, err := m.MergeCalls() + if err != nil { + res.Error = err.Error() + res.Failed = true + return res + } + defer func() { + m.responses <- returnValuesByDest + }() + m.logger.Info().Interface("msgs", msgs).Msgf("Sending %d ccip-send calls", len(msgs)) + startTime := time.Now().UTC() + // for now we are using all ccip-sends with native + sendTx, err := contracts.MultiCallCCIP(m.client, m.MultiCall, msgs, true) + if err != nil { + res.Error = err.Error() + res.Failed = true + return res + } + + lggr := m.logger.With().Str("Msg Tx", sendTx.Hash().String()).Logger() + txConfirmationTime := time.Now().UTC() + rcpt, err1 := bind.WaitMined(context.Background(), m.client.DeployBackend(), sendTx) + if err1 == nil { + hdr, err1 := m.client.HeaderByNumber(context.Background(), rcpt.BlockNumber) + if err1 == nil { + txConfirmationTime = hdr.Timestamp + } + } + var gasUsed uint64 + if rcpt != nil { + gasUsed = rcpt.GasUsed + } + for _, rValues := range returnValuesByDest { + if len(rValues.Stats) != len(rValues.Msgs) { + res.Error = fmt.Sprintf("number of stats and msgs should be same") + res.Failed = true + return res + } + for i, stat := range rValues.Stats { + msg := rValues.Msgs[i] + stat.UpdateState(lggr, 0, testreporters.TX, startTime.Sub(txConfirmationTime), testreporters.Success, + testreporters.TransactionStats{ + Fee: msg.Fee.String(), + GasUsed: gasUsed, + TxHash: sendTx.Hash().Hex(), + NoOfTokensSent: len(msg.Msg.TokenAmounts), + MessageBytesLength: len(msg.Msg.Data), + }) + } + } + + validateGrp := errgroup.Group{} + // wait for + // - CCIPSendRequested Event log to be generated, + for _, rValues := range returnValuesByDest { + key := fmt.Sprintf("%s-%s", rValues.Stats[0].SourceNetwork, rValues.Stats[0].DestNetwork) + c, ok := m.E2ELoads[key] + if !ok { + res.Error = fmt.Sprintf("load for %s not found", key) + res.Failed = true + return res + } + + lggr = lggr.With().Str("Source Network", c.Lane.SourceChain.GetNetworkName()).Str("Dest Network", c.Lane.DestChain.GetNetworkName()).Logger() + stats := rValues.Stats + txConfirmationTime := txConfirmationTime + sendTx := sendTx + lggr := lggr + validateGrp.Go(func() error { + return c.Validate(lggr, sendTx, txConfirmationTime, stats) + }) + } + err = validateGrp.Wait() + if err != nil { + res.Error = err.Error() + res.Failed = true + return res + } + + return res +} + +func (m *CCIPMultiCallLoadGenerator) MergeCalls() ([]contracts.CCIPMsgData, map[string]MultiCallReturnValues, error) { + var ccipMsgs []contracts.CCIPMsgData + statDetails := make(map[string]MultiCallReturnValues) + + for _, e2eLoad := range m.E2ELoads { + destChainSelector, err := chain_selectors.SelectorFromChainId(e2eLoad.Lane.Source.DestinationChainId) + if err != nil { + return ccipMsgs, statDetails, err + } + + allFee := big.NewInt(0) + var allStatsForDest []*testreporters.RequestStat + var allMsgsForDest []contracts.CCIPMsgData + for i := int64(0); i < m.NoOfRequestsPerUnitTime; i++ { + msg, stats := e2eLoad.CCIPMsg() + msg.FeeToken = common.Address{} + fee, err := e2eLoad.Lane.Source.Common.Router.GetFee(destChainSelector, msg) + if err != nil { + return ccipMsgs, statDetails, err + } + // transfer fee to the multicall address + if msg.FeeToken != (common.Address{}) { + allFee = new(big.Int).Add(allFee, fee) + } + msgData := contracts.CCIPMsgData{ + RouterAddr: e2eLoad.Lane.Source.Common.Router.EthAddress, + ChainSelector: destChainSelector, + Msg: msg, + Fee: fee, + } + ccipMsgs = append(ccipMsgs, msgData) + + allStatsForDest = append(allStatsForDest, stats) + allMsgsForDest = append(allMsgsForDest, msgData) + } + statDetails[e2eLoad.Lane.DestNetworkName] = MultiCallReturnValues{ + Stats: allStatsForDest, + Msgs: allMsgsForDest, + } + // transfer fee to the multicall address + if allFee.Cmp(big.NewInt(0)) > 0 { + if err := e2eLoad.Lane.Source.Common.FeeToken.Transfer(e2eLoad.Lane.Source.Common.MulticallContract.Hex(), allFee); err != nil { + return ccipMsgs, statDetails, err + } + } + } + return ccipMsgs, statDetails, nil +} diff --git a/integration-tests/ccip-tests/load/ccip_test.go b/integration-tests/ccip-tests/load/ccip_test.go index be3f1fb37e..6c17322803 100644 --- a/integration-tests/ccip-tests/load/ccip_test.go +++ b/integration-tests/ccip-tests/load/ccip_test.go @@ -28,19 +28,15 @@ func TestLoadCCIPStableRPS(t *testing.T) { log.Info().Msg("Tearing down the environment") require.NoError(t, testArgs.TestSetupArgs.TearDown()) }) - testArgs.TriggerLoad() + testArgs.TriggerLoadByLane() testArgs.Wait() } -func TestLoadCCIPSequentialLaneAdd(t *testing.T) { +func TestLoadCCIPStableRPSTriggerBySource(t *testing.T) { t.Parallel() - t.Skipf("test needs maintenance") lggr := logging.GetTestLogger(t) testArgs := NewLoadArgs(t, lggr, context.Background()) - testArgs.TestCfg.TestGroupInput.SequentialLaneAddition = ptr.Ptr(true) - if len(testArgs.TestCfg.NetworkPairs) <= 1 { - t.Skip("Skipping the test as there are not enough network pairs to run the test") - } + testArgs.TestCfg.TestGroupInput.MulticallInOneTx = ptr.Ptr(true) testArgs.Setup() // if the test runs on remote runner if len(testArgs.TestSetupArgs.Lanes) == 0 { @@ -50,8 +46,7 @@ func TestLoadCCIPSequentialLaneAdd(t *testing.T) { log.Info().Msg("Tearing down the environment") require.NoError(t, testArgs.TestSetupArgs.TearDown()) }) - testArgs.TriggerLoad() - testArgs.AddMoreLanesToRun() + testArgs.TriggerLoadBySource() testArgs.Wait() } @@ -95,7 +90,7 @@ func TestLoadCCIPStableRequestTriggeringWithNetworkChaos(t *testing.T) { }) // now trigger the load - testArgs.TriggerLoad() + testArgs.TriggerLoadByLane() testArgs.Wait() } @@ -148,7 +143,7 @@ func TestLoadCCIPStableWithMajorityNodeFailure(t *testing.T) { require.NotNil(t, testEnv) require.NotNil(t, testEnv.K8Env) - testArgs.TriggerLoad() + testArgs.TriggerLoadByLane() testArgs.ApplyChaos() testArgs.Wait() } @@ -202,7 +197,7 @@ func TestLoadCCIPStableWithMinorityNodeFailure(t *testing.T) { require.NotNil(t, testEnv) require.NotNil(t, testEnv.K8Env) - testArgs.TriggerLoad() + testArgs.TriggerLoadByLane() testArgs.ApplyChaos() testArgs.Wait() } @@ -266,7 +261,7 @@ func TestLoadCCIPStableWithPodChaosDiffCommitAndExec(t *testing.T) { require.NoError(t, testArgs.TestSetupArgs.TearDown()) }) testArgs.SanityCheck() - testArgs.TriggerLoad() + testArgs.TriggerLoadByLane() testArgs.ApplyChaos() testArgs.Wait() }) diff --git a/integration-tests/ccip-tests/load/helper.go b/integration-tests/ccip-tests/load/helper.go index 0e00c71b4a..90772b845d 100644 --- a/integration-tests/ccip-tests/load/helper.go +++ b/integration-tests/ccip-tests/load/helper.go @@ -5,7 +5,6 @@ import ( "fmt" "math" "math/big" - "sync" "testing" "time" @@ -13,7 +12,6 @@ import ( "github.com/rs/zerolog" "github.com/smartcontractkit/chainlink-testing-framework/k8s/chaos" "github.com/smartcontractkit/wasp" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -23,10 +21,6 @@ import ( "github.com/smartcontractkit/chainlink/integration-tests/ccip-tests/testsetups" ) -type laneLoadCfg struct { - lane *actions.CCIPLane -} - type ChaosConfig struct { ChaosName string ChaosFunc chaos.ManifestFunc @@ -35,18 +29,15 @@ type ChaosConfig struct { } type loadArgs struct { - t *testing.T - lggr zerolog.Logger - ctx context.Context - ccipLoad []*CCIPE2ELoad - schedules []*wasp.Segment - loadRunner []*wasp.Generator - LaneLoadCfg chan laneLoadCfg - RunnerWg *errgroup.Group // to wait on individual load generators run - LoadStarterWg *sync.WaitGroup // waits for all the runners to start - TestCfg *testsetups.CCIPTestConfig - TestSetupArgs *testsetups.CCIPTestSetUpOutputs - ChaosExps []ChaosConfig + t *testing.T + lggr zerolog.Logger + ctx context.Context + schedules []*wasp.Segment + RunnerWg *errgroup.Group // to wait on individual load generators run + TestCfg *testsetups.CCIPTestConfig + TestSetupArgs *testsetups.CCIPTestSetUpOutputs + ChaosExps []ChaosConfig + LoadgenTearDowns []func() } func (l *loadArgs) Setup() { @@ -63,6 +54,7 @@ func (l *loadArgs) setSchedule() { var segments []*wasp.Segment var segmentDuration time.Duration require.Greater(l.t, len(l.TestCfg.TestGroupInput.RequestPerUnitTime), 0, "RequestPerUnitTime must be set") + if len(l.TestCfg.TestGroupInput.RequestPerUnitTime) > 1 { for i, req := range l.TestCfg.TestGroupInput.RequestPerUnitTime { duration := l.TestCfg.TestGroupInput.StepDuration[i].Duration() @@ -90,78 +82,60 @@ func (l *loadArgs) SanityCheck() { } } -func (l *loadArgs) TriggerLoad(schedule ...*wasp.Segment) { - l.Start() - if len(schedule) == 0 { - l.setSchedule() - } else { - l.schedules = schedule - } - for _, lane := range l.TestSetupArgs.Lanes { - if lane.LaneDeployed { - l.LaneLoadCfg <- laneLoadCfg{ - lane: lane.ForwardLane, - } - if lane.ReverseLane != nil { - l.LaneLoadCfg <- laneLoadCfg{ - lane: lane.ReverseLane, - } - } - } - } +func (l *loadArgs) TriggerLoadByLane() { + l.setSchedule() l.TestSetupArgs.Reporter.SetDuration(l.TestCfg.TestGroupInput.TestDuration.Duration()) -} + namespace := l.TestCfg.TestGroupInput.ExistingEnv -func (l *loadArgs) AddMoreLanesToRun() { - require.Len(l.t, l.TestSetupArgs.Lanes, 1, "lane for first network pair should be deployed already") - if len(l.TestSetupArgs.Lanes) == len(l.TestCfg.NetworkPairs) { - l.lggr.Info().Msg("All lanes are already deployed, no need to add more lanes") - return + // start load for a lane + startLoad := func(lane *actions.CCIPLane) { + lane.Logger.Info(). + Str("Source Network", lane.SourceNetworkName). + Str("Destination Network", lane.DestNetworkName). + Msg("Starting load for lane") + + ccipLoad := NewCCIPLoad(l.TestCfg.Test, lane, l.TestCfg.TestGroupInput.PhaseTimeout.Duration(), 100000) + ccipLoad.BeforeAllCall(l.TestCfg.TestGroupInput.MsgType) + if lane.TestEnv != nil && lane.TestEnv.K8Env != nil && lane.TestEnv.K8Env.Cfg != nil { + namespace = lane.TestEnv.K8Env.Cfg.Namespace + } + + loadRunner, err := wasp.NewGenerator(&wasp.Config{ + T: l.TestCfg.Test, + GenName: fmt.Sprintf("lane %s-> %s", lane.SourceNetworkName, lane.DestNetworkName), + Schedule: l.schedules, + LoadType: wasp.RPS, + RateLimitUnitDuration: l.TestCfg.TestGroupInput.TimeUnit.Duration(), + CallResultBufLen: 10, // we keep the last 10 call results for each generator, as the detailed report is generated at the end of the test + CallTimeout: (l.TestCfg.TestGroupInput.PhaseTimeout.Duration()) * 5, + Gun: ccipLoad, + Logger: ccipLoad.Lane.Logger, + SharedData: l.TestCfg.TestGroupInput.MsgType, + LokiConfig: wasp.NewEnvLokiConfig(), + Labels: map[string]string{ + "test_group": "load", + "cluster": "sdlc", + "namespace": namespace, + "test_id": "ccip", + "source_chain": lane.SourceNetworkName, + "dest_chain": lane.DestNetworkName, + }, + }) + require.NoError(l.TestCfg.Test, err, "initiating loadgen for lane %s --> %s", + lane.SourceNetworkName, lane.DestNetworkName) + loadRunner.Run(false) + l.AddToRunnerGroup(loadRunner) } - transferAmounts := []*big.Int{big.NewInt(1)} - // set the ticker duration based on number of network pairs and the total test duration - noOfPair := int64(len(l.TestCfg.NetworkPairs)) - step := l.TestCfg.TestGroupInput.TestDuration.Duration().Nanoseconds() / noOfPair - ticker := time.NewTicker(time.Duration(step)) - l.setSchedule() - // Lane for the first network pair is already deployed - netIndex := 1 - for { - select { - case <-ticker.C: - n := l.TestCfg.NetworkPairs[netIndex] - l.lggr.Info(). - Str("Network 1", n.NetworkA.Name). - Str("Network 2", n.NetworkB.Name). - Msg("Adding lanes for network pair") - err := l.TestSetupArgs.AddLanesForNetworkPair( - l.lggr, n.NetworkA, n.NetworkB, - n.ChainClientA, n.ChainClientB, - transferAmounts, 5, true, - true) - assert.NoError(l.t, err) - l.LaneLoadCfg <- laneLoadCfg{ - lane: l.TestSetupArgs.Lanes[netIndex].ForwardLane, - } - if l.TestSetupArgs.Lanes[netIndex].ReverseLane != nil { - l.LaneLoadCfg <- laneLoadCfg{ - lane: l.TestSetupArgs.Lanes[netIndex].ReverseLane, - } - } - netIndex++ - if netIndex >= len(l.TestCfg.NetworkPairs) { - ticker.Stop() - return - } + for _, lane := range l.TestSetupArgs.Lanes { + startLoad(lane.ForwardLane) + if pointer.GetBool(l.TestSetupArgs.Cfg.TestGroupInput.BiDirectionalLane) { + startLoad(lane.ReverseLane) } } } -// Start polls the LaneLoadCfg channel for new lanes and starts the load runner. -// LaneLoadCfg channel should receive a lane whenever the deployment is complete. -func (l *loadArgs) Start() { - l.LoadStarterWg.Add(1) - waitForLoadRun := func(gen *wasp.Generator, ccipLoad *CCIPE2ELoad) error { +func (l *loadArgs) AddToRunnerGroup(gen *wasp.Generator) { + l.RunnerWg.Go(func() error { _, failed := gen.Wait() if failed { return fmt.Errorf("load run is failed") @@ -170,72 +144,15 @@ func (l *loadArgs) Start() { return fmt.Errorf("error in load sequence call %v", gen.Errors()) } return nil - } - go func() { - defer l.LoadStarterWg.Done() - loadCount := 0 - namespace := l.TestCfg.TestGroupInput.ExistingEnv - for { - select { - case cfg := <-l.LaneLoadCfg: - loadCount++ - lane := cfg.lane - l.lggr.Info(). - Str("Source Network", lane.SourceNetworkName). - Str("Destination Network", lane.DestNetworkName). - Msg("Starting load for lane") - - ccipLoad := NewCCIPLoad(l.TestCfg.Test, lane, l.TestCfg.TestGroupInput.PhaseTimeout.Duration(), 100000) - ccipLoad.BeforeAllCall(l.TestCfg.TestGroupInput.MsgType) - if lane.TestEnv != nil && lane.TestEnv.K8Env != nil && lane.TestEnv.K8Env.Cfg != nil { - namespace = lane.TestEnv.K8Env.Cfg.Namespace - } - - loadRunner, err := wasp.NewGenerator(&wasp.Config{ - T: l.TestCfg.Test, - GenName: fmt.Sprintf("lane %s-> %s", lane.SourceNetworkName, lane.DestNetworkName), - Schedule: l.schedules, - LoadType: wasp.RPS, - RateLimitUnitDuration: l.TestCfg.TestGroupInput.TimeUnit.Duration(), - CallResultBufLen: 10, // we keep the last 10 call results for each generator, as the detailed report is generated at the end of the test - CallTimeout: (l.TestCfg.TestGroupInput.PhaseTimeout.Duration()) * 5, - Gun: ccipLoad, - Logger: ccipLoad.Lane.Logger, - SharedData: l.TestCfg.TestGroupInput.MsgType, - LokiConfig: wasp.NewEnvLokiConfig(), - Labels: map[string]string{ - "test_group": "load", - "cluster": "sdlc", - "namespace": namespace, - "test_id": "ccip", - "source_chain": lane.SourceNetworkName, - "dest_chain": lane.DestNetworkName, - }, - }) - require.NoError(l.TestCfg.Test, err, "initiating loadgen for lane %s --> %s", - lane.SourceNetworkName, lane.DestNetworkName) - loadRunner.Run(false) - l.ccipLoad = append(l.ccipLoad, ccipLoad) - l.loadRunner = append(l.loadRunner, loadRunner) - l.RunnerWg.Go(func() error { - return waitForLoadRun(loadRunner, ccipLoad) - }) - if loadCount == len(l.TestCfg.NetworkPairs)*2 { - l.lggr.Info().Msg("load is running for all lanes now") - return - } - } - } - }() + }) } func (l *loadArgs) Wait() { - // wait for load runner to start on all lanes - l.LoadStarterWg.Wait() l.lggr.Info().Msg("Waiting for load to finish on all lanes") // wait for load runner to finish err := l.RunnerWg.Wait() require.NoError(l.t, err, "load run is failed") + l.lggr.Info().Msg("Load finished on all lanes") } func (l *loadArgs) ApplyChaos() { @@ -269,21 +186,74 @@ func (l *loadArgs) ApplyChaos() { } func (l *loadArgs) TearDown() { + for _, tearDn := range l.LoadgenTearDowns { + tearDn() + } if l.TestSetupArgs.TearDown != nil { require.NoError(l.t, l.TestSetupArgs.TearDown()) } } +func (l *loadArgs) TriggerLoadBySource() { + require.NotNil(l.t, l.TestCfg.TestGroupInput.TestDuration, "test duration input is nil") + require.GreaterOrEqual(l.t, 1, len(l.TestCfg.TestGroupInput.RequestPerUnitTime), "time unit input must be specified") + l.TestSetupArgs.Reporter.SetDuration(l.TestCfg.TestGroupInput.TestDuration.Duration()) + namespace := l.TestCfg.TestGroupInput.ExistingEnv + + var laneBySource = make(map[string][]*actions.CCIPLane) + for _, lane := range l.TestSetupArgs.Lanes { + laneBySource[lane.ForwardLane.SourceNetworkName] = append(laneBySource[lane.ForwardLane.SourceNetworkName], lane.ForwardLane) + if lane.ReverseLane != nil { + laneBySource[lane.ReverseLane.SourceNetworkName] = append(laneBySource[lane.ReverseLane.SourceNetworkName], lane.ReverseLane) + } + } + for source, lanes := range laneBySource { + l.lggr.Info(). + Str("Source Network", source). + Msg("Starting load for source") + if lanes[0].TestEnv != nil && lanes[0].TestEnv.K8Env != nil && lanes[0].TestEnv.K8Env.Cfg != nil { + namespace = lanes[0].TestEnv.K8Env.Cfg.Namespace + } + allLabels := map[string]string{ + "test_group": "load", + "cluster": "sdlc", + "namespace": namespace, + "test_id": "ccip", + "source_chain": source, + } + multiCallGen, err := NewMultiCallLoadGenerator(l.TestCfg, lanes, l.TestCfg.TestGroupInput.RequestPerUnitTime[0], allLabels) + require.NoError(l.t, err) + + loadRunner, err := wasp.NewGenerator(&wasp.Config{ + T: l.TestCfg.Test, + GenName: fmt.Sprintf("Source %s", source), + Schedule: wasp.Plain(1, l.TestCfg.TestGroupInput.TestDuration.Duration()), // hardcoded request per unit time to 1 as we are using multiCallGen + LoadType: wasp.RPS, + RateLimitUnitDuration: l.TestCfg.TestGroupInput.TimeUnit.Duration(), + CallResultBufLen: 10, // we keep the last 10 call results for each generator, as the detailed report is generated at the end of the test + CallTimeout: (l.TestCfg.TestGroupInput.PhaseTimeout.Duration()) * 5, + Gun: multiCallGen, + Logger: multiCallGen.logger, + LokiConfig: wasp.NewEnvLokiConfig(), + Labels: allLabels, + }) + require.NoError(l.TestCfg.Test, err, "initiating loadgen for source %s", source) + loadRunner.Run(false) + l.AddToRunnerGroup(loadRunner) + l.LoadgenTearDowns = append(l.LoadgenTearDowns, func() { + require.NoError(l.t, multiCallGen.Stop()) + }) + } +} + func NewLoadArgs(t *testing.T, lggr zerolog.Logger, parent context.Context, chaosExps ...ChaosConfig) *loadArgs { wg, ctx := errgroup.WithContext(parent) return &loadArgs{ - t: t, - lggr: lggr, - RunnerWg: wg, - ctx: ctx, - TestCfg: testsetups.NewCCIPTestConfig(t, lggr, testconfig.Load), - LaneLoadCfg: make(chan laneLoadCfg), - LoadStarterWg: &sync.WaitGroup{}, - ChaosExps: chaosExps, + t: t, + lggr: lggr, + RunnerWg: wg, + ctx: ctx, + TestCfg: testsetups.NewCCIPTestConfig(t, lggr, testconfig.Load), + ChaosExps: chaosExps, } } diff --git a/integration-tests/ccip-tests/testconfig/ccip.go b/integration-tests/ccip-tests/testconfig/ccip.go index 555116e15d..9a25b6929a 100644 --- a/integration-tests/ccip-tests/testconfig/ccip.go +++ b/integration-tests/ccip-tests/testconfig/ccip.go @@ -20,7 +20,6 @@ type CCIPTestConfig struct { ExistingDeployment *bool `toml:",omitempty"` ExistingEnv string `toml:",omitempty"` ReuseContracts *bool `toml:",omitempty"` - SequentialLaneAddition *bool `toml:",omitempty"` NodeFunding float64 `toml:",omitempty"` RequestPerUnitTime []int64 `toml:",omitempty"` TimeUnit *models.Duration `toml:",omitempty"` @@ -71,9 +70,7 @@ func (c *CCIPTestConfig) ApplyOverrides(fromCfg *CCIPTestConfig) error { if fromCfg.ReuseContracts != nil { c.ReuseContracts = fromCfg.ReuseContracts } - if fromCfg.SequentialLaneAddition != nil { - c.SequentialLaneAddition = fromCfg.SequentialLaneAddition - } + if fromCfg.NodeFunding != 0 { c.NodeFunding = fromCfg.NodeFunding } diff --git a/integration-tests/ccip-tests/testconfig/override/prod_testnet.toml b/integration-tests/ccip-tests/testconfig/override/prod_testnet.toml index e8dd95eca2..5dda086631 100644 --- a/integration-tests/ccip-tests/testconfig/override/prod_testnet.toml +++ b/integration-tests/ccip-tests/testconfig/override/prod_testnet.toml @@ -11,6 +11,7 @@ Data = """ "router": "0xF694E193200268f9a4868e4Aa017A0118C9a8177", "price_registry": "0x19e157E5fb1DAec1aE4BaB113fdf077F980704AA", "wrapped_native": "0xd00ae08403B9bbb9124bB305C09058E32C39A48c", + "multicall": "0x88b9782c2b271cf3d016867454e493144098E8AA", "src_contracts": { "BSC Testnet": { "on_ramp": "0xF25ECF1Aad9B2E43EDc2960cF66f325783245535", @@ -73,6 +74,7 @@ Data = """ "router": "0xE1053aE1857476f36A3C62580FF9b016E8EE8F6f", "price_registry": "0xCCDf022c9d31DC26Ebab4FB92432724a5b79809a", "wrapped_native": "0xae13d989daC2f0dEbFf460aC112a837C89BAa7cd", + "multicall": "0x4E63F845A1F42C318bC151cc2c9C7B2144b5dB86", "src_contracts": { "Avalanche Fuji": { "on_ramp": "0xa2515683E99F50ADbE177519A46bb20FfdBaA5de", @@ -122,6 +124,7 @@ Data = """ "router": "0x80AF2F44ed0469018922c9F483dc5A909862fdc2", "price_registry": "0x6d875a24c43974935b4F3D79AA6CD242BDfcD70F", "wrapped_native": "0x4200000000000000000000000000000000000006", + "multicall": "0xa1b97250844B089160a2Cd8dabEaCc4843993f24", "src_contracts": { "Avalanche Fuji": { "on_ramp": "0x2665A5aa729DBD384c804d0640f235D997698363", @@ -175,6 +178,7 @@ Data = """ "router": "0xA8C0c11bf64AF62CDCA6f93D3769B88BdD7cb93D", "price_registry": "0xa1ed3A3aA29166C9c8448654A8cA6b7916BC8379", "wrapped_native": "0x4200000000000000000000000000000000000001", + "multicall": "0x260AF9b83e0d2Bb6C9015fC9f0BfF8858A0CCE68", "src_contracts": { "WeMix Testnet": { "on_ramp": "0x6ea155Fc77566D9dcE01B8aa5D7968665dc4f0C5", @@ -197,6 +201,7 @@ Data = """ "router": "0xcc5a0B910D9E9504A7561934bed294c51285a78D", "price_registry": "0x9fdF51832A473D874057a2fcE768F0bc1c0F6F75", "wrapped_native": "0x4200000000000000000000000000000000000006", + "multicall": "0xA98DFA0e11e5B63C6fcbaEA5Fff49CfDf5765950", "src_contracts": { "Avalanche Fuji": { "on_ramp": "0xE7234D8d807D61821A6bB90735Aa238E229cF167", @@ -246,6 +251,7 @@ Data = """ "router": "0x1035CabC275068e0F4b745A29CEDf38E13aF41b1", "price_registry": "0x6322B0AD5467C12fc11393D479b0452C7a55aD81", "wrapped_native": "0x9c3C9283D3e44854697Cd22D3Faa240Cfb032889", + "multicall": "0xE8C13Eee68212cAD0eCD4E02ca1D34776296Fd27", "src_contracts": { "Avalanche Fuji": { "on_ramp": "0x5e0AD6D742983Ca464Fef0c28fD2D788a320B1c3", @@ -295,6 +301,7 @@ Data = """ "router": "0x0BF3dE8c5D3e8A2B34D2BEeB17ABfCeBaf363A59", "price_registry": "0x9EF7D57a4ea30b9e37794E55b0C75F2A70275dCc", "wrapped_native": "0x097D90c9d3E0B50Ca60e1ae45F6A81010f9FB534", + "multicall": "0x21CA8aC8df982eC644E7623463adAD52F993C5Aa", "src_contracts": { "Avalanche Fuji": { "on_ramp": "0x0477cA0a35eE05D3f9f424d88bC0977ceCf339D4", @@ -366,6 +373,7 @@ Data = """ "router": "0xA8C0c11bf64AF62CDCA6f93D3769B88BdD7cb93D", "price_registry": "0x89D17571DB7C9540eeB36760E3c749C8fb984569", "wrapped_native": "0x765277EebeCA2e31912C9946eAe1021199B39C61", + "multicall": "0x8A06DB818EED9e37c712Fc00b30975b5941Df1a0", "src_contracts": { "Kroma Sepolia": { "on_ramp": "0x428C4dc89b6Bf908B82d77C9CBceA786ea8cc7D0", @@ -404,7 +412,7 @@ TimeUnit = '5m' NoOfTokensPerChain = 1 NoOfTokensInMsg = 1 AmountPerToken = 1 -ExistingEnv = 'prod-testnet-2.7.1-ccip1.2.1-beta' +ExistingEnv = 'prod-testnet-2.7.1-ccip1.2.2-beta' [CCIP.Groups.smoke] MsgType = 'WithoutToken' diff --git a/integration-tests/ccip-tests/testconfig/tomls/default.toml b/integration-tests/ccip-tests/testconfig/tomls/default.toml index a1e8ffe74b..b7e316b0fe 100644 --- a/integration-tests/ccip-tests/testconfig/tomls/default.toml +++ b/integration-tests/ccip-tests/testconfig/tomls/default.toml @@ -179,7 +179,6 @@ ExistingDeployment = false # true if the tests are run on existing envi # the newly deployed contracts to send and verify ccip requests. ReuseContracts = true # Whether to reuse the contracts deployed in the previous run. Default value is true unless specified -SequentialLaneAddition = false # if true, the test will add lanes sequentially, otherwise it will add lanes in parallel NodeFunding = 1.0 # Amount of native currency to fund the chainlink node with for each network. Default value is 1 for smoke and 20 for load unless specified NoOfRoutersPerPair = 1 # denotes the number of routers to be deployed per network. mostly required for scalability tests. NoOfTokensPerChain = 2 # number of bridge tokens to be deployed per network; if MsgType = 'WithToken' @@ -220,7 +219,6 @@ ExistingDeployment = false # same as above # ExistingEnv = __ i.e prod-testnet-2.7.1-ccip1.2.1-beta ReuseContracts = true # same as above -SequentialLaneAddition = false # same as above NodeFunding = 20.0 # same as above NoOfRoutersPerPair = 1 # same as above NoOfTokensPerChain = 2 # same as above @@ -255,7 +253,6 @@ TestDuration = '10m' # load test duration, not used for smoke tes LocalCluster = false ExistingDeployment = false ReuseContracts = true -SequentialLaneAddition = false NodeFunding = 20.0 NoOfRoutersPerPair = 1 NoOfTokensPerChain = 2 diff --git a/integration-tests/ccip-tests/testreporters/ccip.go b/integration-tests/ccip-tests/testreporters/ccip.go index cff1fd04c2..0e65628173 100644 --- a/integration-tests/ccip-tests/testreporters/ccip.go +++ b/integration-tests/ccip-tests/testreporters/ccip.go @@ -60,6 +60,8 @@ type PhaseStat struct { type RequestStat struct { ReqNo int64 SeqNum uint64 + SourceNetwork string + DestNetwork string StatusByPhase map[Phase]PhaseStat `json:"status_by_phase,omitempty"` } @@ -106,10 +108,12 @@ func (stat *RequestStat) UpdateState(lggr zerolog.Logger, seqNum uint64, step Ph } } -func NewCCIPRequestStats(reqNo int64) *RequestStat { +func NewCCIPRequestStats(reqNo int64, source, dest string) *RequestStat { return &RequestStat{ ReqNo: reqNo, StatusByPhase: make(map[Phase]PhaseStat), + SourceNetwork: source, + DestNetwork: dest, } } diff --git a/integration-tests/ccip-tests/testsetups/ccip.go b/integration-tests/ccip-tests/testsetups/ccip.go index 792bda7916..5a0246727c 100644 --- a/integration-tests/ccip-tests/testsetups/ccip.go +++ b/integration-tests/ccip-tests/testsetups/ccip.go @@ -247,11 +247,10 @@ func NewCCIPTestConfig(t *testing.T, lggr zerolog.Logger, tType string) *CCIPTes } type BiDirectionalLaneConfig struct { - NetworkA blockchain.EVMNetwork - NetworkB blockchain.EVMNetwork - ForwardLane *actions.CCIPLane - ReverseLane *actions.CCIPLane - LaneDeployed bool + NetworkA blockchain.EVMNetwork + NetworkB blockchain.EVMNetwork + ForwardLane *actions.CCIPLane + ReverseLane *actions.CCIPLane } type CCIPTestSetUpOutputs struct { @@ -393,10 +392,9 @@ func (o *CCIPTestSetUpOutputs) AddLanesForNetworkPair( networkA.Name, networkB.Name), ccipLaneA2B.Logger) bidirectionalLane := &BiDirectionalLaneConfig{ - NetworkA: networkA, - NetworkB: networkB, - ForwardLane: ccipLaneA2B, - LaneDeployed: true, + NetworkA: networkA, + NetworkB: networkB, + ForwardLane: ccipLaneA2B, } var ccipLaneB2A *actions.CCIPLane @@ -813,13 +811,7 @@ func CCIPDefaultTestSetUp( n.NetworkA.URLs = inputs.NetworkPairs[i].ChainClientA.GetNetworkConfig().URLs n.NetworkB.HTTPURLs = inputs.NetworkPairs[i].ChainClientB.GetNetworkConfig().HTTPURLs n.NetworkB.URLs = inputs.NetworkPairs[i].ChainClientB.GetNetworkConfig().URLs - - // if sequential lane addition is true, continue after adding the first bidirectional lane - // and add rest of the lanes later, after the previously added lane(s) starts getting requests - // this is mainly used for testing the new lane addition functionality while other lanes are running - if i > 0 && pointer.GetBool(inputs.TestGroupInput.SequentialLaneAddition) { - continue - } + laneAddGrp.Go(func() error { return setUpArgs.AddLanesForNetworkPair( lggr, n.NetworkA, n.NetworkB, diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 8423a2d76b..0de8fb9619 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -19,13 +19,14 @@ require ( github.com/onsi/gomega v1.27.8 github.com/pelletier/go-toml/v2 v2.1.0 github.com/pkg/errors v0.9.1 + github.com/prometheus/common v0.45.0 github.com/rs/zerolog v1.30.0 github.com/scylladb/go-reflectx v1.0.1 github.com/segmentio/ksuid v1.0.4 github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chain-selectors v1.0.5 github.com/smartcontractkit/chainlink-common v0.1.7-0.20231117021201-6814387d8d3e - github.com/smartcontractkit/chainlink-testing-framework v1.20.1 + github.com/smartcontractkit/chainlink-testing-framework v1.20.2 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20231107151413-13e0202ae8d7 github.com/smartcontractkit/ocr2keepers v0.7.28 @@ -387,7 +388,6 @@ require ( github.com/prometheus/alertmanager v0.25.1 // indirect github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/common/sigv4 v0.1.0 // indirect github.com/prometheus/exporter-toolkit v0.10.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 9ed4afed51..53c8bd9837 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -2382,8 +2382,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20231023133638-72f4e799ab0 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20231023133638-72f4e799ab05/go.mod h1:o0Pn1pbaUluboaK6/yhf8xf7TiFCkyFl6WUOdwqamuU= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20231024133459-1ef3a11319eb h1:HiluOfEVGOQTM6BTDImOqYdMZZ7qq7fkZ3TJdmItNr8= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20231024133459-1ef3a11319eb/go.mod h1:/30flFG4L/iCYAFeA3DUzR0xuHSxAMONiWTzyzvsNwo= -github.com/smartcontractkit/chainlink-testing-framework v1.20.1 h1:0hxLRts4yIum52MaE95RuM2Xi1S/R0r4UFExpp00iK4= -github.com/smartcontractkit/chainlink-testing-framework v1.20.1/go.mod h1:+FVgkz6phTc+piVT57AcQfr3I8xvZgZ1lOpRPOu/dLQ= +github.com/smartcontractkit/chainlink-testing-framework v1.20.2 h1:IF85y6GJRNPrU0GoJgGR7d8A8alPzTwVV1fNpjHXeKk= +github.com/smartcontractkit/chainlink-testing-framework v1.20.2/go.mod h1:+FVgkz6phTc+piVT57AcQfr3I8xvZgZ1lOpRPOu/dLQ= github.com/smartcontractkit/go-plugin v0.0.0-20231003134350-e49dad63b306 h1:ko88+ZznniNJZbZPWAvHQU8SwKAdHngdDZ+pvVgB5ss= github.com/smartcontractkit/go-plugin v0.0.0-20231003134350-e49dad63b306/go.mod h1:w1sAEES3g3PuV/RzUrgow20W2uErMly84hhD3um1WL4= github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJif132UCdjo8u43i7iPN1/MFnu49hv7lFGFftCHKU=