diff --git a/core/services/ocr3/plugins/ccip_integration_tests/helpers.go b/core/services/ocr3/plugins/ccip_integration_tests/helpers.go index 8526c62b27..18bcf6469c 100644 --- a/core/services/ocr3/plugins/ccip_integration_tests/helpers.go +++ b/core/services/ocr3/plugins/ccip_integration_tests/helpers.go @@ -1,9 +1,13 @@ package ccip_integration_tests import ( + "bytes" "encoding/hex" "math/big" + "sort" + "strconv" "testing" + "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" @@ -11,15 +15,13 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config" - - chainsel "github.com/smartcontractkit/chain-selectors" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/arm_proxy_contract" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/evm_2_evm_multi_offramp" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/evm_2_evm_multi_onramp" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/mock_arm_contract" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/nonce_manager" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ocr3_config_encoder" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/price_registry" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/token_admin_registry" @@ -27,6 +29,12 @@ import ( kcr "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/shared/generated/link_token" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + cctypes "github.com/smartcontractkit/chainlink/v2/core/services/ccipcapability/types" + + confighelper2 "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" + + chainsel "github.com/smartcontractkit/chain-selectors" "github.com/stretchr/testify/require" ) @@ -38,13 +46,14 @@ var ( const ( CapabilityLabelledName = "ccip" CapabilityVersion = "v1.0.0" + NodeOperatorID = 1 ) func e18Mult(amount uint64) *big.Int { - return new(big.Int).Mul(uintBigInt(amount), uintBigInt(1e18)) + return new(big.Int).Mul(uBigInt(amount), uBigInt(1e18)) } -func uintBigInt(i uint64) *big.Int { +func uBigInt(i uint64) *big.Int { return new(big.Int).SetUint64(i) } @@ -53,7 +62,7 @@ type homeChain struct { owner *bind.TransactOpts chainID uint64 capabilityRegistry *kcr.CapabilitiesRegistry - ccipConfigContract common.Address + ccipConfig *ccip_config.CCIPConfig } type onchainUniverse struct { @@ -259,15 +268,149 @@ func setupHomeChain(t *testing.T, owner *bind.TransactOpts, backend *backends.Si require.NoError(t, err, "failed to add capabilities to the capability registry") backend.Commit() + // Add NodeOperator, for simplicity we'll add one NodeOperator only + // First NodeOperator will have NodeOperatorId = 1 + _, err = capabilityRegistry.AddNodeOperators(owner, []kcr.CapabilitiesRegistryNodeOperator{ + { + Admin: owner.From, + Name: "NodeOperator", + }, + }) + require.NoError(t, err, "failed to add node operator to the capability registry") + backend.Commit() + return homeChain{ backend: backend, owner: owner, chainID: homeChainID, capabilityRegistry: capabilityRegistry, - ccipConfigContract: capabilityConfig.Address(), + ccipConfig: capabilityConfig, } } +func sortP2PIDS(p2pIDs [][32]byte) { + sort.Slice(p2pIDs, func(i, j int) bool { + return bytes.Compare(p2pIDs[i][:], p2pIDs[j][:]) < 0 + }) +} + +func (h *homeChain) AddNodes( + t *testing.T, + p2pIDs [][32]byte, + capabilityIDs [][32]byte, +) { + // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail + sortP2PIDS(p2pIDs) + var nodeParams []kcr.CapabilitiesRegistryNodeParams + for _, p2pID := range p2pIDs { + nodeParam := kcr.CapabilitiesRegistryNodeParams{ + NodeOperatorId: NodeOperatorID, + Signer: p2pID, // Not used in tests + P2pId: p2pID, + HashedCapabilityIds: capabilityIDs, + } + nodeParams = append(nodeParams, nodeParam) + } + _, err := h.capabilityRegistry.AddNodes(h.owner, nodeParams) + require.NoError(t, err, "failed to add node operator oracles") + h.backend.Commit() +} + +func (h *homeChain) AddDON( + t *testing.T, + ccipCapabilityID [32]byte, + chainSelector uint64, + OfframpAddress []byte, + f uint8, + bootstrapP2PID [32]byte, + p2pIDs [][32]byte, + oracles []confighelper2.OracleIdentityExtra, +) { + // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail + sortP2PIDS(p2pIDs) + // First Add ChainConfig that includes all p2pIDs as readers + chainConfig := SetupConfigInfo(chainSelector, p2pIDs, FChainA, []byte(strconv.FormatUint(chainSelector, 10))) + inputConfig := []ccip_config.CCIPConfigTypesChainConfigInfo{ + chainConfig, + } + _, err := h.ccipConfig.ApplyChainConfigUpdates(h.owner, nil, inputConfig) + require.NoError(t, err) + h.backend.Commit() + + // Get OCR3 Config from helper + var schedule []int + for range oracles { + schedule = append(schedule, 1) + } + signers, transmitters, f, _, offchainConfigVersion, offchainConfig, err := ocr3confighelper.ContractSetConfigArgsForTests( + 30*time.Second, // deltaProgress + 10*time.Second, // deltaResend + 20*time.Second, // deltaInitial + 2*time.Second, // deltaRound + 20*time.Second, // deltaGrace + 10*time.Second, // deltaCertifiedCommitRequest + 10*time.Second, // deltaStage + 3, // rmax + schedule, + oracles, + []byte{}, // empty offchain config + 50*time.Millisecond, // maxDurationQuery + 5*time.Second, // maxDurationObservation + 10*time.Second, // maxDurationShouldAcceptAttestedReport + 10*time.Second, // maxDurationShouldTransmitAcceptedReport + int(f), + []byte{}) // empty OnChainConfig + require.NoError(t, err, "failed to create contract config") + + tabi, err := ocr3_config_encoder.IOCR3ConfigEncoderMetaData.GetAbi() + require.NoError(t, err) + + signersBytes := make([][]byte, len(signers)) + for i, signer := range signers { + signersBytes[i] = signer + } + + transmittersBytes := make([][]byte, len(transmitters)) + for i, transmitter := range transmitters { + // anotherErr because linting doesn't want to shadow err + parsed, anotherErr := common.ParseHexOrString(string(transmitter)) + require.NoError(t, anotherErr) + transmittersBytes[i] = parsed + } + + // Add DON on capability registry contract + var ocr3Configs []ocr3_config_encoder.CCIPConfigTypesOCR3Config + for _, pluginType := range []cctypes.PluginType{cctypes.PluginTypeCCIPCommit, cctypes.PluginTypeCCIPExec} { + ocr3Configs = append(ocr3Configs, ocr3_config_encoder.CCIPConfigTypesOCR3Config{ + PluginType: uint8(pluginType), + ChainSelector: chainSelector, + F: f, + OffchainConfigVersion: offchainConfigVersion, + OfframpAddress: OfframpAddress, + BootstrapP2PIds: [][32]byte{bootstrapP2PID}, + P2pIds: p2pIDs, + Signers: signersBytes, + Transmitters: transmittersBytes, + OffchainConfig: offchainConfig, + }) + } + + encodedCall, err := tabi.Pack("exposeOCR3Config", ocr3Configs) + require.NoError(t, err) + + // Trim first four bytes to remove function selector. + encodedConfigs := encodedCall[4:] + + _, err = h.capabilityRegistry.AddDON(h.owner, p2pIDs, []kcr.CapabilitiesRegistryCapabilityConfiguration{ + { + CapabilityId: ccipCapabilityID, + Config: encodedConfigs, + }, + }, false, false, f) + require.NoError(t, err) + h.backend.Commit() +} + func connectUniverses( t *testing.T, universes map[uint64]onchainUniverse, @@ -335,6 +478,17 @@ func setupUniverseBasics(t *testing.T, uni onchainUniverse) { uni.backend.Commit() } +func SetupConfigInfo(chainSelector uint64, readers [][32]byte, fChain uint8, cfg []byte) ccip_config.CCIPConfigTypesChainConfigInfo { + return ccip_config.CCIPConfigTypesChainConfigInfo{ + ChainSelector: chainSelector, + ChainConfig: ccip_config.CCIPConfigTypesChainConfig{ + Readers: readers, + FChain: fChain, + Config: cfg, + }, + } +} + // As we can't change router contract. The contract was expecting onRamp and offRamp per lane and not per chain // In the new architecture we have only one onRamp and one offRamp per chain. // hence we add the mapping for all remote chains to the onRamp/offRamp contract of the local chain diff --git a/core/services/ocr3/plugins/ccip_integration_tests/home_chain/home_chain_test.go b/core/services/ocr3/plugins/ccip_integration_tests/home_chain_test.go similarity index 83% rename from core/services/ocr3/plugins/ccip_integration_tests/home_chain/home_chain_test.go rename to core/services/ocr3/plugins/ccip_integration_tests/home_chain_test.go index ab8629f8c1..ab710ff152 100644 --- a/core/services/ocr3/plugins/ccip_integration_tests/home_chain/home_chain_test.go +++ b/core/services/ocr3/plugins/ccip_integration_tests/home_chain_test.go @@ -1,4 +1,4 @@ -package home_chain +package ccip_integration_tests import ( "testing" @@ -10,11 +10,12 @@ import ( libocrtypes "github.com/smartcontractkit/libocr/ragep2p/types" ccipreader "github.com/smartcontractkit/chainlink-ccip/pkg/reader" + cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3" + capcfg "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" "github.com/smartcontractkit/chainlink/v2/core/logger" - it "github.com/smartcontractkit/chainlink/v2/core/services/ocr3/plugins/ccip_integration_tests" "github.com/stretchr/testify/require" ) @@ -22,19 +23,19 @@ import ( func TestHomeChainReader(t *testing.T) { ctx := testutils.Context(t) lggr := logger.TestLogger(t) - uni := it.NewTestUniverse(ctx, t, lggr) + uni := NewTestUniverse(ctx, t, lggr) // We need 3*f + 1 p2pIDs to have enough nodes to bootstrap var arr []int64 - n := int(it.FChainA*3 + 1) + n := int(FChainA*3 + 1) for i := 0; i <= n; i++ { arr = append(arr, int64(i)) } - p2pIDs := it.P2pIDsFromInts(arr) + p2pIDs := P2pIDsFromInts(arr) uni.AddCapability(p2pIDs) //==============================Apply configs to Capability Contract================================= - chainAConf := it.SetupConfigInfo(it.ChainA, p2pIDs, it.FChainA, []byte("ChainA")) - chainBConf := it.SetupConfigInfo(it.ChainB, p2pIDs[1:], it.FChainB, []byte("ChainB")) - chainCConf := it.SetupConfigInfo(it.ChainC, p2pIDs[2:], it.FChainC, []byte("ChainC")) + chainAConf := SetupConfigInfo(ChainA, p2pIDs, FChainA, []byte("ChainA")) + chainBConf := SetupConfigInfo(ChainB, p2pIDs[1:], FChainB, []byte("ChainB")) + chainCConf := SetupConfigInfo(ChainC, p2pIDs[2:], FChainC, []byte("ChainC")) inputConfig := []capcfg.CCIPConfigTypesChainConfigInfo{ chainAConf, chainBConf, @@ -66,13 +67,13 @@ func TestHomeChainReader(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedChainConfigs, configs) //=================================Remove ChainC from OnChainConfig========================================= - _, err = uni.CcipCfg.ApplyChainConfigUpdates(uni.Transactor, []uint64{it.ChainC}, nil) + _, err = uni.CcipCfg.ApplyChainConfigUpdates(uni.Transactor, []uint64{ChainC}, nil) require.NoError(t, err) uni.Backend.Commit() time.Sleep(pollDuration * 5) // Wait for the chain reader to update configs, err = homeChain.GetAllChainConfigs() require.NoError(t, err) - delete(expectedChainConfigs, cciptypes.ChainSelector(it.ChainC)) + delete(expectedChainConfigs, cciptypes.ChainSelector(ChainC)) require.Equal(t, expectedChainConfigs, configs) //================================Close HomeChain Reader=============================== //require.NoError(t, homeChain.Close()) diff --git a/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go b/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go index a595bb2378..93942dbc18 100644 --- a/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go +++ b/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go @@ -294,14 +294,3 @@ func (t *TestUniverse) AddDONToRegistry( require.NoError(t.TestingT, err) t.Backend.Commit() } - -func SetupConfigInfo(chainSelector uint64, readers [][32]byte, fChain uint8, cfg []byte) ccip_config.CCIPConfigTypesChainConfigInfo { - return ccip_config.CCIPConfigTypesChainConfigInfo{ - ChainSelector: chainSelector, - ChainConfig: ccip_config.CCIPConfigTypesChainConfig{ - Readers: readers, - FChain: fChain, - Config: cfg, - }, - } -} diff --git a/core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go b/core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go new file mode 100644 index 0000000000..bd5c12903d --- /dev/null +++ b/core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go @@ -0,0 +1,116 @@ +package ccip_integration_tests + +import ( + "fmt" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/hashicorp/consul/sdk/freeport" + + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey" + + "github.com/smartcontractkit/libocr/commontypes" + confighelper2 "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + "github.com/stretchr/testify/require" +) + +func TestIntegration_OCR3Nodes(t *testing.T) { + numChains := 3 + homeChainUni, universes := createUniverses(t, numChains) + numNodes := 4 + t.Log("creating ocr3 nodes") + var ( + oracles = make(map[uint64][]confighelper2.OracleIdentityExtra) + transmitters = make(map[uint64][]common.Address) + apps []chainlink.Application + nodes []*ocr3Node + p2pIDs [][32]byte + + // The bootstrap node will be the first node (index 0) + bootstrapPort int + bootstrapP2PID p2pkey.PeerID + bootstrappers []commontypes.BootstrapperLocator + ) + + ports := freeport.GetN(t, numNodes) + capabilitiesPorts := freeport.GetN(t, numNodes) + for i := 0; i < numNodes; i++ { + node := setupNodeOCR3(t, ports[i], capabilitiesPorts[i], bootstrappers, universes, homeChainUni) + + apps = append(apps, node.app) + for chainID, transmitter := range node.transmitters { + transmitters[chainID] = append(transmitters[chainID], transmitter) + identity := confighelper2.OracleIdentityExtra{ + OracleIdentity: confighelper2.OracleIdentity{ + OnchainPublicKey: node.keybundle.PublicKey(), + TransmitAccount: ocrtypes.Account(transmitter.Hex()), + OffchainPublicKey: node.keybundle.OffchainPublicKey(), + PeerID: node.peerID, + }, + ConfigEncryptionPublicKey: node.keybundle.ConfigEncryptionPublicKey(), + } + oracles[chainID] = append(oracles[chainID], identity) + } + nodes = append(nodes, node) + peerID, err := p2pkey.MakePeerID(node.peerID) + require.NoError(t, err) + p2pIDs = append(p2pIDs, peerID) + + // First Node is the bootstrap node + if i == 0 { + bootstrapPort = ports[i] + bootstrapP2PID = peerID + bootstrappers = []commontypes.BootstrapperLocator{ + {PeerID: node.peerID, Addrs: []string{ + fmt.Sprintf("127.0.0.1:%d", bootstrapPort), + }}, + } + } + } + + // Start committing periodically in the background for all the chains + tick := time.NewTicker(100 * time.Millisecond) + defer tick.Stop() + commitBlocksBackground(t, universes, tick) + + ctx := testutils.Context(t) + t.Log("creating ocr3 jobs") + for i := 0; i < len(nodes); i++ { + err := nodes[i].app.Start(ctx) + require.NoError(t, err) + tApp := apps[i] + t.Cleanup(func() { + require.NoError(t, tApp.Stop()) + }) + //TODO: Create Jobs and add them to the app + } + + ccipCapabilityID, err := homeChainUni.capabilityRegistry.GetHashedCapabilityId(&bind.CallOpts{ + Context: ctx, + }, CapabilityLabelledName, CapabilityVersion) + require.NoError(t, err, "failed to get hashed capability id for ccip") + require.NotEqual(t, [32]byte{}, ccipCapabilityID, "ccip capability id is empty") + + // Need to Add nodes and assign capabilities to them before creating DONS + homeChainUni.AddNodes(t, p2pIDs, [][32]byte{ccipCapabilityID}) + // Create a DON for each chain + for _, uni := range universes { + // Add nodes and give them the capability + t.Log("AddingDON for universe: ", uni.chainID) + homeChainUni.AddDON(t, + ccipCapabilityID, + uni.chainID, + uni.offramp.Address().Bytes(), + 1, // f + bootstrapP2PID, + p2pIDs, + oracles[uni.chainID], + ) + } +} diff --git a/core/services/ocr3/plugins/ccip_integration_tests/ocr_node_helper.go b/core/services/ocr3/plugins/ccip_integration_tests/ocr_node_helper.go new file mode 100644 index 0000000000..0163388183 --- /dev/null +++ b/core/services/ocr3/plugins/ccip_integration_tests/ocr_node_helper.go @@ -0,0 +1,294 @@ +package ccip_integration_tests + +import ( + "context" + "fmt" + "math/big" + "net/http" + "strconv" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/common" + gethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/jmoiron/sqlx" + "go.uber.org/zap/zapcore" + + "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink-common/pkg/loop" + "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" + "github.com/smartcontractkit/chainlink/v2/core/services/relay" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + v2toml "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" + evmutils "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" + "github.com/smartcontractkit/chainlink/v2/core/chains/legacyevm" + "github.com/smartcontractkit/chainlink/v2/core/internal/cltest/heavyweight" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/logger/audit" + "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/ocr2key" + "github.com/smartcontractkit/chainlink/v2/core/utils" + "github.com/smartcontractkit/chainlink/v2/plugins" + + "github.com/smartcontractkit/libocr/commontypes" + "github.com/stretchr/testify/require" +) + +type ocr3Node struct { + app chainlink.Application + peerID string + transmitters map[uint64]common.Address + keybundle ocr2key.KeyBundle + db *sqlx.DB +} + +// setupNodeOCR3 creates a chainlink node and any associated keys in order to run +// ccip. +func setupNodeOCR3( + t *testing.T, + port int, + capabilitiesPort int, + p2pV2Bootstrappers []commontypes.BootstrapperLocator, + universes map[uint64]onchainUniverse, + homeChainUniverse homeChain, +) *ocr3Node { + // Do not want to load fixtures as they contain a dummy chainID. + cfg, db := heavyweight.FullTestDBNoFixturesV2(t, func(c *chainlink.Config, s *chainlink.Secrets) { + c.Insecure.OCRDevelopmentMode = ptr(true) // Disables ocr spec validation so we can have fast polling for the test. + + c.Feature.LogPoller = ptr(true) + + // P2P V2 configs. + c.P2P.V2.Enabled = ptr(true) + c.P2P.V2.DeltaDial = config.MustNewDuration(500 * time.Millisecond) + c.P2P.V2.DeltaReconcile = config.MustNewDuration(5 * time.Second) + c.P2P.V2.ListenAddresses = &[]string{fmt.Sprintf("127.0.0.1:%d", port)} + if len(p2pV2Bootstrappers) > 0 { + c.P2P.V2.DefaultBootstrappers = &p2pV2Bootstrappers + } + + // Enable Capabilities, This is a pre-requisite for registrySyncer to work. + // Same values as P2P.V2 except for the listen address. + c.Capabilities.Peering.V2 = c.P2P.V2 + c.Capabilities.Peering.V2.ListenAddresses = &[]string{fmt.Sprintf("127.0.0.1:%d", capabilitiesPort)} + c.Capabilities.ExternalRegistry.NetworkID = ptr(relay.NetworkEVM) + c.Capabilities.ExternalRegistry.ChainID = ptr(strconv.FormatUint(homeChainUniverse.chainID, 10)) + c.Capabilities.ExternalRegistry.Address = ptr(homeChainUniverse.capabilityRegistry.Address().String()) + + // OCR configs + c.OCR.Enabled = ptr(false) + c.OCR.DefaultTransactionQueueDepth = ptr(uint32(200)) + c.OCR2.Enabled = ptr(true) + c.OCR2.ContractPollInterval = config.MustNewDuration(5 * time.Second) + + var chains v2toml.EVMConfigs + for chainID := range universes { + chains = append(chains, createConfigV2Chain(uBigInt(chainID))) + } + c.EVM = chains + }) + + lggr := logger.TestLogger(t) + lggr.SetLogLevel(zapcore.InfoLevel) + ctx := testutils.Context(t) + clients := make(map[uint64]client.Client) + + for chainID, uni := range universes { + clients[chainID] = client.NewSimulatedBackendClient(t, uni.backend, uBigInt(chainID)) + } + + master := keystore.New(db, utils.FastScryptParams, lggr) + + kStore := KeystoreSim{ + eks: &EthKeystoreSim{ + Eth: master.Eth(), + t: t, + }, + csa: master.CSA(), + } + mailMon := mailbox.NewMonitor("ccip", lggr.Named("mailbox")) + evmOpts := chainlink.EVMFactoryConfig{ + ChainOpts: legacyevm.ChainOpts{ + AppConfig: cfg, + GenEthClient: func(i *big.Int) client.Client { + t.Log("genning eth client for chain id:", i.String()) + client, ok := clients[i.Uint64()] + if !ok { + t.Fatal("no backend for chainID", i) + } + return client + }, + MailMon: mailMon, + DS: db, + }, + CSAETHKeystore: kStore, + } + relayerFactory := chainlink.RelayerFactory{ + Logger: lggr, + LoopRegistry: plugins.NewLoopRegistry(lggr.Named("LoopRegistry"), cfg.Tracing()), + GRPCOpts: loop.GRPCOpts{}, + } + initOps := []chainlink.CoreRelayerChainInitFunc{chainlink.InitEVM(testutils.Context(t), relayerFactory, evmOpts)} + rci, err := chainlink.NewCoreRelayerChainInteroperators(initOps...) + require.NoError(t, err) + + app, err := chainlink.NewApplication(chainlink.ApplicationOpts{ + Config: cfg, + DS: db, + KeyStore: master, + RelayerChainInteroperators: rci, + Logger: lggr, + ExternalInitiatorManager: nil, + CloseLogger: lggr.Sync, + UnrestrictedHTTPClient: &http.Client{}, + RestrictedHTTPClient: &http.Client{}, + AuditLogger: audit.NoopLogger, + MailMon: mailMon, + LoopRegistry: plugins.NewLoopRegistry(lggr, cfg.Tracing()), + }) + require.NoError(t, err) + require.NoError(t, app.GetKeyStore().Unlock(ctx, "password")) + _, err = app.GetKeyStore().P2P().Create(ctx) + require.NoError(t, err) + + p2pIDs, err := app.GetKeyStore().P2P().GetAll() + require.NoError(t, err) + require.Len(t, p2pIDs, 1) + peerID := p2pIDs[0].PeerID() + // create a transmitter for each chain + transmitters := make(map[uint64]common.Address) + for chainID, uni := range universes { + backend := uni.backend + owner := uni.owner + cID := uBigInt(chainID) + addrs, err2 := app.GetKeyStore().Eth().EnabledAddressesForChain(testutils.Context(t), cID) + require.NoError(t, err2) + if len(addrs) == 1 { + // just fund the address + fundAddress(t, owner, addrs[0], assets.Ether(10).ToInt(), backend) + transmitters[chainID] = addrs[0] + } else { + // create key and fund it + _, err3 := app.GetKeyStore().Eth().Create(testutils.Context(t), cID) + require.NoError(t, err3, "failed to create key for chain", chainID) + sendingKeys, err3 := app.GetKeyStore().Eth().EnabledAddressesForChain(testutils.Context(t), cID) + require.NoError(t, err3) + require.Len(t, sendingKeys, 1) + fundAddress(t, owner, sendingKeys[0], assets.Ether(10).ToInt(), backend) + transmitters[chainID] = sendingKeys[0] + } + } + require.Len(t, transmitters, len(universes)) + + keybundle, err := app.GetKeyStore().OCR2().Create(ctx, chaintype.EVM) + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, db.Close()) + }) + + return &ocr3Node{ + // can't use this app because it doesn't have the right toml config + // missing bootstrapp + app: app, + peerID: peerID.Raw(), + transmitters: transmitters, + keybundle: keybundle, + db: db, + } +} + +func ptr[T any](v T) *T { return &v } + +var _ keystore.Eth = &EthKeystoreSim{} + +type EthKeystoreSim struct { + keystore.Eth + t *testing.T +} + +// override +func (e *EthKeystoreSim) SignTx(ctx context.Context, address common.Address, tx *gethtypes.Transaction, chainID *big.Int) (*gethtypes.Transaction, error) { + // always sign with chain id 1337 for the simulated backend + e.t.Log("always signing tx for chain id:", chainID.String(), "with chain id 1337, tx hash:", tx.Hash()) + return e.Eth.SignTx(ctx, address, tx, big.NewInt(1337)) +} + +type KeystoreSim struct { + eks keystore.Eth + csa keystore.CSA +} + +func (e KeystoreSim) Eth() keystore.Eth { + return e.eks +} + +func (e KeystoreSim) CSA() keystore.CSA { + return e.csa +} + +func fundAddress(t *testing.T, from *bind.TransactOpts, to common.Address, amount *big.Int, backend *backends.SimulatedBackend) { + nonce, err := backend.PendingNonceAt(testutils.Context(t), from.From) + require.NoError(t, err) + gp, err := backend.SuggestGasPrice(testutils.Context(t)) + require.NoError(t, err) + rawTx := gethtypes.NewTx(&gethtypes.LegacyTx{ + Nonce: nonce, + GasPrice: gp, + Gas: 21000, + To: &to, + Value: amount, + }) + signedTx, err := from.Signer(from.From, rawTx) + require.NoError(t, err) + err = backend.SendTransaction(testutils.Context(t), signedTx) + require.NoError(t, err) + backend.Commit() +} + +func createConfigV2Chain(chainID *big.Int) *v2toml.EVMConfig { + chain := v2toml.Defaults((*evmutils.Big)(chainID)) + chain.GasEstimator.LimitDefault = ptr(uint64(5e6)) + chain.LogPollInterval = config.MustNewDuration(100 * time.Millisecond) + chain.Transactions.ForwardersEnabled = ptr(false) + chain.FinalityDepth = ptr(uint32(2)) + return &v2toml.EVMConfig{ + ChainID: (*evmutils.Big)(chainID), + Enabled: ptr(true), + Chain: chain, + Nodes: v2toml.EVMNodes{&v2toml.Node{}}, + } +} + +// Commit blocks periodically in the background for all chains +func commitBlocksBackground(t *testing.T, universes map[uint64]onchainUniverse, tick *time.Ticker) { + t.Log("starting ticker to commit blocks") + tickCtx, tickCancel := context.WithCancel(testutils.Context(t)) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-tick.C: + for _, uni := range universes { + uni.backend.Commit() + } + case <-tickCtx.Done(): + return + } + } + }() + t.Cleanup(func() { + tickCancel() + wg.Wait() + }) +}