From d6a5f16a2b13527a88d3c1cc720818c0e8b73a47 Mon Sep 17 00:00:00 2001 From: "Abdelrahman Soliman (Boda)" <2677789+asoliman92@users.noreply.github.com> Date: Thu, 22 Aug 2024 17:49:35 +0400 Subject: [PATCH] Remove old integration tests for v1.6 (#1346) To fully rely on integration-tests/deployment/ccip --- .../ccip_integration_tests/ocr3_node_test.go | 281 ---------------- .../ccip_integration_tests/ocr_node_helper.go | 314 ------------------ 2 files changed, 595 deletions(-) delete mode 100644 core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go delete mode 100644 core/capabilities/ccip/ccip_integration_tests/ocr_node_helper.go diff --git a/core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go b/core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go deleted file mode 100644 index 5914db8082..0000000000 --- a/core/capabilities/ccip/ccip_integration_tests/ocr3_node_test.go +++ /dev/null @@ -1,281 +0,0 @@ -package ccip_integration_tests - -import ( - "fmt" - "math/big" - "sync" - "testing" - "time" - - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/hashicorp/consul/sdk/freeport" - "go.uber.org/zap/zapcore" - - "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/offramp" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" - "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey" - - confighelper2 "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/stretchr/testify/require" -) - -const STATE_SUCCESS = uint8(2) - -/* -* If you want to debug, set log level to info and use the following commands for easier logs filtering. -* -* // Run the test and redirect logs to logs.txt -* go test -v -run "^TestIntegration_OCR3Nodes" ./core/capabilities/ccip/ccip_integration_tests 2>&1 > logs.txt -* -* // Reads logs.txt as a stream and apply filters using grep -* tail -fn0 logs.txt | grep "CCIPExecPlugin" - */ -func TestIntegration_OCR3Nodes(t *testing.T) { - const ( - numChains = 3 // number of chains that this test will run on - numNodes = 4 // number of OCR3 nodes, test assumes that every node supports every chain - - simulatedBackendBlockTime = 900 * time.Millisecond // Simulated backend blocks committing interval - oraclesBootWaitTime = 30 * time.Second // Time to wait for oracles to come up (HACK) - fChain = 1 // fChain value for all the chains - oracleLogLevel = zapcore.InfoLevel // Log level for the oracle / plugins. - ) - - t.Logf("creating %d universes", numChains) - homeChainUni, universes := createUniverses(t, numChains) - - var ( - oracles = make(map[uint64][]confighelper2.OracleIdentityExtra) - apps []chainlink.Application - nodes []*ocr3Node - p2pIDs [][32]byte - - // The bootstrap node will be: nodes[0] - bootstrapPort int - bootstrapP2PID p2pkey.PeerID - ) - - ports := freeport.GetN(t, numNodes) - ctx := testutils.Context(t) - callCtx := &bind.CallOpts{Context: ctx} - - for i := 0; i < numNodes; i++ { - t.Logf("Setting up ocr3 node:%d at port:%d", i, ports[i]) - node := setupNodeOCR3(t, ports[i], universes, homeChainUni, oracleLogLevel) - - for chainID, transmitter := range node.transmitters { - identity := confighelper2.OracleIdentityExtra{ - OracleIdentity: confighelper2.OracleIdentity{ - OnchainPublicKey: node.keybundle.PublicKey(), // Different for each chain - TransmitAccount: ocrtypes.Account(transmitter.Hex()), - OffchainPublicKey: node.keybundle.OffchainPublicKey(), // Same for each family - PeerID: node.peerID, - }, - ConfigEncryptionPublicKey: node.keybundle.ConfigEncryptionPublicKey(), // Different for each chain - } - oracles[chainID] = append(oracles[chainID], identity) - } - - apps = append(apps, node.app) - nodes = append(nodes, node) - - peerID, err := p2pkey.MakePeerID(node.peerID) - require.NoError(t, err) - p2pIDs = append(p2pIDs, peerID) - } - - bootstrapPort = ports[0] - bootstrapP2PID = p2pIDs[0] - bootstrapAddr := fmt.Sprintf("127.0.0.1:%d", bootstrapPort) - t.Logf("[bootstrap node] peerID:%s p2pID:%d address:%s", nodes[0].peerID, bootstrapP2PID, bootstrapAddr) - - // Start committing periodically in the background for all the chains - tick := time.NewTicker(simulatedBackendBlockTime) - defer tick.Stop() - commitBlocksBackground(t, universes, tick) - - ccipCapabilityID, err := homeChainUni.capabilityRegistry.GetHashedCapabilityId( - callCtx, CapabilityLabelledName, CapabilityVersion) - require.NoError(t, err, "failed to get hashed capability id for ccip") - require.NotEqual(t, [32]byte{}, ccipCapabilityID, "ccip capability id is empty") - - // Need to Add nodes and assign capabilities to them before creating DONS - homeChainUni.AddNodes(t, p2pIDs, [][32]byte{ccipCapabilityID}) - - for _, uni := range universes { - t.Logf("Adding chainconfig for chain %d", uni.chainID) - AddChainConfig(t, homeChainUni, getSelector(uni.chainID), p2pIDs, fChain) - } - - cfgs, err := homeChainUni.ccipConfig.GetAllChainConfigs(callCtx, big.NewInt(0), big.NewInt(100)) - require.NoError(t, err) - require.Len(t, cfgs, numChains) - - // Create a DON for each chain - for _, uni := range universes { - // Add nodes and give them the capability - t.Log("Adding DON for universe: ", uni.chainID) - chainSelector := getSelector(uni.chainID) - homeChainUni.AddDON( - t, - ccipCapabilityID, - chainSelector, - uni, - fChain, - bootstrapP2PID, - p2pIDs, - oracles[uni.chainID], - ) - } - - t.Log("Creating ocr3 jobs, starting oracles") - for i := 0; i < len(nodes); i++ { - err1 := nodes[i].app.Start(ctx) - require.NoError(t, err1) - tApp := apps[i] - t.Cleanup(func() { require.NoError(t, tApp.Stop()) }) - - jb := mustGetJobSpec(t, bootstrapP2PID, bootstrapPort, nodes[i].peerID, nodes[i].keybundle.ID()) - require.NoErrorf(t, tApp.AddJobV2(ctx, &jb), "Wasn't able to create ccip job for node %d", i) - } - - t.Logf("Sending ccip requests from each chain to all other chains") - for _, uni := range universes { - requests := genRequestData(uni.chainID, universes) - uni.SendCCIPRequests(t, requests) - } - - // Wait for the oracles to come up. - // TODO: We need some data driven way to do this e.g. wait until LP filters to be registered. - time.Sleep(oraclesBootWaitTime) - - // Replay the log poller on all the chains so that the logs are in the db. - // otherwise the plugins won't pick them up. - for _, node := range nodes { - for chainID := range universes { - t.Logf("Replaying logs for chain %d from block %d", chainID, 1) - require.NoError(t, node.app.ReplayFromBlock(big.NewInt(int64(chainID)), 1, false), "failed to replay logs") - } - } - - // with only one request sent from each chain to each other chain, - // and with sequence numbers on incrementing by 1 on a per-dest chain - // basis, we expect the min sequence number to be 1 on all chains. - expectedSeqNrRange := ccipocr3.NewSeqNumRange(1, 1) - var wg sync.WaitGroup - for _, uni := range universes { - for remoteSelector := range universes { - if remoteSelector == uni.chainID { - continue - } - wg.Add(1) - go func(uni onchainUniverse, remoteSelector uint64) { - defer wg.Done() - waitForCommitWithInterval(t, uni, getSelector(remoteSelector), expectedSeqNrRange) - }(uni, remoteSelector) - } - } - - start := time.Now() - wg.Wait() - t.Logf("All chains received the expected commit report in %s", time.Since(start)) - - // with only one request sent from each chain to each other chain, - // all ExecutionStateChanged events should have the sequence number 1. - expectedSeqNr := uint64(1) - for _, uni := range universes { - for remoteSelector := range universes { - if remoteSelector == uni.chainID { - continue - } - wg.Add(1) - go func(uni onchainUniverse, remoteSelector uint64) { - defer wg.Done() - waitForExecWithSeqNr(t, uni, getSelector(remoteSelector), expectedSeqNr) - }(uni, remoteSelector) - } - } - - start = time.Now() - wg.Wait() - t.Logf("All chains received the expected ExecutionStateChanged event in %s", time.Since(start)) -} - -func genRequestData(chainID uint64, universes map[uint64]onchainUniverse) []requestData { - var res []requestData - for destChainID, destUni := range universes { - if destChainID == chainID { - continue - } - res = append(res, requestData{ - destChainSelector: getSelector(destChainID), - receiverAddress: destUni.receiver.Address(), - data: []byte(fmt.Sprintf("msg from chain %d to chain %d", chainID, destChainID)), - }) - } - return res -} - -func waitForCommitWithInterval( - t *testing.T, - uni onchainUniverse, - expectedSourceChainSelector uint64, - expectedSeqNumRange ccipocr3.SeqNumRange, -) { - sink := make(chan *offramp.OffRampCommitReportAccepted) - subscription, err := uni.offramp.WatchCommitReportAccepted(&bind.WatchOpts{ - Context: testutils.Context(t), - }, sink) - require.NoError(t, err) - - for { - select { - case <-time.After(10 * time.Second): - t.Logf("Waiting for commit report on chain id %d (selector %d) from source selector %d expected seq nr range %s", - uni.chainID, getSelector(uni.chainID), expectedSourceChainSelector, expectedSeqNumRange.String()) - case subErr := <-subscription.Err(): - t.Fatalf("Subscription error: %+v", subErr) - case report := <-sink: - if len(report.Report.MerkleRoots) > 0 { - // Check the interval of sequence numbers and make sure it matches - // the expected range. - for _, mr := range report.Report.MerkleRoots { - if mr.SourceChainSelector == expectedSourceChainSelector && - uint64(expectedSeqNumRange.Start()) == mr.Interval.Min && - uint64(expectedSeqNumRange.End()) == mr.Interval.Max { - t.Logf("Received commit report on chain id %d (selector %d) from source selector %d expected seq nr range %s", - uni.chainID, getSelector(uni.chainID), expectedSourceChainSelector, expectedSeqNumRange.String()) - return - } - } - } - } - } -} - -func waitForExecWithSeqNr(t *testing.T, uni onchainUniverse, expectedSourceChainSelector, expectedSeqNr uint64) { - for { - scc, err := uni.offramp.GetSourceChainConfig(nil, expectedSourceChainSelector) - require.NoError(t, err) - t.Logf("Waiting for ExecutionStateChanged on chain %d (selector %d) from chain %d with expected sequence number %d, current onchain minSeqNr: %d", - uni.chainID, getSelector(uni.chainID), expectedSourceChainSelector, expectedSeqNr, scc.MinSeqNr) - iter, err := uni.offramp.FilterExecutionStateChanged(nil, []uint64{expectedSourceChainSelector}, []uint64{expectedSeqNr}, nil) - require.NoError(t, err) - var count int - for iter.Next() { - if iter.Event.SequenceNumber == expectedSeqNr && iter.Event.SourceChainSelector == expectedSourceChainSelector { - count++ - } - } - if count == 1 { - t.Logf("Received ExecutionStateChanged on chain %d (selector %d) from chain %d with expected sequence number %d", - uni.chainID, getSelector(uni.chainID), expectedSourceChainSelector, expectedSeqNr) - return - } - time.Sleep(5 * time.Second) - } -} diff --git a/core/capabilities/ccip/ccip_integration_tests/ocr_node_helper.go b/core/capabilities/ccip/ccip_integration_tests/ocr_node_helper.go deleted file mode 100644 index 9ac8ecb2fb..0000000000 --- a/core/capabilities/ccip/ccip_integration_tests/ocr_node_helper.go +++ /dev/null @@ -1,314 +0,0 @@ -package ccip_integration_tests - -import ( - "context" - "fmt" - "math/big" - "net/http" - "strconv" - "sync" - "testing" - "time" - - "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/validate" - - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" - "github.com/ethereum/go-ethereum/common" - gethtypes "github.com/ethereum/go-ethereum/core/types" - "github.com/jmoiron/sqlx" - - "github.com/smartcontractkit/chainlink/v2/core/services/job" - "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey" - - "github.com/smartcontractkit/chainlink-common/pkg/config" - "github.com/smartcontractkit/chainlink-common/pkg/loop" - "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" - "github.com/smartcontractkit/chainlink/v2/core/services/relay" - - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" - v2toml "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" - evmutils "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" - "github.com/smartcontractkit/chainlink/v2/core/chains/legacyevm" - configv2 "github.com/smartcontractkit/chainlink/v2/core/config/toml" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/logger/audit" - "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" - "github.com/smartcontractkit/chainlink/v2/core/services/keystore" - "github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype" - "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/ocr2key" - "github.com/smartcontractkit/chainlink/v2/core/utils" - "github.com/smartcontractkit/chainlink/v2/core/utils/testutils/heavyweight" - "github.com/smartcontractkit/chainlink/v2/plugins" - - "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" -) - -type ocr3Node struct { - app chainlink.Application - peerID string - transmitters map[uint64]common.Address - keybundle ocr2key.KeyBundle - db *sqlx.DB -} - -// setupNodeOCR3 creates a chainlink node and any associated keys in order to run -// ccip. -func setupNodeOCR3( - t *testing.T, - port int, - universes map[uint64]onchainUniverse, - homeChainUniverse homeChain, - logLevel zapcore.Level, -) *ocr3Node { - // Do not want to load fixtures as they contain a dummy chainID. - cfg, db := heavyweight.FullTestDBNoFixturesV2(t, func(c *chainlink.Config, s *chainlink.Secrets) { - c.Insecure.OCRDevelopmentMode = ptr(true) // Disables ocr spec validation so we can have fast polling for the test. - - c.Feature.LogPoller = ptr(true) - - // P2P V2 configs. - c.P2P.V2.Enabled = ptr(true) - c.P2P.V2.DeltaDial = config.MustNewDuration(500 * time.Millisecond) - c.P2P.V2.DeltaReconcile = config.MustNewDuration(5 * time.Second) - c.P2P.V2.ListenAddresses = &[]string{fmt.Sprintf("127.0.0.1:%d", port)} - - // Enable Capabilities, This is a pre-requisite for registrySyncer to work. - c.Capabilities.ExternalRegistry.NetworkID = ptr(relay.NetworkEVM) - c.Capabilities.ExternalRegistry.ChainID = ptr(strconv.FormatUint(homeChainUniverse.chainID, 10)) - c.Capabilities.ExternalRegistry.Address = ptr(homeChainUniverse.capabilityRegistry.Address().String()) - - // OCR configs - c.OCR.Enabled = ptr(false) - c.OCR.DefaultTransactionQueueDepth = ptr(uint32(200)) - c.OCR2.Enabled = ptr(true) - c.OCR2.ContractPollInterval = config.MustNewDuration(5 * time.Second) - - c.Log.Level = ptr(configv2.LogLevel(logLevel)) - - var chains v2toml.EVMConfigs - for chainID := range universes { - chains = append(chains, createConfigV2Chain(uBigInt(chainID))) - } - c.EVM = chains - }) - - lggr := logger.TestLogger(t) - lggr.SetLogLevel(logLevel) - ctx := testutils.Context(t) - clients := make(map[uint64]client.Client) - - for chainID, uni := range universes { - clients[chainID] = client.NewSimulatedBackendClient(t, uni.backend, uBigInt(chainID)) - } - - master := keystore.New(db, utils.FastScryptParams, lggr) - - kStore := KeystoreSim{ - eks: &EthKeystoreSim{ - Eth: master.Eth(), - t: t, - }, - csa: master.CSA(), - } - mailMon := mailbox.NewMonitor("ccip", lggr.Named("mailbox")) - evmOpts := chainlink.EVMFactoryConfig{ - ChainOpts: legacyevm.ChainOpts{ - AppConfig: cfg, - GenEthClient: func(i *big.Int) client.Client { - client, ok := clients[i.Uint64()] - if !ok { - t.Fatal("no backend for chainID", i) - } - return client - }, - MailMon: mailMon, - DS: db, - }, - CSAETHKeystore: kStore, - } - relayerFactory := chainlink.RelayerFactory{ - Logger: lggr, - LoopRegistry: plugins.NewLoopRegistry(lggr.Named("LoopRegistry"), cfg.Tracing()), - GRPCOpts: loop.GRPCOpts{}, - } - initOps := []chainlink.CoreRelayerChainInitFunc{chainlink.InitEVM(testutils.Context(t), relayerFactory, evmOpts)} - rci, err := chainlink.NewCoreRelayerChainInteroperators(initOps...) - require.NoError(t, err) - - app, err := chainlink.NewApplication(chainlink.ApplicationOpts{ - Config: cfg, - DS: db, - KeyStore: master, - RelayerChainInteroperators: rci, - Logger: lggr, - ExternalInitiatorManager: nil, - CloseLogger: lggr.Sync, - UnrestrictedHTTPClient: &http.Client{}, - RestrictedHTTPClient: &http.Client{}, - AuditLogger: audit.NoopLogger, - MailMon: mailMon, - LoopRegistry: plugins.NewLoopRegistry(lggr, cfg.Tracing()), - }) - require.NoError(t, err) - require.NoError(t, app.GetKeyStore().Unlock(ctx, "password")) - _, err = app.GetKeyStore().P2P().Create(ctx) - require.NoError(t, err) - - p2pIDs, err := app.GetKeyStore().P2P().GetAll() - require.NoError(t, err) - require.Len(t, p2pIDs, 1) - peerID := p2pIDs[0].PeerID() - // create a transmitter for each chain - transmitters := make(map[uint64]common.Address) - for chainID, uni := range universes { - backend := uni.backend - owner := uni.owner - cID := uBigInt(chainID) - addrs, err2 := app.GetKeyStore().Eth().EnabledAddressesForChain(testutils.Context(t), cID) - require.NoError(t, err2) - if len(addrs) == 1 { - // just fund the address - fundAddress(t, owner, addrs[0], assets.Ether(10).ToInt(), backend) - transmitters[chainID] = addrs[0] - } else { - // create key and fund it - _, err3 := app.GetKeyStore().Eth().Create(testutils.Context(t), cID) - require.NoError(t, err3, "failed to create key for chain", chainID) - sendingKeys, err3 := app.GetKeyStore().Eth().EnabledAddressesForChain(testutils.Context(t), cID) - require.NoError(t, err3) - require.Len(t, sendingKeys, 1) - fundAddress(t, owner, sendingKeys[0], assets.Ether(10).ToInt(), backend) - transmitters[chainID] = sendingKeys[0] - } - } - require.Len(t, transmitters, len(universes)) - - keybundle, err := app.GetKeyStore().OCR2().Create(ctx, chaintype.EVM) - require.NoError(t, err) - - t.Cleanup(func() { - require.NoError(t, db.Close()) - }) - - return &ocr3Node{ - // can't use this app because it doesn't have the right toml config - // missing bootstrapp - app: app, - peerID: peerID.Raw(), - transmitters: transmitters, - keybundle: keybundle, - db: db, - } -} - -func ptr[T any](v T) *T { return &v } - -var _ keystore.Eth = &EthKeystoreSim{} - -type EthKeystoreSim struct { - keystore.Eth - t *testing.T -} - -// override -func (e *EthKeystoreSim) SignTx(ctx context.Context, address common.Address, tx *gethtypes.Transaction, chainID *big.Int) (*gethtypes.Transaction, error) { - // always sign with chain id 1337 for the simulated backend - return e.Eth.SignTx(ctx, address, tx, big.NewInt(1337)) -} - -type KeystoreSim struct { - eks keystore.Eth - csa keystore.CSA -} - -func (e KeystoreSim) Eth() keystore.Eth { - return e.eks -} - -func (e KeystoreSim) CSA() keystore.CSA { - return e.csa -} - -func fundAddress(t *testing.T, from *bind.TransactOpts, to common.Address, amount *big.Int, backend *backends.SimulatedBackend) { - nonce, err := backend.PendingNonceAt(testutils.Context(t), from.From) - require.NoError(t, err) - gp, err := backend.SuggestGasPrice(testutils.Context(t)) - require.NoError(t, err) - rawTx := gethtypes.NewTx(&gethtypes.LegacyTx{ - Nonce: nonce, - GasPrice: gp, - Gas: 21000, - To: &to, - Value: amount, - }) - signedTx, err := from.Signer(from.From, rawTx) - require.NoError(t, err) - err = backend.SendTransaction(testutils.Context(t), signedTx) - require.NoError(t, err) - backend.Commit() -} - -func createConfigV2Chain(chainID *big.Int) *v2toml.EVMConfig { - chain := v2toml.Defaults((*evmutils.Big)(chainID)) - chain.GasEstimator.LimitDefault = ptr(uint64(5e6)) - chain.LogPollInterval = config.MustNewDuration(100 * time.Millisecond) - chain.Transactions.ForwardersEnabled = ptr(false) - chain.FinalityDepth = ptr(uint32(2)) - return &v2toml.EVMConfig{ - ChainID: (*evmutils.Big)(chainID), - Enabled: ptr(true), - Chain: chain, - Nodes: v2toml.EVMNodes{&v2toml.Node{}}, - } -} - -// Commit blocks periodically in the background for all chains -func commitBlocksBackground(t *testing.T, universes map[uint64]onchainUniverse, tick *time.Ticker) { - t.Log("starting ticker to commit blocks") - tickCtx, tickCancel := context.WithCancel(testutils.Context(t)) - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - for { - select { - case <-tick.C: - for _, uni := range universes { - uni.backend.Commit() - } - case <-tickCtx.Done(): - return - } - } - }() - t.Cleanup(func() { - tickCancel() - wg.Wait() - }) -} - -// p2pKeyID: nodes p2p id -// ocrKeyBundleID: nodes ocr key bundle id -func mustGetJobSpec(t *testing.T, bootstrapP2PID p2pkey.PeerID, bootstrapPort int, p2pKeyID string, ocrKeyBundleID string) job.Job { - specArgs := validate.SpecArgs{ - P2PV2Bootstrappers: []string{ - fmt.Sprintf("%s@127.0.0.1:%d", bootstrapP2PID.Raw(), bootstrapPort), - }, - CapabilityVersion: CapabilityVersion, - CapabilityLabelledName: CapabilityLabelledName, - OCRKeyBundleIDs: map[string]string{ - relay.NetworkEVM: ocrKeyBundleID, - }, - P2PKeyID: p2pKeyID, - PluginConfig: map[string]any{}, - } - specToml, err := validate.NewCCIPSpecToml(specArgs) - require.NoError(t, err) - jb, err := validate.ValidatedCCIPSpec(specToml) - require.NoError(t, err) - return jb -}