From 3f61b86e0cc155df19a8ee61ad9962dec9ec2073 Mon Sep 17 00:00:00 2001 From: Lazar Date: Thu, 7 Nov 2024 17:04:44 +0100 Subject: [PATCH 1/4] run two validators --- container/container.go | 115 ++++++++++++++++++++++++++++++++---- harness/app.go | 14 ++--- harness/babylonclient.go | 4 +- harness/bitcoindsetup.go | 4 +- harness/finalityprovider.go | 8 +-- harness/manager.go | 104 ++++++++++++++++++++------------ 6 files changed, 185 insertions(+), 64 deletions(-) diff --git a/container/container.go b/container/container.go index c776b8f..f1ead10 100644 --- a/container/container.go +++ b/container/container.go @@ -21,8 +21,9 @@ import ( ) const ( - bitcoindContainerName = "bitcoind" - babylondContainerName = "babylond" + bitcoindContainerName = "bitcoind" + babylondNode0ContainerName = "babylond-node0" + babylondNode1ContainerName = "babylond-node1" ) var ( @@ -39,6 +40,7 @@ type Manager struct { pool *dockertest.Pool resources map[string]*dockertest.Resource rawDcClient *rawDc.Client + network *docker.Network } // NewManager creates a new Manager instance and initializes @@ -148,12 +150,11 @@ func (m *Manager) ExecCmd(ctx context.Context, containerName string, command []s // RunBitcoindResource starts a bitcoind docker container func (m *Manager) RunBitcoindResource( - name string, bitcoindCfgPath string, ) (*dockertest.Resource, error) { bitcoindResource, err := m.pool.RunWithOptions( &dockertest.RunOptions{ - Name: fmt.Sprintf("%s-%s", bitcoindContainerName, name), + Name: bitcoindContainerName, Repository: m.cfg.BitcoindRepository, Tag: m.cfg.BitcoindVersion, User: "root:root", @@ -194,15 +195,28 @@ func (m *Manager) RunBitcoindResource( // RunBabylondResource starts a babylond container func (m *Manager) RunBabylondResource( - name string, mounthPath string, baseHeaderHex string, slashingPkScript string, epochInterval uint, -) (*dockertest.Resource, error) { +) (*dockertest.Resource, *dockertest.Resource, error) { + network, err := m.pool.Client.CreateNetwork(docker.CreateNetworkOptions{ + Name: "babylon", + Driver: "bridge", + IPAM: &docker.IPAMOptions{ + Config: []docker.IPAMConfig{ + { + Subnet: "192.168.10.0/24", + }, + }, + }, + }) + if err != nil { + return nil, nil, err + } cmd := []string{ "sh", "-c", fmt.Sprintf( - "babylond testnet --v=1 --output-dir=/home --starting-ip-address=192.168.10.2 "+ + "babylond testnet --v=2 --output-dir=/home --starting-ip-address=192.168.10.2 "+ "--keyring-backend=test --chain-id=chain-test --btc-finalization-timeout=4 "+ "--btc-confirmation-depth=2 --additional-sender-account --btc-network=regtest "+ "--min-staking-time-blocks=200 --min-staking-amount-sat=10000 "+ @@ -217,9 +231,9 @@ func (m *Manager) RunBabylondResource( epochInterval, slashingPkScript, baseHeaderHex, bbn.NewBIP340PubKeyFromBTCPK(CovenantPubKey).MarshalHex()), } - resource, err := m.pool.RunWithOptions( + resourceFirstNode, err := m.pool.RunWithOptions( &dockertest.RunOptions{ - Name: fmt.Sprintf("%s-%s", babylondContainerName, name), + Name: babylondNode0ContainerName, Repository: m.cfg.BabylonRepository, Tag: m.cfg.BabylonVersion, Labels: map[string]string{ @@ -246,12 +260,85 @@ func (m *Manager) RunBabylondResource( noRestart, ) if err != nil { - return nil, err + return nil, nil, err + } + + err = m.pool.Client.ConnectNetwork(network.ID, docker.NetworkConnectionOptions{ + Container: resourceFirstNode.Container.ID, + EndpointConfig: &docker.EndpointConfig{ + IPAMConfig: &docker.EndpointIPAMConfig{ + IPv4Address: "192.168.10.2", + }, + }, + }) + + if err != nil { + return nil, nil, err + } + + cmd2 := []string{ + "sh", "-c", fmt.Sprintf( + "chmod -R 777 /home && ls -la &&" + + "sed -i -e 's/iavl-cache-size = 781250/iavl-cache-size = 0/' /home/node1/babylond/config/app.toml && " + // disable the cache otherwise we go OOM + "sed -i -e 's/iavl-disable-fastnode = false/iavl-disable-fastnode = true/' /home/node1/babylond/config/app.toml && " + + `sed -i -e 's/timeout_commit = "5s"/timeout_commit = "2s"/' /home/node1/babylond/config/config.toml &&` + + "babylond start --home=/home/node1/babylond --rpc.pprof_laddr=0.0.0.0:6060", + ), + } + + time.Sleep(2 * time.Second) // todo(lazar): do a query on that file path to see if testnet cmd is done + + resourceSecondNode, err := m.pool.RunWithOptions( + &dockertest.RunOptions{ + Name: babylondNode1ContainerName, + Repository: m.cfg.BabylonRepository, + Tag: m.cfg.BabylonVersion, + Labels: map[string]string{ + "e2e": "babylond", + }, + User: "root:root", + Mounts: []string{ + fmt.Sprintf("%s/:/home/", mounthPath), + }, + ExposedPorts: []string{ + "9090/tcp", // only expose what we need + "26657/tcp", + "6060/tcp", + }, + Cmd: cmd2, + }, + func(config *docker.HostConfig) { + config.PortBindings = map[docker.Port][]docker.PortBinding{ + "9090/tcp": {{HostIP: "", HostPort: "9091"}}, + "26657/tcp": {{HostIP: "", HostPort: "26658"}}, + "6060/tcp": {{HostIP: "", HostPort: "6061"}}, + } + }, + noRestart, + ) + if err != nil { + return nil, nil, err + } + + err = m.pool.Client.ConnectNetwork(network.ID, docker.NetworkConnectionOptions{ + Container: resourceSecondNode.Container.ID, + EndpointConfig: &docker.EndpointConfig{ + IPAMConfig: &docker.EndpointIPAMConfig{ + IPv4Address: "192.168.10.3", + }, + }, + }) + + if err != nil { + return nil, nil, err } - m.resources[babylondContainerName] = resource + m.resources[babylondNode0ContainerName] = resourceFirstNode + m.resources[babylondNode1ContainerName] = resourceSecondNode - return resource, nil + m.network = network + + return resourceFirstNode, resourceSecondNode, nil } func (m *Manager) MemoryUsage(ctx context.Context, containerName string) (uint64, error) { @@ -282,6 +369,10 @@ func (m *Manager) ClearResources() error { } } + if err := m.pool.Client.RemoveNetwork(m.network.ID); err != nil { + return err + } + return nil } diff --git a/harness/app.go b/harness/app.go index e22d57a..a6143ae 100644 --- a/harness/app.go +++ b/harness/app.go @@ -36,20 +36,20 @@ func startHarness(cmdCtx context.Context, cfg config.Config) error { // bold text fmt.Printf("🟢 Starting with \033[1m%d\033[0m stakers, \u001B[1m%d\u001B[0m finality providers.\n", numStakers, numFinalityProviders) - cpSender, err := NewSenderWithBabylonClient(ctx, "node0", tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr) + cpSender, err := NewSenderWithBabylonClient(ctx, "node0", tm.Config.Babylon0.RPCAddr, tm.Config.Babylon0.GRPCAddr) if err != nil { return err } - headerSender, err := NewSenderWithBabylonClient(ctx, "headerreporter", tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr) + headerSender, err := NewSenderWithBabylonClient(ctx, "headerreporter", tm.Config.Babylon0.RPCAddr, tm.Config.Babylon0.GRPCAddr) if err != nil { return err } - vigilanteSender, err := NewSenderWithBabylonClient(ctx, "vigilante", tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr) + vigilanteSender, err := NewSenderWithBabylonClient(ctx, "vigilante", tm.Config.Babylon0.RPCAddr, tm.Config.Babylon0.GRPCAddr) if err != nil { return err } - fpmSender, err := NewSenderWithBabylonClient(ctx, "fpmsender", tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr) + fpmSender, err := NewSenderWithBabylonClient(ctx, "fpmsender", tm.Config.Babylon0.RPCAddr, tm.Config.Babylon0.GRPCAddr) if err != nil { return err } @@ -83,7 +83,7 @@ func startHarness(cmdCtx context.Context, cfg config.Config) error { var stakers []*BTCStaker for i := 0; i < numStakers; i++ { - stakerSender, err := NewSenderWithBabylonClient(ctx, fmt.Sprintf("staker-%d", i), tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr) + stakerSender, err := NewSenderWithBabylonClient(ctx, fmt.Sprintf("staker-%d", i), tm.Config.Babylon1.RPCAddr, tm.Config.Babylon1.GRPCAddr) if err != nil { return err } @@ -106,7 +106,7 @@ func startHarness(cmdCtx context.Context, cfg config.Config) error { go printStatsForever(ctx, tm, stopChan, cfg) - covenantSender, err := NewSenderWithBabylonClient(ctx, "covenant", tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr) + covenantSender, err := NewSenderWithBabylonClient(ctx, "covenant", tm.Config.Babylon0.RPCAddr, tm.Config.Babylon0.GRPCAddr) if err != nil { return err } @@ -150,7 +150,7 @@ func printStatsForever(ctx context.Context, tm *TestManager, stopChan chan struc close(stopChan) } - mem, err := tm.manger.MemoryUsage(ctx, "babylond") + mem, err := tm.manger.MemoryUsage(ctx, "babylond-node0") if err != nil { fmt.Printf("err getting memory usage for bbn node %v\n", err) } diff --git a/harness/babylonclient.go b/harness/babylonclient.go index 06a5132..85c2092 100644 --- a/harness/babylonclient.go +++ b/harness/babylonclient.go @@ -72,7 +72,7 @@ func New( cp := provider.(*cosmos.CosmosProvider) cp.PCfg.KeyDirectory = cfg.KeyDirectory - // Create tmp Babylon app to retrieve and register codecs + // Create tmp Babylon0 app to retrieve and register codecs // Need to override this manually as otherwise option from config is ignored encCfg := bbn.GetEncodingConfig() cp.Cdc = cosmos.Codec{ @@ -85,7 +85,7 @@ func New( // initialise Cosmos provider // NOTE: this will create a RPC client. The RPC client will be used for // submitting txs and making ad hoc queries. It won't create WebSocket - // connection with Babylon node + // connection with Babylon0 node err = cp.Init(ctx) if err != nil { return nil, err diff --git a/harness/bitcoindsetup.go b/harness/bitcoindsetup.go index ac5dc3f..0d2d424 100644 --- a/harness/bitcoindsetup.go +++ b/harness/bitcoindsetup.go @@ -38,13 +38,13 @@ func NewBitcoindHandler(manager *container.Manager) *BitcoindTestHandler { } } -func (h *BitcoindTestHandler) Start(ctx context.Context, containerName string) (*dockertest.Resource, error) { +func (h *BitcoindTestHandler) Start(ctx context.Context) (*dockertest.Resource, error) { tempPath, err := os.MkdirTemp("", "bbn-benchmark-test-*") if err != nil { return nil, err } - bitcoinResource, err := h.m.RunBitcoindResource(containerName, tempPath) + bitcoinResource, err := h.m.RunBitcoindResource(tempPath) if err != nil { return nil, err } diff --git a/harness/finalityprovider.go b/harness/finalityprovider.go index 829537c..ab62d96 100644 --- a/harness/finalityprovider.go +++ b/harness/finalityprovider.go @@ -87,7 +87,7 @@ func (fpm *FinalityProviderManager) Initialize(ctx context.Context, numPubRand u fpis := make([]*FinalityProviderInstance, fpm.fpCount) - res, err := fpm.tm.BabylonClient.CurrentEpoch() + res, err := fpm.tm.BabylonClientNode0.CurrentEpoch() if err != nil { return err } @@ -95,7 +95,7 @@ func (fpm *FinalityProviderManager) Initialize(ctx context.Context, numPubRand u for i := 0; i < fpm.fpCount; i++ { keyName := lib.GenRandomHexStr(r, 10) - finalitySender, err := NewSenderWithBabylonClient(ctx, keyName, fpm.tm.Config.Babylon.RPCAddr, fpm.tm.Config.Babylon.GRPCAddr) + finalitySender, err := NewSenderWithBabylonClient(ctx, keyName, fpm.tm.Config.Babylon0.RPCAddr, fpm.tm.Config.Babylon0.GRPCAddr) if err != nil { return err } @@ -385,7 +385,7 @@ func (fpi *FinalityProviderInstance) signFinalitySig(b *BlockInfo, btcPk *bbntyp return bbntypes.NewSchnorrEOTSSigFromModNScalar(sig), nil } -// SubmitFinalitySig submits the finality signature via a MsgAddVote to Babylon +// SubmitFinalitySig submits the finality signature via a MsgAddVote to Babylon0 func (fpi *FinalityProviderInstance) SubmitFinalitySig( ctx context.Context, fpPk *btcec.PublicKey, @@ -427,7 +427,7 @@ func (fpi *FinalityProviderInstance) SubmitFinalitySig( func (fpm *FinalityProviderManager) waitUntilFinalized(ctx context.Context, epoch uint64) error { err := lib.Eventually(ctx, func() bool { - lastFinalizedCkpt, err := fpm.tm.BabylonClient.LatestEpochFromStatus(ckpttypes.Finalized) + lastFinalizedCkpt, err := fpm.tm.BabylonClientNode0.LatestEpochFromStatus(ckpttypes.Finalized) if err != nil { return false } diff --git a/harness/manager.go b/harness/manager.go index cedc15f..7f0088f 100644 --- a/harness/manager.go +++ b/harness/manager.go @@ -41,14 +41,16 @@ const ( ) type Config struct { - BTC benchcfg.BTCConfig `mapstructure:"btc"` - Babylon bbncfg.BabylonConfig `mapstructure:"babylon"` + BTC benchcfg.BTCConfig `mapstructure:"btc"` + Babylon0 bbncfg.BabylonConfig `mapstructure:"babylon"` + Babylon1 bbncfg.BabylonConfig `mapstructure:"babylon"` } func DefaultConfig() *Config { return &Config{ - BTC: benchcfg.DefaultBTCConfig(), - Babylon: bbncfg.DefaultBabylonConfig(), + BTC: benchcfg.DefaultBTCConfig(), + Babylon0: bbncfg.DefaultBabylonConfig(), + Babylon1: bbncfg.DefaultBabylonConfig(), } } @@ -65,14 +67,15 @@ func defaultConfig() *Config { } type TestManager struct { - TestRpcClient *rpcclient.Client - BitcoindHandler *BitcoindTestHandler - BabylonClient *bbnclient.Client - Config *Config - WalletPrivKey *btcec.PrivateKey - manger *container.Manager - babylonDir string - benchConfig benchcfg.Config + TestRpcClient *rpcclient.Client + BitcoindHandler *BitcoindTestHandler + BabylonClientNode0 *bbnclient.Client + BabylonClientNode1 *bbnclient.Client + Config *Config + WalletPrivKey *btcec.PrivateKey + manger *container.Manager + babylonDir string + benchConfig benchcfg.Config } // StartManager creates a test manager @@ -83,7 +86,7 @@ func StartManager(ctx context.Context, outputsInWallet uint32, epochInterval uin } btcHandler := NewBitcoindHandler(manager) - bitcoind, err := btcHandler.Start(ctx, "bitcoind") + bitcoind, err := btcHandler.Start(ctx) if err != nil { return nil, err } @@ -135,7 +138,7 @@ func StartManager(ctx context.Context, outputsInWallet uint32, epochInterval uin return nil, err } - // start Babylon node + // start Babylon0 node babylonDir, err := tempDir() if err != nil { return nil, err @@ -145,28 +148,51 @@ func StartManager(ctx context.Context, outputsInWallet uint32, epochInterval uin babylonDir = runCfg.BabylonPath // override with cfg } - babylond, err := manager.RunBabylondResource("main", babylonDir, baseHeaderHex, hex.EncodeToString(pkScript), epochInterval) + babylond, babylondNode1, err := manager.RunBabylondResource(babylonDir, baseHeaderHex, hex.EncodeToString(pkScript), epochInterval) if err != nil { return nil, err } - // create a Babylon client - cfg.Babylon.KeyDirectory = filepath.Join(babylonDir, "node0", "babylond") - cfg.Babylon.Key = "test-spending-key" // keyring to bbn node - cfg.Babylon.GasAdjustment = 3.0 + _ = babylondNode1 + + // create a Babylon0 client + cfg.Babylon0.KeyDirectory = filepath.Join(babylonDir, "node0", "babylond") + cfg.Babylon0.Key = "test-spending-key" // keyring to bbn node + cfg.Babylon0.GasAdjustment = 3.0 + + // update port with the dynamically allocated one from docker + cfg.Babylon0.RPCAddr = fmt.Sprintf("http://localhost:%s", babylond.GetPort("26657/tcp")) + cfg.Babylon0.GRPCAddr = fmt.Sprintf("https://localhost:%s", babylond.GetPort("9090/tcp")) + + // create a Babylon1 client + cfg.Babylon1.KeyDirectory = filepath.Join(babylonDir, "node0", "babylond") + cfg.Babylon1.Key = "test-spending-key" // keyring to bbn node + cfg.Babylon1.GasAdjustment = 3.0 // update port with the dynamically allocated one from docker - cfg.Babylon.RPCAddr = fmt.Sprintf("http://localhost:%s", babylond.GetPort("26657/tcp")) - cfg.Babylon.GRPCAddr = fmt.Sprintf("https://localhost:%s", babylond.GetPort("9090/tcp")) + cfg.Babylon1.RPCAddr = fmt.Sprintf("http://localhost:%s", babylondNode1.GetPort("26657/tcp")) + cfg.Babylon1.GRPCAddr = fmt.Sprintf("https://localhost:%s", babylondNode1.GetPort("9090/tcp")) + + babylonClientNode0, err := bbnclient.New(&cfg.Babylon0, nil) + if err != nil { + return nil, err + } - babylonClient, err := bbnclient.New(&cfg.Babylon, nil) + babylonClientNode1, err := bbnclient.New(&cfg.Babylon1, nil) if err != nil { return nil, err } - // wait until Babylon is ready + // wait until Babylon0 is ready err = lib.Eventually(ctx, func() bool { - _, err := babylonClient.CurrentEpoch() + _, err := babylonClientNode0.CurrentEpoch() + + return err == nil + }, eventuallyWaitTimeOut, eventuallyPollTime, "err waiting current epoch") + + // wait until Babylon1 is ready + err = lib.Eventually(ctx, func() bool { + _, err := babylonClientNode1.CurrentEpoch() return err == nil }, eventuallyWaitTimeOut, eventuallyPollTime, "err waiting current epoch") @@ -175,20 +201,24 @@ func StartManager(ctx context.Context, outputsInWallet uint32, epochInterval uin return nil, err } return &TestManager{ - TestRpcClient: testRpcClient, - BabylonClient: babylonClient, - BitcoindHandler: btcHandler, - Config: cfg, - WalletPrivKey: walletPrivKey, - manger: manager, - babylonDir: babylonDir, - benchConfig: runCfg, + TestRpcClient: testRpcClient, + BabylonClientNode0: babylonClientNode0, + BabylonClientNode1: babylonClientNode1, + BitcoindHandler: btcHandler, + Config: cfg, + WalletPrivKey: walletPrivKey, + manger: manager, + babylonDir: babylonDir, + benchConfig: runCfg, }, nil } func (tm *TestManager) Stop() { - if tm.BabylonClient.IsRunning() { - err := tm.BabylonClient.Stop() + if tm.BabylonClientNode0.IsRunning() { + err := tm.BabylonClientNode0.Stop() + fmt.Printf("🚫 Rrr stopping client %v\n", err) + + err = tm.BabylonClientNode0.Stop() fmt.Printf("🚫 Rrr stopping client %v\n", err) } @@ -304,7 +334,7 @@ func (tm *TestManager) fundAllParties( senders []*SenderWithBabylonClient, ) error { - fundingAccount := tm.BabylonClient.MustGetAddr() + fundingAccount := tm.BabylonClientNode0.MustGetAddr() fundingAddress := sdk.MustAccAddressFromBech32(fundingAccount) var msgs []sdk.Msg @@ -314,7 +344,7 @@ func (tm *TestManager) fundAllParties( msgs = append(msgs, msg) } - resp, err := tm.BabylonClient.ReliablySendMsgs( + resp, err := tm.BabylonClientNode0.ReliablySendMsgs( ctx, msgs, []*errors.Error{}, @@ -340,7 +370,7 @@ func (tm *TestManager) listBlocksForever(ctx context.Context) { case <-ctx.Done(): return case <-lt.C: - resp, err := tm.BabylonClient.ListBlocks(finalitytypes.QueriedBlockStatus_NON_FINALIZED, nil) + resp, err := tm.BabylonClientNode0.ListBlocks(finalitytypes.QueriedBlockStatus_NON_FINALIZED, nil) if err != nil { fmt.Printf("🚫 Failed to list blocks: %v\n", err) continue From 2416ea4c5dc80e204fea5348157824220f707bf6 Mon Sep 17 00:00:00 2001 From: Lazar Date: Thu, 7 Nov 2024 17:36:04 +0100 Subject: [PATCH 2/4] lint --- harness/manager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/harness/manager.go b/harness/manager.go index 7f0088f..0d689bf 100644 --- a/harness/manager.go +++ b/harness/manager.go @@ -190,6 +190,10 @@ func StartManager(ctx context.Context, outputsInWallet uint32, epochInterval uin return err == nil }, eventuallyWaitTimeOut, eventuallyPollTime, "err waiting current epoch") + if err != nil { + return nil, err + } + // wait until Babylon1 is ready err = lib.Eventually(ctx, func() bool { _, err := babylonClientNode1.CurrentEpoch() From 90509ace7f885cd361f5f1c0dba1853d456c813b Mon Sep 17 00:00:00 2001 From: Lazar Date: Fri, 8 Nov 2024 09:23:13 +0100 Subject: [PATCH 3/4] cleanup --- harness/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harness/manager.go b/harness/manager.go index 0d689bf..e0343a7 100644 --- a/harness/manager.go +++ b/harness/manager.go @@ -222,7 +222,7 @@ func (tm *TestManager) Stop() { err := tm.BabylonClientNode0.Stop() fmt.Printf("🚫 Rrr stopping client %v\n", err) - err = tm.BabylonClientNode0.Stop() + err = tm.BabylonClientNode1.Stop() fmt.Printf("🚫 Rrr stopping client %v\n", err) } From c68c286b7e81aea0249d433541283321ea81a830 Mon Sep 17 00:00:00 2001 From: Lazar Date: Fri, 8 Nov 2024 10:26:14 +0100 Subject: [PATCH 4/4] log err --- harness/btcstaker.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/harness/btcstaker.go b/harness/btcstaker.go index 3bf9f5c..47d05ce 100644 --- a/harness/btcstaker.go +++ b/harness/btcstaker.go @@ -81,7 +81,10 @@ func (s *BTCStaker) runForever(ctx context.Context, stakerAddress btcutil.Addres fmt.Printf("🚫 Err getting staking params %v\n", err) continue } - _ = s.buildAndSendStakingTransaction(ctx, stakerAddress, stakerPk, ¶msResp.Params) + err = s.buildAndSendStakingTransaction(ctx, stakerAddress, stakerPk, ¶msResp.Params) + if err != nil { + fmt.Printf("🚫 Err in BTC Staker: %v\n", err) + } } } }