chore(*): run multiple validators

container/container.go

@@ -21,8 +21,9 @@ import (
 )
 
 const (
-	bitcoindContainerName = "bitcoind"
-	babylondContainerName = "babylond"
+	bitcoindContainerName      = "bitcoind"
+	babylondNode0ContainerName = "babylond-node0"
+	babylondNode1ContainerName = "babylond-node1"
 )
 
 var (
@@ -39,6 +40,7 @@ type Manager struct {
 	pool        *dockertest.Pool
 	resources   map[string]*dockertest.Resource
 	rawDcClient *rawDc.Client
+	network     *docker.Network
 }
 
 // NewManager creates a new Manager instance and initializes
@@ -148,12 +150,11 @@ func (m *Manager) ExecCmd(ctx context.Context, containerName string, command []s
 
 // RunBitcoindResource starts a bitcoind docker container
 func (m *Manager) RunBitcoindResource(
-	name string,
 	bitcoindCfgPath string,
 ) (*dockertest.Resource, error) {
 	bitcoindResource, err := m.pool.RunWithOptions(
 		&dockertest.RunOptions{
-			Name:       fmt.Sprintf("%s-%s", bitcoindContainerName, name),
+			Name:       bitcoindContainerName,
 			Repository: m.cfg.BitcoindRepository,
 			Tag:        m.cfg.BitcoindVersion,
 			User:       "root:root",
@@ -194,15 +195,28 @@ func (m *Manager) RunBitcoindResource(
 
 // RunBabylondResource starts a babylond container
 func (m *Manager) RunBabylondResource(
-	name string,
 	mounthPath string,
 	baseHeaderHex string,
 	slashingPkScript string,
 	epochInterval uint,
-) (*dockertest.Resource, error) {
+) (*dockertest.Resource, *dockertest.Resource, error) {
+	network, err := m.pool.Client.CreateNetwork(docker.CreateNetworkOptions{
+		Name:   "babylon",
+		Driver: "bridge",
+		IPAM: &docker.IPAMOptions{
+			Config: []docker.IPAMConfig{
+				{
+					Subnet: "192.168.10.0/24",
+				},
+			},
+		},
+	})
+	if err != nil {
+		return nil, nil, err
+	}
 	cmd := []string{
 		"sh", "-c", fmt.Sprintf(
-			"babylond testnet --v=1 --output-dir=/home --starting-ip-address=192.168.10.2 "+
+			"babylond testnet --v=2 --output-dir=/home --starting-ip-address=192.168.10.2 "+
 				"--keyring-backend=test --chain-id=chain-test --btc-finalization-timeout=4 "+
 				"--btc-confirmation-depth=2 --additional-sender-account --btc-network=regtest "+
 				"--min-staking-time-blocks=200 --min-staking-amount-sat=10000 "+
@@ -217,9 +231,9 @@ func (m *Manager) RunBabylondResource(
 			epochInterval, slashingPkScript, baseHeaderHex, bbn.NewBIP340PubKeyFromBTCPK(CovenantPubKey).MarshalHex()),
 	}
 
-	resource, err := m.pool.RunWithOptions(
+	resourceFirstNode, err := m.pool.RunWithOptions(
 		&dockertest.RunOptions{
-			Name:       fmt.Sprintf("%s-%s", babylondContainerName, name),
+			Name:       babylondNode0ContainerName,
 			Repository: m.cfg.BabylonRepository,
 			Tag:        m.cfg.BabylonVersion,
 			Labels: map[string]string{
@@ -246,12 +260,85 @@ func (m *Manager) RunBabylondResource(
 		noRestart,
 	)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
+	}
+
+	err = m.pool.Client.ConnectNetwork(network.ID, docker.NetworkConnectionOptions{
+		Container: resourceFirstNode.Container.ID,
+		EndpointConfig: &docker.EndpointConfig{
+			IPAMConfig: &docker.EndpointIPAMConfig{
+				IPv4Address: "192.168.10.2",
+			},
+		},
+	})
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cmd2 := []string{
+		"sh", "-c", fmt.Sprintf(
+			"chmod -R 777 /home && ls -la &&" +
+				"sed -i -e 's/iavl-cache-size = 781250/iavl-cache-size = 0/' /home/node1/babylond/config/app.toml && " + // disable the cache otherwise we go OOM
+				"sed -i -e 's/iavl-disable-fastnode = false/iavl-disable-fastnode = true/' /home/node1/babylond/config/app.toml && " +
+				`sed -i -e 's/timeout_commit = "5s"/timeout_commit = "2s"/' /home/node1/babylond/config/config.toml &&` +
+				"babylond start --home=/home/node1/babylond --rpc.pprof_laddr=0.0.0.0:6060",
+		),
+	}
+
+	time.Sleep(2 * time.Second) // todo(lazar): do a query on that file path to see if testnet cmd is done
+
+	resourceSecondNode, err := m.pool.RunWithOptions(
+		&dockertest.RunOptions{
+			Name:       babylondNode1ContainerName,
+			Repository: m.cfg.BabylonRepository,
+			Tag:        m.cfg.BabylonVersion,
+			Labels: map[string]string{
+				"e2e": "babylond",
+			},
+			User: "root:root",
+			Mounts: []string{
+				fmt.Sprintf("%s/:/home/", mounthPath),
+			},
+			ExposedPorts: []string{
+				"9090/tcp", // only expose what we need
+				"26657/tcp",
+				"6060/tcp",
+			},
+			Cmd: cmd2,
+		},
+		func(config *docker.HostConfig) {
+			config.PortBindings = map[docker.Port][]docker.PortBinding{
+				"9090/tcp":  {{HostIP: "", HostPort: "9091"}},
+				"26657/tcp": {{HostIP: "", HostPort: "26658"}},
+				"6060/tcp":  {{HostIP: "", HostPort: "6061"}},
+			}
+		},
+		noRestart,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	err = m.pool.Client.ConnectNetwork(network.ID, docker.NetworkConnectionOptions{
+		Container: resourceSecondNode.Container.ID,
+		EndpointConfig: &docker.EndpointConfig{
+			IPAMConfig: &docker.EndpointIPAMConfig{
+				IPv4Address: "192.168.10.3",
+			},
+		},
+	})
+
+	if err != nil {
+		return nil, nil, err
 	}
 
-	m.resources[babylondContainerName] = resource
+	m.resources[babylondNode0ContainerName] = resourceFirstNode
+	m.resources[babylondNode1ContainerName] = resourceSecondNode
 
-	return resource, nil
+	m.network = network
+
+	return resourceFirstNode, resourceSecondNode, nil
 }
 
 func (m *Manager) MemoryUsage(ctx context.Context, containerName string) (uint64, error) {
@@ -282,6 +369,10 @@ func (m *Manager) ClearResources() error {
 		}
 	}
 
+	if err := m.pool.Client.RemoveNetwork(m.network.ID); err != nil {
+		return err
+	}
+
 	return nil
 }
 

harness/app.go

@@ -36,20 +36,20 @@ func startHarness(cmdCtx context.Context, cfg config.Config) error {
 	// bold text
 	fmt.Printf("🟢 Starting with \033[1m%d\033[0m stakers, \u001B[1m%d\u001B[0m finality providers.\n", numStakers, numFinalityProviders)
 
-	cpSender, err := NewSenderWithBabylonClient(ctx, "node0", tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr)
+	cpSender, err := NewSenderWithBabylonClient(ctx, "node0", tm.Config.Babylon0.RPCAddr, tm.Config.Babylon0.GRPCAddr)
 	if err != nil {
 		return err
 	}
-	headerSender, err := NewSenderWithBabylonClient(ctx, "headerreporter", tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr)
+	headerSender, err := NewSenderWithBabylonClient(ctx, "headerreporter", tm.Config.Babylon0.RPCAddr, tm.Config.Babylon0.GRPCAddr)
 	if err != nil {
 		return err
 	}
-	vigilanteSender, err := NewSenderWithBabylonClient(ctx, "vigilante", tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr)
+	vigilanteSender, err := NewSenderWithBabylonClient(ctx, "vigilante", tm.Config.Babylon0.RPCAddr, tm.Config.Babylon0.GRPCAddr)
 	if err != nil {
 		return err
 	}
 
-	fpmSender, err := NewSenderWithBabylonClient(ctx, "fpmsender", tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr)
+	fpmSender, err := NewSenderWithBabylonClient(ctx, "fpmsender", tm.Config.Babylon0.RPCAddr, tm.Config.Babylon0.GRPCAddr)
 	if err != nil {
 		return err
 	}
@@ -83,7 +83,7 @@ func startHarness(cmdCtx context.Context, cfg config.Config) error {
 
 	var stakers []*BTCStaker
 	for i := 0; i < numStakers; i++ {
-		stakerSender, err := NewSenderWithBabylonClient(ctx, fmt.Sprintf("staker-%d", i), tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr)
+		stakerSender, err := NewSenderWithBabylonClient(ctx, fmt.Sprintf("staker-%d", i), tm.Config.Babylon1.RPCAddr, tm.Config.Babylon1.GRPCAddr)
 		if err != nil {
 			return err
 		}
@@ -106,7 +106,7 @@ func startHarness(cmdCtx context.Context, cfg config.Config) error {
 
 	go printStatsForever(ctx, tm, stopChan, cfg)
 
-	covenantSender, err := NewSenderWithBabylonClient(ctx, "covenant", tm.Config.Babylon.RPCAddr, tm.Config.Babylon.GRPCAddr)
+	covenantSender, err := NewSenderWithBabylonClient(ctx, "covenant", tm.Config.Babylon0.RPCAddr, tm.Config.Babylon0.GRPCAddr)
 	if err != nil {
 		return err
 	}
@@ -150,7 +150,7 @@ func printStatsForever(ctx context.Context, tm *TestManager, stopChan chan struc
 				close(stopChan)
 			}
 
-			mem, err := tm.manger.MemoryUsage(ctx, "babylond")
+			mem, err := tm.manger.MemoryUsage(ctx, "babylond-node0")
 			if err != nil {
 				fmt.Printf("err getting memory usage for bbn node %v\n", err)
 			}

harness/babylonclient.go

@@ -72,7 +72,7 @@ func New(
 	cp := provider.(*cosmos.CosmosProvider)
 	cp.PCfg.KeyDirectory = cfg.KeyDirectory
 
-	// Create tmp Babylon app to retrieve and register codecs
+	// Create tmp Babylon0 app to retrieve and register codecs
 	// Need to override this manually as otherwise option from config is ignored
 	encCfg := bbn.GetEncodingConfig()
 	cp.Cdc = cosmos.Codec{
@@ -85,7 +85,7 @@ func New(
 	// initialise Cosmos provider
 	// NOTE: this will create a RPC client. The RPC client will be used for
 	// submitting txs and making ad hoc queries. It won't create WebSocket
-	// connection with Babylon node
+	// connection with Babylon0 node
 	err = cp.Init(ctx)
 	if err != nil {
 		return nil, err

harness/bitcoindsetup.go

@@ -38,13 +38,13 @@ func NewBitcoindHandler(manager *container.Manager) *BitcoindTestHandler {
 	}
 }
 
-func (h *BitcoindTestHandler) Start(ctx context.Context, containerName string) (*dockertest.Resource, error) {
+func (h *BitcoindTestHandler) Start(ctx context.Context) (*dockertest.Resource, error) {
 	tempPath, err := os.MkdirTemp("", "bbn-benchmark-test-*")
 	if err != nil {
 		return nil, err
 	}
 
-	bitcoinResource, err := h.m.RunBitcoindResource(containerName, tempPath)
+	bitcoinResource, err := h.m.RunBitcoindResource(tempPath)
 	if err != nil {
 		return nil, err
 	}

harness/finalityprovider.go

@@ -87,15 +87,15 @@ func (fpm *FinalityProviderManager) Initialize(ctx context.Context, numPubRand u
 
 	fpis := make([]*FinalityProviderInstance, fpm.fpCount)
 
-	res, err := fpm.tm.BabylonClient.CurrentEpoch()
+	res, err := fpm.tm.BabylonClientNode0.CurrentEpoch()
 	if err != nil {
 		return err
 	}
 
 	for i := 0; i < fpm.fpCount; i++ {
 		keyName := lib.GenRandomHexStr(r, 10)
 
-		finalitySender, err := NewSenderWithBabylonClient(ctx, keyName, fpm.tm.Config.Babylon.RPCAddr, fpm.tm.Config.Babylon.GRPCAddr)
+		finalitySender, err := NewSenderWithBabylonClient(ctx, keyName, fpm.tm.Config.Babylon0.RPCAddr, fpm.tm.Config.Babylon0.GRPCAddr)
 		if err != nil {
 			return err
 		}
@@ -385,7 +385,7 @@ func (fpi *FinalityProviderInstance) signFinalitySig(b *BlockInfo, btcPk *bbntyp
 	return bbntypes.NewSchnorrEOTSSigFromModNScalar(sig), nil
 }
 
-// SubmitFinalitySig submits the finality signature via a MsgAddVote to Babylon
+// SubmitFinalitySig submits the finality signature via a MsgAddVote to Babylon0
 func (fpi *FinalityProviderInstance) SubmitFinalitySig(
 	ctx context.Context,
 	fpPk *btcec.PublicKey,
@@ -427,7 +427,7 @@ func (fpi *FinalityProviderInstance) SubmitFinalitySig(
 
 func (fpm *FinalityProviderManager) waitUntilFinalized(ctx context.Context, epoch uint64) error {
 	err := lib.Eventually(ctx, func() bool {
-		lastFinalizedCkpt, err := fpm.tm.BabylonClient.LatestEpochFromStatus(ckpttypes.Finalized)
+		lastFinalizedCkpt, err := fpm.tm.BabylonClientNode0.LatestEpochFromStatus(ckpttypes.Finalized)
 		if err != nil {
 			return false
 		}