From 08ec9ab44d572bf68e9542a4b1075d7a03ffbbbf Mon Sep 17 00:00:00 2001 From: Cirrus Gai Date: Wed, 11 Dec 2024 22:24:29 +0800 Subject: [PATCH] chore: add multiple fpd connecting to one eotsd in e2e tests (#216) Closes #206 --- CHANGELOG.md | 1 + docs/finality-provider-operation.md | 6 +- finality-provider/service/app.go | 7 +- finality-provider/service/fp_instance.go | 4 + itest/e2e_test.go | 56 +++++--- itest/eotsmanager_handler.go | 7 +- itest/test_manager.go | 173 ++++++++++++----------- 7 files changed, 146 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2543f9f..496dabbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) * [#208](https://github.com/babylonlabs-io/finality-provider/pull/208) Remove sync fp status loop * [#211](https://github.com/babylonlabs-io/finality-provider/pull/211) Clean up unused cmd * [#214](https://github.com/babylonlabs-io/finality-provider/pull/214) Gradual benchmark +* [#216](https://github.com/babylonlabs-io/finality-provider/pull/216) Add multiple fpd connecting to one eotsd in e2e tests ## v0.13.1 diff --git a/docs/finality-provider-operation.md b/docs/finality-provider-operation.md index b2174899..0eced53b 100644 --- a/docs/finality-provider-operation.md +++ b/docs/finality-provider-operation.md @@ -474,10 +474,12 @@ as a centralized key management system. When starting a finality provider instan you specify which EOTS key to use through the `--eots-pk` flag. This allows you to run different finality provider instances using different keys from the same EOTS Manager. Note that someone having access to your EOTS Manager -RPC will have access to all the EOTS keys held within it +RPC will have access to all the EOTS keys held within it. For example, after registering a finality provider, you can start its daemon by -providing the EOTS public key `fpd start --eots-pk ` +providing the EOTS public key `fpd start --eots-pk `. +Note that a single finality provider daemon can only run with a single +finality provider instance at a time. ## 5. Finality Provider Operations diff --git a/finality-provider/service/app.go b/finality-provider/service/app.go index c9dd81c9..84f5cff6 100644 --- a/finality-provider/service/app.go +++ b/finality-provider/service/app.go @@ -304,7 +304,7 @@ func (app *FinalityProviderApp) Stop() error { app.logger.Info("stopping finality provider", zap.String("pk", pkHex)) if err := app.fpIns.Stop(); err != nil { - stopErr = err + stopErr = fmt.Errorf("failed to close the fp instance: %w", err) return } @@ -313,7 +313,7 @@ func (app *FinalityProviderApp) Stop() error { app.logger.Debug("Stopping EOTS manager") if err := app.eotsManager.Close(); err != nil { - stopErr = err + stopErr = fmt.Errorf("failed to close the EOTS manager: %w", err) return } @@ -473,6 +473,9 @@ func (app *FinalityProviderApp) startFinalityProviderInstance( } app.fpIns = fpIns + } else if !pk.Equals(app.fpIns.btcPk) { + return fmt.Errorf("the finality provider daemon is already bonded with the finality provider %s,"+ + "please restart the daemon to switch to another instance", app.fpIns.btcPk.MarshalHex()) } return app.fpIns.Start() diff --git a/finality-provider/service/fp_instance.go b/finality-provider/service/fp_instance.go index 1c688f4a..b30cf9d8 100644 --- a/finality-provider/service/fp_instance.go +++ b/finality-provider/service/fp_instance.go @@ -157,6 +157,10 @@ func (fp *FinalityProviderInstance) Stop() error { return nil } +func (fp *FinalityProviderInstance) GetConfig() *fpcfg.Config { + return fp.cfg +} + func (fp *FinalityProviderInstance) IsRunning() bool { return fp.isStarted.Load() } diff --git a/itest/e2e_test.go b/itest/e2e_test.go index 93b0b4e5..08525980 100644 --- a/itest/e2e_test.go +++ b/itest/e2e_test.go @@ -13,6 +13,7 @@ import ( "time" bbntypes "github.com/babylonlabs-io/babylon/types" + bstypes "github.com/babylonlabs-io/babylon/x/btcstaking/types" sdkmath "cosmossdk.io/math" "github.com/babylonlabs-io/babylon/testutil/datagen" @@ -32,29 +33,38 @@ var ( // creation -> registration -> randomness commitment -> // activation with BTC delegation and Covenant sig -> // vote submission -> block finalization +// The test runs 2 finality providers connecting to +// a single EOTS manager func TestFinalityProviderLifeCycle(t *testing.T) { - tm, fpIns := StartManagerWithFinalityProvider(t) + n := 2 + tm, fps := StartManagerWithFinalityProvider(t, n) defer tm.Stop(t) // check the public randomness is committed - tm.WaitForFpPubRandTimestamped(t, fpIns) + tm.WaitForFpPubRandTimestamped(t, fps[0]) // send a BTC delegation - _ = tm.InsertBTCDelegation(t, []*btcec.PublicKey{fpIns.GetBtcPk()}, stakingTime, stakingAmount) + for _, fp := range fps { + _ = tm.InsertBTCDelegation(t, []*btcec.PublicKey{fp.GetBtcPk()}, stakingTime, stakingAmount) + } // check the BTC delegation is pending - delsResp := tm.WaitForNPendingDels(t, 1) - del, err := ParseRespBTCDelToBTCDel(delsResp[0]) - require.NoError(t, err) - - // send covenant sigs - tm.InsertCovenantSigForDelegation(t, del) + delsResp := tm.WaitForNPendingDels(t, n) + var dels []*bstypes.BTCDelegation + for _, delResp := range delsResp { + del, err := ParseRespBTCDelToBTCDel(delResp) + require.NoError(t, err) + dels = append(dels, del) + // send covenant sigs + tm.InsertCovenantSigForDelegation(t, del) + } // check the BTC delegation is active - _ = tm.WaitForNActiveDels(t, 1) + _ = tm.WaitForNActiveDels(t, n) // check the last voted block is finalized - lastVotedHeight := tm.WaitForFpVoteCast(t, fpIns) + lastVotedHeight := tm.WaitForFpVoteCast(t, fps[0]) + tm.CheckBlockFinalization(t, lastVotedHeight, 1) t.Logf("the block at height %v is finalized", lastVotedHeight) } @@ -63,9 +73,11 @@ func TestFinalityProviderLifeCycle(t *testing.T) { // sends a finality vote over a conflicting block // in this case, the BTC private key should be extracted by Babylon func TestDoubleSigning(t *testing.T) { - tm, fpIns := StartManagerWithFinalityProvider(t) + tm, fps := StartManagerWithFinalityProvider(t, 1) defer tm.Stop(t) + fpIns := fps[0] + // check the public randomness is committed tm.WaitForFpPubRandTimestamped(t, fpIns) @@ -121,9 +133,11 @@ func TestDoubleSigning(t *testing.T) { // TestCatchingUp tests if a fp can catch up after restarted func TestCatchingUp(t *testing.T) { - tm, fpIns := StartManagerWithFinalityProvider(t) + tm, fps := StartManagerWithFinalityProvider(t, 1) defer tm.Stop(t) + fpIns := fps[0] + // check the public randomness is committed tm.WaitForFpPubRandTimestamped(t, fpIns) @@ -168,9 +182,11 @@ func TestCatchingUp(t *testing.T) { } func TestFinalityProviderEditCmd(t *testing.T) { - tm, fpIns := StartManagerWithFinalityProvider(t) + tm, fps := StartManagerWithFinalityProvider(t, 1) defer tm.Stop(t) + fpIns := fps[0] + cmd := daemon.CommandEditFinalityDescription() const ( @@ -192,7 +208,7 @@ func TestFinalityProviderEditCmd(t *testing.T) { args := []string{ fpIns.GetBtcPkHex(), - "--" + fpdDaemonAddressFlag, tm.FpConfig.RPCListener, + "--" + fpdDaemonAddressFlag, fpIns.GetConfig().RPCListener, "--" + monikerFlag, moniker, "--" + websiteFlag, website, "--" + securityContactFlag, securityContact, @@ -223,7 +239,7 @@ func TestFinalityProviderEditCmd(t *testing.T) { moniker = "test2-moniker" args = []string{ fpIns.GetBtcPkHex(), - "--" + fpdDaemonAddressFlag, tm.FpConfig.RPCListener, + "--" + fpdDaemonAddressFlag, fpIns.GetConfig().RPCListener, "--" + monikerFlag, moniker, } @@ -248,9 +264,11 @@ func TestFinalityProviderEditCmd(t *testing.T) { } func TestFinalityProviderCreateCmd(t *testing.T) { - tm, _ := StartManagerWithFinalityProvider(t) + tm, fps := StartManagerWithFinalityProvider(t, 1) defer tm.Stop(t) + fpIns := fps[0] + cmd := daemon.CommandCreateFP() eotsKeyName := "eots-key-2" @@ -271,7 +289,7 @@ func TestFinalityProviderCreateCmd(t *testing.T) { Details string `json:"details"` EotsPK string `json:"eotsPK"` }{ - KeyName: tm.FpConfig.BabylonConfig.Key, + KeyName: fpIns.GetConfig().BabylonConfig.Key, ChainID: testChainID, Passphrase: passphrase, CommissionRate: "0.10", @@ -297,7 +315,7 @@ func TestFinalityProviderCreateCmd(t *testing.T) { cmd.SetArgs([]string{ "--from-file=" + file.Name(), - "--daemon-address=" + tm.FpConfig.RPCListener, + "--daemon-address=" + fpIns.GetConfig().RPCListener, }) // Run the command diff --git a/itest/eotsmanager_handler.go b/itest/eotsmanager_handler.go index 18e13148..5bc3f039 100644 --- a/itest/eotsmanager_handler.go +++ b/itest/eotsmanager_handler.go @@ -16,12 +16,16 @@ type EOTSServerHandler struct { t *testing.T interceptor *signal.Interceptor eotsServer *service.Server + Cfg *config.Config } func NewEOTSServerHandler(t *testing.T, cfg *config.Config, eotsHomeDir string, shutdownInterceptor signal.Interceptor) *EOTSServerHandler { dbBackend, err := cfg.DatabaseConfig.GetDBBackend() require.NoError(t, err) - logger := zap.NewNop() + loggerConfig := zap.NewDevelopmentConfig() + loggerConfig.Level = zap.NewAtomicLevelAt(zap.InfoLevel) + logger, err := loggerConfig.Build() + require.NoError(t, err) eotsManager, err := eotsmanager.NewLocalEOTSManager(eotsHomeDir, cfg.KeyringBackend, dbBackend, logger) require.NoError(t, err) @@ -31,6 +35,7 @@ func NewEOTSServerHandler(t *testing.T, cfg *config.Config, eotsHomeDir string, t: t, eotsServer: eotsServer, interceptor: &shutdownInterceptor, + Cfg: cfg, } } diff --git a/itest/test_manager.go b/itest/test_manager.go index 4d93ae6d..a2740a00 100644 --- a/itest/test_manager.go +++ b/itest/test_manager.go @@ -6,12 +6,12 @@ import ( "math/rand" "os" "path/filepath" - "strings" "sync" "testing" "time" "github.com/lightningnetwork/lnd/signal" + "github.com/ory/dockertest/v3" "github.com/babylonlabs-io/finality-provider/itest/container" "github.com/babylonlabs-io/finality-provider/testutil" @@ -60,14 +60,16 @@ type TestManager struct { Wg sync.WaitGroup EOTSServerHandler *EOTSServerHandler FpConfig *fpcfg.Config - EOTSConfig *eotsconfig.Config - Fpa *service.FinalityProviderApp + Fps []*service.FinalityProviderApp EOTSClient *client.EOTSManagerGRpcClient BBNClient *fpcc.BabylonController StakingParams *types.StakingParams CovenantPrivKeys []*btcec.PrivateKey baseDir string manager *container.Manager + logger *zap.Logger + interceptor signal.Interceptor + babylond *dockertest.Resource } type TestDelegationData struct { @@ -92,7 +94,10 @@ func StartManager(t *testing.T) *TestManager { testDir, err := tempDir(t, "fp-e2e-test-*") require.NoError(t, err) - logger := zap.NewNop() + loggerConfig := zap.NewDevelopmentConfig() + loggerConfig.Level = zap.NewAtomicLevelAt(zap.ErrorLevel) + logger, err := loggerConfig.Build() + require.NoError(t, err) // 1. generate covenant committee covenantQuorum := 2 @@ -105,8 +110,8 @@ func StartManager(t *testing.T) *TestManager { babylond, err := manager.RunBabylondResource(t, babylonDir, covenantQuorum, covenantPubKeys) require.NoError(t, err) - fpHomeDir := filepath.Join(testDir, "fp-home") keyDir := filepath.Join(babylonDir, "node0", "babylond") + fpHomeDir := filepath.Join(testDir, "fp-home") cfg := defaultFpConfig(keyDir, fpHomeDir) // update ports with the dynamically allocated ones from docker cfg.BabylonConfig.RPCAddr = fmt.Sprintf("http://localhost:%s", babylond.GetPort("26657/tcp")) @@ -118,16 +123,6 @@ func StartManager(t *testing.T) *TestManager { return err == nil }, 5*time.Second, eventuallyPollTime) - var currentEpoch uint64 - require.Eventually(t, func() bool { - currentEpoch, err = bc.QueryCurrentEpoch() - if err != nil { - return false - } - return currentEpoch > 0 - }, eventuallyWaitTimeOut, eventuallyPollTime) - t.Logf("current epoch is %d", currentEpoch) - shutdownInterceptor, err := signal.Intercept() require.NoError(t, err) @@ -141,30 +136,17 @@ func StartManager(t *testing.T) *TestManager { eotsCli, err := client.NewEOTSManagerGRpcClient(eotsCfg.RPCListener) require.NoError(t, err) - // 4. prepare finality-provider - fpdb, err := cfg.DatabaseConfig.GetDBBackend() - require.NoError(t, err) - fpApp, err := service.NewFinalityProviderApp(cfg, bc, eotsCli, fpdb, logger) - require.NoError(t, err) - err = fpApp.Start() - require.NoError(t, err) - - fpServer := service.NewFinalityProviderServer(cfg, logger, fpApp, fpdb, shutdownInterceptor) - go func() { - err = fpServer.RunUntilShutdown() - require.NoError(t, err) - }() - tm := &TestManager{ EOTSServerHandler: eh, FpConfig: cfg, - EOTSConfig: eotsCfg, - Fpa: fpApp, EOTSClient: eotsCli, BBNClient: bc, CovenantPrivKeys: covenantPrivKeys, baseDir: testDir, manager: manager, + logger: logger, + interceptor: shutdownInterceptor, + babylond: babylond, } tm.WaitForServicesStart(t) @@ -172,6 +154,67 @@ func StartManager(t *testing.T) *TestManager { return tm } +func (tm *TestManager) AddFinalityProvider(t *testing.T) *service.FinalityProviderInstance { + r := rand.New(rand.NewSource(time.Now().Unix())) + + // create eots key + eotsKeyName := fmt.Sprintf("eots-key-%s", datagen.GenRandomHexStr(r, 4)) + eotsPkBz, err := tm.EOTSClient.CreateKey(eotsKeyName, passphrase, hdPath) + require.NoError(t, err) + eotsPk, err := bbntypes.NewBIP340PubKey(eotsPkBz) + require.NoError(t, err) + + // create fp babylon key + fpKeyName := fmt.Sprintf("fp-key-%s", datagen.GenRandomHexStr(r, 4)) + fpHomeDir := filepath.Join(tm.baseDir, fmt.Sprintf("fp-%s", datagen.GenRandomHexStr(r, 4))) + cfg := defaultFpConfig(tm.baseDir, fpHomeDir) + cfg.BabylonConfig.Key = fpKeyName + cfg.BabylonConfig.RPCAddr = fmt.Sprintf("http://localhost:%s", tm.babylond.GetPort("26657/tcp")) + cfg.BabylonConfig.GRPCAddr = fmt.Sprintf("https://localhost:%s", tm.babylond.GetPort("9090/tcp")) + fpBbnKeyInfo, err := testutil.CreateChainKey(cfg.BabylonConfig.KeyDirectory, cfg.BabylonConfig.ChainID, cfg.BabylonConfig.Key, cfg.BabylonConfig.KeyringBackend, passphrase, hdPath, "") + require.NoError(t, err) + + // add some funds for new fp pay for fees '-' + _, _, err = tm.manager.BabylondTxBankSend(t, fpBbnKeyInfo.AccAddress.String(), "1000000ubbn", "node0") + require.NoError(t, err) + + // create and start finality provider app + eotsCli, err := client.NewEOTSManagerGRpcClient(tm.EOTSServerHandler.Cfg.RPCListener) + require.NoError(t, err) + cc, err := fpcc.NewClientController(cfg.ChainType, cfg.BabylonConfig, &cfg.BTCNetParams, tm.logger) + require.NoError(t, err) + fpdb, err := cfg.DatabaseConfig.GetDBBackend() + require.NoError(t, err) + fpApp, err := service.NewFinalityProviderApp(cfg, cc, eotsCli, fpdb, tm.logger) + require.NoError(t, err) + err = fpApp.Start() + require.NoError(t, err) + + // create and register the finality provider + commission := sdkmath.LegacyZeroDec() + desc := newDescription(testMoniker) + _, err = fpApp.CreateFinalityProvider(cfg.BabylonConfig.Key, testChainID, passphrase, eotsPk, desc, &commission) + require.NoError(t, err) + err = fpApp.StartFinalityProvider(eotsPk, passphrase) + require.NoError(t, err) + + cfg.RPCListener = fmt.Sprintf("127.0.0.1:%d", testutil.AllocateUniquePort(t)) + cfg.Metrics.Port = testutil.AllocateUniquePort(t) + + fpServer := service.NewFinalityProviderServer(cfg, tm.logger, fpApp, fpdb, tm.interceptor) + go func() { + err = fpServer.RunUntilShutdown() + require.NoError(t, err) + }() + + tm.Fps = append(tm.Fps, fpApp) + + fpIns, err := fpApp.GetFinalityProviderInstance() + require.NoError(t, err) + + return fpIns +} + func (tm *TestManager) WaitForServicesStart(t *testing.T) { // wait for Babylon node starts require.Eventually(t, func() bool { @@ -186,33 +229,16 @@ func (tm *TestManager) WaitForServicesStart(t *testing.T) { t.Logf("Babylon node is started") } -func StartManagerWithFinalityProvider(t *testing.T) (*TestManager, *service.FinalityProviderInstance) { +func StartManagerWithFinalityProvider(t *testing.T, n int) (*TestManager, []*service.FinalityProviderInstance) { tm := StartManager(t) - app := tm.Fpa - cfg := app.GetConfig() - - commission := sdkmath.LegacyZeroDec() - desc := newDescription(testMoniker) - - eotsKeyName := "eots-key" - eotsPkBz, err := tm.EOTSClient.CreateKey(eotsKeyName, passphrase, hdPath) - require.NoError(t, err) - eotsPk, err := bbntypes.NewBIP340PubKey(eotsPkBz) - require.NoError(t, err) - res, err := app.CreateFinalityProvider(cfg.BabylonConfig.Key, testChainID, passphrase, eotsPk, desc, &commission) - require.NoError(t, err) - fpPk, err := bbntypes.NewBIP340PubKeyFromHex(res.FpInfo.BtcPkHex) - require.NoError(t, err) - err = app.StartFinalityProvider(fpPk, passphrase) - require.NoError(t, err) - fpIns, err := app.GetFinalityProviderInstance() - require.NoError(t, err) - require.True(t, fpIns.IsRunning()) - require.NoError(t, err) + var runningFps []*service.FinalityProviderInstance + for i := 0; i < n; i++ { + fpIns := tm.AddFinalityProvider(t) + runningFps = append(runningFps, fpIns) + } // check finality providers on Babylon side - var fpRes *bstypes.FinalityProviderResponse require.Eventually(t, func() bool { fps, err := tm.BBNClient.QueryFinalityProviders() if err != nil { @@ -220,35 +246,24 @@ func StartManagerWithFinalityProvider(t *testing.T) (*TestManager, *service.Fina return false } - if len(fps) != 1 { - return false - } - - fpRes = fps[0] - - if !strings.Contains(fpRes.Description.Moniker, testMoniker) { - return false - } - if !fpRes.Commission.Equal(sdkmath.LegacyZeroDec()) { - return false - } - - return true + return len(fps) == n }, eventuallyWaitTimeOut, eventuallyPollTime) - t.Logf("the test manager is running with a finality provider") + t.Logf("the test manager is running with %d finality provider", n) - return tm, fpIns + return tm, runningFps } func (tm *TestManager) Stop(t *testing.T) { - err := tm.Fpa.Stop() - require.NoError(t, err) - err = tm.manager.ClearResources() + for _, fp := range tm.Fps { + err := fp.Stop() + require.NoError(t, err) + } + err := tm.manager.ClearResources() require.NoError(t, err) err = os.RemoveAll(tm.baseDir) require.NoError(t, err) - tm.EOTSServerHandler.Stop() + tm.interceptor.RequestShutdown() } func (tm *TestManager) WaitForFpPubRandTimestamped(t *testing.T, fpIns *service.FinalityProviderInstance) { @@ -390,15 +405,6 @@ func (tm *TestManager) WaitForNFinalizedBlocks(t *testing.T, n int) []*types.Blo return blocks } -func (tm *TestManager) WaitForFpShutDown(t *testing.T) { - require.Eventually(t, func() bool { - _, err := tm.Fpa.GetFinalityProviderInstance() - return err != nil - }, eventuallyWaitTimeOut, eventuallyPollTime) - - t.Logf("the finality-provider instance is shutdown") -} - func (tm *TestManager) StopAndRestartFpAfterNBlocks(t *testing.T, n int, fpIns *service.FinalityProviderInstance) { blockBeforeStop, err := tm.BBNClient.QueryBestBlock() require.NoError(t, err) @@ -416,7 +422,6 @@ func (tm *TestManager) StopAndRestartFpAfterNBlocks(t *testing.T, n int, fpIns * t.Log("restarting the finality-provider instance") - tm.FpConfig.PollerConfig.AutoChainScanningMode = true err = fpIns.Start() require.NoError(t, err) }