feat/parallel pruning

baseapp/abci.go

@@ -444,8 +444,15 @@ func (app *BaseApp) Commit() abci.ResponseCommit {
 	// Write the DeliverTx state into branched storage and commit the MultiStore.
 	// The write to the DeliverTx state writes all state transitions to the root
 	// MultiStore (app.cms) so when Commit() is called is persists those values.
+	startWriteTime := time.Now()
 	app.deliverState.ms.Write()
+	app.logger.Debug("ms.Write", "height", header.Height, "cost of time",
+		fmt.Sprintf(" %.2fms", float64(time.Since(startWriteTime).Microseconds())/1000))
+
+	startCommitTime := time.Now()
 	commitID := app.cms.Commit()
+	app.logger.Debug("cms.Commit", "height", header.Height, "cost of time",
+		fmt.Sprintf(" %.2fms", float64(time.Since(startCommitTime).Microseconds())/1000))
 
 	res := abci.ResponseCommit{
 		Data:         commitID.Hash,

go.mod

@@ -203,3 +203,5 @@ retract (
 	// do not use
 	v0.43.0
 )
+
+replace github.com/cosmos/iavl => ../iavl

store/iavl/store.go

@@ -11,7 +11,7 @@
 	"github.com/cometbft/cometbft/libs/log"
 	tmcrypto "github.com/cometbft/cometbft/proto/tendermint/crypto"
 	ics23 "github.com/confio/ics23/go"
 	"github.com/cosmos/iavl"
 
 	"github.com/cosmos/cosmos-sdk/store/cachekv"
 	pruningtypes "github.com/cosmos/cosmos-sdk/store/pruning/types"
@@ -52,7 +52,7 @@
 // provided DB. An error is returned if the version fails to load, or if called with a positive
 // version on an empty tree.
 func LoadStoreWithInitialVersion(db dbm.DB, logger log.Logger, key types.StoreKey, id types.CommitID, lazyLoading bool, initialVersion uint64, cacheSize int, disableFastNode bool) (types.CommitKVStore, error) {
-	tree, err := iavl.NewMutableTreeWithOpts(db, cacheSize, &iavl.Options{InitialVersion: initialVersion}, disableFastNode)
+	tree, err := iavl.NewMutableTreeWithOpts(db, cacheSize, &iavl.Options{InitialVersion: initialVersion}, disableFastNode, key.Name())
 	if err != nil {
 		return nil, err
 	}
@@ -120,7 +120,7 @@
 	}
 
 	return &Store{
 		tree: &immutableTree{iTree},
 	}, nil
 }
 
@@ -274,11 +274,11 @@
 	if err != nil {
 		return nil, fmt.Errorf("iavl export failed for version %v: %w", version, err)
 	}
 	tree, ok := istore.tree.(*immutableTree)
 	if !ok || tree == nil {
 		return nil, fmt.Errorf("iavl export failed: unable to fetch tree for version %v", version)
 	}
 	return tree.Export()
 }
 
 // Import imports an IAVL tree at the given version, returning an iavl.Importer for importing.

store/rootmulti/store.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"time"
 
 	dbm "github.com/cometbft/cometbft-db"
 	abci "github.com/cometbft/cometbft/abci/types"
@@ -627,6 +628,9 @@ func (rs *Store) PruneStores(clearPruningManager bool, pruningHeights []int64) (
 
 	rs.logger.Debug("pruning store", "heights", pruningHeights)
 
+	startPruneTime := time.Now()
+	var wg sync.WaitGroup
+	errCh := make(chan error)
 	for key, store := range rs.stores {
 		rs.logger.Debug("pruning store", "key", key) // Also log store.name (a private variable)?
 
@@ -638,16 +642,44 @@ func (rs *Store) PruneStores(clearPruningManager bool, pruningHeights []int64) (
 
 		store = rs.GetCommitKVStore(key)
 
-		err := store.(*iavl.Store).DeleteVersions(pruningHeights...)
-		if err == nil {
-			continue
-		}
+		wg.Add(1)
+		go func(key types.StoreKey, store types.CommitStore) {
+			rs.logger.Debug("DeleteVersions", "store", key.Name())
+			startDeleteTime := time.Now()
+			err := store.(*iavl.Store).DeleteVersions(pruningHeights...)
+			if err == nil {
+				rs.logger.Debug("DeleteVersions", "store", key.Name(), "cost of time",
+					fmt.Sprintf("%.2fms", float64(time.Since(startDeleteTime).Microseconds())/1000))
+				wg.Done()
+				return
+			}
 
-		if errCause := errors.Cause(err); errCause != nil && errCause != iavltree.ErrVersionDoesNotExist {
-			return err
-		}
+			if errCause := errors.Cause(err); errCause != nil && errCause != iavltree.ErrVersionDoesNotExist {
+				errCh <- err
+				return
+			}
+			wg.Done()
+		}(key, store)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		done <- struct{}{}
+	}()
+
+	select {
+	case <-done:
+		rs.logger.Debug(
+			"PruneStores",
+			"from", pruningHeights[0],
+			"to", pruningHeights[len(pruningHeights)-1],
+			"cost of time", fmt.Sprintf("%.2fms", float64(time.Since(startPruneTime).Microseconds())/1000),
+		)
+		return nil
+	case err := <-errCh:
+		return err
 	}
-	return nil
 }
 
 // getStoreByName performs a lookup of a StoreKey given a store name typically