Trie storage statistics component

common/interface.go

@@ -56,6 +56,7 @@ type Trie interface {
 	VerifyProof(rootHash []byte, key []byte, proof [][]byte) (bool, error)
 	GetStorageManager() StorageManager
 	IsMigratedToLatestVersion() (bool, error)
+	GetStorageStats() string
 	Close() error
 	IsInterfaceNil() bool
 }
@@ -120,6 +121,7 @@ type StorageManager interface {
 type TrieStorageInteractor interface {
 	BaseStorer
 	GetIdentifier() string
+	GetStateStatsHandler() StateStatisticsHandler
 }
 
 // BaseStorer define the base methods needed for a storer
@@ -218,6 +220,19 @@ type TriesStatisticsCollector interface {
 	GetNumNodes() uint64
 }
 
+// StateStatisticsHandler defines the behaviour of a storage statistics handler
+type StateStatisticsHandler interface {
+	Reset()
+	IncrCacheOp()
+	CacheOp() uint64
+	IncrPersisterOp()
+	PersisterOp() uint64
+	IncrTrieOp()
+	TrieOp() uint64
+	ToString() string
+	IsInterfaceNil() bool
+}
+
 // ProcessStatusHandler defines the behavior of a component able to hold the current status of the node and
 // able to tell if the node is idle or processing/committing a block
 type ProcessStatusHandler interface {

common/statistics/disabled/stateStatistics.go

@@ -0,0 +1,49 @@
+package disabled
+
+type stateStatistics struct{}
+
+// NewStateStatistics will create a new disabled statistics component
+func NewStateStatistics() *stateStatistics {
+	return &stateStatistics{}
+}
+
+// Reset does nothing
+func (s *stateStatistics) Reset() {
+}
+
+// IncrCacheOp does nothing
+func (s *stateStatistics) IncrCacheOp() {
+}
+
+// CacheOp returns zero
+func (s *stateStatistics) CacheOp() uint64 {
+	return 0
+}
+
+// IncrPersisterOp does nothing
+func (s *stateStatistics) IncrPersisterOp() {
+}
+
+// PersisterOp returns zero
+func (s *stateStatistics) PersisterOp() uint64 {
+	return 0
+}
+
+// IncrTrieOp does nothing
+func (s *stateStatistics) IncrTrieOp() {
+}
+
+// TrieOp returns zero
+func (s *stateStatistics) TrieOp() uint64 {
+	return 0
+}
+
+// ToString returns empty string
+func (s *stateStatistics) ToString() string {
+	return ""
+}
+
+// IsInterfaceNil returns true if there is no value under the interface
+func (s *stateStatistics) IsInterfaceNil() bool {
+	return s == nil
+}

common/statistics/disabled/stateStatistics_test.go

@@ -0,0 +1 @@
+package disabled

common/statistics/errors.go

@@ -9,3 +9,6 @@ var ErrNilNetworkStatisticsProvider = errors.New("nil network statistics provide
 
 // ErrInvalidRefreshIntervalValue signals that an invalid value for the refresh interval was provided
 var ErrInvalidRefreshIntervalValue = errors.New("invalid refresh interval value")
+
+// ErrNilStateStatsHandler signals that a nil state statistics handler was provided
+var ErrNilStateStatsHandler = errors.New("nil state statistics handler")

common/statistics/stateStatistics.go

@@ -0,0 +1,71 @@
+package statistics
+
+import (
+	"fmt"
+	"strings"
+	"sync/atomic"
+)
+
+type stateStatistics struct {
+	numCache     uint64
+	numPersister uint64
+	numTrie      uint64
+}
+
+// NewStateStatistics returns a structure able to collect statistics for state
+func NewStateStatistics() *stateStatistics {
+	return &stateStatistics{}
+}
+
+// Reset will reset the contained values to 0
+func (ss *stateStatistics) Reset() {
+	atomic.StoreUint64(&ss.numCache, 0)
+	atomic.StoreUint64(&ss.numPersister, 0)
+	atomic.StoreUint64(&ss.numTrie, 0)
+}
+
+// IncrCacheOp will increment cache counter
+func (ss *stateStatistics) IncrCacheOp() {
+	atomic.AddUint64(&ss.numCache, 1)
+}
+
+// CacheOp returns the number of cached operations
+func (ss *stateStatistics) CacheOp() uint64 {
+	return atomic.LoadUint64(&ss.numCache)
+}
+
+// IncrPersisterOp will increment persister counter
+func (ss *stateStatistics) IncrPersisterOp() {
+	atomic.AddUint64(&ss.numPersister, 1)
+}
+
+// PersisterOp returns the number of persister operations
+func (ss *stateStatistics) PersisterOp() uint64 {
+	return atomic.LoadUint64(&ss.numPersister)
+}
+
+// IncrTrieOp will increment trie counter
+func (ss *stateStatistics) IncrTrieOp() {
+	atomic.AddUint64(&ss.numTrie, 1)
+}
+
+// TrieOp returns the number of trie operations
+func (ss *stateStatistics) TrieOp() uint64 {
+	return atomic.LoadUint64(&ss.numTrie)
+}
+
+// ToString returns collected statistics as string
+func (ss *stateStatistics) ToString() string {
+	stats := make([]string, 0)
+
+	stats = append(stats, fmt.Sprintf("num cache op = %v", atomic.LoadUint64(&ss.numCache)))
+	stats = append(stats, fmt.Sprintf("num persister op = %v", atomic.LoadUint64(&ss.numPersister)))
+	stats = append(stats, fmt.Sprintf("max trie op = %v", atomic.LoadUint64(&ss.numTrie)))
+
+	return strings.Join(stats, " ")
+}
+
+// IsInterfaceNil returns true if there is no value under the interface
+func (ss *stateStatistics) IsInterfaceNil() bool {
+	return ss == nil
+}

common/statistics/stateStatistics_test.go

@@ -0,0 +1,121 @@
+package statistics
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/multiversx/mx-chain-core-go/core/check"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewStateStatistics_ShouldWork(t *testing.T) {
+	t.Parallel()
+
+	ss := NewStateStatistics()
+
+	assert.False(t, check.IfNil(ss))
+}
+
+func TestStateStatistics_Operations(t *testing.T) {
+	t.Parallel()
+
+	t.Run("trie operations", func(t *testing.T) {
+		t.Parallel()
+
+		ss := NewStateStatistics()
+
+		assert.Equal(t, uint64(0), ss.TrieOp())
+
+		ss.IncrTrieOp()
+		ss.IncrTrieOp()
+		assert.Equal(t, uint64(2), ss.TrieOp())
+
+		ss.IncrTrieOp()
+		assert.Equal(t, uint64(3), ss.TrieOp())
+
+		ss.Reset()
+		assert.Equal(t, uint64(0), ss.TrieOp())
+	})
+
+	t.Run("persister operations", func(t *testing.T) {
+		t.Parallel()
+
+		ss := NewStateStatistics()
+
+		assert.Equal(t, uint64(0), ss.PersisterOp())
+
+		ss.IncrPersisterOp()
+		ss.IncrPersisterOp()
+		assert.Equal(t, uint64(2), ss.PersisterOp())
+
+		ss.IncrPersisterOp()
+		assert.Equal(t, uint64(3), ss.PersisterOp())
+
+		ss.Reset()
+		assert.Equal(t, uint64(0), ss.PersisterOp())
+	})
+
+	t.Run("cache operations", func(t *testing.T) {
+		t.Parallel()
+
+		ss := NewStateStatistics()
+
+		assert.Equal(t, uint64(0), ss.CacheOp())
+
+		ss.IncrCacheOp()
+		ss.IncrCacheOp()
+		assert.Equal(t, uint64(2), ss.CacheOp())
+
+		ss.IncrCacheOp()
+		assert.Equal(t, uint64(3), ss.CacheOp())
+
+		ss.Reset()
+		assert.Equal(t, uint64(0), ss.CacheOp())
+	})
+}
+
+func TestStateStatistics_ConcurrenyOperations(t *testing.T) {
+	t.Parallel()
+
+	defer func() {
+		r := recover()
+		if r != nil {
+			assert.Fail(t, fmt.Sprintf("should have not panicked: %v", r))
+		}
+	}()
+
+	numIterations := 10000
+
+	ss := NewStateStatistics()
+
+	wg := sync.WaitGroup{}
+	wg.Add(numIterations)
+
+	for i := 0; i < numIterations; i++ {
+		go func(idx int) {
+			switch idx % 11 {
+			case 0:
+				ss.Reset()
+			case 1:
+				ss.IncrCacheOp()
+			case 2:
+				ss.IncrPersisterOp()
+			case 3:
+				ss.IncrTrieOp()
+			case 7:
+				_ = ss.CacheOp()
+			case 8:
+				_ = ss.PersisterOp()
+			case 9:
+				_ = ss.TrieOp()
+			case 10:
+				_ = ss.ToString()
+			}
+
+			wg.Done()
+		}(i)
+	}
+
+	wg.Wait()
+}

config/config.go

@@ -299,6 +299,7 @@ type StateTriesConfig struct {
 	PeerStatePruningEnabled     bool
 	MaxStateTrieLevelInMemory   uint
 	MaxPeerTrieLevelInMemory    uint
+	StateStatisticsEnabled      bool
 }
 
 // TrieStorageManagerConfig will hold config information about trie storage manager

dataRetriever/factory/storageRequestersContainer/args.go

@@ -28,4 +28,5 @@ type FactoryArgs struct {
 	ManualEpochStartNotifier dataRetriever.ManualEpochStartNotifier
 	ChanGracefullyClose      chan endProcess.ArgEndProcess
 	EnableEpochsHandler      common.EnableEpochsHandler
+	StateStatsHandler        common.StateStatisticsHandler
 }

dataRetriever/factory/storageRequestersContainer/baseRequestersContainerFactory.go

@@ -11,10 +11,11 @@ import (
 	"github.com/multiversx/mx-chain-core-go/marshal"
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/common/disabled"
+	"github.com/multiversx/mx-chain-go/common/statistics"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/dataRetriever"
 	disabledRequesters "github.com/multiversx/mx-chain-go/dataRetriever/requestHandlers/requesters/disabled"
-	"github.com/multiversx/mx-chain-go/dataRetriever/storageRequesters"
+	storagerequesters "github.com/multiversx/mx-chain-go/dataRetriever/storageRequesters"
 	"github.com/multiversx/mx-chain-go/errors"
 	"github.com/multiversx/mx-chain-go/p2p"
 	"github.com/multiversx/mx-chain-go/process/factory"
@@ -37,6 +38,7 @@ type baseRequestersContainerFactory struct {
 	dataPacker               dataRetriever.DataPacker
 	manualEpochStartNotifier dataRetriever.ManualEpochStartNotifier
 	enableEpochsHandler      common.EnableEpochsHandler
+	stateStatsHandler        common.StateStatisticsHandler
 	chanGracefullyClose      chan endProcess.ArgEndProcess
 	generalConfig            config.Config
 	shardIDForTries          uint32
@@ -76,6 +78,9 @@ func (brcf *baseRequestersContainerFactory) checkParams() error {
 	if check.IfNil(brcf.enableEpochsHandler) {
 		return errors.ErrNilEnableEpochsHandler
 	}
+	if check.IfNil(brcf.stateStatsHandler) {
+		return statistics.ErrNilStateStatsHandler
+	}
 
 	return nil
 }
@@ -239,6 +244,7 @@ func (brcf *baseRequestersContainerFactory) newImportDBTrieStorage(
 	checkpointsStorer storage.Storer,
 	storageIdentifier dataRetriever.UnitType,
 	handler common.EnableEpochsHandler,
+	stateStatsHandler common.StateStatisticsHandler,
 ) (common.StorageManager, dataRetriever.TrieDataGetter, error) {
 	pathManager, err := storageFactory.CreatePathManager(
 		storageFactory.ArgCreatePathManager{
@@ -271,6 +277,7 @@ func (brcf *baseRequestersContainerFactory) newImportDBTrieStorage(
 		IdleProvider:        disabled.NewProcessStatusHandler(),
 		Identifier:          storageIdentifier.String(),
 		EnableEpochsHandler: handler,
+		StatsCollector:      stateStatsHandler,
 	}
 	return trieFactoryInstance.Create(args)
 }

dataRetriever/factory/storageRequestersContainer/metaRequestersContainerFactory.go

@@ -6,7 +6,7 @@ import (
 	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-go/dataRetriever"
 	"github.com/multiversx/mx-chain-go/dataRetriever/factory/containers"
-	"github.com/multiversx/mx-chain-go/dataRetriever/storageRequesters"
+	storagerequesters "github.com/multiversx/mx-chain-go/dataRetriever/storageRequesters"
 	"github.com/multiversx/mx-chain-go/process/factory"
 )
 
@@ -38,6 +38,7 @@ func NewMetaRequestersContainerFactory(
 		workingDir:               args.WorkingDirectory,
 		snapshotsEnabled:         args.GeneralConfig.StateTriesConfig.SnapshotsEnabled,
 		enableEpochsHandler:      args.EnableEpochsHandler,
+		stateStatsHandler:        args.StateStatsHandler,
 	}
 
 	err := base.checkParams()
@@ -197,6 +198,7 @@ func (mrcf *metaRequestersContainerFactory) generateTrieNodesRequesters() error
 		userAccountsCheckpointStorer,
 		dataRetriever.UserAccountsUnit,
 		mrcf.enableEpochsHandler,
+		mrcf.stateStatsHandler,
 	)
 	if err != nil {
 		return fmt.Errorf("%w while creating user accounts data trie storage getter", err)
@@ -235,6 +237,7 @@ func (mrcf *metaRequestersContainerFactory) generateTrieNodesRequesters() error
 		peerAccountsCheckpointStorer,
 		dataRetriever.PeerAccountsUnit,
 		mrcf.enableEpochsHandler,
+		mrcf.stateStatsHandler,
 	)
 	if err != nil {
 		return fmt.Errorf("%w while creating peer accounts data trie storage getter", err)

dataRetriever/factory/storageRequestersContainer/shardRequestersContainerFactory.go

@@ -172,6 +172,7 @@ func (srcf *shardRequestersContainerFactory) generateTrieNodesRequesters() error
 		userAccountsCheckpointStorer,
 		dataRetriever.UserAccountsUnit,
 		srcf.enableEpochsHandler,
+		srcf.stateStatsHandler,
 	)
 	if err != nil {
 		return fmt.Errorf("%w while creating user accounts data trie storage getter", err)