From 8050945b83a566e092f0b141c661988194aa3252 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Thu, 1 Feb 2024 11:27:34 +0200 Subject: [PATCH 1/6] add state statistics field to config file --- cmd/node/config/config.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index 85fde2e08cf..6523fd6a9bf 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -658,6 +658,7 @@ PeerStatePruningEnabled = true MaxStateTrieLevelInMemory = 5 MaxPeerTrieLevelInMemory = 5 + StateStatisticsEnabled = false [BlockSizeThrottleConfig] MinSizeInBytes = 104857 # 104857 is 10% from 1MB From d8ca65622fbe98402405d69850619cd2918c24ca Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 9 Feb 2024 12:58:52 +0200 Subject: [PATCH 2/6] reset processing stats on new epoch --- state/accountsDB.go | 1 + 1 file changed, 1 insertion(+) diff --git a/state/accountsDB.go b/state/accountsDB.go index 06fb88eac3a..7f02197adbb 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -787,6 +787,7 @@ func (adb *AccountsDB) CommitInEpoch(currentEpoch uint32, epochToCommit uint32) adb.mainTrie.GetStorageManager().SetEpochForPutOperation(currentEpoch) adb.mutOp.Unlock() adb.loadCodeMeasurements.resetAndPrint() + adb.mainTrie.GetStorageManager().GetStateStatsHandler().Reset() }() adb.mainTrie.GetStorageManager().SetEpochForPutOperation(epochToCommit) From 65de2fe5cdcc87f0835dac220aaa6a6db4ced171 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 9 Feb 2024 13:21:05 +0200 Subject: [PATCH 3/6] reset processing stats on new epoch - move under protection --- state/accountsDB.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state/accountsDB.go b/state/accountsDB.go index 7f02197adbb..249dd64f471 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -785,9 +785,9 @@ func (adb *AccountsDB) CommitInEpoch(currentEpoch uint32, epochToCommit uint32) adb.mutOp.Lock() defer func() { adb.mainTrie.GetStorageManager().SetEpochForPutOperation(currentEpoch) + adb.mainTrie.GetStorageManager().GetStateStatsHandler().Reset() adb.mutOp.Unlock() adb.loadCodeMeasurements.resetAndPrint() - adb.mainTrie.GetStorageManager().GetStateStatsHandler().Reset() }() adb.mainTrie.GetStorageManager().SetEpochForPutOperation(epochToCommit) From e865ea91d947b508847fd1adcfd7b178092e4dc4 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 16 Feb 2024 09:58:24 +0200 Subject: [PATCH 4/6] rename incr to increment --- common/interface.go | 10 +++--- common/statistics/disabled/stateStatistics.go | 20 +++++------ .../disabled/stateStatistics_test.go | 12 +++---- common/statistics/stateStatistics.go | 20 +++++------ common/statistics/stateStatistics_test.go | 36 +++++++++---------- storage/interface.go | 8 ++--- storage/pruning/pruningStorer.go | 4 +-- storage/pruning/triePruningStorer.go | 4 +-- trie/node.go | 2 +- 9 files changed, 58 insertions(+), 58 deletions(-) diff --git a/common/interface.go b/common/interface.go index 38efb0a082b..84e4be9f055 100644 --- a/common/interface.go +++ b/common/interface.go @@ -223,17 +223,17 @@ type StateStatisticsHandler interface { Reset() ResetSnapshot() - IncrCache() + IncrementCache() Cache() uint64 - IncrSnapshotCache() + IncrementSnapshotCache() SnapshotCache() uint64 - IncrPersister(epoch uint32) + IncrementPersister(epoch uint32) Persister(epoch uint32) uint64 - IncrSnapshotPersister(epoch uint32) + IncrementSnapshotPersister(epoch uint32) SnapshotPersister(epoch uint32) uint64 - IncrTrie() + IncrementTrie() Trie() uint64 ProcessingStats() []string diff --git a/common/statistics/disabled/stateStatistics.go b/common/statistics/disabled/stateStatistics.go index d10d310129a..c3bdf12420d 100644 --- a/common/statistics/disabled/stateStatistics.go +++ b/common/statistics/disabled/stateStatistics.go @@ -19,8 +19,8 @@ func (s *stateStatistics) Reset() { func (s *stateStatistics) ResetSnapshot() { } -// IncrCache does nothing -func (s *stateStatistics) IncrCache() { +// IncrementCache does nothing +func (s *stateStatistics) IncrementCache() { } // Cache returns zero @@ -28,8 +28,8 @@ func (s *stateStatistics) Cache() uint64 { return 0 } -// IncrSnapshotCache does nothing -func (ss *stateStatistics) IncrSnapshotCache() { +// IncrementSnapshotCache does nothing +func (ss *stateStatistics) IncrementSnapshotCache() { } // SnapshotCache returns the number of cached operations @@ -37,8 +37,8 @@ func (ss *stateStatistics) SnapshotCache() uint64 { return 0 } -// IncrPersister does nothing -func (s *stateStatistics) IncrPersister(epoch uint32) { +// IncrementPersister does nothing +func (s *stateStatistics) IncrementPersister(epoch uint32) { } // Persister returns zero @@ -46,8 +46,8 @@ func (s *stateStatistics) Persister(epoch uint32) uint64 { return 0 } -// IncrSnapshotPersister does nothing -func (ss *stateStatistics) IncrSnapshotPersister(epoch uint32) { +// IncrementSnapshotPersister does nothing +func (ss *stateStatistics) IncrementSnapshotPersister(epoch uint32) { } // SnapshotPersister returns the number of persister operations @@ -55,8 +55,8 @@ func (ss *stateStatistics) SnapshotPersister(epoch uint32) uint64 { return 0 } -// IncrTrie does nothing -func (s *stateStatistics) IncrTrie() { +// IncrementTrie does nothing +func (s *stateStatistics) IncrementTrie() { } // Trie returns zero diff --git a/common/statistics/disabled/stateStatistics_test.go b/common/statistics/disabled/stateStatistics_test.go index 7d17aa689d1..725ec3ee6a1 100644 --- a/common/statistics/disabled/stateStatistics_test.go +++ b/common/statistics/disabled/stateStatistics_test.go @@ -31,12 +31,12 @@ func TestStateStatistics_MethodsShouldNotPanic(t *testing.T) { stats.ResetSnapshot() stats.ResetAll() - stats.IncrCache() - stats.IncrSnapshotCache() - stats.IncrSnapshotCache() - stats.IncrPersister(1) - stats.IncrSnapshotPersister(1) - stats.IncrTrie() + stats.IncrementCache() + stats.IncrementSnapshotCache() + stats.IncrementSnapshotCache() + stats.IncrementPersister(1) + stats.IncrementSnapshotPersister(1) + stats.IncrementTrie() require.Equal(t, uint64(0), stats.Cache()) require.Equal(t, uint64(0), stats.SnapshotCache()) diff --git a/common/statistics/stateStatistics.go b/common/statistics/stateStatistics.go index c41040ab933..474dc6d47d1 100644 --- a/common/statistics/stateStatistics.go +++ b/common/statistics/stateStatistics.go @@ -51,8 +51,8 @@ func (ss *stateStatistics) ResetSnapshot() { ss.mutPersisters.Unlock() } -// IncrCache will increment cache counter -func (ss *stateStatistics) IncrCache() { +// IncrementCache will increment cache counter +func (ss *stateStatistics) IncrementCache() { atomic.AddUint64(&ss.numCache, 1) } @@ -61,8 +61,8 @@ func (ss *stateStatistics) Cache() uint64 { return atomic.LoadUint64(&ss.numCache) } -// IncrSnapshotCache will increment snapshot cache counter -func (ss *stateStatistics) IncrSnapshotCache() { +// IncrementSnapshotCache will increment snapshot cache counter +func (ss *stateStatistics) IncrementSnapshotCache() { atomic.AddUint64(&ss.numSnapshotCache, 1) } @@ -71,8 +71,8 @@ func (ss *stateStatistics) SnapshotCache() uint64 { return atomic.LoadUint64(&ss.numSnapshotCache) } -// IncrPersister will increment persister counter -func (ss *stateStatistics) IncrPersister(epoch uint32) { +// IncrementPersister will increment persister counter +func (ss *stateStatistics) IncrementPersister(epoch uint32) { ss.mutPersisters.Lock() defer ss.mutPersisters.Unlock() @@ -87,8 +87,8 @@ func (ss *stateStatistics) Persister(epoch uint32) uint64 { return ss.numPersister[epoch] } -// IncrSnapshotPersister will increment snapshot persister counter -func (ss *stateStatistics) IncrSnapshotPersister(epoch uint32) { +// IncrementSnapshotPersister will increment snapshot persister counter +func (ss *stateStatistics) IncrementSnapshotPersister(epoch uint32) { ss.mutPersisters.Lock() defer ss.mutPersisters.Unlock() @@ -103,8 +103,8 @@ func (ss *stateStatistics) SnapshotPersister(epoch uint32) uint64 { return ss.numSnapshotPersister[epoch] } -// IncrTrie will increment trie counter -func (ss *stateStatistics) IncrTrie() { +// IncrementTrie will increment trie counter +func (ss *stateStatistics) IncrementTrie() { atomic.AddUint64(&ss.numTrie, 1) } diff --git a/common/statistics/stateStatistics_test.go b/common/statistics/stateStatistics_test.go index e1beaf9d35b..674b3d8ea6b 100644 --- a/common/statistics/stateStatistics_test.go +++ b/common/statistics/stateStatistics_test.go @@ -27,11 +27,11 @@ func TestStateStatistics_Processing(t *testing.T) { assert.Equal(t, uint64(0), ss.Trie()) - ss.IncrTrie() - ss.IncrTrie() + ss.IncrementTrie() + ss.IncrementTrie() assert.Equal(t, uint64(2), ss.Trie()) - ss.IncrTrie() + ss.IncrementTrie() assert.Equal(t, uint64(3), ss.Trie()) ss.Reset() @@ -47,11 +47,11 @@ func TestStateStatistics_Processing(t *testing.T) { assert.Equal(t, uint64(0), ss.Persister(epoch)) - ss.IncrPersister(epoch) - ss.IncrPersister(epoch) + ss.IncrementPersister(epoch) + ss.IncrementPersister(epoch) assert.Equal(t, uint64(2), ss.Persister(epoch)) - ss.IncrPersister(epoch) + ss.IncrementPersister(epoch) assert.Equal(t, uint64(3), ss.Persister(epoch)) ss.Reset() @@ -65,11 +65,11 @@ func TestStateStatistics_Processing(t *testing.T) { assert.Equal(t, uint64(0), ss.Cache()) - ss.IncrCache() - ss.IncrCache() + ss.IncrementCache() + ss.IncrementCache() assert.Equal(t, uint64(2), ss.Cache()) - ss.IncrCache() + ss.IncrementCache() assert.Equal(t, uint64(3), ss.Cache()) ss.Reset() @@ -89,11 +89,11 @@ func TestStateStatistics_Snapshot(t *testing.T) { assert.Equal(t, uint64(0), ss.SnapshotPersister(epoch)) - ss.IncrSnapshotPersister(epoch) - ss.IncrSnapshotPersister(epoch) + ss.IncrementSnapshotPersister(epoch) + ss.IncrementSnapshotPersister(epoch) assert.Equal(t, uint64(2), ss.SnapshotPersister(epoch)) - ss.IncrSnapshotPersister(epoch) + ss.IncrementSnapshotPersister(epoch) assert.Equal(t, uint64(3), ss.SnapshotPersister(epoch)) ss.ResetSnapshot() @@ -107,11 +107,11 @@ func TestStateStatistics_Snapshot(t *testing.T) { assert.Equal(t, uint64(0), ss.Cache()) - ss.IncrSnapshotCache() - ss.IncrSnapshotCache() + ss.IncrementSnapshotCache() + ss.IncrementSnapshotCache() assert.Equal(t, uint64(2), ss.SnapshotCache()) - ss.IncrSnapshotCache() + ss.IncrementSnapshotCache() assert.Equal(t, uint64(3), ss.SnapshotCache()) ss.ResetSnapshot() @@ -144,11 +144,11 @@ func TestStateStatistics_ConcurrenyOperations(t *testing.T) { case 0: ss.Reset() case 1: - ss.IncrCache() + ss.IncrementCache() case 2: - ss.IncrPersister(epoch) + ss.IncrementPersister(epoch) case 3: - ss.IncrTrie() + ss.IncrementTrie() case 7: _ = ss.Cache() case 8: diff --git a/storage/interface.go b/storage/interface.go index 328eb86c4ed..c3e5aa3826d 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -216,8 +216,8 @@ type PersisterFactoryHandler interface { // StateStatsHandler defines the behaviour needed to handler storage statistics type StateStatsHandler interface { - IncrCache() - IncrSnapshotCache() - IncrPersister(epoch uint32) - IncrSnapshotPersister(epoch uint32) + IncrementCache() + IncrementSnapshotCache() + IncrementPersister(epoch uint32) + IncrementSnapshotPersister(epoch uint32) } diff --git a/storage/pruning/pruningStorer.go b/storage/pruning/pruningStorer.go index f90f1c75aaa..2007454a7c8 100644 --- a/storage/pruning/pruningStorer.go +++ b/storage/pruning/pruningStorer.go @@ -434,7 +434,7 @@ func (ps *PruningStorer) createAndInitPersister(pd *persisterData) (storage.Pers func (ps *PruningStorer) Get(key []byte) ([]byte, error) { v, ok := ps.cacher.Get(key) if ok { - ps.stateStatsHandler.IncrCache() + ps.stateStatsHandler.IncrementCache() return v.([]byte), nil } @@ -457,7 +457,7 @@ func (ps *PruningStorer) Get(key []byte) ([]byte, error) { // if found in persistence unit, add it to cache and return _ = ps.cacher.Put(key, val, len(val)) - ps.stateStatsHandler.IncrPersister(ps.activePersisters[idx].epoch) + ps.stateStatsHandler.IncrementPersister(ps.activePersisters[idx].epoch) return val, nil } diff --git a/storage/pruning/triePruningStorer.go b/storage/pruning/triePruningStorer.go index 1eb290023c6..e013820db65 100644 --- a/storage/pruning/triePruningStorer.go +++ b/storage/pruning/triePruningStorer.go @@ -95,7 +95,7 @@ func (ps *triePruningStorer) PutInEpochWithoutCache(key []byte, data []byte, epo func (ps *triePruningStorer) GetFromOldEpochsWithoutAddingToCache(key []byte) ([]byte, core.OptionalUint32, error) { v, ok := ps.cacher.Get(key) if ok && !bytes.Equal([]byte(common.ActiveDBKey), key) { - ps.stateStatsHandler.IncrSnapshotCache() + ps.stateStatsHandler.IncrementSnapshotCache() return v.([]byte), core.OptionalUint32{}, nil } @@ -118,7 +118,7 @@ func (ps *triePruningStorer) GetFromOldEpochsWithoutAddingToCache(key []byte) ([ HasValue: true, } - ps.stateStatsHandler.IncrSnapshotPersister(epoch.Value) + ps.stateStatsHandler.IncrementSnapshotPersister(epoch.Value) return val, epoch, nil } diff --git a/trie/node.go b/trie/node.go index 6d82a238e95..754b3b3548d 100644 --- a/trie/node.go +++ b/trie/node.go @@ -152,7 +152,7 @@ func resolveIfCollapsed(n node, pos byte, db common.TrieStorageInteractor) error func handleStorageInteractorStats(db common.TrieStorageInteractor) { if db != nil { - db.GetStateStatsHandler().IncrTrie() + db.GetStateStatsHandler().IncrementTrie() } } From 8e483e0609317d22bf61e56cd23ef36b159e2cef Mon Sep 17 00:00:00 2001 From: ssd04 Date: Fri, 16 Feb 2024 12:57:50 +0200 Subject: [PATCH 5/6] improve test coverage --- storage/pruning/triePruningStorer_test.go | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/storage/pruning/triePruningStorer_test.go b/storage/pruning/triePruningStorer_test.go index 4d9a7c83227..28dc5c93f8e 100644 --- a/storage/pruning/triePruningStorer_test.go +++ b/storage/pruning/triePruningStorer_test.go @@ -76,6 +76,31 @@ func TestTriePruningStorer_GetFromOldEpochsWithoutCacheSearchesOnlyOldEpochsAndR assert.True(t, strings.Contains(err.Error(), "not found")) } +func TestTriePruningStorer_GetFromOldEpochsWithCache(t *testing.T) { + t.Parallel() + + args := getDefaultArgs() + ps, _ := pruning.NewTriePruningStorer(args) + cacher := testscommon.NewCacherMock() + ps.SetCacher(cacher) + + testKey1 := []byte("key1") + testVal1 := []byte("value1") + + err := ps.PutInEpoch(testKey1, testVal1, 0) + assert.Nil(t, err) + + err = ps.ChangeEpochSimple(1) + assert.Nil(t, err) + ps.SetEpochForPutOperation(1) + + res, epoch, err := ps.GetFromOldEpochsWithoutAddingToCache(testKey1) + assert.Equal(t, testVal1, res) + assert.Nil(t, err) + assert.False(t, epoch.HasValue) + assert.Equal(t, uint32(0), epoch.Value) +} + func TestTriePruningStorer_GetFromOldEpochsWithoutCacheLessActivePersisters(t *testing.T) { t.Parallel() From cb25f66ec31d2dc7216671fe5ddf61188cd6a963 Mon Sep 17 00:00:00 2001 From: ssd04 Date: Wed, 21 Feb 2024 11:44:02 +0200 Subject: [PATCH 6/6] fix after merge - update stub --- testscommon/stateStatisticsHandlerStub.go | 69 +++++++++++------------ 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/testscommon/stateStatisticsHandlerStub.go b/testscommon/stateStatisticsHandlerStub.go index 970aceedfda..bc13bea90d4 100644 --- a/testscommon/stateStatisticsHandlerStub.go +++ b/testscommon/stateStatisticsHandlerStub.go @@ -2,20 +2,20 @@ package testscommon // StateStatisticsHandlerStub - type StateStatisticsHandlerStub struct { - ResetCalled func() - ResetSnapshotCalled func() - IncrCacheCalled func() - CacheCalled func() uint64 - IncrSnapshotCacheCalled func() - SnapshotCacheCalled func() uint64 - IncrPersisterCalled func(epoch uint32) - PersisterCalled func(epoch uint32) uint64 - IncrSnapshotPersisterCalled func(epoch uint32) - SnapshotPersisterCalled func(epoch uint32) uint64 - IncrTrieCalled func() - TrieCalled func() uint64 - ProcessingStatsCalled func() []string - SnapshotStatsCalled func() []string + ResetCalled func() + ResetSnapshotCalled func() + IncrementCacheCalled func() + CacheCalled func() uint64 + IncrementSnapshotCacheCalled func() + SnapshotCacheCalled func() uint64 + IncrementPersisterCalled func(epoch uint32) + PersisterCalled func(epoch uint32) uint64 + IncrementSnapshotPersisterCalled func(epoch uint32) + SnapshotPersisterCalled func(epoch uint32) uint64 + IncrementTrieCalled func() + TrieCalled func() uint64 + ProcessingStatsCalled func() []string + SnapshotStatsCalled func() []string } // Reset - @@ -32,11 +32,10 @@ func (stub *StateStatisticsHandlerStub) ResetSnapshot() { } } -// IncrCache - -// TODO: replace Incr with Increment on all usages in this file + rename the interface and the other 2 implementations -func (stub *StateStatisticsHandlerStub) IncrCache() { - if stub.IncrCacheCalled != nil { - stub.IncrCacheCalled() +// IncrementCache - +func (stub *StateStatisticsHandlerStub) IncrementCache() { + if stub.IncrementCacheCalled != nil { + stub.IncrementCacheCalled() } } @@ -49,10 +48,10 @@ func (stub *StateStatisticsHandlerStub) Cache() uint64 { return 0 } -// IncrSnapshotCache - -func (stub *StateStatisticsHandlerStub) IncrSnapshotCache() { - if stub.IncrSnapshotCacheCalled != nil { - stub.IncrSnapshotCacheCalled() +// IncrementSnapshotCache - +func (stub *StateStatisticsHandlerStub) IncrementSnapshotCache() { + if stub.IncrementSnapshotCacheCalled != nil { + stub.IncrementSnapshotCacheCalled() } } @@ -65,10 +64,10 @@ func (stub *StateStatisticsHandlerStub) SnapshotCache() uint64 { return 0 } -// IncrPersister - -func (stub *StateStatisticsHandlerStub) IncrPersister(epoch uint32) { - if stub.IncrPersisterCalled != nil { - stub.IncrPersisterCalled(epoch) +// IncrementPersister - +func (stub *StateStatisticsHandlerStub) IncrementPersister(epoch uint32) { + if stub.IncrementPersisterCalled != nil { + stub.IncrementPersisterCalled(epoch) } } @@ -81,10 +80,10 @@ func (stub *StateStatisticsHandlerStub) Persister(epoch uint32) uint64 { return 0 } -// IncrSnapshotPersister - -func (stub *StateStatisticsHandlerStub) IncrSnapshotPersister(epoch uint32) { - if stub.IncrSnapshotPersisterCalled != nil { - stub.IncrSnapshotPersisterCalled(epoch) +// IncrementSnapshotPersister - +func (stub *StateStatisticsHandlerStub) IncrementSnapshotPersister(epoch uint32) { + if stub.IncrementSnapshotPersisterCalled != nil { + stub.IncrementSnapshotPersisterCalled(epoch) } } @@ -97,10 +96,10 @@ func (stub *StateStatisticsHandlerStub) SnapshotPersister(epoch uint32) uint64 { return 0 } -// IncrTrie - -func (stub *StateStatisticsHandlerStub) IncrTrie() { - if stub.IncrTrieCalled != nil { - stub.IncrTrieCalled() +// IncrementTrie - +func (stub *StateStatisticsHandlerStub) IncrementTrie() { + if stub.IncrementTrieCalled != nil { + stub.IncrementTrieCalled() } }