From 353692799711feec685bc34be07de751ccc992b7 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Tue, 10 Dec 2024 16:28:31 +0200 Subject: [PATCH 1/5] remove old persisters when new already existing --- storage/pruning/pruningStorer.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/storage/pruning/pruningStorer.go b/storage/pruning/pruningStorer.go index 2007454a7c8..aab0b744e54 100644 --- a/storage/pruning/pruningStorer.go +++ b/storage/pruning/pruningStorer.go @@ -779,7 +779,7 @@ func (ps *PruningStorer) changeEpoch(header data.HeaderHandler) error { } log.Debug("change epoch pruning storer success", "persister", ps.identifier, "epoch", epoch) - return nil + return ps.removeOldPersistersIfNeeded(header, epoch) } shardID := core.GetShardIDString(ps.shardCoordinator.SelfId()) @@ -802,6 +802,10 @@ func (ps *PruningStorer) changeEpoch(header data.HeaderHandler) error { ps.activePersisters = append(singleItemPersisters, ps.activePersisters...) ps.persistersMapByEpoch[epoch] = newPersister + return ps.removeOldPersistersIfNeeded(header, epoch) +} + +func (ps *PruningStorer) removeOldPersistersIfNeeded(header data.HeaderHandler, epoch uint32) error { wasExtended := ps.extendSavedEpochsIfNeeded(header) if wasExtended { if len(ps.activePersisters) > int(ps.numOfActivePersisters) { @@ -814,11 +818,12 @@ func (ps *PruningStorer) changeEpoch(header data.HeaderHandler) error { return nil } - err = ps.closeAndDestroyPersisters(epoch) + err := ps.closeAndDestroyPersisters(epoch) if err != nil { log.Warn("closing persisters", "error", err.Error()) return err } + return nil } From aa66f652193d15790ce8dfb72ccf6b31cf088be7 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Tue, 10 Dec 2024 17:07:43 +0200 Subject: [PATCH 2/5] add log trace --- storage/pruning/fullHistoryPruningStorer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/pruning/fullHistoryPruningStorer.go b/storage/pruning/fullHistoryPruningStorer.go index 71213b1dcdd..97852aa3bcd 100644 --- a/storage/pruning/fullHistoryPruningStorer.go +++ b/storage/pruning/fullHistoryPruningStorer.go @@ -184,6 +184,7 @@ func (fhps *FullHistoryPruningStorer) getOrOpenPersister(epoch uint32) (storage. } fhps.oldEpochsActivePersistersCache.Put([]byte(epochString), newPdata, 0) + log.Trace("full history pruning storer - init new storer", "epoch", epoch) fhps.persistersMapByEpoch[epoch] = newPdata return newPdata.getPersister(), nil From 0bdd1c3f50446c8db3384fba8d2059f55b2cfa5e Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Wed, 11 Dec 2024 13:38:28 +0200 Subject: [PATCH 3/5] add unit tests --- .../pruning/fullHistoryPruningStorer_test.go | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/storage/pruning/fullHistoryPruningStorer_test.go b/storage/pruning/fullHistoryPruningStorer_test.go index 0e0d43877e8..80b0290511d 100644 --- a/storage/pruning/fullHistoryPruningStorer_test.go +++ b/storage/pruning/fullHistoryPruningStorer_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "fmt" + "github.com/multiversx/mx-chain-go/testscommon" "math" "path/filepath" "sync" @@ -399,3 +400,33 @@ func TestFullHistoryPruningStorer_IsInterfaceNil(t *testing.T) { fhps, _ = pruning.NewFullHistoryPruningStorer(fhArgs) require.False(t, fhps.IsInterfaceNil()) } + +func TestFullHistoryPruningStorer_changeEpochClosesOldDbs(t *testing.T) { + t.Parallel() + + shouldCleanCalled := false + args := getDefaultArgs() + fhArgs := pruning.FullHistoryStorerArgs{ + StorerArgs: args, + NumOfOldActivePersisters: 2, + } + fhArgs.OldDataCleanerProvider = &testscommon.OldDataCleanerProviderStub{ + ShouldCleanCalled: func() bool { + shouldCleanCalled = true + return true + }, + } + fhps, err := pruning.NewFullHistoryPruningStorer(fhArgs) + require.Nil(t, err) + + numEpochsChanged := 10 + startEpoch := uint32(0) + for i := 0; i < numEpochsChanged; i++ { + startEpoch++ + key := []byte(fmt.Sprintf("key-%d", i)) + _, _ = fhps.GetFromEpoch(key, startEpoch) + err = fhps.ChangeEpochSimple(startEpoch) + require.Nil(t, err) + } + require.True(t, shouldCleanCalled) +} From 1beb4ca88778ac0dda512234f45df59107e3c3a5 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Wed, 11 Dec 2024 13:41:15 +0200 Subject: [PATCH 4/5] sort imports --- storage/pruning/fullHistoryPruningStorer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/pruning/fullHistoryPruningStorer_test.go b/storage/pruning/fullHistoryPruningStorer_test.go index 80b0290511d..d1274499bb9 100644 --- a/storage/pruning/fullHistoryPruningStorer_test.go +++ b/storage/pruning/fullHistoryPruningStorer_test.go @@ -4,7 +4,6 @@ import ( "context" "crypto/rand" "fmt" - "github.com/multiversx/mx-chain-go/testscommon" "math" "path/filepath" "sync" @@ -19,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/pathmanager" "github.com/multiversx/mx-chain-go/storage/pruning" + "github.com/multiversx/mx-chain-go/testscommon" logger "github.com/multiversx/mx-chain-logger-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" From fee796fb44c15303b33de173a46151c118f8154e Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Wed, 11 Dec 2024 15:03:36 +0200 Subject: [PATCH 5/5] fix after review --- storage/pruning/pruningStorer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/storage/pruning/pruningStorer.go b/storage/pruning/pruningStorer.go index aab0b744e54..d40680e5c87 100644 --- a/storage/pruning/pruningStorer.go +++ b/storage/pruning/pruningStorer.go @@ -779,7 +779,7 @@ func (ps *PruningStorer) changeEpoch(header data.HeaderHandler) error { } log.Debug("change epoch pruning storer success", "persister", ps.identifier, "epoch", epoch) - return ps.removeOldPersistersIfNeeded(header, epoch) + return ps.removeOldPersistersIfNeeded(header) } shardID := core.GetShardIDString(ps.shardCoordinator.SelfId()) @@ -802,10 +802,11 @@ func (ps *PruningStorer) changeEpoch(header data.HeaderHandler) error { ps.activePersisters = append(singleItemPersisters, ps.activePersisters...) ps.persistersMapByEpoch[epoch] = newPersister - return ps.removeOldPersistersIfNeeded(header, epoch) + return ps.removeOldPersistersIfNeeded(header) } -func (ps *PruningStorer) removeOldPersistersIfNeeded(header data.HeaderHandler, epoch uint32) error { +func (ps *PruningStorer) removeOldPersistersIfNeeded(header data.HeaderHandler) error { + epoch := header.GetEpoch() wasExtended := ps.extendSavedEpochsIfNeeded(header) if wasExtended { if len(ps.activePersisters) > int(ps.numOfActivePersisters) {