From 91fcc09b83f9b02a8f80051e3621dd962369674c Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Wed, 16 Oct 2024 23:11:22 +0200 Subject: [PATCH] make state pruner faster (#8734) * make state pruner faster --- .../services/chainstorage/StorageService.java | 13 ++++- .../chainstorage/StorageServiceTest.java | 56 +++++++++++++------ .../storage/server/pruner/StatePruner.java | 6 ++ 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java index 329211b237b..53f43bdc332 100644 --- a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java +++ b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java @@ -18,6 +18,7 @@ import com.google.common.annotations.VisibleForTesting; import java.nio.file.Path; +import java.time.Duration; import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -53,6 +54,7 @@ import tech.pegasys.teku.storage.server.pruner.StatePruner; public class StorageService extends Service implements StorageServiceFacade { + public static final Duration STATE_PRUNING_INTERVAL = Duration.ofMinutes(1); private final StorageConfiguration config; private volatile ChainStorage chainStorage; private final ServiceConfig serviceConfig; @@ -134,12 +136,20 @@ protected SafeFuture doStart() { configureStatePruner( config.getRetainedSlots(), storagePrunerAsyncRunner, + config.getStatePruningInterval(), pruningTimingsLabelledGauge, pruningActiveLabelledGauge); } else if (!config.getDataStorageMode().storesFinalizedStates()) { + final Duration statePruningInterval = + config + .getStatePruningInterval() + .equals(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL) + ? STATE_PRUNING_INTERVAL + : config.getStatePruningInterval(); configureStatePruner( StorageConfiguration.DEFAULT_STORAGE_RETAINED_SLOTS, storagePrunerAsyncRunner, + statePruningInterval, pruningTimingsLabelledGauge, pruningActiveLabelledGauge); } @@ -221,6 +231,7 @@ protected SafeFuture doStart() { void configureStatePruner( final long slotsToRetain, final AsyncRunner storagePrunerAsyncRunner, + final Duration pruningInterval, final SettableLabelledGauge pruningTimingsLabelledGauge, final SettableLabelledGauge pruningActiveLabelledGauge) { if (config.getDataStorageCreateDbVersion() == DatabaseVersion.LEVELDB_TREE) { @@ -240,7 +251,7 @@ void configureStatePruner( config.getSpec(), database, storagePrunerAsyncRunner, - config.getStatePruningInterval(), + pruningInterval, slotsToRetain, config.getStatePruningLimit(), "state", diff --git a/services/chainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/StorageServiceTest.java b/services/chainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/StorageServiceTest.java index 0d19a70753e..87176ec3319 100644 --- a/services/chainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/StorageServiceTest.java +++ b/services/chainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/StorageServiceTest.java @@ -20,11 +20,14 @@ import static org.mockito.Mockito.when; import java.nio.file.Path; +import java.time.Duration; import java.util.Optional; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import tech.pegasys.teku.ethereum.execution.types.Eth1Address; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.async.StubAsyncRunner; @@ -59,6 +62,10 @@ void setUp(@TempDir final Path tempDir) { .thenReturn(StorageConfiguration.DEFAULT_MAX_KNOWN_NODE_CACHE_SIZE); when(storageConfiguration.getDataStorageFrequency()) .thenReturn(StorageConfiguration.DEFAULT_STORAGE_FREQUENCY); + when(storageConfiguration.getStatePruningLimit()) + .thenReturn(StorageConfiguration.DEFAULT_STATE_PRUNING_LIMIT); + when(storageConfiguration.getStatePruningInterval()) + .thenReturn(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL); when(storageConfiguration.getEth1DepositContract()).thenReturn(eth1DepositContract); when(storageConfiguration.isStoreNonCanonicalBlocksEnabled()).thenReturn(false); when(storageConfiguration.getSpec()).thenReturn(spec); @@ -80,9 +87,9 @@ void setUp(@TempDir final Path tempDir) { void shouldNotSetupStatePrunerWhenArchiveMode() { when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.ARCHIVE); final SafeFuture future = storageService.doStart(); - final Optional statePruner = storageService.getStatePruner(); + final Optional maybeStatePruner = storageService.getStatePruner(); assertThat(future).isCompleted(); - assertThat(statePruner).isEmpty(); + assertThat(maybeStatePruner).isEmpty(); } @Test @@ -90,29 +97,44 @@ void shouldSetupStatePrunerWhenArchiveModeAndRetentionSlotsEnabled() { when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.ARCHIVE); when(storageConfiguration.getRetainedSlots()).thenReturn(5L); final SafeFuture future = storageService.doStart(); - final Optional statePruner = storageService.getStatePruner(); + final Optional maybeStatePruner = storageService.getStatePruner(); assertThat(future).isCompleted(); - assertThat(statePruner).isPresent(); - assertThat(storageService.getStatePruner().get().isRunning()).isTrue(); + assertThat(maybeStatePruner).isPresent(); + final StatePruner statePruner = maybeStatePruner.get(); + assertThat(statePruner.isRunning()).isTrue(); + assertThat(statePruner.getPruneInterval()) + .isEqualTo(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL); } - @Test - void shouldSetupStatePrunerWhenPruneMode() { - when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.PRUNE); + @ParameterizedTest + @EnumSource( + value = StateStorageMode.class, + names = {"PRUNE", "MINIMAL"}) + void shouldSetupStatePrunerWhenPruneMode(final StateStorageMode stateStorageMode) { + when(storageConfiguration.getDataStorageMode()).thenReturn(stateStorageMode); final SafeFuture future = storageService.doStart(); - final Optional statePruner = storageService.getStatePruner(); + final Optional maybeStatePruner = storageService.getStatePruner(); assertThat(future).isCompleted(); - assertThat(statePruner).isPresent(); - assertThat(storageService.getStatePruner().get().isRunning()).isTrue(); + assertThat(maybeStatePruner).isPresent(); + final StatePruner statePruner = maybeStatePruner.get(); + assertThat(statePruner.isRunning()).isTrue(); + assertThat(statePruner.getPruneInterval()).isEqualTo(StorageService.STATE_PRUNING_INTERVAL); } - @Test - void shouldSetupStatePrunerWhenMinimalMode() { - when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.MINIMAL); + @ParameterizedTest + @EnumSource( + value = StateStorageMode.class, + names = {"PRUNE", "MINIMAL"}) + void shouldSetupStatePrunerWithCustomInterval(final StateStorageMode stateStorageMode) { + when(storageConfiguration.getDataStorageMode()).thenReturn(stateStorageMode); + final Duration customPruningInterval = Duration.ofSeconds(8); + when(storageConfiguration.getStatePruningInterval()).thenReturn(customPruningInterval); final SafeFuture future = storageService.doStart(); - final Optional statePruner = storageService.getStatePruner(); + final Optional maybeStatePruner = storageService.getStatePruner(); assertThat(future).isCompleted(); - assertThat(statePruner).isPresent(); - assertThat(storageService.getStatePruner().get().isRunning()).isTrue(); + assertThat(maybeStatePruner).isPresent(); + final StatePruner statePruner = maybeStatePruner.get(); + assertThat(statePruner.isRunning()).isTrue(); + assertThat(statePruner.getPruneInterval()).isEqualTo(customPruningInterval); } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/StatePruner.java b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/StatePruner.java index 49e41da20c0..2a6536822b0 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/StatePruner.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/StatePruner.java @@ -13,6 +13,7 @@ package tech.pegasys.teku.storage.server.pruner; +import com.google.common.annotations.VisibleForTesting; import java.time.Duration; import java.util.Optional; import java.util.concurrent.RejectedExecutionException; @@ -114,4 +115,9 @@ private void pruneStates() { LOG.debug("Shutting down", ex); } } + + @VisibleForTesting + public Duration getPruneInterval() { + return pruneInterval; + } }