make state pruner faster (#8734)

* make state pruner faster
Consensys · Oct 16, 2024 · 91fcc09 · 91fcc09
1 parent afacaae
commit 91fcc09
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 18 deletions.
diff --git a/...es/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java b/...es/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java
@@ -18,6 +18,7 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Optional;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -53,6 +54,7 @@
 import tech.pegasys.teku.storage.server.pruner.StatePruner;
 
 public class StorageService extends Service implements StorageServiceFacade {
+  public static final Duration STATE_PRUNING_INTERVAL = Duration.ofMinutes(1);
   private final StorageConfiguration config;
   private volatile ChainStorage chainStorage;
   private final ServiceConfig serviceConfig;
@@ -134,12 +136,20 @@ protected SafeFuture<?> doStart() {
                 configureStatePruner(
                     config.getRetainedSlots(),
                     storagePrunerAsyncRunner,
+                    config.getStatePruningInterval(),
                     pruningTimingsLabelledGauge,
                     pruningActiveLabelledGauge);
               } else if (!config.getDataStorageMode().storesFinalizedStates()) {
+                final Duration statePruningInterval =
+                    config
+                            .getStatePruningInterval()
+                            .equals(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL)
+                        ? STATE_PRUNING_INTERVAL
+                        : config.getStatePruningInterval();
                 configureStatePruner(
                     StorageConfiguration.DEFAULT_STORAGE_RETAINED_SLOTS,
                     storagePrunerAsyncRunner,
+                    statePruningInterval,
                     pruningTimingsLabelledGauge,
                     pruningActiveLabelledGauge);
               }
@@ -221,6 +231,7 @@ protected SafeFuture<?> doStart() {
   void configureStatePruner(
       final long slotsToRetain,
       final AsyncRunner storagePrunerAsyncRunner,
+      final Duration pruningInterval,
       final SettableLabelledGauge pruningTimingsLabelledGauge,
       final SettableLabelledGauge pruningActiveLabelledGauge) {
     if (config.getDataStorageCreateDbVersion() == DatabaseVersion.LEVELDB_TREE) {
@@ -240,7 +251,7 @@ void configureStatePruner(
                 config.getSpec(),
                 database,
                 storagePrunerAsyncRunner,
-                config.getStatePruningInterval(),
+                pruningInterval,
                 slotsToRetain,
                 config.getStatePruningLimit(),
                 "state",

diff --git a/...hainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/StorageServiceTest.java b/...hainstorage/src/test/java/tech/pegasys/teku/services/chainstorage/StorageServiceTest.java
@@ -20,11 +20,14 @@
 import static org.mockito.Mockito.when;
 
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Optional;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
 import tech.pegasys.teku.ethereum.execution.types.Eth1Address;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.async.StubAsyncRunner;
@@ -59,6 +62,10 @@ void setUp(@TempDir final Path tempDir) {
         .thenReturn(StorageConfiguration.DEFAULT_MAX_KNOWN_NODE_CACHE_SIZE);
     when(storageConfiguration.getDataStorageFrequency())
         .thenReturn(StorageConfiguration.DEFAULT_STORAGE_FREQUENCY);
+    when(storageConfiguration.getStatePruningLimit())
+        .thenReturn(StorageConfiguration.DEFAULT_STATE_PRUNING_LIMIT);
+    when(storageConfiguration.getStatePruningInterval())
+        .thenReturn(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL);
     when(storageConfiguration.getEth1DepositContract()).thenReturn(eth1DepositContract);
     when(storageConfiguration.isStoreNonCanonicalBlocksEnabled()).thenReturn(false);
     when(storageConfiguration.getSpec()).thenReturn(spec);
@@ -80,39 +87,54 @@ void setUp(@TempDir final Path tempDir) {
   void shouldNotSetupStatePrunerWhenArchiveMode() {
     when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.ARCHIVE);
     final SafeFuture<?> future = storageService.doStart();
-    final Optional<StatePruner> statePruner = storageService.getStatePruner();
+    final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
     assertThat(future).isCompleted();
-    assertThat(statePruner).isEmpty();
+    assertThat(maybeStatePruner).isEmpty();
   }
 
   @Test
   void shouldSetupStatePrunerWhenArchiveModeAndRetentionSlotsEnabled() {
     when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.ARCHIVE);
     when(storageConfiguration.getRetainedSlots()).thenReturn(5L);
     final SafeFuture<?> future = storageService.doStart();
-    final Optional<StatePruner> statePruner = storageService.getStatePruner();
+    final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
     assertThat(future).isCompleted();
-    assertThat(statePruner).isPresent();
-    assertThat(storageService.getStatePruner().get().isRunning()).isTrue();
+    assertThat(maybeStatePruner).isPresent();
+    final StatePruner statePruner = maybeStatePruner.get();
+    assertThat(statePruner.isRunning()).isTrue();
+    assertThat(statePruner.getPruneInterval())
+        .isEqualTo(StorageConfiguration.DEFAULT_STATE_PRUNING_INTERVAL);
   }
 
-  @Test
-  void shouldSetupStatePrunerWhenPruneMode() {
-    when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.PRUNE);
+  @ParameterizedTest
+  @EnumSource(
+      value = StateStorageMode.class,
+      names = {"PRUNE", "MINIMAL"})
+  void shouldSetupStatePrunerWhenPruneMode(final StateStorageMode stateStorageMode) {
+    when(storageConfiguration.getDataStorageMode()).thenReturn(stateStorageMode);
     final SafeFuture<?> future = storageService.doStart();
-    final Optional<StatePruner> statePruner = storageService.getStatePruner();
+    final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
     assertThat(future).isCompleted();
-    assertThat(statePruner).isPresent();
-    assertThat(storageService.getStatePruner().get().isRunning()).isTrue();
+    assertThat(maybeStatePruner).isPresent();
+    final StatePruner statePruner = maybeStatePruner.get();
+    assertThat(statePruner.isRunning()).isTrue();
+    assertThat(statePruner.getPruneInterval()).isEqualTo(StorageService.STATE_PRUNING_INTERVAL);
   }
 
-  @Test
-  void shouldSetupStatePrunerWhenMinimalMode() {
-    when(storageConfiguration.getDataStorageMode()).thenReturn(StateStorageMode.MINIMAL);
+  @ParameterizedTest
+  @EnumSource(
+      value = StateStorageMode.class,
+      names = {"PRUNE", "MINIMAL"})
+  void shouldSetupStatePrunerWithCustomInterval(final StateStorageMode stateStorageMode) {
+    when(storageConfiguration.getDataStorageMode()).thenReturn(stateStorageMode);
+    final Duration customPruningInterval = Duration.ofSeconds(8);
+    when(storageConfiguration.getStatePruningInterval()).thenReturn(customPruningInterval);
     final SafeFuture<?> future = storageService.doStart();
-    final Optional<StatePruner> statePruner = storageService.getStatePruner();
+    final Optional<StatePruner> maybeStatePruner = storageService.getStatePruner();
     assertThat(future).isCompleted();
-    assertThat(statePruner).isPresent();
-    assertThat(storageService.getStatePruner().get().isRunning()).isTrue();
+    assertThat(maybeStatePruner).isPresent();
+    final StatePruner statePruner = maybeStatePruner.get();
+    assertThat(statePruner.isRunning()).isTrue();
+    assertThat(statePruner.getPruneInterval()).isEqualTo(customPruningInterval);
   }
 }
diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/StatePruner.java b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/StatePruner.java
@@ -13,6 +13,7 @@
 
 package tech.pegasys.teku.storage.server.pruner;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.time.Duration;
 import java.util.Optional;
 import java.util.concurrent.RejectedExecutionException;
@@ -114,4 +115,9 @@ private void pruneStates() {
       LOG.debug("Shutting down", ex);
     }
   }
+
+  @VisibleForTesting
+  public Duration getPruneInterval() {
+    return pruneInterval;
+  }
 }