Merge branch 'main' into fee-history-calculation

CHANGELOG.md

@@ -41,6 +41,9 @@
 ---
 
 ### Download Links
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.2/besu-23.10.2.zip / sha256: 597ab71898d379180106baf24878239ed49acefea5772344fd359b0ff13fe19f
+
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.2/besu-23.10.2.tar.gz / sha256: 255818a5c6067a38aa8b565d8f32a49a172a7536a1d370673bbb75f548263c2c
 
 ## 23.10.1
 

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -294,7 +294,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
   private final ChainPruningOptions unstableChainPruningOptions = ChainPruningOptions.create();
 
   // stable CLI options
-  private final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
+  final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
   private final EthstatsOptions ethstatsOptions = EthstatsOptions.create();
   private final NodePrivateKeyFileOption nodePrivateKeyFileOption =
       NodePrivateKeyFileOption.create();
@@ -1785,6 +1785,7 @@ private void validateOptions() {
     validateChainDataPruningParams();
     validatePostMergeCheckpointBlockRequirements();
     validateTransactionPoolOptions();
+    validateDataStorageOptions();
     p2pTLSConfigOptions.checkP2PTLSOptionsDependencies(logger, commandLine);
     pkiBlockCreationOptions.checkPkiBlockCreationOptionsDependencies(logger, commandLine);
   }
@@ -1793,6 +1794,10 @@ private void validateTransactionPoolOptions() {
     transactionPoolOptions.validate(commandLine, getActualGenesisConfigOptions());
   }
 
+  private void validateDataStorageOptions() {
+    dataStorageOptions.validate(commandLine);
+  }
+
   private void validateRequiredOptions() {
     commandLine
         .getCommandSpec()
@@ -3466,6 +3471,14 @@ private String generateConfigurationOverview() {
       builder.setHighSpecEnabled();
     }
 
+    if (dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningEnabled()) {
+      builder.setTrieLogPruningEnabled();
+      builder.setTrieLogRetentionThreshold(
+          dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogRetentionThreshold());
+      builder.setTrieLogPruningLimit(
+          dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningLimit());
+    }
+
     builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation());
     builder.setWorldStateUpdateMode(unstableEvmOptions.toDomainObject().worldUpdaterMode());
 

besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java

@@ -50,6 +50,9 @@ public class ConfigurationOverviewBuilder {
   private Collection<String> engineApis;
   private String engineJwtFilePath;
   private boolean isHighSpec = false;
+  private boolean isTrieLogPruningEnabled = false;
+  private long trieLogRetentionThreshold = 0;
+  private Integer trieLogPruningLimit = null;
   private TransactionPoolConfiguration.Implementation txPoolImplementation;
   private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode;
   private Map<String, String> environment;
@@ -171,6 +174,38 @@ public ConfigurationOverviewBuilder setHighSpecEnabled() {
     return this;
   }
 
+  /**
+   * Sets trie log pruning enabled
+   *
+   * @return the builder
+   */
+  public ConfigurationOverviewBuilder setTrieLogPruningEnabled() {
+    isTrieLogPruningEnabled = true;
+    return this;
+  }
+
+  /**
+   * Sets trie log retention threshold
+   *
+   * @param threshold the number of blocks to retain trie logs for
+   * @return the builder
+   */
+  public ConfigurationOverviewBuilder setTrieLogRetentionThreshold(final long threshold) {
+    trieLogRetentionThreshold = threshold;
+    return this;
+  }
+
+  /**
+   * Sets trie log pruning limit
+   *
+   * @param limit the max number of blocks to load and prune trie logs for at startup
+   * @return the builder
+   */
+  public ConfigurationOverviewBuilder setTrieLogPruningLimit(final int limit) {
+    trieLogPruningLimit = limit;
+    return this;
+  }
+
   /**
    * Sets the txpool implementation in use.
    *
@@ -266,13 +301,25 @@ public String build() {
       lines.add("Engine JWT: " + engineJwtFilePath);
     }
 
+    lines.add("Using " + txPoolImplementation + " transaction pool implementation");
+
     if (isHighSpec) {
       lines.add("Experimental high spec configuration enabled");
     }
 
-    lines.add("Using " + txPoolImplementation + " transaction pool implementation");
     lines.add("Using " + worldStateUpdateMode + " worldstate update mode");
 
+    if (isTrieLogPruningEnabled) {
+      final StringBuilder trieLogPruningString = new StringBuilder();
+      trieLogPruningString
+          .append("Trie log pruning enabled: retention: ")
+          .append(trieLogRetentionThreshold);
+      if (trieLogPruningLimit != null) {
+        trieLogPruningString.append("; prune limit: ").append(trieLogPruningLimit);
+      }
+      lines.add(trieLogPruningString.toString());
+    }
+
     lines.add("");
     lines.add("Host:");
 

besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java

@@ -17,15 +17,21 @@
 package org.hyperledger.besu.cli.options.stable;
 
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
 
 import org.hyperledger.besu.cli.options.CLIOptions;
+import org.hyperledger.besu.cli.util.CommandLineUtils;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
 
 import java.util.List;
 
 import org.apache.commons.lang3.StringUtils;
+import picocli.CommandLine;
 import picocli.CommandLine.Option;
 
 /** The Data storage CLI options. */
@@ -42,16 +48,41 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
       description =
           "Format to store trie data in.  Either FOREST or BONSAI (default: ${DEFAULT-VALUE}).",
       arity = "1")
-  private final DataStorageFormat dataStorageFormat = DataStorageFormat.FOREST;
+  private DataStorageFormat dataStorageFormat = DataStorageFormat.FOREST;
 
   @Option(
       names = {BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"},
       paramLabel = "<LONG>",
       description =
           "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}).",
       arity = "1")
-  private final Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
+  private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
 
+  @CommandLine.ArgGroup(validate = false)
+  private final DataStorageOptions.Unstable unstableOptions = new Unstable();
+
+  static class Unstable {
+
+    @CommandLine.Option(
+        hidden = true,
+        names = {"--Xbonsai-trie-log-pruning-enabled"},
+        description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})")
+    private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED;
+
+    @CommandLine.Option(
+        hidden = true,
+        names = {"--Xbonsai-trie-log-retention-threshold"},
+        description =
+            "The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})")
+    private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
+
+    @CommandLine.Option(
+        hidden = true,
+        names = {"--Xbonsai-trie-log-pruning-limit"},
+        description =
+            "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})")
+    private int bonsaiTrieLogPruningLimit = DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT;
+  }
   /**
    * Create data storage options.
    *
@@ -61,21 +92,62 @@ public static DataStorageOptions create() {
     return new DataStorageOptions();
   }
 
+  /**
+   * Validates the data storage options
+   *
+   * @param commandLine the full commandLine to check all the options specified by the user
+   */
+  public void validate(final CommandLine commandLine) {
+    if (unstableOptions.bonsaiTrieLogPruningEnabled) {
+      if (unstableOptions.bonsaiTrieLogRetentionThreshold
+          < MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD) {
+        throw new CommandLine.ParameterException(
+            commandLine,
+            String.format(
+                "--Xbonsai-trie-log-retention-threshold minimum value is %d",
+                MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD));
+      }
+      if (unstableOptions.bonsaiTrieLogPruningLimit <= 0) {
+        throw new CommandLine.ParameterException(
+            commandLine,
+            String.format(
+                "--Xbonsai-trie-log-pruning-limit=%d must be greater than 0",
+                unstableOptions.bonsaiTrieLogPruningLimit));
+      }
+    }
+  }
+
+  static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject) {
+    final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
+    dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat();
+    dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad();
+    dataStorageOptions.unstableOptions.bonsaiTrieLogPruningEnabled =
+        domainObject.getUnstable().getBonsaiTrieLogPruningEnabled();
+    dataStorageOptions.unstableOptions.bonsaiTrieLogRetentionThreshold =
+        domainObject.getUnstable().getBonsaiTrieLogRetentionThreshold();
+    dataStorageOptions.unstableOptions.bonsaiTrieLogPruningLimit =
+        domainObject.getUnstable().getBonsaiTrieLogPruningLimit();
+
+    return dataStorageOptions;
+  }
+
   @Override
   public DataStorageConfiguration toDomainObject() {
     return ImmutableDataStorageConfiguration.builder()
         .dataStorageFormat(dataStorageFormat)
         .bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad)
+        .unstable(
+            ImmutableDataStorageConfiguration.Unstable.builder()
+                .bonsaiTrieLogPruningEnabled(unstableOptions.bonsaiTrieLogPruningEnabled)
+                .bonsaiTrieLogRetentionThreshold(unstableOptions.bonsaiTrieLogRetentionThreshold)
+                .bonsaiTrieLogPruningLimit(unstableOptions.bonsaiTrieLogPruningLimit)
+                .build())
         .build();
   }
 
   @Override
   public List<String> getCLIOptions() {
-    return List.of(
-        DATA_STORAGE_FORMAT,
-        dataStorageFormat.toString(),
-        BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD,
-        bonsaiMaxLayersToLoad.toString());
+    return CommandLineUtils.getCLIOptions(this, new DataStorageOptions());
   }
 
   /**

besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java

@@ -19,6 +19,8 @@
 import org.hyperledger.besu.cli.BesuCommand;
 import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
 import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoaderModule;
+import org.hyperledger.besu.ethereum.eth.transactions.BlobCache;
+import org.hyperledger.besu.ethereum.eth.transactions.BlobCacheModule;
 import org.hyperledger.besu.metrics.MetricsSystemModule;
 import org.hyperledger.besu.metrics.ObservableMetricsSystem;
 import org.hyperledger.besu.services.BesuPluginContextImpl;
@@ -36,7 +38,8 @@
       BesuCommandModule.class,
       MetricsSystemModule.class,
       CachedMerkleTrieLoaderModule.class,
-      BesuPluginContextModule.class
+      BesuPluginContextModule.class,
+      BlobCacheModule.class
     })
 public interface BesuComponent {
 
@@ -72,8 +75,15 @@ public interface BesuComponent {
   /**
    * Besu plugin context for doing plugin service discovery.
    *
-   * @return BesuComponent
+   * @return BesuPluginContextImpl
    */
   @Named("besuPluginContext")
   BesuPluginContextImpl getBesuPluginContext();
+
+  /**
+   * Cache to store blobs in for re-use after reorgs.
+   *
+   * @return BlobCache
+   */
+  BlobCache getBlobCache();
 }

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -34,6 +34,7 @@
 import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider;
 import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
 import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
+import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
 import org.hyperledger.besu.ethereum.chain.ChainDataPruner;
@@ -73,6 +74,7 @@
 import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.ImmutableCheckpoint;
 import org.hyperledger.besu.ethereum.eth.sync.fullsync.SyncTerminationCondition;
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
+import org.hyperledger.besu.ethereum.eth.transactions.BlobCache;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolFactory;
@@ -722,7 +724,8 @@ public BesuController build() {
             syncState,
             miningParameters,
             transactionPoolConfiguration,
-            pluginTransactionValidatorFactory);
+            pluginTransactionValidatorFactory,
+            besuComponent.map(BesuComponent::getBlobCache).orElse(new BlobCache()));
 
     final List<PeerValidator> peerValidators = createPeerValidators(protocolSchedule);
 
@@ -1064,14 +1067,30 @@ WorldStateArchive createWorldStateArchive(
       final Blockchain blockchain,
       final CachedMerkleTrieLoader cachedMerkleTrieLoader) {
     return switch (dataStorageConfiguration.getDataStorageFormat()) {
-      case BONSAI -> new BonsaiWorldStateProvider(
-          (BonsaiWorldStateKeyValueStorage) worldStateStorage,
-          blockchain,
-          Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
-          cachedMerkleTrieLoader,
-          metricsSystem,
-          besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null),
-          evmConfiguration);
+      case BONSAI -> {
+        final GenesisConfigOptions genesisConfigOptions = configOptionsSupplier.get();
+        final boolean isProofOfStake =
+            genesisConfigOptions.getTerminalTotalDifficulty().isPresent();
+        final TrieLogPruner trieLogPruner =
+            dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningEnabled()
+                ? new TrieLogPruner(
+                    (BonsaiWorldStateKeyValueStorage) worldStateStorage,
+                    blockchain,
+                    dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold(),
+                    dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit(),
+                    isProofOfStake)
+                : TrieLogPruner.noOpTrieLogPruner();
+        trieLogPruner.initialize();
+        yield new BonsaiWorldStateProvider(
+            (BonsaiWorldStateKeyValueStorage) worldStateStorage,
+            blockchain,
+            Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
+            cachedMerkleTrieLoader,
+            metricsSystem,
+            besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null),
+            evmConfiguration,
+            trieLogPruner);
+      }
       case FOREST -> {
         final WorldStatePreimageStorage preimageStorage =
             storageProvider.createWorldStatePreimageStorage();

besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java

@@ -33,6 +33,7 @@
 import org.hyperledger.besu.cli.config.EthNetworkConfig;
 import org.hyperledger.besu.cli.options.MiningOptions;
 import org.hyperledger.besu.cli.options.TransactionPoolOptions;
+import org.hyperledger.besu.cli.options.stable.DataStorageOptions;
 import org.hyperledger.besu.cli.options.stable.EthstatsOptions;
 import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
 import org.hyperledger.besu.cli.options.unstable.MetricsCLIOptions;
@@ -568,6 +569,10 @@ public TransactionPoolOptions getTransactionPoolOptions() {
       return transactionPoolOptions;
     }
 
+    public DataStorageOptions getDataStorageOptions() {
+      return dataStorageOptions;
+    }
+
     public MetricsCLIOptions getMetricsCLIOptions() {
       return unstableMetricsCLIOptions;
     }