Merge branch 'main' of github.com:hyperledger/besu into CheckForSnapS…

…erver

CHANGELOG.md

@@ -1,6 +1,21 @@
 # Changelog
 
-## Next Release
+## Next release
+
+### Breaking Changes
+
+### Additions and Improvements
+
+### Bug fixes
+
+## 24.7.0
+
+### Upcoming Breaking Changes
+- Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version.
+- PKI-backed QBFT will be removed in a future version of Besu. Other forms of QBFT will remain unchanged.
+- --Xbonsai-limit-trie-logs-enabled is deprecated, use --bonsai-limit-trie-logs-enabled instead
+- --Xbonsai-trie-logs-pruning-window-size is deprecated, use --bonsai-trie-logs-pruning-window-size instead
+- `besu storage x-trie-log` subcommand is deprecated, use `besu storage trie-log` instead
 
 ### Breaking Changes
 - `Xp2p-peer-lower-bound` has been removed. [#7247](https://github.com/hyperledger/besu/pull/7247)
@@ -15,13 +30,19 @@
 - Update Docker base image to Ubuntu 24.04 [#7251](https://github.com/hyperledger/besu/pull/7251)
 - Add LUKSO as predefined network name [#7223](https://github.com/hyperledger/besu/pull/7223)
 - Refactored how code, initcode, and max stack size are configured in forks. [#7245](https://github.com/hyperledger/besu/pull/7245)
+- Nodes in a permissioned chain maintain (and retry) connections to bootnodes [#7257](https://github.com/hyperledger/besu/pull/7257)
+- Promote experimental `besu storage x-trie-log` subcommand to production-ready [#7278](https://github.com/hyperledger/besu/pull/7278)
+- Enhanced BFT round-change diagnostics [#7271](https://github.com/hyperledger/besu/pull/7271)
+- `--Xsnapsync-bft-enabled` option enables experimental support for snap sync with IBFT/QBFT permissioned Bonsai-DB chains [#7140](https://github.com/hyperledger/besu/pull/7140)
 
 ### Bug fixes
 - Validation errors ignored in accounts-allowlist and empty list [#7138](https://github.com/hyperledger/besu/issues/7138)
 - Fix "Invalid block detected" for BFT chains using Bonsai DB [#7204](https://github.com/hyperledger/besu/pull/7204)
 - Fix "Could not confirm best peer had pivot block" [#7109](https://github.com/hyperledger/besu/issues/7109)
 - Fix "Chain Download Halt" [#6884](https://github.com/hyperledger/besu/issues/6884)
 
+
+
 ## 24.6.0
 
 ### Breaking Changes

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningAcceptanceTest.java

@@ -38,17 +38,17 @@ public void setUp() {
     permissionedCluster.start(bootnode, forbiddenNode, allowedNode, permissionedNode);
 
     // updating permissioning smart contract with allowed nodes
-
+    permissionedNode.verify(nodeIsForbidden(bootnode));
     permissionedNode.execute(allowNode(bootnode));
     permissionedNode.verify(nodeIsAllowed(bootnode));
+    permissionedNode.verify(admin.hasPeer(bootnode));
 
     permissionedNode.execute(allowNode(allowedNode));
     permissionedNode.verify(nodeIsAllowed(allowedNode));
 
     permissionedNode.execute(allowNode(permissionedNode));
     permissionedNode.verify(nodeIsAllowed(permissionedNode));
 
-    permissionedNode.verify(admin.addPeer(bootnode));
     permissionedNode.verify(admin.addPeer(allowedNode));
 
     allowedNode.verify(eth.syncingStatus(false));

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java

@@ -62,7 +62,7 @@ public void permissionedNodeShouldPeerOnlyWithAllowedNodes() {
 
   @Test
   public void permissionedNodeShouldDisconnectFromNodeNotPermittedAnymore() {
-    permissionedNode.verify(admin.addPeer(bootnode));
+    permissionedNode.verify(admin.hasPeer(bootnode));
     permissionedNode.verify(admin.addPeer(allowedNode));
     permissionedNode.verify(net.awaitPeerCount(2));
 
@@ -74,7 +74,7 @@ public void permissionedNodeShouldDisconnectFromNodeNotPermittedAnymore() {
 
   @Test
   public void permissionedNodeShouldConnectToNewlyPermittedNode() {
-    permissionedNode.verify(admin.addPeer(bootnode));
+    permissionedNode.verify(admin.hasPeer(bootnode));
     permissionedNode.verify(admin.addPeer(allowedNode));
     permissionedNode.verify(net.awaitPeerCount(2));
 
@@ -89,7 +89,7 @@ public void permissionedNodeShouldConnectToNewlyPermittedNode() {
 
   @Test
   public void permissioningUpdatesPropagateThroughNetwork() {
-    permissionedNode.verify(admin.addPeer(bootnode));
+    permissionedNode.verify(admin.hasPeer(bootnode));
     permissionedNode.verify(admin.addPeer(allowedNode));
     permissionedNode.verify(net.awaitPeerCount(2));
 

besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java

@@ -792,7 +792,20 @@ public Runner build() {
       LOG.debug("added ethash observer: {}", stratumServer.get());
     }
 
-    sanitizePeers(network, staticNodes)
+    final Stream<EnodeURL> maintainedPeers;
+    if (besuController.getGenesisConfigOptions().isPoa()) {
+      // In a permissioned chain Besu should maintain connections to both static nodes and
+      // bootnodes, which includes retries periodically
+      maintainedPeers =
+          sanitizePeers(
+              network,
+              Stream.concat(staticNodes.stream(), bootnodes.stream()).collect(Collectors.toList()));
+      LOG.debug("Added bootnodes to the maintained peer list");
+    } else {
+      // In a public chain only maintain connections to static nodes
+      maintainedPeers = sanitizePeers(network, staticNodes);
+    }
+    maintainedPeers
         .map(DefaultPeer::fromEnodeURL)
         .forEach(peerNetwork::addMaintainedConnectionPeer);
 

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -1552,10 +1552,11 @@ private void validateOptions() {
   }
 
   private void validateConsensusSyncCompatibilityOptions() {
-    // snap and checkpoint can't be used with BFT but can for clique
-    if (genesisConfigOptionsSupplier.get().isIbftLegacy()
-        || genesisConfigOptionsSupplier.get().isIbft2()
-        || genesisConfigOptionsSupplier.get().isQbft()) {
+    // snap and checkpoint are experimental for BFT
+    if ((genesisConfigOptionsSupplier.get().isIbftLegacy()
+            || genesisConfigOptionsSupplier.get().isIbft2()
+            || genesisConfigOptionsSupplier.get().isQbft())
+        && !unstableSynchronizerOptions.isSnapSyncBftEnabled()) {
       final String errorSuffix = "can't be used with BFT networks";
       if (SyncMode.CHECKPOINT.equals(syncMode) || SyncMode.X_CHECKPOINT.equals(syncMode)) {
         throw new ParameterException(
@@ -2181,7 +2182,7 @@ private SynchronizerConfiguration buildSyncConfig() {
     return unstableSynchronizerOptions
         .toDomainObject()
         .syncMode(syncMode)
-        .fastSyncMinimumPeerCount(syncMinPeerCount)
+        .syncMinimumPeerCount(syncMinPeerCount)
         .build();
   }
 
@@ -2194,14 +2195,14 @@ private TransactionPoolConfiguration buildTransactionPoolConfiguration() {
             .saveFile((dataPath.resolve(txPoolConf.getSaveFile().getPath()).toFile()));
 
     if (genesisConfigOptionsSupplier.get().isZeroBaseFee()) {
-      logger.info(
+      logger.warn(
           "Forcing price bump for transaction replacement to 0, since we are on a zero basefee network");
       txPoolConfBuilder.priceBump(Percentage.ZERO);
     }
 
     if (miningParametersSupplier.get().getMinTransactionGasPrice().equals(Wei.ZERO)
         && !transactionPoolOptions.isPriceBumpSet(commandLine)) {
-      logger.info(
+      logger.warn(
           "Forcing price bump for transaction replacement to 0, since min-gas-price is set to 0");
       txPoolConfBuilder.priceBump(Percentage.ZERO);
     }
@@ -2810,6 +2811,7 @@ && getDataStorageConfiguration().getBonsaiLimitTrieLogsEnabled()) {
     }
 
     builder.setSnapServerEnabled(this.unstableSynchronizerOptions.isSnapsyncServerEnabled());
+    builder.setSnapSyncBftEnabled(this.unstableSynchronizerOptions.isSnapSyncBftEnabled());
 
     builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation());
     builder.setWorldStateUpdateMode(unstableEvmOptions.toDomainObject().worldUpdaterMode());

besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java

@@ -56,6 +56,7 @@ public class ConfigurationOverviewBuilder {
   private long trieLogRetentionLimit = 0;
   private Integer trieLogsPruningWindowSize = null;
   private boolean isSnapServerEnabled = false;
+  private boolean isSnapSyncBftEnabled = false;
   private TransactionPoolConfiguration.Implementation txPoolImplementation;
   private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode;
   private Map<String, String> environment;
@@ -233,6 +234,17 @@ public ConfigurationOverviewBuilder setSnapServerEnabled(final boolean snapServe
     return this;
   }
 
+  /**
+   * Sets snap sync BFT enabled/disabled
+   *
+   * @param snapSyncBftEnabled bool to indicate if snap sync for BFT is enabled
+   * @return the builder
+   */
+  public ConfigurationOverviewBuilder setSnapSyncBftEnabled(final boolean snapSyncBftEnabled) {
+    isSnapSyncBftEnabled = snapSyncBftEnabled;
+    return this;
+  }
+
   /**
    * Sets trie logs pruning window size
    *
@@ -357,6 +369,10 @@ public String build() {
       lines.add("Experimental Snap Sync server enabled");
     }
 
+    if (isSnapSyncBftEnabled) {
+      lines.add("Experimental Snap Sync for BFT enabled");
+    }
+
     if (isBonsaiLimitTrieLogsEnabled) {
       final StringBuilder trieLogPruningString = new StringBuilder();
       trieLogPruningString

besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java

@@ -85,6 +85,8 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
 
   private static final String CHECKPOINT_POST_MERGE_FLAG = "--Xcheckpoint-post-merge-enabled";
 
+  private static final String SNAP_SYNC_BFT_ENABLED_FLAG = "--Xsnapsync-bft-enabled";
+
   /**
    * Parse block propagation range.
    *
@@ -304,6 +306,14 @@ public void parseBlockPropagationRange(final String arg) {
   private Boolean checkpointPostMergeSyncEnabled =
       SynchronizerConfiguration.DEFAULT_CHECKPOINT_POST_MERGE_ENABLED;
 
+  @CommandLine.Option(
+      names = SNAP_SYNC_BFT_ENABLED_FLAG,
+      hidden = true,
+      paramLabel = "<Boolean>",
+      arity = "0..1",
+      description = "Snap sync enabled for BFT chains (default: ${DEFAULT-VALUE})")
+  private Boolean snapsyncBftEnabled = SnapSyncConfiguration.DEFAULT_SNAP_SYNC_BFT_ENABLED;
+
   private SynchronizerOptions() {}
 
   /**
@@ -315,6 +325,15 @@ public boolean isSnapsyncServerEnabled() {
     return snapsyncServerEnabled;
   }
 
+  /**
+   * Flag to know whether the Snap sync should be enabled for a BFT chain
+   *
+   * @return true if snap sync for BFT is enabled
+   */
+  public boolean isSnapSyncBftEnabled() {
+    return snapsyncBftEnabled;
+  }
+
   /**
    * Create synchronizer options.
    *
@@ -342,7 +361,7 @@ public static SynchronizerOptions fromConfig(final SynchronizerConfiguration con
     options.downloaderParallelism = config.getDownloaderParallelism();
     options.transactionsParallelism = config.getTransactionsParallelism();
     options.computationParallelism = config.getComputationParallelism();
-    options.fastSyncPivotDistance = config.getFastSyncPivotDistance();
+    options.fastSyncPivotDistance = config.getSyncPivotDistance();
     options.fastSyncFullValidationRate = config.getFastSyncFullValidationRate();
     options.worldStateHashCountPerRequest = config.getWorldStateHashCountPerRequest();
     options.worldStateRequestParallelism = config.getWorldStateRequestParallelism();
@@ -365,6 +384,7 @@ public static SynchronizerOptions fromConfig(final SynchronizerConfiguration con
         config.getSnapSyncConfiguration().getLocalFlatStorageCountToHealPerRequest();
     options.checkpointPostMergeSyncEnabled = config.isCheckpointPostMergeEnabled();
     options.snapsyncServerEnabled = config.getSnapSyncConfiguration().isSnapServerEnabled();
+    options.snapsyncBftEnabled = config.getSnapSyncConfiguration().isSnapSyncBftEnabled();
     return options;
   }
 
@@ -380,7 +400,7 @@ public SynchronizerConfiguration.Builder toDomainObject() {
     builder.downloaderParallelism(downloaderParallelism);
     builder.transactionsParallelism(transactionsParallelism);
     builder.computationParallelism(computationParallelism);
-    builder.fastSyncPivotDistance(fastSyncPivotDistance);
+    builder.syncPivotDistance(fastSyncPivotDistance);
     builder.fastSyncFullValidationRate(fastSyncFullValidationRate);
     builder.worldStateHashCountPerRequest(worldStateHashCountPerRequest);
     builder.worldStateRequestParallelism(worldStateRequestParallelism);
@@ -397,6 +417,7 @@ public SynchronizerConfiguration.Builder toDomainObject() {
             .localFlatAccountCountToHealPerRequest(snapsyncFlatAccountHealedCountPerRequest)
             .localFlatStorageCountToHealPerRequest(snapsyncFlatStorageHealedCountPerRequest)
             .isSnapServerEnabled(snapsyncServerEnabled)
+            .isSnapSyncBftEnabled(snapsyncBftEnabled)
             .build());
     builder.checkpointPostMergeEnabled(checkpointPostMergeSyncEnabled);
 
@@ -454,7 +475,9 @@ public List<String> getCLIOptions() {
             SNAP_FLAT_STORAGE_HEALED_COUNT_PER_REQUEST_FLAG,
             OptionParser.format(snapsyncFlatStorageHealedCountPerRequest),
             SNAP_SERVER_ENABLED_FLAG,
-            OptionParser.format(snapsyncServerEnabled));
+            OptionParser.format(snapsyncServerEnabled),
+            SNAP_SYNC_BFT_ENABLED_FLAG,
+            OptionParser.format(snapsyncBftEnabled));
     return value;
   }
 }

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java

@@ -49,7 +49,8 @@
 
 /** The Trie Log subcommand. */
 @Command(
-    name = "x-trie-log",
+    name = "trie-log",
+    aliases = "x-trie-log",
     description = "Manipulate trie logs",
     mixinStandardHelpOptions = true,
     versionProvider = VersionProvider.class,

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -22,6 +22,7 @@
 import org.hyperledger.besu.config.GenesisConfigOptions;
 import org.hyperledger.besu.consensus.merge.MergeContext;
 import org.hyperledger.besu.consensus.merge.UnverifiedForkchoiceSupplier;
+import org.hyperledger.besu.consensus.qbft.BFTPivotSelectorFromPeers;
 import org.hyperledger.besu.consensus.qbft.pki.PkiBlockCreationConfiguration;
 import org.hyperledger.besu.cryptoservices.NodeKey;
 import org.hyperledger.besu.datatypes.Hash;
@@ -106,6 +107,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalLong;
@@ -692,7 +694,7 @@ public BesuController build() {
 
     final PivotBlockSelector pivotBlockSelector =
         createPivotSelector(
-            protocolSchedule, protocolContext, ethContext, syncState, metricsSystem);
+            protocolSchedule, protocolContext, ethContext, syncState, metricsSystem, blockchain);
 
     final DefaultSynchronizer synchronizer =
         createSynchronizer(
@@ -856,9 +858,22 @@ private PivotBlockSelector createPivotSelector(
       final ProtocolContext protocolContext,
       final EthContext ethContext,
       final SyncState syncState,
-      final MetricsSystem metricsSystem) {
+      final MetricsSystem metricsSystem,
+      final Blockchain blockchain) {
 
-    if (genesisConfigOptions.getTerminalTotalDifficulty().isPresent()) {
+    if (genesisConfigOptions.isQbft() || genesisConfigOptions.isIbft2()) {
+      LOG.info(
+          "{} is configured, creating initial sync for BFT",
+          genesisConfigOptions.getConsensusEngine().toUpperCase(Locale.ROOT));
+      return new BFTPivotSelectorFromPeers(
+          ethContext,
+          syncConfig,
+          syncState,
+          metricsSystem,
+          protocolContext,
+          nodeKey,
+          blockchain.getChainHeadHeader());
+    } else if (genesisConfigOptions.getTerminalTotalDifficulty().isPresent()) {
       LOG.info("TTD difficulty is present, creating initial sync for PoS");
 
       final MergeContext mergeContext = protocolContext.getConsensusContext(MergeContext.class);

besu/src/test/java/org/hyperledger/besu/RunnerTest.java

@@ -225,8 +225,8 @@ private void syncFromGenesis(final SyncMode mode, final GenesisConfigFile genesi
       final SynchronizerConfiguration syncConfigBehind =
           SynchronizerConfiguration.builder()
               .syncMode(mode)
-              .fastSyncPivotDistance(5)
-              .fastSyncMinimumPeerCount(1)
+              .syncPivotDistance(5)
+              .syncMinimumPeerCount(1)
               .build();
       final Path dataDirBehind = Files.createTempDirectory(temp, "db-behind");
 

besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java

@@ -1167,7 +1167,7 @@ public void checkValidDefaultFastSyncMinPeers() {
 
     final SynchronizerConfiguration syncConfig = syncConfigurationCaptor.getValue();
     assertThat(syncConfig.getSyncMode()).isEqualTo(SyncMode.FAST);
-    assertThat(syncConfig.getFastSyncMinimumPeerCount()).isEqualTo(5);
+    assertThat(syncConfig.getSyncMinimumPeerCount()).isEqualTo(5);
     assertThat(commandOutput.toString(UTF_8)).isEmpty();
     assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
   }
@@ -1179,7 +1179,7 @@ public void parsesValidFastSyncMinPeersOption() {
 
     final SynchronizerConfiguration syncConfig = syncConfigurationCaptor.getValue();
     assertThat(syncConfig.getSyncMode()).isEqualTo(SyncMode.FAST);
-    assertThat(syncConfig.getFastSyncMinimumPeerCount()).isEqualTo(11);
+    assertThat(syncConfig.getSyncMinimumPeerCount()).isEqualTo(11);
     assertThat(commandOutput.toString(UTF_8)).isEmpty();
     assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
   }
@@ -1191,7 +1191,7 @@ public void parsesValidSnapSyncMinPeersOption() {
 
     final SynchronizerConfiguration syncConfig = syncConfigurationCaptor.getValue();
     assertThat(syncConfig.getSyncMode()).isEqualTo(SyncMode.SNAP);
-    assertThat(syncConfig.getFastSyncMinimumPeerCount()).isEqualTo(11);
+    assertThat(syncConfig.getSyncMinimumPeerCount()).isEqualTo(11);
     assertThat(commandOutput.toString(UTF_8)).isEmpty();
     assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
   }
@@ -1203,7 +1203,7 @@ public void parsesValidSyncMinPeersOption() {
 
     final SynchronizerConfiguration syncConfig = syncConfigurationCaptor.getValue();
     assertThat(syncConfig.getSyncMode()).isEqualTo(SyncMode.FAST);
-    assertThat(syncConfig.getFastSyncMinimumPeerCount()).isEqualTo(11);
+    assertThat(syncConfig.getSyncMinimumPeerCount()).isEqualTo(11);
     assertThat(commandOutput.toString(UTF_8)).isEmpty();
     assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
   }