From 264e7d9facffe1c881d6de090ad99e807f4d836b Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Mon, 1 Jul 2024 16:32:12 -0600 Subject: [PATCH 1/6] Deeper tracing of self-destructed accounts (#7284) Consider previously self-destructed accounts when creating accounts. Signed-off-by: Danno Ferrin --- .../evm/worldstate/UpdateTrackingAccount.java | 1 - .../besu/evm/worldstate/WorldUpdater.java | 27 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/evm/src/main/java/org/hyperledger/besu/evm/worldstate/UpdateTrackingAccount.java b/evm/src/main/java/org/hyperledger/besu/evm/worldstate/UpdateTrackingAccount.java index f571ff2ff03..9f065aee98b 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/worldstate/UpdateTrackingAccount.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/worldstate/UpdateTrackingAccount.java @@ -128,7 +128,6 @@ public A getWrappedAccount() { public void setWrappedAccount(final A account) { if (this.account == null) { this.account = account; - storageWasCleared = false; } else { throw new IllegalStateException("Already tracking a wrapped account"); } diff --git a/evm/src/main/java/org/hyperledger/besu/evm/worldstate/WorldUpdater.java b/evm/src/main/java/org/hyperledger/besu/evm/worldstate/WorldUpdater.java index 9b57b6ab9e3..4cbda732798 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/worldstate/WorldUpdater.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/worldstate/WorldUpdater.java @@ -24,6 +24,8 @@ import java.util.Collection; import java.util.Optional; +import org.apache.tuweni.bytes.Bytes; + /** * An object that buffers updates made over a particular {@link WorldView}. * @@ -73,8 +75,29 @@ default MutableAccount createAccount(final Address address) { * #createAccount(Address)} (and thus all his fields will be zero/empty). */ default MutableAccount getOrCreate(final Address address) { - final MutableAccount account = getAccount(address); - return account == null ? createAccount(address) : account; + MutableAccount account = getAccount(address); + if (account == null) { + account = createAccount(address); + if (parentUpdater().isPresent() && parentUpdater().get().isDeleted(address)) { + account.clearStorage(); + account.setCode(Bytes.EMPTY); + } + } + return account; + } + + /** + * Check this and parent updaters to see if an address has been deleted since the last persist + * + * @param address address to check + * @return true if any updaters have marked the address as deleted. + */ + default boolean isDeleted(final Address address) { + if (getDeletedAccountAddresses().contains(address)) { + return true; + } else { + return parentUpdater().map(wu -> wu.isDeleted(address)).orElse(false); + } } /** From 3a73dccd619ed5ab1caf1548ec3a84aeb30a2d58 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Mon, 1 Jul 2024 16:06:55 -0700 Subject: [PATCH 2/6] next release version after 24.7.0 (#7285) Signed-off-by: garyschulte --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 819bb05a7cd..d14b3e0fc82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Changelog -## Next Release +## Next release + +### Breaking Changes + +### Additions and Improvements + +### Bug fixes + +## 24.7.0 ### Upcoming Breaking Changes - Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version. @@ -32,6 +40,8 @@ - Fix "Could not confirm best peer had pivot block" [#7109](https://github.com/hyperledger/besu/issues/7109) - Fix "Chain Download Halt" [#6884](https://github.com/hyperledger/besu/issues/6884) + + ## 24.6.0 ### Breaking Changes From 8ca7129b0be3ba26b60ceb8920c01f717d5f5aca Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Tue, 2 Jul 2024 09:39:59 +0100 Subject: [PATCH 3/6] Add experimental `--Xsnapsync-bft-enabled` which enables snap sync for BFT chains (#7140) * Create a BFT-specific pivot block handler Signed-off-by: Matthew Whitehead * Change visibility Signed-off-by: Matthew Whitehead * Refactor sync-peer-count internal variable to match name, add experimental flag to enabled snap + BFT Signed-off-by: Matthew Whitehead * Merge with main Signed-off-by: Matthew Whitehead * Fix uppercase Signed-off-by: Matthew Whitehead * Address synchronization issue with trie pruning. Create BFT-specific account range handler. Add pipeline name and logs Signed-off-by: Matthew Whitehead * Remove debug log Signed-off-by: Matthew Whitehead * fixing snapsync for empty state Signed-off-by: Karim Taam * Don't queue up events we can't handle Signed-off-by: Matthew Whitehead * Fix timing window where a validator with an empty data dir sometimes falls back to full sync if peer status isn't received quickly enough Signed-off-by: Matthew Whitehead * Remove BFT-specific account request class. Not needed Signed-off-by: Matthew Whitehead * Refactor some more 'fast' sync variables that are common to all pivot-based sync types Signed-off-by: Matthew Whitehead * In FULL sync mode, disable bonsai-limit-trie-logs-enabled instead of failing to start Signed-off-by: Matthew Whitehead * Add javadoc comments, clarify overriding bonsai-limit-trie-logs-enabled Signed-off-by: Matthew Whitehead * Add BFT pivot block selector tests Signed-off-by: Matthew Whitehead * Fix failure error message Signed-off-by: Matthew Whitehead * Remove the unnamed Pipe constructor and update tests to set a pipe name Signed-off-by: Matthew Whitehead * Revert some info logs back to debug given the feedback on noise in the logs syncing with holesky Signed-off-by: Matthew Whitehead * Refactor fastSyncPivotDistance to syncPivotDistance Signed-off-by: Matthew Whitehead * Incomplete refactoring Signed-off-by: Matthew Whitehead * Update BFT event queueing tests Signed-off-by: Matthew Whitehead * Event queue test fixes Signed-off-by: Matthew Whitehead * Remove automatic setting of bonsai-limit-trie-logs-enabled to false if sync-mode = FULL (moving to another PR) Signed-off-by: Matthew Whitehead --------- Signed-off-by: Matthew Whitehead Signed-off-by: Karim Taam Signed-off-by: Matt Whitehead Co-authored-by: Karim Taam --- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 16 +- .../cli/ConfigurationOverviewBuilder.java | 16 ++ .../options/unstable/SynchronizerOptions.java | 29 +++- .../controller/BesuControllerBuilder.java | 21 ++- .../java/org/hyperledger/besu/RunnerTest.java | 4 +- .../hyperledger/besu/cli/BesuCommandTest.java | 8 +- .../cli/CascadingDefaultProviderTest.java | 4 +- .../cli/options/SynchronizerOptionsTest.java | 6 +- .../consensus/common/bft/BftEventQueue.java | 27 ++- .../consensus/common/bft/BftProcessor.java | 3 + .../common/validator/ValidatorProvider.java | 12 ++ .../common/bft/BftEventQueueTest.java | 69 +++++++- .../common/bft/BftProcessorTest.java | 1 + .../consensus/common/bft/RoundTimerTest.java | 1 + .../BftMiningCoordinatorTest.java | 1 + .../qbft/BFTPivotSelectorFromPeers.java | 160 ++++++++++++++++++ .../qbft/sync/QbftPivotSelectorTest.java | 159 +++++++++++++++++ .../common/trielog/TrieLogPruner.java | 4 +- .../ethereum/eth/manager/snap/SnapServer.java | 18 +- .../eth/sync/DefaultSynchronizer.java | 29 ++-- .../eth/sync/SynchronizerConfiguration.java | 44 ++--- .../eth/sync/fastsync/FastSyncActions.java | 4 +- .../eth/sync/fastsync/FastSyncDownloader.java | 10 +- .../fastsync/NoSyncRequiredException.java | 17 ++ .../sync/fastsync/NoSyncRequiredState.java | 17 ++ .../sync/fastsync/PivotSelectorFromPeers.java | 16 +- .../sync/snapsync/SnapSyncConfiguration.java | 7 + .../request/AccountRangeDataRequest.java | 16 +- .../sync/fastsync/FastSyncActionsTest.java | 42 ++--- .../worldstate/LoadLocalDataStepTest.java | 2 +- .../sync/snapsync/LoadLocalDataStepTest.java | 2 +- .../trie/patricia/TrieNodeDecoder.java | 4 + .../pipeline/AsyncOperationProcessor.java | 2 +- .../besu/services/pipeline/Pipe.java | 19 ++- .../besu/services/pipeline/Pipeline.java | 13 ++ .../services/pipeline/PipelineBuilder.java | 3 +- .../pipeline/BatchingReadPipeTest.java | 3 +- .../services/pipeline/CompleterStageTest.java | 3 +- .../pipeline/FlatMapProcessorTest.java | 6 +- .../pipeline/IteratorSourceStageTest.java | 3 +- .../services/pipeline/MapProcessorTest.java | 6 +- .../besu/services/pipeline/PipeTest.java | 3 +- .../pipeline/ProcessingStageTest.java | 4 +- 44 files changed, 708 insertions(+), 127 deletions(-) create mode 100644 consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/BFTPivotSelectorFromPeers.java create mode 100644 consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/sync/QbftPivotSelectorTest.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/NoSyncRequiredException.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/NoSyncRequiredState.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d14b3e0fc82..4d29b8f8af8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ - Nodes in a permissioned chain maintain (and retry) connections to bootnodes [#7257](https://github.com/hyperledger/besu/pull/7257) - Promote experimental `besu storage x-trie-log` subcommand to production-ready [#7278](https://github.com/hyperledger/besu/pull/7278) - Enhanced BFT round-change diagnostics [#7271](https://github.com/hyperledger/besu/pull/7271) +- `--Xsnapsync-bft-enabled` option enables experimental support for snap sync with IBFT/QBFT permissioned Bonsai-DB chains [#7140](https://github.com/hyperledger/besu/pull/7140) ### Bug fixes - Validation errors ignored in accounts-allowlist and empty list [#7138](https://github.com/hyperledger/besu/issues/7138) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 7e1be15fb65..12752f2e9a1 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1552,10 +1552,11 @@ private void validateOptions() { } private void validateConsensusSyncCompatibilityOptions() { - // snap and checkpoint can't be used with BFT but can for clique - if (genesisConfigOptionsSupplier.get().isIbftLegacy() - || genesisConfigOptionsSupplier.get().isIbft2() - || genesisConfigOptionsSupplier.get().isQbft()) { + // snap and checkpoint are experimental for BFT + if ((genesisConfigOptionsSupplier.get().isIbftLegacy() + || genesisConfigOptionsSupplier.get().isIbft2() + || genesisConfigOptionsSupplier.get().isQbft()) + && !unstableSynchronizerOptions.isSnapSyncBftEnabled()) { final String errorSuffix = "can't be used with BFT networks"; if (SyncMode.CHECKPOINT.equals(syncMode) || SyncMode.X_CHECKPOINT.equals(syncMode)) { throw new ParameterException( @@ -2181,7 +2182,7 @@ private SynchronizerConfiguration buildSyncConfig() { return unstableSynchronizerOptions .toDomainObject() .syncMode(syncMode) - .fastSyncMinimumPeerCount(syncMinPeerCount) + .syncMinimumPeerCount(syncMinPeerCount) .build(); } @@ -2194,14 +2195,14 @@ private TransactionPoolConfiguration buildTransactionPoolConfiguration() { .saveFile((dataPath.resolve(txPoolConf.getSaveFile().getPath()).toFile())); if (genesisConfigOptionsSupplier.get().isZeroBaseFee()) { - logger.info( + logger.warn( "Forcing price bump for transaction replacement to 0, since we are on a zero basefee network"); txPoolConfBuilder.priceBump(Percentage.ZERO); } if (miningParametersSupplier.get().getMinTransactionGasPrice().equals(Wei.ZERO) && !transactionPoolOptions.isPriceBumpSet(commandLine)) { - logger.info( + logger.warn( "Forcing price bump for transaction replacement to 0, since min-gas-price is set to 0"); txPoolConfBuilder.priceBump(Percentage.ZERO); } @@ -2810,6 +2811,7 @@ && getDataStorageConfiguration().getBonsaiLimitTrieLogsEnabled()) { } builder.setSnapServerEnabled(this.unstableSynchronizerOptions.isSnapsyncServerEnabled()); + builder.setSnapSyncBftEnabled(this.unstableSynchronizerOptions.isSnapSyncBftEnabled()); builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation()); builder.setWorldStateUpdateMode(unstableEvmOptions.toDomainObject().worldUpdaterMode()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java index 2d74ae8d26f..b2f89a349e2 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java @@ -56,6 +56,7 @@ public class ConfigurationOverviewBuilder { private long trieLogRetentionLimit = 0; private Integer trieLogsPruningWindowSize = null; private boolean isSnapServerEnabled = false; + private boolean isSnapSyncBftEnabled = false; private TransactionPoolConfiguration.Implementation txPoolImplementation; private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode; private Map environment; @@ -233,6 +234,17 @@ public ConfigurationOverviewBuilder setSnapServerEnabled(final boolean snapServe return this; } + /** + * Sets snap sync BFT enabled/disabled + * + * @param snapSyncBftEnabled bool to indicate if snap sync for BFT is enabled + * @return the builder + */ + public ConfigurationOverviewBuilder setSnapSyncBftEnabled(final boolean snapSyncBftEnabled) { + isSnapSyncBftEnabled = snapSyncBftEnabled; + return this; + } + /** * Sets trie logs pruning window size * @@ -357,6 +369,10 @@ public String build() { lines.add("Experimental Snap Sync server enabled"); } + if (isSnapSyncBftEnabled) { + lines.add("Experimental Snap Sync for BFT enabled"); + } + if (isBonsaiLimitTrieLogsEnabled) { final StringBuilder trieLogPruningString = new StringBuilder(); trieLogPruningString diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java index e52839cceb4..95bbe0a2b1f 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java @@ -85,6 +85,8 @@ public class SynchronizerOptions implements CLIOptions