From 657a78e62b68d121dae6ed61f65c7a8da5183fad Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Tue, 28 Nov 2023 11:37:38 +0100 Subject: [PATCH 01/59] worldstate storage refactor Signed-off-by: Karim TAAM --- .../org/hyperledger/besu/cli/BesuCommand.java | 2 +- .../cli/subcommands/operator/BackupState.java | 14 +- .../subcommands/operator/RestoreState.java | 16 +- .../controller/BesuControllerBuilder.java | 38 ++-- ...onsensusScheduleBesuControllerBuilder.java | 2 +- .../TransitionBesuControllerBuilder.java | 10 +- .../hyperledger/besu/cli/BesuCommandTest.java | 2 +- .../controller/BesuControllerBuilderTest.java | 24 +-- .../MergeBesuControllerBuilderTest.java | 14 +- .../QbftBesuControllerBuilderTest.java | 14 +- .../qbft/support/TestContextBuilder.java | 4 +- .../api/query/StateBackupService.java | 16 +- .../worldstate/PrunerIntegrationTest.java | 21 ++- .../ethereum/WorldStateKeyValueStorage.java | 39 ++++ .../bonsai/BonsaiWorldStateProvider.java | 37 ++-- .../bonsai/cache/CachedBonsaiWorldView.java | 18 +- .../bonsai/cache/CachedMerkleTrieLoader.java | 28 +-- .../cache/CachedWorldStorageManager.java | 10 +- ...nsaiSnapshotWorldStateKeyValueStorage.java | 11 +- .../BonsaiWorldStateKeyValueStorage.java | 95 ++-------- .../bonsai/trielog/TrieLogManager.java | 8 +- .../bonsai/worldview/BonsaiWorldState.java | 74 ++++---- .../besu/ethereum/chain/GenesisState.java | 10 +- .../besu/ethereum/core/PrivacyParameters.java | 10 +- .../ForestWorldStateArchive.java} | 34 ++-- .../pruner}/MarkSweepPruner.java | 39 ++-- .../{worldstate => forest/pruner}/Pruner.java | 8 +- .../pruner}/PrunerConfiguration.java | 2 +- .../ForestWorldStateKeyValueStorage.java} | 88 ++++----- .../worldview/ForestMutableWorldState.java} | 65 ++++--- .../storage/PrivacyStorageProvider.java | 7 +- .../PrivacyKeyValueStorageProvider.java | 14 +- .../proof/WorldStateProofProvider.java | 14 +- .../ethereum/storage/StorageProvider.java | 8 +- .../keyvalue/KeyValueStorageProvider.java | 15 +- .../worldstate/WorldStateStorage.java | 126 ------------- .../WorldStateStorageCoordinator.java | 137 ++++++++++++++ .../core/InMemoryKeyValueStorageProvider.java | 16 +- .../core/InMemoryPrivacyStorageProvider.java | 25 ++- .../besu/ethereum/core/TrieGenerator.java | 62 ++++--- .../BlockImportExceptionHandlingTest.java | 10 +- .../bonsai/AbstractIsolationTests.java | 10 +- .../bonsai/BonsaiSnapshotIsolationTests.java | 2 +- ...java => BonsaiWorldStateProviderTest.java} | 14 +- .../bonsai/CachedMerkleTrieLoaderTest.java | 5 +- .../besu/ethereum/bonsai/RollingImport.java | 4 +- .../BonsaiWorldStateKeyValueStorageTest.java | 52 +++--- .../proof/WorldStateProofProviderTest.java | 32 ++-- .../WorldStateRangeProofProviderTest.java | 27 +-- .../KeyValueStorageWorldStateStorageTest.java | 104 ++++++----- .../DefaultMutableWorldStateTest.java | 18 +- .../worldstate/MarkSweepPrunerTest.java | 34 ++-- .../besu/ethereum/worldstate/PrunerTest.java | 3 + .../WorldStateDownloaderBenchmark.java | 14 +- .../eth/sync/DefaultSynchronizer.java | 12 +- .../CheckpointDownloaderFactory.java | 12 +- .../checkpointsync/CheckpointSyncActions.java | 8 +- .../CheckpointSyncChainDownloader.java | 6 +- .../eth/sync/fastsync/FastSyncActions.java | 10 +- .../fastsync/FastSyncChainDownloader.java | 6 +- .../eth/sync/fastsync/FastSyncDownloader.java | 23 ++- .../sync/fastsync/FastSyncTargetManager.java | 10 +- .../AccountTrieNodeDataRequest.java | 36 ++-- .../worldstate/CodeNodeDataRequest.java | 35 +++- .../worldstate/FastDownloaderFactory.java | 37 ++-- .../worldstate/FastWorldDownloadState.java | 21 ++- .../worldstate/FastWorldStateDownloader.java | 16 +- .../worldstate/LoadLocalDataStep.java | 14 +- .../fastsync/worldstate/NodeDataRequest.java | 15 +- .../fastsync/worldstate/PersistDataStep.java | 14 +- .../StorageTrieNodeDataRequest.java | 50 ++++-- .../worldstate/TrieNodeDataRequest.java | 12 +- .../eth/sync/snapsync/LoadLocalDataStep.java | 23 ++- .../eth/sync/snapsync/PersistDataStep.java | 28 +-- .../eth/sync/snapsync/RequestDataStep.java | 48 +++-- .../sync/snapsync/SnapDownloaderFactory.java | 10 +- .../eth/sync/snapsync/SnapSyncDownloader.java | 6 +- .../sync/snapsync/SnapWorldDownloadState.java | 44 +++-- .../snapsync/SnapWorldStateDownloader.java | 42 +++-- .../request/AccountRangeDataRequest.java | 44 +++-- .../snapsync/request/BytecodeRequest.java | 23 ++- .../snapsync/request/SnapDataRequest.java | 25 +-- .../request/StorageRangeDataRequest.java | 45 +++-- ...ccountFlatDatabaseHealingRangeRequest.java | 11 +- .../heal/AccountTrieNodeHealingRequest.java | 42 +++-- ...torageFlatDatabaseHealingRangeRequest.java | 12 +- .../heal/StorageTrieNodeHealingRequest.java | 42 +++-- .../request/heal/TrieNodeHealingRequest.java | 31 ++-- .../sync/worldstate/WorldDownloadState.java | 8 +- .../CheckPointSyncChainDownloaderTest.java | 9 +- .../fastsync/FastDownloaderFactoryTest.java | 16 +- .../sync/fastsync/FastSyncActionsTest.java | 7 +- .../fastsync/FastSyncChainDownloaderTest.java | 9 +- .../sync/fastsync/FastSyncDownloaderTest.java | 13 +- .../FastWorldDownloadStateTest.java | 25 ++- .../FastWorldStateDownloaderTest.java | 167 ++++++++++-------- .../worldstate/LoadLocalDataStepTest.java | 16 +- .../worldstate/PersistDataStepTest.java | 22 +-- .../snapsync/AccountHealingTrackingTest.java | 28 +-- .../sync/snapsync/LoadLocalDataStepTest.java | 24 ++- .../sync/snapsync/PersistDataStepTest.java | 27 ++- .../eth/sync/snapsync/RangeManagerTest.java | 25 +-- .../snapsync/SnapWorldDownloadStateTest.java | 45 +++-- .../eth/sync/snapsync/StackTrieTest.java | 45 ++--- .../eth/sync/snapsync/TaskGenerator.java | 29 +-- ...ntFlatDatabaseHealingRangeRequestTest.java | 63 ++++--- ...geFlatDatabaseHealingRangeRequestTest.java | 36 ++-- .../StorageTrieNodeHealingRequestTest.java | 23 ++- .../besu/evmtool/BlockchainModule.java | 25 +-- .../BonsaiReferenceTestWorldState.java | 13 +- .../DefaultReferenceTestWorldState.java | 8 +- .../ethereum/retesteth/RetestethContext.java | 10 +- .../besu/ethereum/rlp/RLPOutput.java | 2 +- .../RangeStorageEntriesCollectorTest.java | 15 +- 114 files changed, 1722 insertions(+), 1307 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/WorldStateKeyValueStorage.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/{worldstate/DefaultWorldStateArchive.java => forest/ForestWorldStateArchive.java} (72%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/{worldstate => forest/pruner}/MarkSweepPruner.java (89%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/{worldstate => forest/pruner}/Pruner.java (97%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/{worldstate => forest/pruner}/PrunerConfiguration.java (96%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/{storage/keyvalue/WorldStateKeyValueStorage.java => forest/storage/ForestWorldStateKeyValueStorage.java} (68%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/{worldstate/DefaultMutableWorldState.java => forest/worldview/ForestMutableWorldState.java} (86%) delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/{BonsaiWorldStateArchiveTest.java => BonsaiWorldStateProviderTest.java} (98%) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index a909b1994ac..70cfeb18b8d 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -129,6 +129,7 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.mainnet.FrontierTargetingGasLimitCalculator; import org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration; import org.hyperledger.besu.ethereum.p2p.peers.EnodeDnsConfiguration; @@ -145,7 +146,6 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; -import org.hyperledger.besu.ethereum.worldstate.PrunerConfiguration; import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract; import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract; import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/BackupState.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/BackupState.java index 0c5c4c0e32e..0b25a3a7d14 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/BackupState.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/BackupState.java @@ -26,8 +26,8 @@ import org.hyperledger.besu.ethereum.api.query.StateBackupService.BackupStatus; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; -import org.hyperledger.besu.ethereum.worldstate.DefaultWorldStateArchive; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.forest.ForestWorldStateArchive; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.io.File; @@ -81,15 +81,19 @@ public void run() { final BesuController besuController = createBesuController(); final MutableBlockchain blockchain = besuController.getProtocolContext().getBlockchain(); - final WorldStateStorage worldStateStorage = - ((DefaultWorldStateArchive) besuController.getProtocolContext().getWorldStateArchive()) + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + ((ForestWorldStateArchive) besuController.getProtocolContext().getWorldStateArchive()) .getWorldStateStorage(); final EthScheduler scheduler = new EthScheduler(1, 1, 1, 1, new NoOpMetricsSystem()); try { final long targetBlock = Math.min(blockchain.getChainHeadBlockNumber(), this.block); final StateBackupService backup = new StateBackupService( - BesuInfo.version(), blockchain, backupDir.toPath(), scheduler, worldStateStorage); + BesuInfo.version(), + blockchain, + backupDir.toPath(), + scheduler, + worldStateKeyValueStorage); final BackupStatus status = backup.requestBackup(targetBlock, compress, Optional.empty()); final double refValue = Math.pow(2, 256) / 100.0d; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java index 47452b6eb32..c8470f701f5 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java @@ -30,15 +30,15 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions; import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.forest.ForestWorldStateArchive; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.trie.Node; import org.hyperledger.besu.ethereum.trie.PersistVisitor; import org.hyperledger.besu.ethereum.trie.RestoreVisitor; -import org.hyperledger.besu.ethereum.worldstate.DefaultWorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.util.io.RollingFileReader; import java.io.IOException; @@ -83,7 +83,7 @@ public class RestoreState implements Runnable { private long trieNodeCount; private boolean compressed; private BesuController besuController; - private WorldStateStorage.Updater updater; + private ForestWorldStateKeyValueStorage.Updater updater; private Path accountFileName(final int fileNumber, final boolean compressed) { return StateBackupService.accountFileName(backupDir, targetBlock, fileNumber, compressed); @@ -249,10 +249,10 @@ private void newWorldStateUpdater() { if (updater != null) { updater.commit(); } - final WorldStateStorage worldStateStorage = - ((DefaultWorldStateArchive) besuController.getProtocolContext().getWorldStateArchive()) + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + ((ForestWorldStateArchive) besuController.getProtocolContext().getWorldStateArchive()) .getWorldStateStorage(); - updater = worldStateStorage.updater(); + updater = worldStateKeyValueStorage.updater(); } private void maybeCommitUpdater() { @@ -269,14 +269,14 @@ private void updateCode(final Bytes code) { private void updateAccountState(final Bytes32 key, final Bytes value) { maybeCommitUpdater(); // restore by path not supported - updater.putAccountStateTrieNode(null, key, value); + updater.putAccountStateTrieNode(key, value); trieNodeCount++; } private void updateAccountStorage(final Bytes32 key, final Bytes value) { maybeCommitUpdater(); // restore by path not supported - updater.putAccountStorageTrieNode(null, null, key, value); + updater.putAccountStorageTrieNode(key, value); trieNodeCount++; } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 643aed546cb..791fe5af1b3 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -78,6 +78,10 @@ import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolFactory; +import org.hyperledger.besu.ethereum.forest.ForestWorldStateArchive; +import org.hyperledger.besu.ethereum.forest.pruner.MarkSweepPruner; +import org.hyperledger.besu.ethereum.forest.pruner.Pruner; +import org.hyperledger.besu.ethereum.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; @@ -86,13 +90,9 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.DefaultWorldStateArchive; -import org.hyperledger.besu.ethereum.worldstate.MarkSweepPruner; -import org.hyperledger.besu.ethereum.worldstate.Pruner; -import org.hyperledger.besu.ethereum.worldstate.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -595,8 +595,9 @@ public BesuController build() { final VariablesStorage variablesStorage = storageProvider.createVariablesStorage(); - final WorldStateStorage worldStateStorage = - storageProvider.createWorldStateStorage(dataStorageConfiguration.getDataStorageFormat()); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + storageProvider.createWorldStateStorageCoordinator( + dataStorageConfiguration.getDataStorageFormat()); final BlockchainStorage blockchainStorage = storageProvider.createBlockchainStorage(protocolSchedule, variablesStorage); @@ -616,7 +617,7 @@ public BesuController build() { .orElseGet(() -> new CachedMerkleTrieLoader(metricsSystem)); final WorldStateArchive worldStateArchive = - createWorldStateArchive(worldStateStorage, blockchain, cachedMerkleTrieLoader); + createWorldStateArchive(worldStateStorageCoordinator, blockchain, cachedMerkleTrieLoader); if (blockchain.getChainHeadBlockNumber() < 1) { genesisState.writeStateTo(worldStateArchive.getMutable()); @@ -657,7 +658,7 @@ public BesuController build() { Optional.of( new Pruner( new MarkSweepPruner( - ((DefaultWorldStateArchive) worldStateArchive).getWorldStateStorage(), + ((ForestWorldStateArchive) worldStateArchive).getWorldStateStorage(), blockchain, storageProvider.getStorageBySegmentIdentifier( KeyValueSegmentIdentifier.PRUNING_STATE), @@ -751,7 +752,7 @@ public BesuController build() { final Synchronizer synchronizer = createSynchronizer( protocolSchedule, - worldStateStorage, + worldStateStorageCoordinator, protocolContext, maybePruner, ethContext, @@ -810,7 +811,7 @@ public BesuController build() { * Create synchronizer synchronizer. * * @param protocolSchedule the protocol schedule - * @param worldStateStorage the world state storage + * @param worldStateStorageCoordinator the world state storage * @param protocolContext the protocol context * @param maybePruner the maybe pruner * @param ethContext the eth context @@ -821,7 +822,7 @@ public BesuController build() { */ protected Synchronizer createSynchronizer( final ProtocolSchedule protocolSchedule, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolContext protocolContext, final Optional maybePruner, final EthContext ethContext, @@ -833,7 +834,7 @@ protected Synchronizer createSynchronizer( syncConfig, protocolSchedule, protocolContext, - worldStateStorage, + worldStateStorageCoordinator, ethProtocolManager.getBlockBroadcaster(), maybePruner, ethContext, @@ -1062,7 +1063,7 @@ private Optional createSnapProtocolManager( } WorldStateArchive createWorldStateArchive( - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final Blockchain blockchain, final CachedMerkleTrieLoader cachedMerkleTrieLoader) { return switch (dataStorageConfiguration.getDataStorageFormat()) { @@ -1070,10 +1071,12 @@ WorldStateArchive createWorldStateArchive( final GenesisConfigOptions genesisConfigOptions = configOptionsSupplier.get(); final boolean isProofOfStake = genesisConfigOptions.getTerminalTotalDifficulty().isPresent(); + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = + worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class); final TrieLogPruner trieLogPruner = dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningEnabled() ? new TrieLogPruner( - (BonsaiWorldStateKeyValueStorage) worldStateStorage, + worldStateKeyValueStorage, blockchain, dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold(), dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit(), @@ -1081,7 +1084,7 @@ WorldStateArchive createWorldStateArchive( : TrieLogPruner.noOpTrieLogPruner(); trieLogPruner.initialize(); yield new BonsaiWorldStateProvider( - (BonsaiWorldStateKeyValueStorage) worldStateStorage, + worldStateKeyValueStorage, blockchain, Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), cachedMerkleTrieLoader, @@ -1093,7 +1096,8 @@ yield new BonsaiWorldStateProvider( case FOREST -> { final WorldStatePreimageStorage preimageStorage = storageProvider.createWorldStatePreimageStorage(); - yield new DefaultWorldStateArchive(worldStateStorage, preimageStorage, evmConfiguration); + yield new ForestWorldStateArchive( + worldStateStorageCoordinator, preimageStorage, evmConfiguration); } }; } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java index 921289c66e5..c8db08ecbd0 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java @@ -53,11 +53,11 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java index 8231f8e26a5..78a252929c9 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java @@ -50,13 +50,13 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.forest.pruner.Pruner; +import org.hyperledger.besu.ethereum.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.Pruner; -import org.hyperledger.besu.ethereum.worldstate.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.permissioning.NodeMessagePermissioningProvider; @@ -223,7 +223,7 @@ protected PluginServiceFactory createAdditionalPluginServices( @Override protected Synchronizer createSynchronizer( final ProtocolSchedule protocolSchedule, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolContext protocolContext, final Optional maybePruner, final EthContext ethContext, @@ -235,7 +235,7 @@ protected Synchronizer createSynchronizer( (DefaultSynchronizer) super.createSynchronizer( protocolSchedule, - worldStateStorage, + worldStateStorageCoordinator, protocolContext, maybePruner, ethContext, diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 19e42e1cd82..77a1c62268d 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -79,12 +79,12 @@ import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; import org.hyperledger.besu.ethereum.permissioning.LocalPermissioningConfiguration; import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; import org.hyperledger.besu.ethereum.permissioning.SmartContractPermissioningConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.PrunerConfiguration; import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract; import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract; import org.hyperledger.besu.metrics.StandardMetricCategory; diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index a24479e8617..209a2284706 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -30,6 +30,7 @@ import org.hyperledger.besu.cryptoservices.NodeKeyUtils; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.GasLimitCalculator; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; @@ -39,6 +40,7 @@ import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; @@ -48,10 +50,9 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -88,7 +89,7 @@ public class BesuControllerBuilderTest { @Mock Clock clock; @Mock StorageProvider storageProvider; @Mock GasLimitCalculator gasLimitCalculator; - @Mock WorldStateStorage worldStateStorage; + @Mock WorldStateStorageCoordinator worldStateStorageCoordinator; @Mock WorldStateArchive worldStateArchive; @Mock BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorage; @Mock WorldStatePreimageStorage worldStatePreimageStorage; @@ -129,14 +130,15 @@ public void setup() { when(synchronizerConfiguration.getBlockPropagationRange()).thenReturn(Range.closed(1L, 2L)); - when(storageProvider.createWorldStateStorage(DataStorageFormat.FOREST)) - .thenReturn(worldStateStorage); + when(storageProvider.createWorldStateStorageCoordinator(DataStorageFormat.FOREST)) + .thenReturn(worldStateStorageCoordinator); when(storageProvider.createWorldStatePreimageStorage()).thenReturn(worldStatePreimageStorage); - when(worldStateStorage.isWorldStateAvailable(any(), any())).thenReturn(true); + when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); when(worldStatePreimageStorage.updater()) .thenReturn(mock(WorldStatePreimageStorage.Updater.class)); - when(worldStateStorage.updater()).thenReturn(mock(WorldStateStorage.Updater.class)); + when(worldStateStorageCoordinator.updater()) + .thenReturn(mock(WorldStateKeyValueStorage.Updater.class)); besuControllerBuilder = spy(visitWithMockConfigs(new MainnetBesuControllerBuilder())); } @@ -165,11 +167,13 @@ public void shouldDisablePruningIfBonsaiIsEnabled() { doReturn(worldStateArchive) .when(besuControllerBuilder) .createWorldStateArchive( - any(WorldStateStorage.class), any(Blockchain.class), any(CachedMerkleTrieLoader.class)); + any(WorldStateStorageCoordinator.class), + any(Blockchain.class), + any(CachedMerkleTrieLoader.class)); doReturn(mockWorldState).when(worldStateArchive).getMutable(); - when(storageProvider.createWorldStateStorage(DataStorageFormat.BONSAI)) - .thenReturn(bonsaiWorldStateStorage); + when(storageProvider.createWorldStateStorageCoordinator(DataStorageFormat.BONSAI)) + .thenReturn(new WorldStateStorageCoordinator(bonsaiWorldStateStorage)); besuControllerBuilder .isPruningEnabled(true) .dataStorageConfiguration( diff --git a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java index 33a539f8bc2..c961aa0c323 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.GasLimitCalculator; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.GenesisState; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -54,7 +55,7 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -98,7 +99,7 @@ public class MergeBesuControllerBuilderTest { @Mock Clock clock; @Mock StorageProvider storageProvider; @Mock GasLimitCalculator gasLimitCalculator; - @Mock WorldStateStorage worldStateStorage; + @Mock WorldStateStorageCoordinator worldStateStorageCoordinator; @Mock WorldStatePreimageStorage worldStatePreimageStorage; BigInteger networkId = BigInteger.ONE; @@ -139,14 +140,15 @@ public void setup() { when(synchronizerConfiguration.getBlockPropagationRange()).thenReturn(Range.closed(1L, 2L)); - when(storageProvider.createWorldStateStorage(DataStorageFormat.FOREST)) - .thenReturn(worldStateStorage); + when(storageProvider.createWorldStateStorageCoordinator(DataStorageFormat.FOREST)) + .thenReturn(worldStateStorageCoordinator); when(storageProvider.createWorldStatePreimageStorage()).thenReturn(worldStatePreimageStorage); - when(worldStateStorage.isWorldStateAvailable(any(), any())).thenReturn(true); + when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); when(worldStatePreimageStorage.updater()) .thenReturn(mock(WorldStatePreimageStorage.Updater.class)); - when(worldStateStorage.updater()).thenReturn(mock(WorldStateStorage.Updater.class)); + when(worldStateStorageCoordinator.updater()) + .thenReturn(mock(WorldStateKeyValueStorage.Updater.class)); when(miningParameters.getTargetGasLimit()).thenReturn(OptionalLong.empty()); besuControllerBuilder = visitWithMockConfigs(new MergeBesuControllerBuilder()); diff --git a/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java index 585f138027e..55b2db4159c 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java @@ -36,6 +36,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.GasLimitCalculator; import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; @@ -49,7 +50,7 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -84,7 +85,7 @@ public class QbftBesuControllerBuilderTest { @Mock private Clock clock; @Mock private StorageProvider storageProvider; @Mock private GasLimitCalculator gasLimitCalculator; - @Mock private WorldStateStorage worldStateStorage; + @Mock private WorldStateStorageCoordinator worldStateStorageCoordinator; @Mock private WorldStatePreimageStorage worldStatePreimageStorage; private static final BigInteger networkId = BigInteger.ONE; private static final NodeKey nodeKey = NodeKeyUtils.generate(); @@ -111,10 +112,11 @@ public void setup() { new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), new MainnetBlockHeaderFunctions())); - when(storageProvider.createWorldStateStorage(DataStorageFormat.FOREST)) - .thenReturn(worldStateStorage); - when(worldStateStorage.isWorldStateAvailable(any(), any())).thenReturn(true); - when(worldStateStorage.updater()).thenReturn(mock(WorldStateStorage.Updater.class)); + when(storageProvider.createWorldStateStorageCoordinator(DataStorageFormat.FOREST)) + .thenReturn(worldStateStorageCoordinator); + when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); + when(worldStateStorageCoordinator.updater()) + .thenReturn(mock(WorldStateKeyValueStorage.Updater.class)); when(worldStatePreimageStorage.updater()) .thenReturn(mock(WorldStatePreimageStorage.Updater.class)); when(storageProvider.createWorldStatePreimageStorage()).thenReturn(worldStatePreimageStorage); diff --git a/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/support/TestContextBuilder.java b/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/support/TestContextBuilder.java index 6d8c64a835f..c7e7f43dd62 100644 --- a/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/support/TestContextBuilder.java +++ b/consensus/qbft/src/integration-test/java/org/hyperledger/besu/consensus/qbft/support/TestContextBuilder.java @@ -103,8 +103,8 @@ import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolMetrics; import org.hyperledger.besu.ethereum.eth.transactions.sorter.GasPricePendingTransactionsSorter; +import org.hyperledger.besu.ethereum.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; -import org.hyperledger.besu.ethereum.worldstate.DefaultWorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -241,7 +241,7 @@ public TestContext build() { } final MutableBlockchain blockChain; - final DefaultWorldStateArchive worldStateArchive = createInMemoryWorldStateArchive(); + final ForestWorldStateArchive worldStateArchive = createInMemoryWorldStateArchive(); if (genesisFile.isPresent()) { try { diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java index 6ae5125bc5f..a3c88950e07 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.trie.Node; @@ -33,7 +34,6 @@ import org.hyperledger.besu.ethereum.trie.TrieIterator.State; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.util.io.RollingFileWriter; import java.io.IOException; @@ -73,7 +73,7 @@ public class StateBackupService { private final Lock submissionLock = new ReentrantLock(); private final EthScheduler scheduler; private final Blockchain blockchain; - private final WorldStateStorage worldStateStorage; + private final ForestWorldStateKeyValueStorage worldStateKeyValueStorage; private final BackupStatus backupStatus = new BackupStatus(); private Path backupDir; @@ -84,12 +84,12 @@ public StateBackupService( final Blockchain blockchain, final Path backupDir, final EthScheduler scheduler, - final WorldStateStorage worldStateStorage) { + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage) { this.besuVersion = besuVersion; this.blockchain = blockchain; this.backupDir = backupDir; this.scheduler = scheduler; - this.worldStateStorage = worldStateStorage; + this.worldStateKeyValueStorage = worldStateKeyValueStorage; } public Path getBackupDir() { @@ -214,7 +214,7 @@ private void backupLeaves() throws IOException { return; } final Optional worldStateRoot = - worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, header.get().getStateRoot()); + worldStateKeyValueStorage.getAccountStateTrieNode(header.get().getStateRoot()); if (worldStateRoot.isEmpty()) { backupStatus.currentAccount = null; return; @@ -226,7 +226,7 @@ private void backupLeaves() throws IOException { final StoredMerklePatriciaTrie accountTrie = new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, + (location, hash) -> worldStateKeyValueStorage.getAccountStateTrieNode(hash), header.get().getStateRoot(), Function.identity(), Function.identity()); @@ -246,7 +246,7 @@ private TrieIterator.State visitAccount(final Bytes32 nodeKey, final Node final StateTrieAccountValue account = StateTrieAccountValue.readFrom(new BytesValueRLPInput(nodeValue, false)); - final Bytes code = worldStateStorage.getCode(account.getCodeHash(), null).orElse(Bytes.EMPTY); + final Bytes code = worldStateKeyValueStorage.getCode(account.getCodeHash()).orElse(Bytes.EMPTY); backupStatus.codeSize.addAndGet(code.size()); final BytesValueRLPOutput accountOutput = new BytesValueRLPOutput(); @@ -266,7 +266,7 @@ private TrieIterator.State visitAccount(final Bytes32 nodeKey, final Node // storage is written for each leaf, otherwise the whole trie would have to fit in memory final StoredMerklePatriciaTrie storageTrie = new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, + (location, hash) -> worldStateKeyValueStorage.getAccountStateTrieNode(hash), account.getStorageRoot(), Function.identity(), Function.identity()); diff --git a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java b/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java index 77f871f1002..782db13cf86 100644 --- a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java +++ b/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java @@ -25,12 +25,16 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.forest.ForestWorldStateArchive; +import org.hyperledger.besu.ethereum.forest.pruner.MarkSweepPruner; +import org.hyperledger.besu.ethereum.forest.pruner.Pruner; +import org.hyperledger.besu.ethereum.forest.pruner.Pruner.PruningPhase; +import org.hyperledger.besu.ethereum.forest.pruner.PrunerConfiguration; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.Pruner.PruningPhase; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -57,10 +61,11 @@ public class PrunerIntegrationTest { private final NoOpMetricsSystem metricsSystem = new NoOpMetricsSystem(); private final Map> hashValueStore = new HashMap<>(); private final InMemoryKeyValueStorage stateStorage = new TestInMemoryStorage(hashValueStore); - private final WorldStateStorage worldStateStorage = new WorldStateKeyValueStorage(stateStorage); + private final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(stateStorage); private final WorldStateArchive worldStateArchive = - new DefaultWorldStateArchive( - worldStateStorage, + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(worldStateKeyValueStorage), new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()), EvmConfiguration.DEFAULT); private final InMemoryKeyValueStorage markStorage = new InMemoryKeyValueStorage(); @@ -106,7 +111,7 @@ private void testPruner( final var markSweepPruner = new MarkSweepPruner( - worldStateStorage, blockchain, markStorage, metricsSystem, opsPerTransaction); + worldStateKeyValueStorage, blockchain, markStorage, metricsSystem, opsPerTransaction); final var pruner = new Pruner( markSweepPruner, @@ -238,7 +243,7 @@ private void collectTrieNodes(final MerkleTrie trie, final Set createStateTrie(final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, + (location, hash) -> worldStateKeyValueStorage.getAccountStateTrieNode(hash), rootHash, Function.identity(), Function.identity()); @@ -246,7 +251,7 @@ private MerkleTrie createStateTrie(final Bytes32 rootHash) { private MerkleTrie createStorageTrie(final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateStorage.getAccountStorageTrieNode(null, location, hash), + (location, hash) -> worldStateKeyValueStorage.getAccountStorageTrieNode(hash), rootHash, Function.identity(), Function.identity()); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/WorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/WorldStateKeyValueStorage.java new file mode 100644 index 00000000000..67946f6181a --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/WorldStateKeyValueStorage.java @@ -0,0 +1,39 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum; + +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; + +import java.util.Collection; + +import org.apache.tuweni.bytes.Bytes32; + +public interface WorldStateKeyValueStorage { + + DataStorageFormat getDataStorageFormat(); + + Updater updater(); + + void clear(); + + interface NodesAddedListener { + void onNodesAdded(Collection nodeHash); + } + + interface Updater { + void commit(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java index 5313977c037..9bdb704300d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java @@ -37,6 +37,7 @@ import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -65,11 +66,11 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { private final CachedWorldStorageManager cachedWorldStorageManager; private final TrieLogManager trieLogManager; private final BonsaiWorldState persistedState; - private final BonsaiWorldStateKeyValueStorage worldStateStorage; + private final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; private final CachedMerkleTrieLoader cachedMerkleTrieLoader; public BonsaiWorldStateProvider( - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final Blockchain blockchain, final Optional maxLayersToLoad, final CachedMerkleTrieLoader cachedMerkleTrieLoader, @@ -78,20 +79,20 @@ public BonsaiWorldStateProvider( final EvmConfiguration evmConfiguration, final TrieLogPruner trieLogPruner) { + this.worldStateKeyValueStorage = worldStateKeyValueStorage; this.cachedWorldStorageManager = - new CachedWorldStorageManager(this, worldStateStorage, metricsSystem); + new CachedWorldStorageManager(this, worldStateKeyValueStorage, metricsSystem); // TODO: de-dup constructors this.trieLogManager = new TrieLogManager( blockchain, - worldStateStorage, + worldStateKeyValueStorage, maxLayersToLoad.orElse(RETAINED_LAYERS), pluginContext, trieLogPruner); this.blockchain = blockchain; - this.worldStateStorage = worldStateStorage; this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; - this.persistedState = new BonsaiWorldState(this, worldStateStorage, evmConfiguration); + this.persistedState = new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration); blockchain .getBlockHeader(persistedState.getWorldStateBlockHash()) .ifPresent( @@ -104,15 +105,15 @@ public BonsaiWorldStateProvider( BonsaiWorldStateProvider( final CachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final Blockchain blockchain, final CachedMerkleTrieLoader cachedMerkleTrieLoader, final EvmConfiguration evmConfiguration) { this.cachedWorldStorageManager = cachedWorldStorageManager; this.trieLogManager = trieLogManager; this.blockchain = blockchain; - this.worldStateStorage = worldStateStorage; - this.persistedState = new BonsaiWorldState(this, worldStateStorage, evmConfiguration); + this.worldStateKeyValueStorage = worldStateKeyValueStorage; + this.persistedState = new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration); this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; blockchain .getBlockHeader(persistedState.getWorldStateBlockHash()) @@ -141,7 +142,7 @@ public Optional get(final Hash rootHash, final Hash blockHash) { public boolean isWorldStateAvailable(final Hash rootHash, final Hash blockHash) { return cachedWorldStorageManager.containWorldStateStorage(blockHash) || persistedState.blockHash().equals(blockHash) - || worldStateStorage.isWorldStateAvailable(rootHash, blockHash); + || worldStateKeyValueStorage.isWorldStateAvailable(rootHash, blockHash); } @Override @@ -290,12 +291,12 @@ public MutableWorldState getMutable() { */ public void prepareStateHealing(final Address address, final Bytes location) { final Set keysToDelete = new HashSet<>(); - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = worldStateStorage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updater = worldStateKeyValueStorage.updater(); final Hash accountHash = address.addressHash(); final StoredMerklePatriciaTrie accountTrie = new StoredMerklePatriciaTrie<>( (l, h) -> { - final Optional node = worldStateStorage.getAccountStateTrieNode(l, h); + final Optional node = worldStateKeyValueStorage.getAccountStateTrieNode(l, h); if (node.isPresent()) { keysToDelete.add(l); } @@ -315,7 +316,8 @@ public void prepareStateHealing(final Address address, final Bytes location) { new StoredMerklePatriciaTrie<>( (l, h) -> { Optional node = - worldStateStorage.getAccountStorageTrieNode(accountHash, l, h); + worldStateKeyValueStorage.getAccountStorageTrieNode( + accountHash, l, h); if (node.isPresent()) { keysToDelete.add(Bytes.concatenate(accountHash, l)); } @@ -335,10 +337,10 @@ public void prepareStateHealing(final Address address, final Bytes location) { LOG.warn("Invalid node for account {} at location {}", address, location); // ignore } - keysToDelete.forEach(bytes -> updater.removeAccountStateTrieNode(bytes, null)); + keysToDelete.forEach(bytes -> updater.removeAccountStateTrieNode(bytes)); updater.commit(); - worldStateStorage.downgradeToPartialFlatDbMode(); + worldStateKeyValueStorage.downgradeToPartialFlatDbMode(); } public TrieLogManager getTrieLogManager() { @@ -366,7 +368,8 @@ public Optional getAccountProof( try (BonsaiWorldState ws = (BonsaiWorldState) getMutable(blockHeader, false).orElse(null)) { if (ws != null) { final WorldStateProofProvider worldStateProofProvider = - new WorldStateProofProvider(ws.getWorldStateStorage()); + new WorldStateProofProvider( + new WorldStateStorageCoordinator(ws.getWorldStateStorage())); return mapper.apply( worldStateProofProvider.getAccountProof( ws.getWorldStateRootHash(), accountAddress, accountStorageKeys)); @@ -385,7 +388,7 @@ public Optional getNodeData(final Hash hash) { @Override public void close() { try { - worldStateStorage.close(); + worldStateKeyValueStorage.close(); } catch (Exception e) { // no op } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedBonsaiWorldView.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedBonsaiWorldView.java index 473933b3959..2624f981c78 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedBonsaiWorldView.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedBonsaiWorldView.java @@ -24,7 +24,7 @@ import org.slf4j.LoggerFactory; public class CachedBonsaiWorldView implements BonsaiStorageSubscriber { - private BonsaiWorldStateKeyValueStorage worldStateStorage; + private BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; private final BlockHeader blockHeader; private long worldViewSubscriberId; private static final Logger LOG = LoggerFactory.getLogger(CachedBonsaiWorldView.class); @@ -32,12 +32,12 @@ public class CachedBonsaiWorldView implements BonsaiStorageSubscriber { public CachedBonsaiWorldView( final BlockHeader blockHeader, final BonsaiWorldStateKeyValueStorage worldView) { this.blockHeader = blockHeader; - this.worldStateStorage = worldView; - this.worldViewSubscriberId = worldStateStorage.subscribe(this); + this.worldStateKeyValueStorage = worldView; + this.worldViewSubscriberId = worldStateKeyValueStorage.subscribe(this); } public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { - return worldStateStorage; + return worldStateKeyValueStorage; } public long getBlockNumber() { @@ -49,9 +49,9 @@ public Hash getBlockHash() { } public synchronized void close() { - worldStateStorage.unSubscribe(this.worldViewSubscriberId); + worldStateKeyValueStorage.unSubscribe(this.worldViewSubscriberId); try { - worldStateStorage.close(); + worldStateKeyValueStorage.close(); } catch (final Exception e) { LOG.warn("Failed to close worldstate storage for block " + blockHeader.toLogString(), e); } @@ -60,9 +60,9 @@ public synchronized void close() { public synchronized void updateWorldStateStorage( final BonsaiWorldStateKeyValueStorage newWorldStateStorage) { long newSubscriberId = newWorldStateStorage.subscribe(this); - this.worldStateStorage.unSubscribe(this.worldViewSubscriberId); - BonsaiWorldStateKeyValueStorage oldWorldStateStorage = this.worldStateStorage; - this.worldStateStorage = newWorldStateStorage; + this.worldStateKeyValueStorage.unSubscribe(this.worldViewSubscriberId); + BonsaiWorldStateKeyValueStorage oldWorldStateStorage = this.worldStateKeyValueStorage; + this.worldStateKeyValueStorage = newWorldStateStorage; this.worldViewSubscriberId = newSubscriberId; try { oldWorldStateStorage.close(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedMerkleTrieLoader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedMerkleTrieLoader.java index f53342d6317..9c1d399af28 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedMerkleTrieLoader.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedMerkleTrieLoader.java @@ -57,24 +57,25 @@ public CachedMerkleTrieLoader(final ObservableMetricsSystem metricsSystem) { } public void preLoadAccount( - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final Hash worldStateRootHash, final Address account) { CompletableFuture.runAsync( - () -> cacheAccountNodes(worldStateStorage, worldStateRootHash, account)); + () -> cacheAccountNodes(worldStateKeyValueStorage, worldStateRootHash, account)); } @VisibleForTesting public void cacheAccountNodes( - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final Hash worldStateRootHash, final Address account) { - final long storageSubscriberId = worldStateStorage.subscribe(this); + final long storageSubscriberId = worldStateKeyValueStorage.subscribe(this); try { final StoredMerklePatriciaTrie accountTrie = new StoredMerklePatriciaTrie<>( (location, hash) -> { - Optional node = getAccountStateTrieNode(worldStateStorage, location, hash); + Optional node = + getAccountStateTrieNode(worldStateKeyValueStorage, location, hash); node.ifPresent(bytes -> accountNodes.put(Hash.hash(bytes), bytes)); return node; }, @@ -85,26 +86,27 @@ public void cacheAccountNodes( } catch (MerkleTrieException e) { // ignore exception for the cache } finally { - worldStateStorage.unSubscribe(storageSubscriberId); + worldStateKeyValueStorage.unSubscribe(storageSubscriberId); } } public void preLoadStorageSlot( - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final Address account, final StorageSlotKey slotKey) { - CompletableFuture.runAsync(() -> cacheStorageNodes(worldStateStorage, account, slotKey)); + CompletableFuture.runAsync( + () -> cacheStorageNodes(worldStateKeyValueStorage, account, slotKey)); } @VisibleForTesting public void cacheStorageNodes( - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final Address account, final StorageSlotKey slotKey) { final Hash accountHash = account.addressHash(); - final long storageSubscriberId = worldStateStorage.subscribe(this); + final long storageSubscriberId = worldStateKeyValueStorage.subscribe(this); try { - worldStateStorage + worldStateKeyValueStorage .getStateTrieNode(Bytes.concatenate(accountHash, Bytes.EMPTY)) .ifPresent( storageRoot -> { @@ -114,7 +116,7 @@ public void cacheStorageNodes( (location, hash) -> { Optional node = getAccountStorageTrieNode( - worldStateStorage, accountHash, location, hash); + worldStateKeyValueStorage, accountHash, location, hash); node.ifPresent(bytes -> storageNodes.put(Hash.hash(bytes), bytes)); return node; }, @@ -127,7 +129,7 @@ public void cacheStorageNodes( } }); } finally { - worldStateStorage.unSubscribe(storageSubscriberId); + worldStateKeyValueStorage.unSubscribe(storageSubscriberId); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java index bf205f05b72..144486aef18 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/cache/CachedWorldStorageManager.java @@ -49,12 +49,12 @@ public class CachedWorldStorageManager implements BonsaiStorageSubscriber { private CachedWorldStorageManager( final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final Map cachedWorldStatesByHash, final ObservableMetricsSystem metricsSystem, final EvmConfiguration evmConfiguration) { - worldStateStorage.subscribe(this); - this.rootWorldStateStorage = worldStateStorage; + worldStateKeyValueStorage.subscribe(this); + this.rootWorldStateStorage = worldStateKeyValueStorage; this.cachedWorldStatesByHash = cachedWorldStatesByHash; this.archive = archive; this.metricsSystem = metricsSystem; @@ -63,11 +63,11 @@ private CachedWorldStorageManager( public CachedWorldStorageManager( final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final ObservableMetricsSystem metricsSystem) { this( archive, - worldStateStorage, + worldStateKeyValueStorage, new ConcurrentHashMap<>(), metricsSystem, EvmConfiguration.DEFAULT); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index 80bd2ce86eb..cdafba0eaa9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -56,12 +56,13 @@ public BonsaiSnapshotWorldStateKeyValueStorage( } public BonsaiSnapshotWorldStateKeyValueStorage( - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final ObservableMetricsSystem metricsSystem) { this( - worldStateStorage, - ((SnappableKeyValueStorage) worldStateStorage.composedWorldStateStorage).takeSnapshot(), - worldStateStorage.trieLogStorage, + worldStateKeyValueStorage, + ((SnappableKeyValueStorage) worldStateKeyValueStorage.composedWorldStateStorage) + .takeSnapshot(), + worldStateKeyValueStorage.trieLogStorage, metricsSystem); } @@ -74,7 +75,7 @@ private boolean isClosedGet() { } @Override - public BonsaiUpdater updater() { + public Updater updater() { return new Updater( ((SnappedKeyValueStorage) composedWorldStateStorage).getSnapshotTransaction(), trieLogStorage.startTransaction(), diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 28e2c5bd7aa..27bda772a63 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -21,6 +21,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.flat.FlatDbStrategy; import org.hyperledger.besu.ethereum.bonsai.storage.flat.FullFlatDbStrategy; import org.hyperledger.besu.ethereum.bonsai.storage.flat.PartialFlatDbStrategy; @@ -30,7 +31,6 @@ import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; @@ -45,7 +45,6 @@ import java.util.NavigableMap; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Stream; @@ -54,8 +53,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@SuppressWarnings("unused") -public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoCloseable { +public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorage, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class); // 0x776f726c64526f6f74 @@ -145,12 +143,10 @@ public DataStorageFormat getDataStorageFormat() { return DataStorageFormat.BONSAI; } - @Override public FlatDbMode getFlatDbMode() { return flatDbMode; } - @Override public Optional getCode(final Bytes32 codeHash, final Hash accountHash) { if (codeHash.equals(Hash.EMPTY)) { return Optional.of(Bytes.EMPTY); @@ -168,7 +164,6 @@ public Optional getAccount(final Hash accountHash) { composedWorldStateStorage); } - @Override public Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { if (nodeHash.equals(MerkleTrie.EMPTY_TRIE_NODE_HASH)) { return Optional.of(MerkleTrie.EMPTY_TRIE_NODE); @@ -180,7 +175,6 @@ public Optional getAccountStateTrieNode(final Bytes location, final Bytes } } - @Override public Optional getAccountStorageTrieNode( final Hash accountHash, final Bytes location, final Bytes32 nodeHash) { if (nodeHash.equals(MerkleTrie.EMPTY_TRIE_NODE_HASH)) { @@ -193,7 +187,6 @@ public Optional getAccountStorageTrieNode( } } - @Override public Optional getTrieNodeUnsafe(final Bytes key) { return composedWorldStateStorage .get(TRIE_BRANCH_STORAGE, Bytes.concatenate(key).toArrayUnsafe()) @@ -253,14 +246,12 @@ public Optional getStorageValueByStorageSlotKey( composedWorldStateStorage); } - @Override public Map streamFlatAccounts( final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { return getFlatDbStrategy() .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); } - @Override public Map streamFlatStorages( final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { return getFlatDbStrategy() @@ -273,12 +264,6 @@ public NavigableMap storageEntriesFrom( throw new RuntimeException("Bonsai Tries does not currently support enumerating storage"); } - @Override - public Optional getNodeData(final Bytes location, final Bytes32 hash) { - return Optional.empty(); - } - - @Override public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { return composedWorldStateStorage .get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY) @@ -315,31 +300,24 @@ public void clear() { loadFlatDbStrategy(); // force reload of flat db reader strategy } - @Override public void clearTrieLog() { subscribers.forEach(BonsaiStorageSubscriber::onClearTrieLog); trieLogStorage.clear(); } - @Override public void clearFlatDatabase() { subscribers.forEach(BonsaiStorageSubscriber::onClearFlatDatabaseStorage); getFlatDbStrategy().resetOnResync(composedWorldStateStorage); } @Override - public BonsaiUpdater updater() { + public Updater updater() { return new Updater( composedWorldStateStorage.startTransaction(), trieLogStorage.startTransaction(), flatDbStrategy); } - @Override - public long prune(final Predicate inUseCheck) { - throw new RuntimeException("Bonsai Tries do not work with pruning."); - } - public boolean pruneTrieLog(final Hash blockHash) { try { return trieLogStorage.tryDelete(blockHash.toArrayUnsafe()); @@ -349,34 +327,7 @@ public boolean pruneTrieLog(final Hash blockHash) { } } - @Override - public long addNodeAddedListener(final NodesAddedListener listener) { - throw new RuntimeException("addNodeAddedListener not available"); - } - - @Override - public void removeNodeAddedListener(final long id) { - throw new RuntimeException("removeNodeAddedListener not available"); - } - - public interface BonsaiUpdater extends WorldStateStorage.Updater { - BonsaiUpdater removeCode(final Hash accountHash); - - BonsaiUpdater removeAccountInfoState(final Hash accountHash); - - BonsaiUpdater putAccountInfoState(final Hash accountHash, final Bytes accountValue); - - BonsaiUpdater putStorageValueBySlotHash( - final Hash accountHash, final Hash slotHash, final Bytes storage); - - void removeStorageValueBySlotHash(final Hash accountHash, final Hash slotHash); - - SegmentedKeyValueStorageTransaction getWorldStateTransaction(); - - KeyValueStorageTransaction getTrieLogStorageTransaction(); - } - - public static class Updater implements BonsaiUpdater { + public static class Updater implements WorldStateKeyValueStorage.Updater { private final SegmentedKeyValueStorageTransaction composedWorldStateTransaction; private final KeyValueStorageTransaction trieLogStorageTransaction; @@ -392,14 +343,18 @@ public Updater( this.flatDbStrategy = flatDbStrategy; } - @Override - public BonsaiUpdater removeCode(final Hash accountHash) { + public Updater removeCode(final Hash accountHash) { flatDbStrategy.removeFlatCode(composedWorldStateTransaction, accountHash); return this; } - @Override - public BonsaiUpdater putCode(final Hash accountHash, final Bytes32 codeHash, final Bytes code) { + public Updater putCode(final Hash accountHash, final Bytes code) { + // Skip the hash calculation for empty code + final Hash codeHash = code.size() == 0 ? Hash.EMPTY : Hash.hash(code); + return putCode(accountHash, codeHash, code); + } + + public Updater putCode(final Hash accountHash, final Bytes32 codeHash, final Bytes code) { if (code.size() == 0) { // Don't save empty values return this; @@ -408,14 +363,12 @@ public BonsaiUpdater putCode(final Hash accountHash, final Bytes32 codeHash, fin return this; } - @Override - public BonsaiUpdater removeAccountInfoState(final Hash accountHash) { + public Updater removeAccountInfoState(final Hash accountHash) { flatDbStrategy.removeFlatAccount(composedWorldStateTransaction, accountHash); return this; } - @Override - public BonsaiUpdater putAccountInfoState(final Hash accountHash, final Bytes accountValue) { + public Updater putAccountInfoState(final Hash accountHash, final Bytes accountValue) { if (accountValue.size() == 0) { // Don't save empty values return this; @@ -424,9 +377,7 @@ public BonsaiUpdater putAccountInfoState(final Hash accountHash, final Bytes acc return this; } - @Override - public WorldStateStorage.Updater saveWorldState( - final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { + public Updater saveWorldState(final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { composedWorldStateTransaction.put( TRIE_BRANCH_STORAGE, Bytes.EMPTY.toArrayUnsafe(), node.toArrayUnsafe()); composedWorldStateTransaction.put( @@ -436,8 +387,7 @@ public WorldStateStorage.Updater saveWorldState( return this; } - @Override - public BonsaiUpdater putAccountStateTrieNode( + public Updater putAccountStateTrieNode( final Bytes location, final Bytes32 nodeHash, final Bytes node) { if (nodeHash.equals(MerkleTrie.EMPTY_TRIE_NODE_HASH)) { // Don't save empty nodes @@ -448,14 +398,12 @@ public BonsaiUpdater putAccountStateTrieNode( return this; } - @Override - public BonsaiUpdater removeAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { + public Updater removeAccountStateTrieNode(final Bytes location) { composedWorldStateTransaction.remove(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()); return this; } - @Override - public synchronized BonsaiUpdater putAccountStorageTrieNode( + public synchronized Updater putAccountStorageTrieNode( final Hash accountHash, final Bytes location, final Bytes32 nodeHash, final Bytes node) { if (nodeHash.equals(MerkleTrie.EMPTY_TRIE_NODE_HASH)) { // Don't save empty nodes @@ -468,27 +416,23 @@ public synchronized BonsaiUpdater putAccountStorageTrieNode( return this; } - @Override - public synchronized BonsaiUpdater putStorageValueBySlotHash( + public synchronized Updater putStorageValueBySlotHash( final Hash accountHash, final Hash slotHash, final Bytes storage) { flatDbStrategy.putFlatAccountStorageValueByStorageSlotHash( composedWorldStateTransaction, accountHash, slotHash, storage); return this; } - @Override public synchronized void removeStorageValueBySlotHash( final Hash accountHash, final Hash slotHash) { flatDbStrategy.removeFlatAccountStorageValueByStorageSlotHash( composedWorldStateTransaction, accountHash, slotHash); } - @Override public SegmentedKeyValueStorageTransaction getWorldStateTransaction() { return composedWorldStateTransaction; } - @Override public KeyValueStorageTransaction getTrieLogStorageTransaction() { return trieLogStorageTransaction; } @@ -500,7 +444,6 @@ public void commit() { composedWorldStateTransaction.commit(); } - @Override public void rollback() { composedWorldStateTransaction.rollback(); trieLogStorageTransaction.rollback(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java index 3a874de6b67..75bb250bffe 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManager.java @@ -51,12 +51,12 @@ public class TrieLogManager { public TrieLogManager( final Blockchain blockchain, - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final long maxLayersToLoad, final BesuContext pluginContext, final TrieLogPruner trieLogPruner) { this.blockchain = blockchain; - this.rootWorldStateStorage = worldStateStorage; + this.rootWorldStateStorage = worldStateKeyValueStorage; this.maxLayersToLoad = maxLayersToLoad; this.trieLogFactory = setupTrieLogFactory(pluginContext); this.trieLogPruner = trieLogPruner; @@ -71,7 +71,7 @@ public synchronized void saveTrieLog( // if it's only in memory we need to save it // for example, in case of reorg we don't replace a trielog layer if (rootWorldStateStorage.getTrieLog(forBlockHeader.getHash()).isEmpty()) { - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater = + final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = forWorldState.getWorldStateStorage().updater(); boolean success = false; try { @@ -109,7 +109,7 @@ private void persistTrieLog( final BlockHeader blockHeader, final Hash worldStateRootHash, final TrieLog trieLog, - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater) { + final BonsaiWorldStateKeyValueStorage.Updater stateUpdater) { LOG.atDebug() .setMessage("Persisting trie log for block hash {} and world state root {}") .addArgument(blockHeader::toLogString) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java index 7c7f7fd5516..3931731fbc4 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; @@ -41,7 +42,6 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.NodeLoader; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; @@ -68,7 +68,7 @@ public class BonsaiWorldState private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldState.class); - protected BonsaiWorldStateKeyValueStorage worldStateStorage; + protected BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; protected final CachedMerkleTrieLoader cachedMerkleTrieLoader; protected final CachedWorldStorageManager cachedWorldStorageManager; @@ -81,10 +81,10 @@ public class BonsaiWorldState public BonsaiWorldState( final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final EvmConfiguration evmConfiguration) { this( - worldStateStorage, + worldStateKeyValueStorage, archive.getCachedMerkleTrieLoader(), archive.getCachedWorldStorageManager(), archive.getTrieLogManager(), @@ -92,17 +92,19 @@ public BonsaiWorldState( } protected BonsaiWorldState( - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, final CachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final EvmConfiguration evmConfiguration) { - this.worldStateStorage = worldStateStorage; + this.worldStateKeyValueStorage = worldStateKeyValueStorage; this.worldStateRootHash = Hash.wrap( - Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH))); + Bytes32.wrap( + worldStateKeyValueStorage.getWorldStateRootHash().orElse(getEmptyTrieHash()))); this.worldStateBlockHash = - Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); + Hash.wrap( + Bytes32.wrap(worldStateKeyValueStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); this.accumulator = new BonsaiWorldStateUpdateAccumulator( this, @@ -147,16 +149,16 @@ public Hash getWorldStateRootHash() { @Override public boolean isPersisted() { - return isPersisted(worldStateStorage); + return isPersisted(worldStateKeyValueStorage); } - private boolean isPersisted(final WorldStateStorage worldStateStorage) { - return !(worldStateStorage instanceof BonsaiSnapshotWorldStateKeyValueStorage); + private boolean isPersisted(final WorldStateKeyValueStorage worldStateKeyValueStorage) { + return !(worldStateKeyValueStorage instanceof BonsaiSnapshotWorldStateKeyValueStorage); } @Override public Optional getCode(@Nonnull final Address address, final Hash codeHash) { - return worldStateStorage.getCode(codeHash, address.addressHash()); + return worldStateKeyValueStorage.getCode(codeHash, address.addressHash()); } /** @@ -171,11 +173,11 @@ public void resetWorldStateTo(final BlockHeader blockHeader) { @Override public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { - return worldStateStorage; + return worldStateKeyValueStorage; } private Hash calculateRootHash( - final Optional maybeStateUpdater, + final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { clearStorage(maybeStateUpdater, worldStateUpdater); @@ -200,7 +202,8 @@ private Hash calculateRootHash( final StoredMerklePatriciaTrie accountTrie = createTrie( (location, hash) -> - cachedMerkleTrieLoader.getAccountStateTrieNode(worldStateStorage, location, hash), + cachedMerkleTrieLoader.getAccountStateTrieNode( + worldStateKeyValueStorage, location, hash), worldStateRootHash); // for manicured tries and composting, collect branches here (not implemented) @@ -222,7 +225,7 @@ private Hash calculateRootHash( } private void updateTheAccounts( - final Optional maybeStateUpdater, + final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater, final StoredMerklePatriciaTrie accountTrie) { for (final Map.Entry> accountUpdate : @@ -253,7 +256,7 @@ private void updateTheAccounts( } private void updateCode( - final Optional maybeStateUpdater, + final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { maybeStateUpdater.ifPresent( bonsaiUpdater -> { @@ -271,7 +274,7 @@ private void updateCode( } private void updateAccountStorageState( - final Optional maybeStateUpdater, + final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater, final Map.Entry>> storageAccountUpdate) { @@ -287,7 +290,7 @@ private void updateAccountStorageState( createTrie( (location, key) -> cachedMerkleTrieLoader.getAccountStorageTrieNode( - worldStateStorage, updatedAddressHash, location, key), + worldStateKeyValueStorage, updatedAddressHash, location, key), storageRoot); // for manicured tries and composting, collect branches here (not implemented) @@ -334,7 +337,7 @@ private void updateAccountStorageState( } private void clearStorage( - final Optional maybeStateUpdater, + final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { maybeStateUpdater.ifPresent( @@ -342,7 +345,7 @@ private void clearStorage( for (final Address address : worldStateUpdater.getStorageToClear()) { // because we are clearing persisted values we need the account root as persisted final BonsaiAccount oldAccount = - worldStateStorage + worldStateKeyValueStorage .getAccount(address.addressHash()) .map(bytes -> fromRLP(BonsaiWorldState.this, address, bytes, true)) .orElse(null); @@ -393,7 +396,8 @@ public void persist(final BlockHeader blockHeader) { boolean success = false; - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater stateUpdater = worldStateStorage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = + worldStateKeyValueStorage.updater(); Runnable saveTrieLog = () -> {}; try { @@ -517,7 +521,7 @@ public Hash frontierRootHash() { return calculateRootHash( Optional.of( new BonsaiWorldStateKeyValueStorage.Updater( - noOpSegmentedTx, noOpTx, worldStateStorage.getFlatDbStrategy())), + noOpSegmentedTx, noOpTx, worldStateKeyValueStorage.getFlatDbStrategy())), accumulator.copy()); } @@ -532,17 +536,17 @@ public Stream streamAccounts(final Bytes32 startKeyHash, fina @Override public Account get(final Address address) { - return worldStateStorage + return worldStateKeyValueStorage .getAccount(address.addressHash()) .map(bytes -> fromRLP(accumulator, address, bytes, true)) .orElse(null); } protected Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { - return worldStateStorage.getAccountStateTrieNode(location, nodeHash); + return worldStateKeyValueStorage.getAccountStateTrieNode(location, nodeHash); } - private void writeTrieNode( + protected void writeTrieNode( final SegmentIdentifier segmentId, final SegmentedKeyValueStorageTransaction tx, final Bytes location, @@ -552,11 +556,11 @@ private void writeTrieNode( protected Optional getStorageTrieNode( final Hash accountHash, final Bytes location, final Bytes32 nodeHash) { - return worldStateStorage.getAccountStorageTrieNode(accountHash, location, nodeHash); + return worldStateKeyValueStorage.getAccountStorageTrieNode(accountHash, location, nodeHash); } private void writeStorageTrieNode( - final WorldStateStorage.Updater stateUpdater, + final BonsaiWorldStateKeyValueStorage.Updater stateUpdater, final Hash accountHash, final Bytes location, final Bytes32 nodeHash, @@ -573,7 +577,7 @@ public UInt256 getStorageValue(final Address address, final UInt256 storageKey) @Override public Optional getStorageValueByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { - return worldStateStorage + return worldStateKeyValueStorage .getStorageValueByStorageSlotKey(address.addressHash(), storageSlotKey) .map(UInt256::fromBytes); } @@ -582,7 +586,7 @@ public Optional getStorageValueByStorageSlotKey( final Supplier> storageRootSupplier, final Address address, final StorageSlotKey storageSlotKey) { - return worldStateStorage + return worldStateKeyValueStorage .getStorageValueByStorageSlotKey(storageRootSupplier, address.addressHash(), storageSlotKey) .map(UInt256::fromBytes); } @@ -603,7 +607,7 @@ public Map getAllAccountStorage(final Address address, final Has @Override public MutableWorldState freeze() { this.isFrozen = true; - this.worldStateStorage = new BonsaiWorldStateLayerStorage(worldStateStorage); + this.worldStateKeyValueStorage = new BonsaiWorldStateLayerStorage(worldStateKeyValueStorage); return this; } @@ -617,7 +621,7 @@ private StoredMerklePatriciaTrie createTrie( public void close() { try { if (!isPersisted()) { - this.worldStateStorage.close(); + this.worldStateKeyValueStorage.close(); if (isFrozen) { closeFrozenStorage(); } @@ -630,7 +634,7 @@ public void close() { private void closeFrozenStorage() { try { final BonsaiWorldStateLayerStorage worldStateLayerStorage = - (BonsaiWorldStateLayerStorage) worldStateStorage; + (BonsaiWorldStateLayerStorage) worldStateKeyValueStorage; if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { worldStateLayerStorage.getParentWorldStateStorage().close(); } @@ -643,4 +647,8 @@ protected Hash hashAndSavePreImage(final Bytes value) { // by default do not save has preImages return Hash.hash(value); } + + protected Hash getEmptyTrieHash() { + return Hash.EMPTY_TRIE_HASH; + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java index a69e920dd20..94d5fe32ef9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java @@ -30,11 +30,11 @@ import org.hyperledger.besu.ethereum.core.Difficulty; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Withdrawal; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.log.LogsBloomFilter; @@ -134,12 +134,12 @@ private static void writeAccountsTo( } private static Hash calculateGenesisStateHash(final List genesisAccounts) { - final WorldStateKeyValueStorage stateStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage stateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStatePreimageKeyValueStorage preimageStorage = new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); final MutableWorldState worldState = - new DefaultMutableWorldState(stateStorage, preimageStorage, EvmConfiguration.DEFAULT); + new ForestMutableWorldState(stateStorage, preimageStorage, EvmConfiguration.DEFAULT); writeAccountsTo(worldState, genesisAccounts, null); return worldState.rootHash(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/PrivacyParameters.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/PrivacyParameters.java index 1d4cfdf6c1b..9c61be6ff39 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/PrivacyParameters.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/PrivacyParameters.java @@ -21,15 +21,15 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.enclave.Enclave; import org.hyperledger.besu.enclave.EnclaveFactory; +import org.hyperledger.besu.ethereum.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.privacy.PrivateStateGenesisAllocator; import org.hyperledger.besu.ethereum.privacy.PrivateStateRootResolver; import org.hyperledger.besu.ethereum.privacy.PrivateWorldStateReader; import org.hyperledger.besu.ethereum.privacy.storage.PrivacyStorageProvider; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; -import org.hyperledger.besu.ethereum.worldstate.DefaultWorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.plugin.services.PrivacyPluginService; import org.hyperledger.besu.plugin.services.privacy.PrivacyGroupGenesisProvider; @@ -335,12 +335,12 @@ public Builder setPrivacyService(final PrivacyPluginService privacyPluginService public PrivacyParameters build() { final PrivacyParameters config = new PrivacyParameters(); if (enabled) { - final WorldStateStorage privateWorldStateStorage = - storageProvider.createWorldStateStorage(); + final WorldStateStorageCoordinator privateWorldStateStorage = + storageProvider.createWorldStateStorageCoordinator(); final WorldStatePreimageStorage privatePreimageStorage = storageProvider.createWorldStatePreimageStorage(); final WorldStateArchive privateWorldStateArchive = - new DefaultWorldStateArchive( + new ForestWorldStateArchive( privateWorldStateStorage, privatePreimageStorage, EvmConfiguration.DEFAULT); final PrivateStateStorage privateStateStorage = storageProvider.createPrivateStateStorage(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/ForestWorldStateArchive.java similarity index 72% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/ForestWorldStateArchive.java index a29bd9b2ff4..adbaf4c9fb5 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/ForestWorldStateArchive.java @@ -13,15 +13,20 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.worldstate; +package org.hyperledger.besu.ethereum.forest; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; +import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldState; @@ -32,21 +37,22 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; -public class DefaultWorldStateArchive implements WorldStateArchive { - private final WorldStateStorage worldStateStorage; +public class ForestWorldStateArchive implements WorldStateArchive { + private final ForestWorldStateKeyValueStorage worldStateKeyValueStorage; private final WorldStatePreimageStorage preimageStorage; private final WorldStateProofProvider worldStateProof; private final EvmConfiguration evmConfiguration; private static final Hash EMPTY_ROOT_HASH = Hash.wrap(MerkleTrie.EMPTY_TRIE_NODE_HASH); - public DefaultWorldStateArchive( - final WorldStateStorage worldStateStorage, + public ForestWorldStateArchive( + final WorldStateStorageCoordinator worldStateStorageCoordinator, final WorldStatePreimageStorage preimageStorage, final EvmConfiguration evmConfiguration) { - this.worldStateStorage = worldStateStorage; + this.worldStateKeyValueStorage = + worldStateStorageCoordinator.getStrategy(ForestWorldStateKeyValueStorage.class); this.preimageStorage = preimageStorage; - this.worldStateProof = new WorldStateProofProvider(worldStateStorage); + this.worldStateProof = new WorldStateProofProvider(worldStateStorageCoordinator); this.evmConfiguration = evmConfiguration; } @@ -57,7 +63,7 @@ public Optional get(final Hash rootHash, final Hash blockHash) { @Override public boolean isWorldStateAvailable(final Hash rootHash, final Hash blockHash) { - return worldStateStorage.isWorldStateAvailable(rootHash, blockHash); + return worldStateKeyValueStorage.isWorldStateAvailable(rootHash); } @Override @@ -68,12 +74,12 @@ public Optional getMutable( @Override public Optional getMutable(final Hash rootHash, final Hash blockHash) { - if (!worldStateStorage.isWorldStateAvailable(rootHash, blockHash)) { + if (!worldStateKeyValueStorage.isWorldStateAvailable(rootHash)) { return Optional.empty(); } return Optional.of( - new DefaultMutableWorldState( - rootHash, worldStateStorage, preimageStorage, evmConfiguration)); + new ForestMutableWorldState( + rootHash, worldStateKeyValueStorage, preimageStorage, evmConfiguration)); } @Override @@ -89,11 +95,11 @@ public void resetArchiveStateTo(final BlockHeader blockHeader) { @Override public Optional getNodeData(final Hash hash) { // query by location is not supported, only query by content - return worldStateStorage.getNodeData(null, hash); + return worldStateKeyValueStorage.getNodeData(hash); } - public WorldStateStorage getWorldStateStorage() { - return worldStateStorage; + public ForestWorldStateKeyValueStorage getWorldStateStorage() { + return worldStateKeyValueStorage; } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/pruner/MarkSweepPruner.java similarity index 89% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPruner.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/pruner/MarkSweepPruner.java index 136e8b01445..4aa47eff84f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/pruner/MarkSweepPruner.java @@ -12,14 +12,16 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.worldstate; +package org.hyperledger.besu.ethereum.forest.pruner; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; @@ -56,7 +58,7 @@ public class MarkSweepPruner { private static final int MAX_MARKING_THREAD_POOL_SIZE = 2; private final int operationsPerTransaction; - private final WorldStateStorage worldStateStorage; + private final ForestWorldStateKeyValueStorage worldStateKeyValueStorage; private final MutableBlockchain blockchain; private final KeyValueStorage markStorage; private final Counter markedNodesCounter; @@ -69,20 +71,25 @@ public class MarkSweepPruner { private final Set pendingMarks = Collections.newSetFromMap(new ConcurrentHashMap<>()); public MarkSweepPruner( - final WorldStateStorage worldStateStorage, + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage, final MutableBlockchain blockchain, final KeyValueStorage markStorage, final ObservableMetricsSystem metricsSystem) { - this(worldStateStorage, blockchain, markStorage, metricsSystem, DEFAULT_OPS_PER_TRANSACTION); + this( + worldStateKeyValueStorage, + blockchain, + markStorage, + metricsSystem, + DEFAULT_OPS_PER_TRANSACTION); } public MarkSweepPruner( - final WorldStateStorage worldStateStorage, + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage, final MutableBlockchain blockchain, final KeyValueStorage markStorage, final ObservableMetricsSystem metricsSystem, final int operationsPerTransaction) { - this.worldStateStorage = worldStateStorage; + this.worldStateKeyValueStorage = worldStateKeyValueStorage; this.markStorage = markStorage; this.blockchain = blockchain; this.operationsPerTransaction = operationsPerTransaction; @@ -123,7 +130,7 @@ public void prepare() { // last time, causing the first sweep to be smaller than it needs to be. clearMarks(); - nodeAddedListenerId = worldStateStorage.addNodeAddedListener(this::markNodes); + nodeAddedListenerId = worldStateKeyValueStorage.addNodeAddedListener(this::markNodes); } /** @@ -194,34 +201,34 @@ public void sweepBefore(final long markedBlockNumber) { // Sweep state roots first, walking backwards until we get to a state root that isn't in the // storage long prunedNodeCount = 0; - WorldStateStorage.Updater updater = worldStateStorage.updater(); + ForestWorldStateKeyValueStorage.Updater updater = worldStateKeyValueStorage.updater(); for (long blockNumber = markedBlockNumber - 1; blockNumber >= 0; blockNumber--) { final BlockHeader blockHeader = blockchain.getBlockHeader(blockNumber).get(); final Hash candidateStateRootHash = blockHeader.getStateRoot(); - if (!worldStateStorage.isWorldStateAvailable(candidateStateRootHash, null)) { + if (!worldStateKeyValueStorage.isWorldStateAvailable(candidateStateRootHash)) { break; } if (!isMarked(candidateStateRootHash)) { - updater.removeAccountStateTrieNode(null, candidateStateRootHash); + updater.removeAccountStateTrieNode(candidateStateRootHash); prunedNodeCount++; if (prunedNodeCount % operationsPerTransaction == 0) { updater.commit(); - updater = worldStateStorage.updater(); + updater = worldStateKeyValueStorage.updater(); } } } updater.commit(); // Sweep non-state-root nodes - prunedNodeCount += worldStateStorage.prune(this::isMarked); + prunedNodeCount += worldStateKeyValueStorage.prune(this::isMarked); sweptNodesCounter.inc(prunedNodeCount); clearMarks(); LOG.debug("Completed sweeping unused nodes"); } public void cleanup() { - worldStateStorage.removeNodeAddedListener(nodeAddedListenerId); + worldStateKeyValueStorage.removeNodeAddedListener(nodeAddedListenerId); clearMarks(); } @@ -240,7 +247,7 @@ private boolean isMarked(final byte[] key) { private MerkleTrie createStateTrie(final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, + (location, hash) -> worldStateKeyValueStorage.getAccountStateTrieNode(hash), rootHash, Function.identity(), Function.identity()); @@ -248,7 +255,7 @@ private MerkleTrie createStateTrie(final Bytes32 rootHash) { private MerkleTrie createStorageTrie(final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateStorage.getAccountStorageTrieNode(null, location, hash), + (location, hash) -> worldStateKeyValueStorage.getAccountStorageTrieNode(hash), rootHash, Function.identity(), Function.identity()); @@ -263,7 +270,7 @@ private void processAccountState(final Bytes value, final ExecutorService execut } @VisibleForTesting - void markNode(final Bytes32 hash) { + public void markNode(final Bytes32 hash) { markThenMaybeFlush(() -> pendingMarks.add(hash), 1); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/Pruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/pruner/Pruner.java similarity index 97% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/Pruner.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/pruner/Pruner.java index 04949b55e12..c1ba086503f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/Pruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/pruner/Pruner.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.worldstate; +package org.hyperledger.besu.ethereum.forest.pruner; import static com.google.common.base.Preconditions.checkArgument; @@ -51,7 +51,7 @@ public class Pruner { private final ExecutorService executorService; @VisibleForTesting - Pruner( + public Pruner( final MarkSweepPruner pruningStrategy, final Blockchain blockchain, final PrunerConfiguration prunerConfiguration, @@ -179,11 +179,11 @@ private void execute(final Runnable action) { } } - PruningPhase getPruningPhase() { + public PruningPhase getPruningPhase() { return pruningPhase.get(); } - enum PruningPhase { + public enum PruningPhase { IDLE, MARK_BLOCK_CONFIRMATIONS_AWAITING, MARKING, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/PrunerConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/pruner/PrunerConfiguration.java similarity index 96% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/PrunerConfiguration.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/pruner/PrunerConfiguration.java index f2336ee9a3f..e9ca4aeb871 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/PrunerConfiguration.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/pruner/PrunerConfiguration.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.worldstate; +package org.hyperledger.besu.ethereum.forest.pruner; public class PrunerConfiguration { public static final int DEFAULT_PRUNING_BLOCKS_RETAINED = 1024; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/storage/ForestWorldStateKeyValueStorage.java similarity index 68% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/storage/ForestWorldStateKeyValueStorage.java index 74051b5e9b7..4a1e6ea21b6 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/storage/ForestWorldStateKeyValueStorage.java @@ -12,13 +12,12 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.storage.keyvalue; +package org.hyperledger.besu.ethereum.forest.storage; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.util.Subscribers; @@ -35,13 +34,14 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -public class WorldStateKeyValueStorage implements WorldStateStorage { +public class ForestWorldStateKeyValueStorage implements WorldStateKeyValueStorage { - private final Subscribers nodeAddedListeners = Subscribers.create(); + private final Subscribers nodeAddedListeners = + Subscribers.create(); private final KeyValueStorage keyValueStorage; private final ReentrantLock lock = new ReentrantLock(); - public WorldStateKeyValueStorage(final KeyValueStorage keyValueStorage) { + public ForestWorldStateKeyValueStorage(final KeyValueStorage keyValueStorage) { this.keyValueStorage = keyValueStorage; } @@ -50,8 +50,7 @@ public DataStorageFormat getDataStorageFormat() { return DataStorageFormat.FOREST; } - @Override - public Optional getCode(final Bytes32 codeHash, final Hash accountHash) { + public Optional getCode(final Bytes32 codeHash) { if (codeHash.equals(Hash.EMPTY)) { return Optional.of(Bytes.EMPTY); } else { @@ -59,14 +58,11 @@ public Optional getCode(final Bytes32 codeHash, final Hash accountHash) { } } - @Override - public Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { + public Optional getAccountStateTrieNode(final Bytes32 nodeHash) { return getTrieNode(nodeHash); } - @Override - public Optional getAccountStorageTrieNode( - final Hash accountHash, final Bytes location, final Bytes32 nodeHash) { + public Optional getAccountStorageTrieNode(final Bytes32 nodeHash) { return getTrieNode(nodeHash); } @@ -78,18 +74,12 @@ private Optional getTrieNode(final Bytes32 nodeHash) { } } - @Override - public Optional getTrieNodeUnsafe(final Bytes key) { - return keyValueStorage.get(key.toArrayUnsafe()).map(Bytes::wrap); - } - - @Override - public FlatDbMode getFlatDbMode() { - return FlatDbMode.NO_FLATTENED; + public boolean contains(final Bytes32 hash) { + // we don't have location info + return getNodeData(hash).isPresent(); } - @Override - public Optional getNodeData(final Bytes location, final Bytes32 hash) { + public Optional getNodeData(final Bytes32 hash) { if (hash.equals(MerkleTrie.EMPTY_TRIE_NODE_HASH)) { return Optional.of(MerkleTrie.EMPTY_TRIE_NODE); } else if (hash.equals(Hash.EMPTY)) { @@ -99,9 +89,8 @@ public Optional getNodeData(final Bytes location, final Bytes32 hash) { } } - @Override - public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { - return getAccountStateTrieNode(Bytes.EMPTY, rootHash).isPresent(); + public boolean isWorldStateAvailable(final Bytes32 rootHash) { + return getAccountStateTrieNode(rootHash).isPresent(); } @Override @@ -109,22 +98,11 @@ public void clear() { keyValueStorage.clear(); } - @Override - public void clearTrieLog() { - // nothing to do for forest - } - - @Override - public void clearFlatDatabase() { - // nothing to do for forest - } - @Override public Updater updater() { return new Updater(lock, keyValueStorage.startTransaction(), nodeAddedListeners); } - @Override public long prune(final Predicate inUseCheck) { final AtomicInteger prunedKeys = new AtomicInteger(0); try (final Stream entry = keyValueStorage.streamKeys()) { @@ -144,17 +122,15 @@ public long prune(final Predicate inUseCheck) { return prunedKeys.get(); } - @Override public long addNodeAddedListener(final NodesAddedListener listener) { return nodeAddedListeners.subscribe(listener); } - @Override public void removeNodeAddedListener(final long id) { nodeAddedListeners.unsubscribe(id); } - public static class Updater implements WorldStateStorage.Updater { + public static class Updater implements WorldStateKeyValueStorage.Updater { private final KeyValueStorageTransaction transaction; private final Subscribers nodeAddedListeners; @@ -170,9 +146,14 @@ public Updater( this.nodeAddedListeners = nodeAddedListeners; } - @Override - public WorldStateStorage.Updater putCode( - final Hash accountHash, final Bytes32 codeHash, final Bytes code) { + public ForestWorldStateKeyValueStorage.Updater putCode(final Bytes code) { + // Skip the hash calculation for empty code + final Hash codeHash = code.size() == 0 ? Hash.EMPTY : Hash.hash(code); + return putCode(codeHash, code); + } + + public ForestWorldStateKeyValueStorage.Updater putCode( + final Bytes32 codeHash, final Bytes code) { if (code.size() == 0) { // Don't save empty values return this; @@ -183,15 +164,13 @@ public WorldStateStorage.Updater putCode( return this; } - @Override - public WorldStateStorage.Updater saveWorldState( - final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { - return putAccountStateTrieNode(null, nodeHash, node); + public ForestWorldStateKeyValueStorage.Updater saveWorldState( + final Bytes32 nodeHash, final Bytes node) { + return putAccountStateTrieNode(nodeHash, node); } - @Override - public Updater putAccountStateTrieNode( - final Bytes location, final Bytes32 nodeHash, final Bytes node) { + public ForestWorldStateKeyValueStorage.Updater putAccountStateTrieNode( + final Bytes32 nodeHash, final Bytes node) { if (nodeHash.equals(MerkleTrie.EMPTY_TRIE_NODE_HASH)) { // Don't save empty nodes return this; @@ -201,16 +180,12 @@ public Updater putAccountStateTrieNode( return this; } - @Override - public WorldStateStorage.Updater removeAccountStateTrieNode( - final Bytes location, final Bytes32 nodeHash) { + public WorldStateKeyValueStorage.Updater removeAccountStateTrieNode(final Bytes32 nodeHash) { transaction.remove(nodeHash.toArrayUnsafe()); return this; } - @Override - public Updater putAccountStorageTrieNode( - final Hash accountHash, final Bytes location, final Bytes32 nodeHash, final Bytes node) { + public Updater putAccountStorageTrieNode(final Bytes32 nodeHash, final Bytes node) { if (nodeHash.equals(MerkleTrie.EMPTY_TRIE_NODE_HASH)) { // Don't save empty nodes return this; @@ -231,7 +206,6 @@ public void commit() { } } - @Override public void rollback() { addedNodes.clear(); transaction.rollback(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/worldview/ForestMutableWorldState.java similarity index 86% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/worldview/ForestMutableWorldState.java index d50f702af31..4746c6d11c3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/worldview/ForestMutableWorldState.java @@ -12,18 +12,22 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.worldstate; +package org.hyperledger.besu.ethereum.forest.worldview; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -47,10 +51,10 @@ import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -public class DefaultMutableWorldState implements MutableWorldState { +public class ForestMutableWorldState implements MutableWorldState { private final EvmConfiguration evmConfiguration; - private final WorldStateStorage worldStateStorage; + private final ForestWorldStateKeyValueStorage worldStateKeyValueStorage; private final WorldStatePreimageStorage preimageStorage; private final MerkleTrie accountStateTrie; @@ -59,34 +63,37 @@ public class DefaultMutableWorldState implements MutableWorldState { private final Map newStorageKeyPreimages = new HashMap<>(); private final Map newAccountKeyPreimages = new HashMap<>(); - public DefaultMutableWorldState( - final WorldStateStorage storage, + public ForestMutableWorldState( + final WorldStateKeyValueStorage worldStateKeyValueStorage, final WorldStatePreimageStorage preimageStorage, final EvmConfiguration evmConfiguration) { - this(MerkleTrie.EMPTY_TRIE_NODE_HASH, storage, preimageStorage, evmConfiguration); + this( + MerkleTrie.EMPTY_TRIE_NODE_HASH, + worldStateKeyValueStorage, + preimageStorage, + evmConfiguration); } - public DefaultMutableWorldState( + public ForestMutableWorldState( final Bytes32 rootHash, - final WorldStateStorage worldStateStorage, + final WorldStateKeyValueStorage worldStateKeyValueStorage, final WorldStatePreimageStorage preimageStorage, final EvmConfiguration evmConfiguration) { - this.worldStateStorage = worldStateStorage; + this.worldStateKeyValueStorage = (ForestWorldStateKeyValueStorage) worldStateKeyValueStorage; this.accountStateTrie = newAccountStateTrie(rootHash); this.preimageStorage = preimageStorage; this.evmConfiguration = evmConfiguration; } - public DefaultMutableWorldState( + public ForestMutableWorldState( final WorldState worldState, final EvmConfiguration evmConfiguration) { // TODO: this is an abstraction leak (and kind of incorrect in that we reuse the underlying // storage), but the reason for this is that the accounts() method is unimplemented below and // can't be until NC-754. - if (!(worldState instanceof DefaultMutableWorldState other)) { + if (!(worldState instanceof ForestMutableWorldState other)) { throw new UnsupportedOperationException(); } - - this.worldStateStorage = other.worldStateStorage; + this.worldStateKeyValueStorage = other.worldStateKeyValueStorage; this.preimageStorage = other.preimageStorage; this.accountStateTrie = newAccountStateTrie(other.accountStateTrie.getRootHash()); this.evmConfiguration = evmConfiguration; @@ -94,12 +101,15 @@ public DefaultMutableWorldState( private MerkleTrie newAccountStateTrie(final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, rootHash, b -> b, b -> b); + (location, hash) -> worldStateKeyValueStorage.getAccountStateTrieNode(hash), + rootHash, + b -> b, + b -> b); } private MerkleTrie newAccountStorageTrie(final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateStorage.getAccountStorageTrieNode(null, location, hash), + (location, hash) -> worldStateKeyValueStorage.getAccountStorageTrieNode(hash), rootHash, b -> b, b -> b); @@ -156,7 +166,7 @@ public int hashCode() { @Override public final boolean equals(final Object other) { - if (!(other instanceof DefaultMutableWorldState that)) { + if (!(other instanceof ForestMutableWorldState that)) { return false; } @@ -165,19 +175,20 @@ public final boolean equals(final Object other) { @Override public void persist(final BlockHeader blockHeader) { - final WorldStateStorage.Updater stateUpdater = worldStateStorage.updater(); + final ForestWorldStateKeyValueStorage.Updater stateUpdater = + worldStateKeyValueStorage.updater(); // Store updated code for (final Bytes code : updatedAccountCode.values()) { - stateUpdater.putCode(null, code); + stateUpdater.putCode(code); } // Commit account storage tries for (final MerkleTrie updatedStorage : updatedStorageTries.values()) { updatedStorage.commit( - (location, hash, value) -> - stateUpdater.putAccountStorageTrieNode(null, location, hash, value)); + (location, hash, value) -> stateUpdater.putAccountStorageTrieNode(hash, value)); } // Commit account updates - accountStateTrie.commit(stateUpdater::putAccountStateTrieNode); + accountStateTrie.commit( + (location, hash, value) -> stateUpdater.putAccountStateTrieNode(hash, value)); // Persist preimages final WorldStatePreimageStorage.Updater preimageUpdater = preimageStorage.updater(); @@ -271,7 +282,7 @@ public Bytes getCode() { if (codeHash.equals(Hash.EMPTY)) { return Bytes.EMPTY; } - return worldStateStorage.getCode(codeHash, null).orElse(Bytes.EMPTY); + return worldStateKeyValueStorage.getCode(codeHash).orElse(Bytes.EMPTY); } @Override @@ -288,7 +299,7 @@ public Hash getCodeHash() { public UInt256 getStorageValue(final UInt256 key) { return storageTrie() .get(Hash.hash(key)) - .map(DefaultMutableWorldState::convertToUInt256) + .map(ForestMutableWorldState::convertToUInt256) .orElse(UInt256.ZERO); } @@ -332,16 +343,16 @@ private Optional getStorageTrieKeyPreimage(final Bytes32 trieKey) { } protected static class Updater - extends AbstractWorldUpdater { + extends AbstractWorldUpdater { protected Updater( - final DefaultMutableWorldState world, final EvmConfiguration evmConfiguration) { + final ForestMutableWorldState world, final EvmConfiguration evmConfiguration) { super(world, evmConfiguration); } @Override protected WorldStateAccount getForMutation(final Address address) { - final DefaultMutableWorldState wrapped = wrappedWorldView(); + final ForestMutableWorldState wrapped = wrappedWorldView(); final Hash addressHash = address.addressHash(); return wrapped .accountStateTrie @@ -368,7 +379,7 @@ public void revert() { @Override public void commit() { - final DefaultMutableWorldState wrapped = wrappedWorldView(); + final ForestMutableWorldState wrapped = wrappedWorldView(); for (final Address address : getDeletedAccounts()) { final Hash addressHash = address.addressHash(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyStorageProvider.java index 2dc167a884b..1ee99f65bdc 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/PrivacyStorageProvider.java @@ -14,14 +14,17 @@ */ package org.hyperledger.besu.ethereum.privacy.storage; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.io.Closeable; public interface PrivacyStorageProvider extends Closeable { - WorldStateStorage createWorldStateStorage(); + WorldStateKeyValueStorage createWorldStateStorage(); + + WorldStateStorageCoordinator createWorldStateStorageCoordinator(); WorldStatePreimageStorage createWorldStatePreimageStorage(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/keyvalue/PrivacyKeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/keyvalue/PrivacyKeyValueStorageProvider.java index d54a4ba164f..eae728de43d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/keyvalue/PrivacyKeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/keyvalue/PrivacyKeyValueStorageProvider.java @@ -14,15 +14,16 @@ */ package org.hyperledger.besu.ethereum.privacy.storage.keyvalue; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateKeyValueStorage; import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateStorage; import org.hyperledger.besu.ethereum.privacy.storage.PrivacyStorageProvider; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import java.io.IOException; @@ -47,8 +48,13 @@ public PrivacyKeyValueStorageProvider( } @Override - public WorldStateStorage createWorldStateStorage() { - return new WorldStateKeyValueStorage(privateWorldStateKeyValueStorage); + public WorldStateKeyValueStorage createWorldStateStorage() { + return new ForestWorldStateKeyValueStorage(privateWorldStateKeyValueStorage); + } + + @Override + public WorldStateStorageCoordinator createWorldStateStorageCoordinator() { + return new WorldStateStorageCoordinator(createWorldStateStorage()); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProvider.java index 16023264b7d..0258ce1034a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProvider.java @@ -26,7 +26,7 @@ import org.hyperledger.besu.ethereum.trie.patricia.SimpleMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.HashMap; import java.util.List; @@ -48,10 +48,10 @@ */ public class WorldStateProofProvider { - private final WorldStateStorage worldStateStorage; + private final WorldStateStorageCoordinator worldStateStorageCoordinator; - public WorldStateProofProvider(final WorldStateStorage worldStateStorage) { - this.worldStateStorage = worldStateStorage; + public WorldStateProofProvider(final WorldStateStorageCoordinator worldStateStorageCoordinator) { + this.worldStateStorageCoordinator = worldStateStorageCoordinator; } public Optional getAccountProof( @@ -59,7 +59,7 @@ public Optional getAccountProof( final Address accountAddress, final List accountStorageKeys) { - if (!worldStateStorage.isWorldStateAvailable(worldStateRoot, null)) { + if (!worldStateStorageCoordinator.isWorldStateAvailable(worldStateRoot, null)) { return Optional.empty(); } else { final Hash accountHash = accountAddress.addressHash(); @@ -122,14 +122,14 @@ public List getStorageProofRelatedNodes( private MerkleTrie newAccountStateTrie(final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, rootHash, b -> b, b -> b); + worldStateStorageCoordinator::getAccountStateTrieNode, rootHash, b -> b, b -> b); } private MerkleTrie newAccountStorageTrie( final Hash accountHash, final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash), + worldStateStorageCoordinator.getAccountStorageTrieNode(accountHash, location, hash), rootHash, b -> b, b -> b); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java index 6cc9741cb22..40bf5fc6652 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java @@ -14,12 +14,13 @@ */ package org.hyperledger.besu.ethereum.storage; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.BlockchainStorage; import org.hyperledger.besu.ethereum.chain.VariablesStorage; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; @@ -34,7 +35,10 @@ public interface StorageProvider extends Closeable { BlockchainStorage createBlockchainStorage( ProtocolSchedule protocolSchedule, VariablesStorage variablesStorage); - WorldStateStorage createWorldStateStorage(DataStorageFormat dataStorageFormat); + WorldStateKeyValueStorage createWorldStateStorage(DataStorageFormat dataStorageFormat); + + WorldStateStorageCoordinator createWorldStateStorageCoordinator( + DataStorageFormat dataStorageFormat); WorldStatePreimageStorage createWorldStatePreimageStorage(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index 8a7eef8ee5d..c65c7fef007 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -14,15 +14,17 @@ */ package org.hyperledger.besu.ethereum.storage.keyvalue; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.BlockchainStorage; import org.hyperledger.besu.ethereum.chain.VariablesStorage; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; @@ -74,15 +76,22 @@ public BlockchainStorage createBlockchainStorage( } @Override - public WorldStateStorage createWorldStateStorage(final DataStorageFormat dataStorageFormat) { + public WorldStateKeyValueStorage createWorldStateStorage( + final DataStorageFormat dataStorageFormat) { if (dataStorageFormat.equals(DataStorageFormat.BONSAI)) { return new BonsaiWorldStateKeyValueStorage(this, metricsSystem); } else { - return new WorldStateKeyValueStorage( + return new ForestWorldStateKeyValueStorage( getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.WORLD_STATE)); } } + @Override + public WorldStateStorageCoordinator createWorldStateStorageCoordinator( + final DataStorageFormat dataStorageFormat) { + return new WorldStateStorageCoordinator(createWorldStateStorage(dataStorageFormat)); + } + @Override public WorldStatePreimageStorage createWorldStatePreimageStorage() { return new WorldStatePreimageKeyValueStorage(worldStatePreimageStorage); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java deleted file mode 100644 index 3bb6a0dfe78..00000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.worldstate; - -import org.hyperledger.besu.datatypes.Hash; - -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; - -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; - -public interface WorldStateStorage { - - Optional getCode(Bytes32 codeHash, Hash accountHash); - - Optional getAccountStateTrieNode(Bytes location, Bytes32 nodeHash); - - Optional getAccountStorageTrieNode(Hash accountHash, Bytes location, Bytes32 nodeHash); - - /** - * This method allows obtaining a TrieNode in an unsafe manner, without verifying the consistency - * of the obtained node. Checks such as node hash verification are not performed here. - * - * @param key of the trie node - * @return value of the trie node - */ - Optional getTrieNodeUnsafe(Bytes key); - - Optional getNodeData(Bytes location, Bytes32 hash); - - FlatDbMode getFlatDbMode(); - - boolean isWorldStateAvailable(Bytes32 rootHash, Hash blockHash); - - default boolean contains(final Bytes32 hash) { - // we don't have location info - return getNodeData(null, hash).isPresent(); - } - - /** - * Streams flat accounts within a specified range. - * - * @param startKeyHash The start key hash of the range. - * @param endKeyHash The end key hash of the range. - * @param max The maximum number of entries to stream. - * @return A map of flat accounts. (Empty map in this default implementation) - */ - default Map streamFlatAccounts( - final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return Collections.emptyMap(); - } - - /** - * Streams flat storages within a specified range. - * - * @param accountHash The account hash. - * @param startKeyHash The start key hash of the range. - * @param endKeyHash The end key hash of the range. - * @param max The maximum number of entries to stream. - * @return A map of flat storages. (Empty map in this default implementation) - */ - default Map streamFlatStorages( - final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return Collections.emptyMap(); - } - - DataStorageFormat getDataStorageFormat(); - - void clear(); - - void clearTrieLog(); - - void clearFlatDatabase(); - - Updater updater(); - - long prune(Predicate inUseCheck); - - long addNodeAddedListener(NodesAddedListener listener); - - void removeNodeAddedListener(long id); - - interface Updater { - - Updater putCode(Hash accountHash, Bytes32 nodeHash, Bytes code); - - default Updater putCode(final Hash accountHash, final Bytes code) { - // Skip the hash calculation for empty code - final Hash codeHash = code.size() == 0 ? Hash.EMPTY : Hash.hash(code); - return putCode(accountHash, codeHash, code); - } - - Updater saveWorldState(Bytes blockHash, Bytes32 nodeHash, Bytes node); - - Updater putAccountStateTrieNode(Bytes location, Bytes32 nodeHash, Bytes node); - - Updater removeAccountStateTrieNode(Bytes location, Bytes32 nodeHash); - - Updater putAccountStorageTrieNode( - Hash accountHash, Bytes location, Bytes32 nodeHash, Bytes node); - - void commit(); - - void rollback(); - } - - interface NodesAddedListener { - void onNodesAdded(Collection nodeHash); - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java new file mode 100644 index 00000000000..041de8f8b2b --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java @@ -0,0 +1,137 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.worldstate; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; + +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +public record WorldStateStorageCoordinator(WorldStateKeyValueStorage worldStateKeyValueStorage) { + + public DataStorageFormat getDataStorageFormat() { + return worldStateKeyValueStorage.getDataStorageFormat(); + } + + public boolean isWorldStateAvailable(final Bytes32 nodeHash, final Hash blockHash) { + return applyForStrategy( + bonsai -> bonsai.isWorldStateAvailable(nodeHash, blockHash), + forest -> forest.isWorldStateAvailable(nodeHash)); + } + + public Optional getTrieNodeUnsafe(final Bytes key) { + return applyForStrategy( + bonsai -> bonsai.getTrieNodeUnsafe(key), + forest -> forest.getAccountStateTrieNode(Bytes32.wrap(key))); + } + + public Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { + return applyForStrategy( + bonsai -> bonsai.getAccountStateTrieNode(location, nodeHash), + forest -> forest.getAccountStateTrieNode(nodeHash)); + } + + public Optional getAccountStorageTrieNode( + final Hash accountHash, final Bytes location, final Bytes32 nodeHash) { + return applyForStrategy( + bonsai -> bonsai.getAccountStorageTrieNode(accountHash, location, nodeHash), + forest -> forest.getAccountStorageTrieNode(nodeHash)); + } + + @SuppressWarnings("unchecked") + public STRATEGY getStrategy( + final Class strategyClass) { + return (STRATEGY) worldStateKeyValueStorage; + } + + public void applyOnMatchingFlatMode( + final FlatDbMode flatDbMode, final Consumer onStrategy) { + applyOnMatchingStrategy( + DataStorageFormat.BONSAI, + worldStateKeyValueStorage -> { + final BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorageStrategy = + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage(); + if (bonsaiWorldStateStorageStrategy.getFlatDbMode().equals(flatDbMode)) { + onStrategy.accept(bonsaiWorldStateStorageStrategy); + } + }); + } + + public void applyWhenFlatModeEnabled(final Consumer onStrategy) { + applyOnMatchingStrategy( + DataStorageFormat.BONSAI, + worldStateKeyValueStorage -> { + final BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorageStrategy = + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage(); + if (!bonsaiWorldStateStorageStrategy.getFlatDbMode().equals(FlatDbMode.NO_FLATTENED)) { + onStrategy.accept(bonsaiWorldStateStorageStrategy); + } + }); + } + + public void applyOnMatchingStrategy( + final DataStorageFormat dataStorageFormat, + final Consumer onStrategy) { + if (getDataStorageFormat().equals(dataStorageFormat)) { + onStrategy.accept(worldStateKeyValueStorage()); + } + } + + public RESPONSE applyForStrategy( + final Function onBonsai, + final Function onForest) { + if (getDataStorageFormat().equals(DataStorageFormat.BONSAI)) { + return onBonsai.apply(((BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage())); + } else { + return onForest.apply(((ForestWorldStateKeyValueStorage) worldStateKeyValueStorage())); + } + } + + public void consumeForStrategy( + final Consumer onBonsai, + final Consumer onForest) { + if (getDataStorageFormat().equals(DataStorageFormat.BONSAI)) { + onBonsai.accept(((BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage())); + } else { + onForest.accept(((ForestWorldStateKeyValueStorage) worldStateKeyValueStorage())); + } + } + + public static void applyForStrategy( + final WorldStateKeyValueStorage.Updater updater, + final Consumer onBonsai, + final Consumer onForest) { + if (updater instanceof BonsaiWorldStateKeyValueStorage.Updater) { + onBonsai.accept(((BonsaiWorldStateKeyValueStorage.Updater) updater)); + } else if (updater instanceof ForestWorldStateKeyValueStorage.Updater) { + onForest.accept(((ForestWorldStateKeyValueStorage.Updater) updater)); + } + } + + public WorldStateKeyValueStorage.Updater updater() { + return worldStateKeyValueStorage().updater(); + } + + public void clear() { + worldStateKeyValueStorage.clear(); + } +} diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index fc8bfe77fba..bfe1860ee9f 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -22,17 +22,18 @@ import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.chain.VariablesStorage; +import org.hyperledger.besu.ethereum.forest.ForestWorldStateArchive; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; -import org.hyperledger.besu.ethereum.worldstate.DefaultWorldStateArchive; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -78,9 +79,10 @@ public static MutableBlockchain createInMemoryBlockchain( 0); } - public static DefaultWorldStateArchive createInMemoryWorldStateArchive() { - return new DefaultWorldStateArchive( - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()), + public static ForestWorldStateArchive createInMemoryWorldStateArchive() { + return new ForestWorldStateArchive( + new WorldStateStorageCoordinator( + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage())), new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()), EvmConfiguration.DEFAULT); } @@ -110,7 +112,7 @@ public static BonsaiWorldStateProvider createBonsaiInMemoryWorldStateArchive( public static MutableWorldState createInMemoryWorldState() { final InMemoryKeyValueStorageProvider provider = new InMemoryKeyValueStorageProvider(); - return new DefaultMutableWorldState( + return new ForestMutableWorldState( provider.createWorldStateStorage(DataStorageFormat.FOREST), provider.createWorldStatePreimageStorage(), EvmConfiguration.DEFAULT); diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryPrivacyStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryPrivacyStorageProvider.java index 91e115cffab..c61ec80f346 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryPrivacyStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryPrivacyStorageProvider.java @@ -14,41 +14,48 @@ */ package org.hyperledger.besu.ethereum.core; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.ForestWorldStateArchive; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateKeyValueStorage; import org.hyperledger.besu.ethereum.privacy.storage.LegacyPrivateStateStorage; import org.hyperledger.besu.ethereum.privacy.storage.PrivacyStorageProvider; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateKeyValueStorage; import org.hyperledger.besu.ethereum.privacy.storage.PrivateStateStorage; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; -import org.hyperledger.besu.ethereum.worldstate.DefaultWorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; public class InMemoryPrivacyStorageProvider implements PrivacyStorageProvider { public static WorldStateArchive createInMemoryWorldStateArchive() { - return new DefaultWorldStateArchive( - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()), + return new ForestWorldStateArchive( + new WorldStateStorageCoordinator( + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage())), new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()), EvmConfiguration.DEFAULT); } public static MutableWorldState createInMemoryWorldState() { final InMemoryPrivacyStorageProvider provider = new InMemoryPrivacyStorageProvider(); - return new DefaultMutableWorldState( + return new ForestMutableWorldState( provider.createWorldStateStorage(), provider.createWorldStatePreimageStorage(), EvmConfiguration.DEFAULT); } @Override - public WorldStateStorage createWorldStateStorage() { - return new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + public WorldStateKeyValueStorage createWorldStateStorage() { + return new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + } + + @Override + public WorldStateStorageCoordinator createWorldStateStorageCoordinator() { + return new WorldStateStorageCoordinator(createWorldStateStorage()); } @Override diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java index 9ee15e0775b..e614cb8d5f3 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java @@ -14,14 +14,17 @@ */ package org.hyperledger.besu.ethereum.core; +import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; + import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.List; import java.util.stream.Collectors; @@ -34,22 +37,23 @@ public class TrieGenerator { public static MerkleTrie generateTrie( - final WorldStateStorage worldStateStorage, final int nbAccounts) { + final WorldStateStorageCoordinator worldStateStorageCoordinator, final int nbAccounts) { return generateTrie( - worldStateStorage, + worldStateStorageCoordinator, IntStream.range(0, nbAccounts) .mapToObj(operand -> Hash.wrap(Bytes32.leftPad(Bytes.of(operand + 1)))) .collect(Collectors.toList())); } public static MerkleTrie generateTrie( - final WorldStateStorage worldStateStorage, final List accounts) { - final MerkleTrie accountStateTrie = emptyAccountStateTrie(worldStateStorage); + final WorldStateStorageCoordinator worldStateStorageCoordinator, final List accounts) { + final MerkleTrie accountStateTrie = + emptyAccountStateTrie(worldStateStorageCoordinator); // Add some storage values for (int i = 0; i < accounts.size(); i++) { - final WorldStateStorage.Updater updater = worldStateStorage.updater(); + final WorldStateKeyValueStorage.Updater updater = worldStateStorageCoordinator.updater(); final MerkleTrie storageTrie = - emptyStorageTrie(worldStateStorage, accounts.get(i)); + emptyStorageTrie(worldStateStorageCoordinator, accounts.get(i)); writeStorageValue(updater, storageTrie, accounts.get(i), UInt256.ONE, UInt256.valueOf(2L)); writeStorageValue( updater, storageTrie, accounts.get(i), UInt256.valueOf(2L), UInt256.valueOf(4L)); @@ -57,20 +61,36 @@ public static MerkleTrie generateTrie( updater, storageTrie, accounts.get(i), UInt256.valueOf(3L), UInt256.valueOf(6L)); int accountIndex = i; storageTrie.commit( - (location, hash, value) -> - updater.putAccountStorageTrieNode(accounts.get(accountIndex), location, hash, value)); + (location, hash, value) -> { + applyForStrategy( + updater, + onBonsai -> { + onBonsai.putAccountStorageTrieNode( + accounts.get(accountIndex), location, hash, value); + }, + onForest -> { + onForest.putAccountStorageTrieNode(hash, value); + }); + }); final Bytes code = Bytes32.leftPad(Bytes.of(i + 10)); final Hash codeHash = Hash.hash(code); final StateTrieAccountValue accountValue = new StateTrieAccountValue(1L, Wei.of(2L), Hash.wrap(storageTrie.getRootHash()), codeHash); accountStateTrie.put(accounts.get(i), RLP.encode(accountValue::writeTo)); - if (worldStateStorage instanceof BonsaiWorldStateKeyValueStorage) { - ((BonsaiWorldStateKeyValueStorage.Updater) updater) - .putAccountInfoState(accounts.get(i), RLP.encode(accountValue::writeTo)); - updater.putCode(accounts.get(i), code); - } - accountStateTrie.commit(updater::putAccountStateTrieNode); - updater.putCode(codeHash, code); + applyForStrategy( + updater, + onBonsai -> { + onBonsai.putAccountInfoState( + accounts.get(accountIndex), RLP.encode(accountValue::writeTo)); + accountStateTrie.commit(onBonsai::putAccountStateTrieNode); + onBonsai.putCode(accounts.get(accountIndex), codeHash, code); + }, + onForest -> { + accountStateTrie.commit( + (location, hash, value) -> onForest.putAccountStateTrieNode(hash, value)); + onForest.putCode(code); + }); + // Persist updates updater.commit(); } @@ -78,7 +98,7 @@ public static MerkleTrie generateTrie( } private static void writeStorageValue( - final WorldStateStorage.Updater updater, + final WorldStateKeyValueStorage.Updater updater, final MerkleTrie storageTrie, final Hash hash, final UInt256 key, @@ -101,17 +121,17 @@ private static Bytes encodeStorageValue(final UInt256 storageValue) { } public static MerkleTrie emptyStorageTrie( - final WorldStateStorage worldStateStorage, final Hash accountHash) { + final WorldStateStorageCoordinator worldStateStorageCoordinator, final Hash accountHash) { return new StoredMerklePatriciaTrie<>( (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash), + worldStateStorageCoordinator.getAccountStorageTrieNode(accountHash, location, hash), b -> b, b -> b); } public static MerkleTrie emptyAccountStateTrie( - final WorldStateStorage worldStateStorage) { + final WorldStateStorageCoordinator worldStateStorageCoordinator) { return new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, b -> b, b -> b); + worldStateStorageCoordinator::getAccountStateTrieNode, b -> b, b -> b); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java index a33f6a2b04e..be270a6035d 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java @@ -45,7 +45,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.exception.StorageException; @@ -79,8 +79,9 @@ class BlockImportExceptionHandlingTest { protected final MutableBlockchain blockchain = mock(MutableBlockchain.class); private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); - private final WorldStateStorage worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + private final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator( + new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem())); private final WorldStateArchive worldStateArchive = // contains a BonsaiWorldState which we need to spy on. @@ -91,7 +92,8 @@ class BlockImportExceptionHandlingTest { spy( new BonsaiWorldState( (BonsaiWorldStateProvider) worldStateArchive, - (BonsaiWorldStateKeyValueStorage) worldStateStorage, + (BonsaiWorldStateKeyValueStorage) + worldStateStorageCoordinator.worldStateKeyValueStorage(), EvmConfiguration.DEFAULT)); private final BadBlockManager badBlockManager = new BadBlockManager(); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java index 4f40afdb437..49dd245490f 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/AbstractIsolationTests.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.BlockProcessingResult; import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.blockcreation.AbstractBlockCreator; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; @@ -93,7 +94,7 @@ public abstract class AbstractIsolationTests { protected BonsaiWorldStateProvider archive; - protected BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorage; + protected WorldStateKeyValueStorage worldStateKeyValueStorage; protected ProtocolContext protocolContext; protected EthContext ethContext; protected EthScheduler ethScheduler = new DeterministicEthScheduler(); @@ -145,12 +146,11 @@ public abstract class AbstractIsolationTests { @BeforeEach public void createStorage() { - bonsaiWorldStateStorage = - (BonsaiWorldStateKeyValueStorage) - createKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.BONSAI); + worldStateKeyValueStorage = + createKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.BONSAI); archive = new BonsaiWorldStateProvider( - bonsaiWorldStateStorage, + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, blockchain, Optional.of(16L), new CachedMerkleTrieLoader(new NoOpMetricsSystem()), diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java index 03407d2868f..3eba3818444 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiSnapshotIsolationTests.java @@ -37,7 +37,7 @@ public void ensureTruncateDoesNotCauseSegfault() { var preTruncatedWorldState = archive.getMutable(genesisState.getBlock().getHeader(), false); assertThat(preTruncatedWorldState) .isPresent(); // really just assert that we have not segfaulted after truncating - bonsaiWorldStateStorage.clear(); + worldStateKeyValueStorage.clear(); var postTruncatedWorldState = archive.getMutable(genesisState.getBlock().getHeader(), false); assertThat(postTruncatedWorldState).isEmpty(); // assert that trying to access pre-worldstate does not segfault after truncating diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProviderTest.java similarity index 98% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProviderTest.java index bfe69546295..c555d879508 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchiveTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProviderTest.java @@ -65,7 +65,7 @@ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) -class BonsaiWorldStateArchiveTest { +class BonsaiWorldStateProviderTest { final BlockHeaderTestFixture blockBuilder = new BlockHeaderTestFixture(); @Mock Blockchain blockchain; @@ -166,14 +166,14 @@ void testGetMutableWithStorageInconsistencyRollbackTheState() { .when(trieLogManager) .getTrieLogLayer(any(Hash.class)); - var worldStateStorage = + var worldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); bonsaiWorldStateArchive = spy( new BonsaiWorldStateProvider( cachedWorldStorageManager, trieLogManager, - worldStateStorage, + worldStateKeyValueStorage, blockchain, new CachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); @@ -192,14 +192,14 @@ void testGetMutableWithStorageInconsistencyRollbackTheState() { @Test void testGetMutableWithStorageConsistencyNotRollbackTheState() { - var worldStateStorage = + var worldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); bonsaiWorldStateArchive = spy( new BonsaiWorldStateProvider( cachedWorldStorageManager, trieLogManager, - worldStateStorage, + worldStateKeyValueStorage, blockchain, new CachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); @@ -228,7 +228,7 @@ void testGetMutableWithStorageConsistencyToRollbackAndRollForwardTheState() { .when(trieLogManager) .getTrieLogLayer(any(Hash.class)); - var worldStateStorage = + var worldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); bonsaiWorldStateArchive = @@ -236,7 +236,7 @@ void testGetMutableWithStorageConsistencyToRollbackAndRollForwardTheState() { new BonsaiWorldStateProvider( cachedWorldStorageManager, trieLogManager, - worldStateStorage, + worldStateKeyValueStorage, blockchain, new CachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoaderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoaderTest.java index dfb247ee29b..fa89b5469ac 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoaderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/CachedMerkleTrieLoaderTest.java @@ -30,6 +30,7 @@ import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.ArrayList; @@ -49,6 +50,8 @@ class CachedMerkleTrieLoaderTest { private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private final BonsaiWorldStateKeyValueStorage inMemoryWorldState = Mockito.spy(new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem())); + private final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(inMemoryWorldState); final List
accounts = List.of(Address.fromHexString("0xdeadbeef"), Address.fromHexString("0xdeadbeee")); @@ -59,7 +62,7 @@ class CachedMerkleTrieLoaderTest { public void setup() { trie = TrieGenerator.generateTrie( - inMemoryWorldState, + worldStateStorageCoordinator, accounts.stream().map(Address::addressHash).collect(Collectors.toList())); merkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java index 5aa0f68daa0..32ffefd4cba 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java @@ -58,7 +58,7 @@ public static void main(final String[] arg) throws IOException { archive, new BonsaiWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); - final SegmentedInMemoryKeyValueStorage worldStateStorage = + final SegmentedInMemoryKeyValueStorage worldStateKeyValueStorage = (SegmentedInMemoryKeyValueStorage) provider.getStorageBySegmentIdentifiers( List.of( @@ -126,7 +126,7 @@ public static void main(final String[] arg) throws IOException { } } System.out.printf("Back to zero!%n"); - worldStateStorage.dump(System.out); + worldStateKeyValueStorage.dump(System.out); trieLogStorage.dump(System.out); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java index 13dbe22a403..7b5e8051ec7 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java @@ -38,6 +38,7 @@ import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; @@ -95,20 +96,6 @@ void getAccountStorageTrieNode_returnsEmptyNode(final FlatDbMode flatDbMode) { .contains(MerkleTrie.EMPTY_TRIE_NODE); } - @ParameterizedTest - @MethodSource("data") - void getNodeData_returnsEmptyValue(final FlatDbMode flatDbMode) { - setUp(flatDbMode); - assertThat(storage.getNodeData(null, null)).isEmpty(); - } - - @ParameterizedTest - @MethodSource("data") - void getNodeData_returnsEmptyNode(final FlatDbMode flatDbMode) { - setUp(flatDbMode); - assertThat(storage.getNodeData(Bytes.EMPTY, MerkleTrie.EMPTY_TRIE_NODE_HASH)).isEmpty(); - } - @ParameterizedTest @MethodSource("data") void getCode_saveAndGetSpecialValues(final FlatDbMode flatDbMode) { @@ -207,15 +194,15 @@ void getAccount_notLoadFromTrieWhenEmptyAndFlatDbFullMode(final FlatDbMode flatD setUp(flatDbMode); Assumptions.assumeTrue(flatDbMode == FlatDbMode.FULL); final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage()); - - final MerkleTrie trie = TrieGenerator.generateTrie(storage, 1); + final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage); + final MerkleTrie trie = TrieGenerator.generateTrie(coordinator, 1); final TreeMap accounts = (TreeMap) trie.entriesFrom(root -> StorageEntriesCollector.collectEntries(root, Hash.ZERO, 1)); // save world state root hash - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updater = storage.updater(); updater .getWorldStateTransaction() .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, trie.getRootHash().toArrayUnsafe()); @@ -239,13 +226,14 @@ void getAccount_loadFromTrieWhenEmptyAndFlatDbPartialMode(final FlatDbMode flatD setUp(flatDbMode); Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL); final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage()); - final MerkleTrie trie = TrieGenerator.generateTrie(storage, 1); + final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage); + final MerkleTrie trie = TrieGenerator.generateTrie(coordinator, 1); final TreeMap accounts = (TreeMap) trie.entriesFrom(root -> StorageEntriesCollector.collectEntries(root, Hash.ZERO, 1)); // save world state root hash - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updater = storage.updater(); updater .getWorldStateTransaction() .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, trie.getRootHash().toArrayUnsafe()); @@ -268,14 +256,14 @@ void shouldUsePartialDBStrategyAfterDowngradingMode(final FlatDbMode flatDbMode) setUp(flatDbMode); Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL); final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage()); - - final MerkleTrie trie = TrieGenerator.generateTrie(storage, 1); + final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage); + final MerkleTrie trie = TrieGenerator.generateTrie(coordinator, 1); final TreeMap accounts = (TreeMap) trie.entriesFrom(root -> StorageEntriesCollector.collectEntries(root, Hash.ZERO, 1)); // save world state root hash - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updater = storage.updater(); updater .getWorldStateTransaction() .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, trie.getRootHash().toArrayUnsafe()); @@ -300,7 +288,8 @@ void getStorage_loadFromTrieWhenEmptyWithPartialMode(final FlatDbMode flatDbMode setUp(flatDbMode); Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL); final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage()); - final MerkleTrie trie = TrieGenerator.generateTrie(storage, 1); + final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage); + final MerkleTrie trie = TrieGenerator.generateTrie(coordinator, 1); final TreeMap accounts = (TreeMap) trie.entriesFrom(root -> StorageEntriesCollector.collectEntries(root, Hash.ZERO, 1)); @@ -322,7 +311,7 @@ void getStorage_loadFromTrieWhenEmptyWithPartialMode(final FlatDbMode flatDbMode root -> StorageEntriesCollector.collectEntries(root, Hash.ZERO, 1)); // save world state root hash - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updater = storage.updater(); updater .getWorldStateTransaction() .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, trie.getRootHash().toArrayUnsafe()); @@ -350,10 +339,11 @@ void getStorage_loadFromTrieWhenEmptyWithFullMode(final FlatDbMode flatDbMode) { Assumptions.assumeTrue(flatDbMode == FlatDbMode.FULL); final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage()); storage.upgradeToFullFlatDbMode(); - final MerkleTrie trie = TrieGenerator.generateTrie(storage, 1); + final WorldStateStorageCoordinator coordinator = new WorldStateStorageCoordinator(storage); + final MerkleTrie trie = TrieGenerator.generateTrie(coordinator, 1); // save world state root hash - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updater = storage.updater(); updater .getWorldStateTransaction() .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, trie.getRootHash().toArrayUnsafe()); @@ -370,7 +360,7 @@ void clear_reloadFlatDbStrategy(final FlatDbMode flatDbMode) { final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage()); // save world state root hash - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updater = storage.updater(); updater.putAccountInfoState(Hash.ZERO, Bytes32.random()).commit(); assertThat(storage.getAccount(Hash.ZERO)).isNotEmpty(); @@ -394,8 +384,8 @@ void reconcilesNonConflictingUpdaters(final FlatDbMode flatDbMode) { final Bytes bytesB = Bytes.fromHexString("0x1234"); final Bytes bytesC = Bytes.fromHexString("0x123456"); - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updaterA = storage.updater(); - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updaterB = storage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updaterA = storage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updaterB = storage.updater(); updaterA.putCode(accountHashA, bytesA); updaterB.putCode(accountHashB, bytesA); @@ -421,7 +411,7 @@ void isWorldStateAvailable_defaultIsFalse(final FlatDbMode flatDbMode) { void isWorldStateAvailable_StateAvailableByRootHash(final FlatDbMode flatDbMode) { setUp(flatDbMode); - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updater = storage.updater(); final Bytes rootHashKey = Bytes32.fromHexString("0x01"); updater .getWorldStateTransaction() @@ -436,7 +426,7 @@ void isWorldStateAvailable_StateAvailableByRootHash(final FlatDbMode flatDbMode) void isWorldStateAvailable_afterCallingSaveWorldstate(final FlatDbMode flatDbMode) { setUp(flatDbMode); - final BonsaiWorldStateKeyValueStorage.BonsaiUpdater updater = storage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updater = storage.updater(); final Bytes blockHash = Bytes32.fromHexString("0x01"); final Bytes32 nodeHashKey = Bytes32.fromHexString("0x02"); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProviderTest.java index b6357954f53..8a9b94912d9 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateProofProviderTest.java @@ -19,12 +19,12 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.ArrayList; @@ -47,14 +47,15 @@ public class WorldStateProofProviderTest { private static final Address address = Address.fromHexString("0x1234567890123456789012345678901234567890"); - private final WorldStateStorage worldStateStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + private final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); private WorldStateProofProvider worldStateProofProvider; @BeforeEach public void setup() { - worldStateProofProvider = new WorldStateProofProvider(worldStateStorage); + worldStateProofProvider = + new WorldStateProofProvider(new WorldStateStorageCoordinator(worldStateKeyValueStorage)); } @Test @@ -68,19 +69,17 @@ public void getProofWhenWorldStateNotAvailable() { @Test public void getProofWhenWorldStateAvailable() { final Hash addressHash = address.addressHash(); - final MerkleTrie worldStateTrie = emptyWorldStateTrie(addressHash); + final MerkleTrie worldStateTrie = emptyWorldStateTrie(); final MerkleTrie storageTrie = emptyStorageTrie(); - final WorldStateStorage.Updater updater = worldStateStorage.updater(); + final ForestWorldStateKeyValueStorage.Updater updater = worldStateKeyValueStorage.updater(); // Add some storage values writeStorageValue(storageTrie, UInt256.ONE, UInt256.valueOf(2L)); writeStorageValue(storageTrie, UInt256.valueOf(2L), UInt256.valueOf(4L)); writeStorageValue(storageTrie, UInt256.valueOf(3L), UInt256.valueOf(6L)); // Save to Storage - storageTrie.commit( - (location, hash, value) -> - updater.putAccountStorageTrieNode(addressHash, location, hash, value)); + storageTrie.commit((location, hash, value) -> updater.putAccountStorageTrieNode(hash, value)); // Define account value final Hash codeHash = Hash.hash(Bytes.fromHexString("0x1122")); @@ -88,7 +87,7 @@ public void getProofWhenWorldStateAvailable() { new StateTrieAccountValue(1L, Wei.of(2L), Hash.wrap(storageTrie.getRootHash()), codeHash); // Save to storage worldStateTrie.put(addressHash, RLP.encode(accountValue::writeTo)); - worldStateTrie.commit(updater::putAccountStateTrieNode); + worldStateTrie.commit((location, hash, value) -> updater.putAccountStateTrieNode(hash, value)); // Persist updates updater.commit(); @@ -121,7 +120,7 @@ public void getProofWhenWorldStateAvailable() { @Test public void getProofWhenStateTrieAccountUnavailable() { - final MerkleTrie worldStateTrie = emptyWorldStateTrie(null); + final MerkleTrie worldStateTrie = emptyWorldStateTrie(); final Optional accountProof = worldStateProofProvider.getAccountProof( @@ -145,13 +144,14 @@ private Bytes encodeStorageValue(final UInt256 storageValue) { private MerkleTrie emptyStorageTrie() { return new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, b -> b, b -> b); + (location, hash) -> worldStateKeyValueStorage.getAccountStateTrieNode(hash), + b -> b, + b -> b); } - private MerkleTrie emptyWorldStateTrie(final Hash accountHash) { + private MerkleTrie emptyWorldStateTrie() { return new StoredMerklePatriciaTrie<>( - (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash), + (location, hash) -> worldStateKeyValueStorage.getAccountStorageTrieNode(hash), b -> b, b -> b); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java index bd4d95321a7..36929435f4a 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/proof/WorldStateRangeProofProviderTest.java @@ -18,11 +18,11 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.TrieGenerator; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.ArrayList; @@ -40,21 +40,22 @@ public class WorldStateRangeProofProviderTest { private static final Hash MAX_RANGE = Hash.fromHexString("0x0fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + private static final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); - private static final WorldStateStorage worldStateStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); - + private WorldStateStorageCoordinator worldStateStorageCoordinator; private static WorldStateProofProvider worldStateProofProvider; @BeforeEach public void setup() { - worldStateProofProvider = new WorldStateProofProvider(worldStateStorage); + worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); + worldStateProofProvider = new WorldStateProofProvider(worldStateStorageCoordinator); } @Test public void rangeProofValidationNominalCase() { final MerkleTrie accountStateTrie = - TrieGenerator.generateTrie(worldStateStorage, 15); + TrieGenerator.generateTrie(worldStateStorageCoordinator, 15); // collect accounts in range final RangeStorageEntriesCollector collector = RangeStorageEntriesCollector.createCollector(Hash.ZERO, MAX_RANGE, 10, Integer.MAX_VALUE); @@ -82,7 +83,8 @@ public void rangeProofValidationNominalCase() { @Test public void rangeProofValidationMissingAccount() { - MerkleTrie accountStateTrie = TrieGenerator.generateTrie(worldStateStorage, 15); + MerkleTrie accountStateTrie = + TrieGenerator.generateTrie(worldStateStorageCoordinator, 15); // collect accounts in range final RangeStorageEntriesCollector collector = RangeStorageEntriesCollector.createCollector(Hash.ZERO, MAX_RANGE, 10, Integer.MAX_VALUE); @@ -119,7 +121,8 @@ public void rangeProofValidationMissingAccount() { @Test public void rangeProofValidationNoMonotonicIncreasing() { - MerkleTrie accountStateTrie = TrieGenerator.generateTrie(worldStateStorage, 15); + MerkleTrie accountStateTrie = + TrieGenerator.generateTrie(worldStateStorageCoordinator, 15); // generate the invalid proof final RangeStorageEntriesCollector collector = @@ -155,7 +158,8 @@ public void rangeProofValidationNoMonotonicIncreasing() { @Test public void rangeProofValidationEmptyProof() { - MerkleTrie accountStateTrie = TrieGenerator.generateTrie(worldStateStorage, 15); + MerkleTrie accountStateTrie = + TrieGenerator.generateTrie(worldStateStorageCoordinator, 15); // generate the invalid proof final RangeStorageEntriesCollector collector = @@ -182,7 +186,8 @@ public void rangeProofValidationEmptyProof() { @Test public void rangeProofValidationInvalidEmptyProof() { - MerkleTrie accountStateTrie = TrieGenerator.generateTrie(worldStateStorage, 15); + MerkleTrie accountStateTrie = + TrieGenerator.generateTrie(worldStateStorageCoordinator, 15); // generate the invalid proof final RangeStorageEntriesCollector collector = diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java index 758866fd6ff..ea6f9d5e3db 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java @@ -17,7 +17,8 @@ import static org.assertj.core.api.Assertions.assertThat; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage.Updater; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage.Updater; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -29,129 +30,126 @@ public class KeyValueStorageWorldStateStorageTest { @Test public void getCode_returnsEmpty() { - final WorldStateKeyValueStorage storage = emptyStorage(); - assertThat(storage.getCode(Hash.EMPTY, null)).contains(Bytes.EMPTY); + final ForestWorldStateKeyValueStorage storage = emptyStorage(); + assertThat(storage.getCode(Hash.EMPTY)).contains(Bytes.EMPTY); } @Test public void getAccountStateTrieNode_returnsEmptyNode() { - final WorldStateKeyValueStorage storage = emptyStorage(); - assertThat(storage.getAccountStateTrieNode(Bytes.EMPTY, MerkleTrie.EMPTY_TRIE_NODE_HASH)) + final ForestWorldStateKeyValueStorage storage = emptyStorage(); + assertThat(storage.getAccountStateTrieNode(MerkleTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerkleTrie.EMPTY_TRIE_NODE); } @Test public void getAccountStorageTrieNode_returnsEmptyNode() { - final WorldStateKeyValueStorage storage = emptyStorage(); - assertThat( - storage.getAccountStorageTrieNode(null, Bytes.EMPTY, MerkleTrie.EMPTY_TRIE_NODE_HASH)) + final ForestWorldStateKeyValueStorage storage = emptyStorage(); + assertThat(storage.getAccountStorageTrieNode(MerkleTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerkleTrie.EMPTY_TRIE_NODE); } @Test public void getNodeData_returnsEmptyValue() { - final WorldStateKeyValueStorage storage = emptyStorage(); - assertThat(storage.getNodeData(null, Hash.EMPTY)).contains(Bytes.EMPTY); + final ForestWorldStateKeyValueStorage storage = emptyStorage(); + assertThat(storage.getNodeData(Hash.EMPTY)).contains(Bytes.EMPTY); } @Test public void getNodeData_returnsEmptyNode() { - final WorldStateKeyValueStorage storage = emptyStorage(); - assertThat(storage.getNodeData(Bytes.EMPTY, MerkleTrie.EMPTY_TRIE_NODE_HASH)) + final ForestWorldStateKeyValueStorage storage = emptyStorage(); + assertThat(storage.getNodeData(MerkleTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerkleTrie.EMPTY_TRIE_NODE); } @Test public void getCode_saveAndGetSpecialValues() { - final WorldStateKeyValueStorage storage = emptyStorage(); - storage.updater().putCode(null, MerkleTrie.EMPTY_TRIE_NODE).putCode(null, Bytes.EMPTY).commit(); + final ForestWorldStateKeyValueStorage storage = emptyStorage(); + storage.updater().putCode(null, MerkleTrie.EMPTY_TRIE_NODE); + storage.updater().putCode(Bytes.EMPTY).commit(); - assertThat(storage.getCode(MerkleTrie.EMPTY_TRIE_NODE_HASH, null)) + assertThat(storage.getCode(MerkleTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerkleTrie.EMPTY_TRIE_NODE); - assertThat(storage.getCode(Hash.EMPTY, null)).contains(Bytes.EMPTY); + assertThat(storage.getCode(Hash.EMPTY)).contains(Bytes.EMPTY); } @Test public void getCode_saveAndGetRegularValue() { final Bytes bytes = Bytes.fromHexString("0x123456"); - final WorldStateKeyValueStorage storage = emptyStorage(); + final ForestWorldStateKeyValueStorage storage = emptyStorage(); storage.updater().putCode(null, bytes).commit(); - assertThat(storage.getCode(Hash.hash(bytes), null)).contains(bytes); + assertThat(storage.getCode(Hash.hash(bytes))).contains(bytes); } @Test public void getAccountStateTrieNode_saveAndGetSpecialValues() { - final WorldStateKeyValueStorage storage = emptyStorage(); + final ForestWorldStateKeyValueStorage storage = emptyStorage(); storage .updater() - .putAccountStateTrieNode( - null, Hash.hash(MerkleTrie.EMPTY_TRIE_NODE), MerkleTrie.EMPTY_TRIE_NODE) - .putAccountStateTrieNode(null, Hash.hash(Bytes.EMPTY), Bytes.EMPTY) + .putAccountStateTrieNode(Hash.hash(MerkleTrie.EMPTY_TRIE_NODE), MerkleTrie.EMPTY_TRIE_NODE) + .putAccountStateTrieNode(Hash.hash(Bytes.EMPTY), Bytes.EMPTY) .commit(); - assertThat(storage.getAccountStateTrieNode(Bytes.EMPTY, MerkleTrie.EMPTY_TRIE_NODE_HASH)) + assertThat(storage.getAccountStateTrieNode(MerkleTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerkleTrie.EMPTY_TRIE_NODE); - assertThat(storage.getAccountStateTrieNode(Bytes.EMPTY, Hash.EMPTY)).contains(Bytes.EMPTY); + assertThat(storage.getAccountStateTrieNode(Hash.EMPTY)).contains(Bytes.EMPTY); } @Test public void getAccountStateTrieNode_saveAndGetRegularValue() { final Bytes bytes = Bytes.fromHexString("0x123456"); - final WorldStateKeyValueStorage storage = emptyStorage(); - storage.updater().putAccountStateTrieNode(null, Hash.hash(bytes), bytes).commit(); + final ForestWorldStateKeyValueStorage storage = emptyStorage(); + storage.updater().putAccountStateTrieNode(Hash.hash(bytes), bytes).commit(); - assertThat(storage.getAccountStateTrieNode(Bytes.EMPTY, Hash.hash(bytes))).contains(bytes); + assertThat(storage.getAccountStateTrieNode(Hash.hash(bytes))).contains(bytes); } @Test public void getAccountStorageTrieNode_saveAndGetSpecialValues() { - final WorldStateKeyValueStorage storage = emptyStorage(); + final ForestWorldStateKeyValueStorage storage = emptyStorage(); storage .updater() .putAccountStorageTrieNode( - null, null, Hash.hash(MerkleTrie.EMPTY_TRIE_NODE), MerkleTrie.EMPTY_TRIE_NODE) - .putAccountStorageTrieNode(null, null, Hash.hash(Bytes.EMPTY), Bytes.EMPTY) + Hash.hash(MerkleTrie.EMPTY_TRIE_NODE), MerkleTrie.EMPTY_TRIE_NODE) + .putAccountStorageTrieNode(Hash.hash(Bytes.EMPTY), Bytes.EMPTY) .commit(); - assertThat( - storage.getAccountStorageTrieNode(null, Bytes.EMPTY, MerkleTrie.EMPTY_TRIE_NODE_HASH)) + assertThat(storage.getAccountStorageTrieNode(MerkleTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerkleTrie.EMPTY_TRIE_NODE); - assertThat(storage.getAccountStorageTrieNode(null, Bytes.EMPTY, Hash.EMPTY)) - .contains(Bytes.EMPTY); + assertThat(storage.getAccountStorageTrieNode(Hash.EMPTY)).contains(Bytes.EMPTY); } @Test public void getAccountStorageTrieNode_saveAndGetRegularValue() { final Bytes bytes = Bytes.fromHexString("0x123456"); - final WorldStateKeyValueStorage storage = emptyStorage(); - storage.updater().putAccountStorageTrieNode(null, null, Hash.hash(bytes), bytes).commit(); + final ForestWorldStateKeyValueStorage storage = emptyStorage(); + storage.updater().putAccountStorageTrieNode(Hash.hash(bytes), bytes).commit(); - assertThat(storage.getAccountStateTrieNode(Bytes.EMPTY, Hash.hash(bytes))).contains(bytes); + assertThat(storage.getAccountStateTrieNode(Hash.hash(bytes))).contains(bytes); } @Test public void getNodeData_saveAndGetSpecialValues() { - final WorldStateKeyValueStorage storage = emptyStorage(); + final ForestWorldStateKeyValueStorage storage = emptyStorage(); storage .updater() .putAccountStorageTrieNode( - null, null, Hash.hash(MerkleTrie.EMPTY_TRIE_NODE), MerkleTrie.EMPTY_TRIE_NODE) - .putAccountStorageTrieNode(null, null, Hash.hash(Bytes.EMPTY), Bytes.EMPTY) + Hash.hash(MerkleTrie.EMPTY_TRIE_NODE), MerkleTrie.EMPTY_TRIE_NODE) + .putAccountStorageTrieNode(Hash.hash(Bytes.EMPTY), Bytes.EMPTY) .commit(); - assertThat(storage.getNodeData(Bytes.EMPTY, MerkleTrie.EMPTY_TRIE_NODE_HASH)) + assertThat(storage.getNodeData(MerkleTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerkleTrie.EMPTY_TRIE_NODE); - assertThat(storage.getNodeData(Bytes.EMPTY, Hash.EMPTY)).contains(Bytes.EMPTY); + assertThat(storage.getNodeData(Hash.EMPTY)).contains(Bytes.EMPTY); } @Test public void getNodeData_saveAndGetRegularValue() { final Bytes bytes = Bytes.fromHexString("0x123456"); - final WorldStateKeyValueStorage storage = emptyStorage(); - storage.updater().putAccountStorageTrieNode(null, null, Hash.hash(bytes), bytes).commit(); + final ForestWorldStateKeyValueStorage storage = emptyStorage(); + storage.updater().putAccountStorageTrieNode(Hash.hash(bytes), bytes).commit(); - assertThat(storage.getNodeData(null, Hash.hash(bytes))).contains(bytes); + assertThat(storage.getNodeData(Hash.hash(bytes))).contains(bytes); } @Test @@ -160,7 +158,7 @@ public void reconcilesNonConflictingUpdaters() { final Bytes bytesB = Bytes.fromHexString("0x1234"); final Bytes bytesC = Bytes.fromHexString("0x123456"); - final WorldStateKeyValueStorage storage = emptyStorage(); + final ForestWorldStateKeyValueStorage storage = emptyStorage(); final Updater updaterA = storage.updater(); final Updater updaterB = storage.updater(); @@ -172,22 +170,22 @@ public void reconcilesNonConflictingUpdaters() { updaterA.commit(); updaterB.commit(); - assertThat(storage.getCode(Hash.hash(bytesA), null)).contains(bytesA); - assertThat(storage.getCode(Hash.hash(bytesB), null)).contains(bytesB); - assertThat(storage.getCode(Hash.hash(bytesC), null)).contains(bytesC); + assertThat(storage.getCode(Hash.hash(bytesA))).contains(bytesA); + assertThat(storage.getCode(Hash.hash(bytesB))).contains(bytesB); + assertThat(storage.getCode(Hash.hash(bytesC))).contains(bytesC); } @Test public void isWorldStateAvailable_defaultIsFalse() { - assertThat(emptyStorage().isWorldStateAvailable(UInt256.valueOf(1), null)).isFalse(); + assertThat(emptyStorage().isWorldStateAvailable(UInt256.valueOf(1))).isFalse(); } @Test public void isWorldStateAvailable_emptyTrieStateAlwaysAvailable() { - assertThat(emptyStorage().isWorldStateAvailable(Hash.EMPTY_TRIE_HASH, null)).isTrue(); + assertThat(emptyStorage().isWorldStateAvailable(Hash.EMPTY_TRIE_HASH)).isTrue(); } - private WorldStateKeyValueStorage emptyStorage() { - return new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + private ForestWorldStateKeyValueStorage emptyStorage() { + return new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java index 4e423563416..8b090990efe 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java @@ -21,7 +21,8 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.evm.account.AccountStorageEntry; @@ -54,10 +55,10 @@ class DefaultMutableWorldStateTest { private static final Address ADDRESS = Address.fromHexString("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"); - private static MutableWorldState createEmpty(final WorldStateKeyValueStorage storage) { + private static MutableWorldState createEmpty(final ForestWorldStateKeyValueStorage storage) { final WorldStatePreimageKeyValueStorage preimageStorage = new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); - return new DefaultMutableWorldState(storage, preimageStorage, EvmConfiguration.DEFAULT); + return new ForestMutableWorldState(storage, preimageStorage, EvmConfiguration.DEFAULT); } private static MutableWorldState createEmpty() { @@ -243,7 +244,8 @@ void streamAccounts_multipleAccounts() { @Test void commitAndPersist() { final KeyValueStorage storage = new InMemoryKeyValueStorage(); - final WorldStateKeyValueStorage kvWorldStateStorage = new WorldStateKeyValueStorage(storage); + final ForestWorldStateKeyValueStorage kvWorldStateStorage = + new ForestWorldStateKeyValueStorage(storage); final MutableWorldState worldState = createEmpty(kvWorldStateStorage); final WorldUpdater updater = worldState.updater(); final Wei newBalance = Wei.of(100000); @@ -262,21 +264,21 @@ void commitAndPersist() { assertThat(worldState.get(ADDRESS).getBalance()).isEqualTo(newBalance); // Check that storage is empty before persisting - assertThat(kvWorldStateStorage.isWorldStateAvailable(worldState.rootHash(), null)).isFalse(); + assertThat(kvWorldStateStorage.isWorldStateAvailable(worldState.rootHash())).isFalse(); // Persist and re-run assertions worldState.persist(null); - assertThat(kvWorldStateStorage.isWorldStateAvailable(worldState.rootHash(), null)).isTrue(); + assertThat(kvWorldStateStorage.isWorldStateAvailable(worldState.rootHash())).isTrue(); assertThat(worldState.rootHash()).isEqualTo(expectedRootHash); assertThat(worldState.get(ADDRESS)).isNotNull(); assertThat(worldState.get(ADDRESS).getBalance()).isEqualTo(newBalance); // Create new world state and check that it can access modified address final MutableWorldState newWorldState = - new DefaultMutableWorldState( + new ForestMutableWorldState( expectedRootHash, - new WorldStateKeyValueStorage(storage), + new ForestWorldStateKeyValueStorage(storage), new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()), EvmConfiguration.DEFAULT); assertThat(newWorldState.rootHash()).isEqualTo(expectedRootHash); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java index 28997591f73..1141a915925 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java @@ -28,8 +28,10 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.forest.ForestWorldStateArchive; +import org.hyperledger.besu.ethereum.forest.pruner.MarkSweepPruner; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; @@ -62,11 +64,11 @@ class MarkSweepPrunerTest { private final NoOpMetricsSystem metricsSystem = new NoOpMetricsSystem(); private final Map> hashValueStore = spy(new HashMap<>()); private final InMemoryKeyValueStorage stateStorage = new TestInMemoryStorage(hashValueStore); - private final WorldStateStorage worldStateStorage = - spy(new WorldStateKeyValueStorage(stateStorage)); + private final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + spy(new ForestWorldStateKeyValueStorage(stateStorage)); private final WorldStateArchive worldStateArchive = - new DefaultWorldStateArchive( - worldStateStorage, + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(worldStateKeyValueStorage), new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()), EvmConfiguration.DEFAULT); private final InMemoryKeyValueStorage markStorage = new InMemoryKeyValueStorage(); @@ -76,7 +78,7 @@ class MarkSweepPrunerTest { @Test void mark_marksAllExpectedNodes() { final MarkSweepPruner pruner = - new MarkSweepPruner(worldStateStorage, blockchain, markStorage, metricsSystem); + new MarkSweepPruner(worldStateKeyValueStorage, blockchain, markStorage, metricsSystem); // Generate accounts and save corresponding state root final int numBlocks = 15; @@ -125,7 +127,7 @@ void mark_marksAllExpectedNodes() { @Test void sweepBefore_shouldSweepStateRootFirst() { final MarkSweepPruner pruner = - new MarkSweepPruner(worldStateStorage, blockchain, markStorage, metricsSystem); + new MarkSweepPruner(worldStateKeyValueStorage, blockchain, markStorage, metricsSystem); // Generate accounts and save corresponding state root final int numBlocks = 15; @@ -150,17 +152,17 @@ void sweepBefore_shouldSweepStateRootFirst() { stateRoots.forEach( stateRoot -> { final InOrder thisRootsOrdering = - inOrder(worldStateStorage, hashValueStore, worldStateStorage); - thisRootsOrdering.verify(worldStateStorage).isWorldStateAvailable(stateRoot, null); + inOrder(worldStateKeyValueStorage, hashValueStore, worldStateKeyValueStorage); + thisRootsOrdering.verify(worldStateKeyValueStorage).isWorldStateAvailable(stateRoot); thisRootsOrdering.verify(hashValueStore).keySet(); - thisRootsOrdering.verify(worldStateStorage).prune(any()); + thisRootsOrdering.verify(worldStateKeyValueStorage).prune(any()); }); } @Test void sweepBefore_shouldNotRemoveMarkedStateRoots() { final MarkSweepPruner pruner = - new MarkSweepPruner(worldStateStorage, blockchain, markStorage, metricsSystem); + new MarkSweepPruner(worldStateKeyValueStorage, blockchain, markStorage, metricsSystem); // Generate accounts and save corresponding state root final int numBlocks = 15; @@ -189,10 +191,10 @@ void sweepBefore_shouldNotRemoveMarkedStateRoots() { stateRoots.forEach( stateRoot -> { final InOrder thisRootsOrdering = - inOrder(worldStateStorage, hashValueStore, worldStateStorage); - thisRootsOrdering.verify(worldStateStorage).isWorldStateAvailable(stateRoot, null); + inOrder(worldStateKeyValueStorage, hashValueStore, worldStateKeyValueStorage); + thisRootsOrdering.verify(worldStateKeyValueStorage).isWorldStateAvailable(stateRoot); thisRootsOrdering.verify(hashValueStore).keySet(); - thisRootsOrdering.verify(worldStateStorage).prune(any()); + thisRootsOrdering.verify(worldStateKeyValueStorage).prune(any()); }); assertThat(stateStorage.containsKey(markedRoot.toArray())).isTrue(); @@ -265,7 +267,7 @@ private void collectTrieNodes(final MerkleTrie trie, final Set createStateTrie(final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, + (location, hash) -> worldStateKeyValueStorage.getAccountStateTrieNode(hash), rootHash, Function.identity(), Function.identity()); @@ -273,7 +275,7 @@ private MerkleTrie createStateTrie(final Bytes32 rootHash) { private MerkleTrie createStorageTrie(final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateStorage.getAccountStorageTrieNode(null, location, hash), + (location, hash) -> worldStateKeyValueStorage.getAccountStorageTrieNode(hash), rootHash, Function.identity(), Function.identity()); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/PrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/PrunerTest.java index 8f1b34bccac..50c9e926799 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/PrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/PrunerTest.java @@ -29,6 +29,9 @@ import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.BlockDataGenerator.BlockOptions; import org.hyperledger.besu.ethereum.core.TransactionReceipt; +import org.hyperledger.besu.ethereum.forest.pruner.MarkSweepPruner; +import org.hyperledger.besu.ethereum.forest.pruner.Pruner; +import org.hyperledger.besu.ethereum.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; diff --git a/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java b/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java index 5f7cbcdc8df..15b8126d70a 100644 --- a/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java +++ b/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java @@ -35,12 +35,13 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState; import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.FastWorldStateDownloader; import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.NodeDataRequest; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBKeyValueStorageFactory; @@ -76,7 +77,7 @@ public class WorldStateDownloaderBenchmark { private BlockHeader blockHeader; private final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); private WorldStateDownloader worldStateDownloader; - private WorldStateStorage worldStateStorage; + private WorldStateStorageCoordinator worldStateStorageCoordinator; private RespondingEthPeer peer; private Responder responder; private InMemoryTasksPriorityQueues pendingRequests; @@ -105,13 +106,14 @@ public void setUpUnchangedState() { final StorageProvider storageProvider = createKeyValueStorageProvider(tempDir, tempDir.resolve("database")); - worldStateStorage = storageProvider.createWorldStateStorage(DataStorageFormat.FOREST); + worldStateStorageCoordinator = + storageProvider.createWorldStateStorageCoordinator(DataStorageFormat.FOREST); pendingRequests = new InMemoryTasksPriorityQueues<>(); worldStateDownloader = new FastWorldStateDownloader( ethContext, - worldStateStorage, + worldStateStorageCoordinator, pendingRequests, syncConfig.getWorldStateHashCountPerRequest(), syncConfig.getWorldStateRequestParallelism(), @@ -151,7 +153,9 @@ public Optional downloadWorldState() { peer.respondWhileOtherThreadsWork(responder, () -> !result.isDone()); result.getNow(null); final Optional rootData = - worldStateStorage.getNodeData(Bytes.EMPTY, blockHeader.getStateRoot()); + worldStateStorageCoordinator + .getStrategy(ForestWorldStateKeyValueStorage.class) + .getNodeData(blockHeader.getStateRoot()); if (rootData.isEmpty()) { throw new IllegalStateException("World state download did not complete."); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index 2f98c1a8314..e3dfc924860 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -33,10 +33,10 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.context.SnapSyncStatePersistenceManager; import org.hyperledger.besu.ethereum.eth.sync.state.PendingBlocksManager; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; +import org.hyperledger.besu.ethereum.forest.pruner.Pruner; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.worldstate.Pruner; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.data.SyncStatus; import org.hyperledger.besu.plugin.services.BesuEvents.SyncStatusListener; @@ -77,7 +77,7 @@ public DefaultSynchronizer( final SynchronizerConfiguration syncConfig, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final BlockBroadcaster blockBroadcaster, final Optional maybePruner, final EthContext ethContext, @@ -139,7 +139,7 @@ public DefaultSynchronizer( protocolContext, metricsSystem, ethContext, - worldStateStorage, + worldStateStorageCoordinator, syncState, clock); } else if (SyncMode.X_CHECKPOINT.equals(syncConfig.getSyncMode())) { @@ -154,7 +154,7 @@ public DefaultSynchronizer( protocolContext, metricsSystem, ethContext, - worldStateStorage, + worldStateStorageCoordinator, syncState, clock); } else { @@ -169,7 +169,7 @@ public DefaultSynchronizer( protocolContext, metricsSystem, ethContext, - worldStateStorage, + worldStateStorageCoordinator, syncState, clock); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java index ff8187d8d21..1177fe42f8b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointDownloaderFactory.java @@ -35,7 +35,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.trie.CompactEncoding; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; @@ -60,7 +60,7 @@ public static Optional> createCheckpointDownloader( final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SyncState syncState, final Clock clock) { @@ -104,7 +104,7 @@ public static Optional> createCheckpointDownloader( fastSyncActions = new FastSyncActions( syncConfig, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, @@ -121,7 +121,7 @@ public static Optional> createCheckpointDownloader( fastSyncActions = new CheckpointSyncActions( syncConfig, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, @@ -142,7 +142,7 @@ public static Optional> createCheckpointDownloader( ethContext, snapContext, protocolContext, - worldStateStorage, + worldStateStorageCoordinator, snapTaskCollection, syncConfig.getSnapSyncConfiguration(), syncConfig.getWorldStateRequestParallelism(), @@ -153,7 +153,7 @@ public static Optional> createCheckpointDownloader( final FastSyncDownloader fastSyncDownloader = new SnapSyncDownloader( fastSyncActions, - worldStateStorage, + worldStateStorageCoordinator, snapWorldStateDownloader, fastSyncStateStorage, snapTaskCollection, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java index 8159ccdaa02..7319e9e44f7 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncActions.java @@ -23,13 +23,13 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.MetricsSystem; public class CheckpointSyncActions extends FastSyncActions { public CheckpointSyncActions( final SynchronizerConfiguration syncConfig, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, @@ -38,7 +38,7 @@ public CheckpointSyncActions( final MetricsSystem metricsSystem) { super( syncConfig, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, @@ -51,7 +51,7 @@ public CheckpointSyncActions( public ChainDownloader createChainDownloader(final FastSyncState currentState) { return CheckpointSyncChainDownloader.create( syncConfig, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java index 854972c39fd..6443bf28249 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckpointSyncChainDownloader.java @@ -23,14 +23,14 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncTargetManager; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.MetricsSystem; public class CheckpointSyncChainDownloader extends FastSyncChainDownloader { public static ChainDownloader create( final SynchronizerConfiguration config, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, @@ -41,7 +41,7 @@ public static ChainDownloader create( final FastSyncTargetManager syncTargetManager = new FastSyncTargetManager( config, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java index d7d3523538b..52d384acf6e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java @@ -26,7 +26,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.eth.sync.tasks.RetryingGetHeaderFromPeerByHashTask; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; @@ -43,7 +43,7 @@ public class FastSyncActions { private static final Logger LOG = LoggerFactory.getLogger(FastSyncActions.class); protected final SynchronizerConfiguration syncConfig; - protected final WorldStateStorage worldStateStorage; + protected final WorldStateStorageCoordinator worldStateStorageCoordinator; protected final ProtocolSchedule protocolSchedule; protected final ProtocolContext protocolContext; protected final EthContext ethContext; @@ -55,7 +55,7 @@ public class FastSyncActions { public FastSyncActions( final SynchronizerConfiguration syncConfig, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, @@ -63,7 +63,7 @@ public FastSyncActions( final PivotBlockSelector pivotBlockSelector, final MetricsSystem metricsSystem) { this.syncConfig = syncConfig; - this.worldStateStorage = worldStateStorage; + this.worldStateStorageCoordinator = worldStateStorageCoordinator; this.protocolSchedule = protocolSchedule; this.protocolContext = protocolContext; this.ethContext = ethContext; @@ -158,7 +158,7 @@ private FastSyncState updateStats(final FastSyncState fastSyncState) { public ChainDownloader createChainDownloader(final FastSyncState currentState) { return FastSyncChainDownloader.create( syncConfig, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java index c4034cb87a0..250e1e8e3ec 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java @@ -21,7 +21,7 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.MetricsSystem; public class FastSyncChainDownloader { @@ -30,7 +30,7 @@ protected FastSyncChainDownloader() {} public static ChainDownloader create( final SynchronizerConfiguration config, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, @@ -41,7 +41,7 @@ public static ChainDownloader create( final FastSyncTargetManager syncTargetManager = new FastSyncTargetManager( config, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java index b73daa28301..7a6387ff264 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java @@ -16,13 +16,14 @@ import static org.hyperledger.besu.util.FutureUtils.exceptionallyCompose; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.manager.exceptions.MaxRetriesReachedException; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; import org.hyperledger.besu.ethereum.eth.sync.TrailingPeerRequirements; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.TaskCollection; import org.hyperledger.besu.util.ExceptionUtils; @@ -45,7 +46,7 @@ public class FastSyncDownloader { private static final Duration FAST_SYNC_RETRY_DELAY = Duration.ofSeconds(5); private static final Logger LOG = LoggerFactory.getLogger(FastSyncDownloader.class); - private final WorldStateStorage worldStateStorage; + private final WorldStateStorageCoordinator worldStateStorageCoordinator; private final WorldStateDownloader worldStateDownloader; private final TaskCollection taskCollection; private final Path fastSyncDataDirectory; @@ -58,14 +59,14 @@ public class FastSyncDownloader { public FastSyncDownloader( final FastSyncActions fastSyncActions, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final WorldStateDownloader worldStateDownloader, final FastSyncStateStorage fastSyncStateStorage, final TaskCollection taskCollection, final Path fastSyncDataDirectory, final FastSyncState initialFastSyncState) { this.fastSyncActions = fastSyncActions; - this.worldStateStorage = worldStateStorage; + this.worldStateStorageCoordinator = worldStateStorageCoordinator; this.worldStateDownloader = worldStateDownloader; this.fastSyncStateStorage = fastSyncStateStorage; this.taskCollection = taskCollection; @@ -82,11 +83,15 @@ public CompletableFuture start() { } protected CompletableFuture start(final FastSyncState fastSyncState) { - if (worldStateStorage.getDataStorageFormat().equals(DataStorageFormat.BONSAI)) { - LOG.info("Clearing bonsai flat account db"); - worldStateStorage.clearFlatDatabase(); - worldStateStorage.clearTrieLog(); - } + worldStateStorageCoordinator.applyOnMatchingStrategy( + DataStorageFormat.BONSAI, + worldStateKeyValueStorage -> { + BonsaiWorldStateKeyValueStorage onBonsai = + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage; + LOG.info("Clearing bonsai flat account db"); + onBonsai.clearFlatDatabase(); + onBonsai.clearTrieLog(); + }); LOG.debug("Start sync with initial sync state {}", fastSyncState); return findPivotBlock(fastSyncState, fss -> downloadChainAndWorldState(fastSyncActions, fss)); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java index 11dc8c1af61..b0482541c58 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java @@ -28,7 +28,7 @@ import org.hyperledger.besu.ethereum.eth.sync.tasks.RetryingGetHeaderFromPeerByNumberTask; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage.DisconnectReason; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.MetricsSystem; import java.util.List; @@ -42,7 +42,7 @@ public class FastSyncTargetManager extends SyncTargetManager { private static final Logger LOG = LoggerFactory.getLogger(FastSyncTargetManager.class); - private final WorldStateStorage worldStateStorage; + private final WorldStateStorageCoordinator worldStateStorageCoordinator; private final ProtocolSchedule protocolSchedule; private final ProtocolContext protocolContext; private final EthContext ethContext; @@ -55,14 +55,14 @@ public class FastSyncTargetManager extends SyncTargetManager { public FastSyncTargetManager( final SynchronizerConfiguration config, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final EthContext ethContext, final MetricsSystem metricsSystem, final FastSyncState fastSyncState) { super(config, protocolSchedule, protocolContext, ethContext, metricsSystem); - this.worldStateStorage = worldStateStorage; + this.worldStateStorageCoordinator = worldStateStorageCoordinator; this.protocolSchedule = protocolSchedule; this.protocolContext = protocolContext; this.ethContext = ethContext; @@ -176,7 +176,7 @@ public boolean shouldContinueDownloading() { return true; } } - return !worldStateStorage.isWorldStateAvailable( + return !worldStateStorageCoordinator.isWorldStateAvailable( pivotBlockHeader.getStateRoot(), pivotBlockHeader.getBlockHash()); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/AccountTrieNodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/AccountTrieNodeDataRequest.java index 90aa59c045a..25ce5cc49ec 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/AccountTrieNodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/AccountTrieNodeDataRequest.java @@ -14,16 +14,16 @@ */ package org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate; +import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; + import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPOutput; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage.Updater; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.Optional; import java.util.stream.Stream; @@ -38,16 +38,24 @@ class AccountTrieNodeDataRequest extends TrieNodeDataRequest { } @Override - protected void doPersist(final Updater updater) { - updater.putAccountStateTrieNode(getLocation().orElse(Bytes.EMPTY), getHash(), getData()); + protected void doPersist(final WorldStateKeyValueStorage.Updater updater) { + applyForStrategy( + updater, + onBonsai -> { + onBonsai.putAccountStateTrieNode(getLocation().orElse(Bytes.EMPTY), getHash(), getData()); + }, + onForest -> { + onForest.putAccountStateTrieNode(getHash(), getData()); + }); } @Override - public Optional getExistingData(final WorldStateStorage worldStateStorage) { + public Optional getExistingData( + final WorldStateStorageCoordinator worldStateKeyValueStorage) { return getLocation() .flatMap( location -> - worldStateStorage + worldStateKeyValueStorage .getAccountStateTrieNode(location, getHash()) .filter(data -> Hash.hash(data).equals(getHash()))); } @@ -60,7 +68,7 @@ protected NodeDataRequest createChildNodeDataRequest( @Override protected Stream getRequestsFromTrieNodeValue( - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateKeyValueStorage, final Optional location, final Bytes path, final Bytes value) { @@ -73,11 +81,11 @@ protected Stream getRequestsFromTrieNodeValue( Bytes32.wrap( CompactEncoding.pathToBytes( Bytes.concatenate(getLocation().orElse(Bytes.EMPTY), path))))); - if (!worldStateStorage.getFlatDbMode().equals(FlatDbMode.NO_FLATTENED)) { - ((BonsaiWorldStateKeyValueStorage.Updater) worldStateStorage.updater()) - .putAccountInfoState(accountHash.get(), value) - .commit(); - } + + worldStateKeyValueStorage.applyWhenFlatModeEnabled( + onBonsai -> { + onBonsai.updater().putAccountInfoState(accountHash.get(), value).commit(); + }); // Add code, if appropriate if (!accountValue.getCodeHash().equals(Hash.EMPTY)) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/CodeNodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/CodeNodeDataRequest.java index 0a8672ef795..327949e50a9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/CodeNodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/CodeNodeDataRequest.java @@ -14,10 +14,12 @@ */ package org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate; +import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; + import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLPOutput; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage.Updater; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.Optional; import java.util.stream.Stream; @@ -34,21 +36,36 @@ class CodeNodeDataRequest extends NodeDataRequest { } @Override - protected void doPersist(final Updater updater) { - updater.putCode(accountHash.orElse(Hash.EMPTY), getHash(), getData()); + protected void doPersist(final WorldStateKeyValueStorage.Updater updater) { + applyForStrategy( + updater, + onBonsai -> { + onBonsai.putCode(accountHash.orElse(Hash.EMPTY), getHash(), getData()); + }, + onForest -> { + onForest.putCode(getHash(), getData()); + }); } @Override - public Stream getChildRequests(final WorldStateStorage worldStateStorage) { + public Stream getChildRequests( + final WorldStateStorageCoordinator worldStateStorageCoordinator) { // Code nodes have nothing further to download return Stream.empty(); } @Override - public Optional getExistingData(final WorldStateStorage worldStateStorage) { - return worldStateStorage - .getCode(getHash(), accountHash.orElse(Hash.EMPTY)) - .filter(codeBytes -> Hash.hash(codeBytes).equals(getHash())); + public Optional getExistingData( + final WorldStateStorageCoordinator worldStateStorageCoordinator) { + return worldStateStorageCoordinator.applyForStrategy( + onBonsai -> { + return onBonsai + .getCode(getHash(), accountHash.orElse(Hash.EMPTY)) + .filter(codeBytes -> Hash.hash(codeBytes).equals(getHash())); + }, + onForest -> { + return onForest.getCode(getHash()); + }); } public Optional getAccountHash() { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java index 467dc0a53d0..bf9f28db6ce 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastDownloaderFactory.java @@ -15,7 +15,6 @@ package org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate; import org.hyperledger.besu.ethereum.ProtocolContext; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector; @@ -29,7 +28,7 @@ import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; @@ -59,7 +58,7 @@ public static Optional> create( final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SyncState syncState, final Clock clock) { @@ -90,23 +89,25 @@ public static Optional> create( return Optional.empty(); } - if (worldStateStorage instanceof BonsaiWorldStateKeyValueStorage) { - worldStateStorage.clearFlatDatabase(); - } else { - final Path queueDataDir = fastSyncDataDirectory.resolve("statequeue"); - if (queueDataDir.toFile().exists()) { - LOG.warn( - "Fast sync is picking up after old fast sync version. Pruning the world state and starting from scratch."); - clearOldFastSyncWorldStateData(worldStateStorage, queueDataDir); - } - } + worldStateStorageCoordinator.consumeForStrategy( + onBonsai -> { + onBonsai.clearFlatDatabase(); + }, + onForest -> { + final Path queueDataDir = fastSyncDataDirectory.resolve("statequeue"); + if (queueDataDir.toFile().exists()) { + LOG.warn( + "Fast sync is picking up after old fast sync version. Pruning the world state and starting from scratch."); + clearOldFastSyncWorldStateData(worldStateStorageCoordinator, queueDataDir); + } + }); final InMemoryTasksPriorityQueues taskCollection = createWorldStateDownloaderTaskCollection( metricsSystem, syncConfig.getWorldStateTaskCacheSize()); final WorldStateDownloader worldStateDownloader = new FastWorldStateDownloader( ethContext, - worldStateStorage, + worldStateStorageCoordinator, taskCollection, syncConfig.getWorldStateHashCountPerRequest(), syncConfig.getWorldStateRequestParallelism(), @@ -118,14 +119,14 @@ public static Optional> create( new FastSyncDownloader<>( new FastSyncActions( syncConfig, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, syncState, pivotBlockSelector, metricsSystem), - worldStateStorage, + worldStateStorageCoordinator, worldStateDownloader, fastSyncStateStorage, taskCollection, @@ -136,8 +137,8 @@ public static Optional> create( } private static void clearOldFastSyncWorldStateData( - final WorldStateStorage worldStateStorage, final Path queueDataDir) { - worldStateStorage.clear(); + final WorldStateStorageCoordinator worldStateKeyValueStorage, final Path queueDataDir) { + worldStateKeyValueStorage.clear(); try (final Stream stream = Files.list(queueDataDir); ) { stream.forEach(FastDownloaderFactory::deleteFile); deleteFile(queueDataDir); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java index 0703c93ef89..8c09a993bee 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadState.java @@ -14,10 +14,12 @@ */ package org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate; +import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; + +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldDownloadState; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage.Updater; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; import java.time.Clock; @@ -31,13 +33,13 @@ public class FastWorldDownloadState extends WorldDownloadState private static final Logger LOG = LoggerFactory.getLogger(FastWorldDownloadState.class); public FastWorldDownloadState( - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final InMemoryTasksPriorityQueues pendingRequests, final int maxRequestsWithoutProgress, final long minMillisBeforeStalling, final Clock clock) { super( - worldStateStorage, + worldStateStorageCoordinator, pendingRequests, maxRequestsWithoutProgress, minMillisBeforeStalling, @@ -53,8 +55,15 @@ public synchronized boolean checkCompletion(final BlockHeader header) { header.getStateRoot(), Optional.of(Bytes.EMPTY))); return false; } - final Updater updater = worldStateStorage.updater(); - updater.saveWorldState(header.getHash(), header.getStateRoot(), rootNodeData); + final WorldStateKeyValueStorage.Updater updater = worldStateStorageCoordinator.updater(); + applyForStrategy( + updater, + onBonsai -> { + onBonsai.saveWorldState(header.getHash(), header.getStateRoot(), rootNodeData); + }, + onForest -> { + onForest.saveWorldState(header.getStateRoot(), rootNodeData); + }); updater.commit(); internalFuture.complete(null); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloader.java index 4bdbae71cee..559ade5900c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloader.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncActions; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; @@ -48,7 +48,7 @@ public class FastWorldStateDownloader implements WorldStateDownloader { private final int hashCountPerRequest; private final int maxOutstandingRequests; private final int maxNodeRequestsWithoutProgress; - private final WorldStateStorage worldStateStorage; + private final WorldStateStorageCoordinator worldStateStorageCoordinator; private final AtomicReference downloadState = new AtomicReference<>(); @@ -56,7 +56,7 @@ public class FastWorldStateDownloader implements WorldStateDownloader { public FastWorldStateDownloader( final EthContext ethContext, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final InMemoryTasksPriorityQueues taskCollection, final int hashCountPerRequest, final int maxOutstandingRequests, @@ -65,7 +65,7 @@ public FastWorldStateDownloader( final Clock clock, final MetricsSystem metricsSystem) { this.ethContext = ethContext; - this.worldStateStorage = worldStateStorage; + this.worldStateStorageCoordinator = worldStateStorageCoordinator; this.taskCollection = taskCollection; this.hashCountPerRequest = hashCountPerRequest; this.maxOutstandingRequests = maxOutstandingRequests; @@ -117,7 +117,7 @@ public CompletableFuture run( final BlockHeader header = fastSyncState.getPivotBlockHeader().get(); final Hash stateRoot = header.getStateRoot(); - if (worldStateStorage.isWorldStateAvailable(stateRoot, header.getHash())) { + if (worldStateStorageCoordinator.isWorldStateAvailable(stateRoot, header.getHash())) { LOG.info( "World state already available for block {} ({}). State root {}", header.getNumber(), @@ -133,7 +133,7 @@ public CompletableFuture run( final FastWorldDownloadState newDownloadState = new FastWorldDownloadState( - worldStateStorage, + worldStateStorageCoordinator, taskCollection, maxNodeRequestsWithoutProgress, minMillisBeforeStalling, @@ -151,9 +151,9 @@ public CompletableFuture run( FastWorldStateDownloadProcess.builder() .hashCountPerRequest(hashCountPerRequest) .maxOutstandingRequests(maxOutstandingRequests) - .loadLocalDataStep(new LoadLocalDataStep(worldStateStorage, metricsSystem)) + .loadLocalDataStep(new LoadLocalDataStep(worldStateStorageCoordinator, metricsSystem)) .requestDataStep(new RequestDataStep(ethContext, metricsSystem)) - .persistDataStep(new PersistDataStep(worldStateStorage)) + .persistDataStep(new PersistDataStep(worldStateStorageCoordinator)) .completeTaskStep(maybeCompleteTask.get()) .downloadState(newDownloadState) .pivotBlockHeader(header) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStep.java index f56780f5259..bd015ba6c07 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStep.java @@ -14,7 +14,8 @@ */ package org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; @@ -28,12 +29,13 @@ public class LoadLocalDataStep { - private final WorldStateStorage worldStateStorage; + private final WorldStateStorageCoordinator worldStateStorageCoordinator; private final Counter existingNodeCounter; public LoadLocalDataStep( - final WorldStateStorage worldStateStorage, final MetricsSystem metricsSystem) { - this.worldStateStorage = worldStateStorage; + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final MetricsSystem metricsSystem) { + this.worldStateStorageCoordinator = worldStateStorageCoordinator; existingNodeCounter = metricsSystem.createCounter( BesuMetricCategory.SYNCHRONIZER, @@ -44,12 +46,12 @@ public LoadLocalDataStep( public Stream> loadLocalData( final Task task, final Pipe> completedTasks) { final NodeDataRequest request = task.getData(); - final Optional existingData = request.getExistingData(worldStateStorage); + final Optional existingData = request.getExistingData(worldStateStorageCoordinator); if (existingData.isPresent()) { existingNodeCounter.inc(); request.setData(existingData.get()); request.setRequiresPersisting(false); - final WorldStateStorage.Updater updater = worldStateStorage.updater(); + final WorldStateKeyValueStorage.Updater updater = worldStateStorageCoordinator.updater(); request.persist(updater); updater.commit(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/NodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/NodeDataRequest.java index 2ff8429316e..18dc943a7c2 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/NodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/NodeDataRequest.java @@ -17,10 +17,11 @@ import static com.google.common.base.Preconditions.checkNotNull; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloaderException; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPOutput; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.TasksPriorityProvider; import java.util.Optional; @@ -103,7 +104,7 @@ public NodeDataRequest setRequiresPersisting(final boolean requiresPersisting) { return this; } - public final void persist(final WorldStateStorage.Updater updater) { + public final void persist(final WorldStateKeyValueStorage.Updater updater) { if (pendingChildren.get() > 0) { return; // we do nothing. Our last child will eventually persist us. } @@ -115,7 +116,7 @@ public final void persist(final WorldStateStorage.Updater updater) { parent -> parent.saveParent(updater), () -> LOG.warn("Missing a parent for {}", this.hash)); } - private void saveParent(final WorldStateStorage.Updater updater) { + private void saveParent(final WorldStateKeyValueStorage.Updater updater) { if (pendingChildren.decrementAndGet() == 0) { persist(updater); } @@ -127,11 +128,13 @@ private int incrementChildren() { protected abstract void writeTo(final RLPOutput out); - protected abstract void doPersist(final WorldStateStorage.Updater updater); + protected abstract void doPersist(final WorldStateKeyValueStorage.Updater updater); - public abstract Stream getChildRequests(WorldStateStorage worldStateStorage); + public abstract Stream getChildRequests( + WorldStateStorageCoordinator worldStateStorageCoordinator); - public abstract Optional getExistingData(final WorldStateStorage worldStateStorage); + public abstract Optional getExistingData( + final WorldStateStorageCoordinator worldStateStorageCoordinator); protected void registerParent(final NodeDataRequest parent) { if (this.possibleParent.isPresent()) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStep.java index 1ab202ee6ce..694e8a9944f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStep.java @@ -18,10 +18,10 @@ import static org.hyperledger.besu.ethereum.eth.sync.StorageExceptionManager.errorCountAtThreshold; import static org.hyperledger.besu.ethereum.eth.sync.StorageExceptionManager.getRetryableErrorCounter; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldDownloadState; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage.Updater; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.services.tasks.Task; @@ -34,10 +34,10 @@ public class PersistDataStep { private static final Logger LOG = LoggerFactory.getLogger(PersistDataStep.class); - private final WorldStateStorage worldStateStorage; + private final WorldStateStorageCoordinator worldStateStorageCoordinator; - public PersistDataStep(final WorldStateStorage worldStateStorage) { - this.worldStateStorage = worldStateStorage; + public PersistDataStep(final WorldStateStorageCoordinator worldStateStorageCoordinator) { + this.worldStateStorageCoordinator = worldStateStorageCoordinator; } public List> persist( @@ -45,7 +45,7 @@ public List> persist( final BlockHeader blockHeader, final WorldDownloadState downloadState) { try { - final Updater updater = worldStateStorage.updater(); + final WorldStateKeyValueStorage.Updater updater = worldStateStorageCoordinator.updater(); tasks.stream() .map( task -> { @@ -88,6 +88,6 @@ private boolean isRootState(final BlockHeader blockHeader, final NodeDataRequest private void enqueueChildren( final Task task, final WorldDownloadState downloadState) { final NodeDataRequest request = task.getData(); - downloadState.enqueueRequests(request.getChildRequests(worldStateStorage)); + downloadState.enqueueRequests(request.getChildRequests(worldStateStorageCoordinator)); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/StorageTrieNodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/StorageTrieNodeDataRequest.java index a46faa06a67..efc2d847d5e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/StorageTrieNodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/StorageTrieNodeDataRequest.java @@ -14,13 +14,13 @@ */ package org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate; +import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; + import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLPOutput; import org.hyperledger.besu.ethereum.trie.CompactEncoding; -import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage.Updater; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.Optional; import java.util.stream.Stream; @@ -40,20 +40,31 @@ class StorageTrieNodeDataRequest extends TrieNodeDataRequest { } @Override - protected void doPersist(final Updater updater) { - updater.putAccountStorageTrieNode( - accountHash.orElse(Hash.EMPTY), getLocation().orElse(Bytes.EMPTY), getHash(), getData()); + protected void doPersist(final WorldStateKeyValueStorage.Updater updater) { + applyForStrategy( + updater, + onBonsai -> { + onBonsai.putAccountStorageTrieNode( + accountHash.orElse(Hash.EMPTY), + getLocation().orElse(Bytes.EMPTY), + getHash(), + getData()); + }, + onForest -> { + onForest.putAccountStorageTrieNode(getHash(), getData()); + }); } @Override - public Optional getExistingData(final WorldStateStorage worldStateStorage) { + public Optional getExistingData( + final WorldStateStorageCoordinator worldStateStorageCoordinator) { return getAccountHash() .flatMap( accountHash -> getLocation() .flatMap( location -> - worldStateStorage + worldStateStorageCoordinator .getAccountStorageTrieNode(accountHash, location, getHash()) .filter(data -> Hash.hash(data).equals(getHash())))); } @@ -66,18 +77,21 @@ protected NodeDataRequest createChildNodeDataRequest( @Override protected Stream getRequestsFromTrieNodeValue( - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final Optional location, final Bytes path, final Bytes value) { - if (!worldStateStorage.getFlatDbMode().equals(FlatDbMode.NO_FLATTENED)) { - ((BonsaiWorldStateKeyValueStorage.Updater) worldStateStorage.updater()) - .putStorageValueBySlotHash( - accountHash.get(), - getSlotHash(location, path), - Bytes32.leftPad(RLP.decodeValue(value))) - .commit(); - } + + worldStateStorageCoordinator.applyWhenFlatModeEnabled( + worldStateKeyValueStorage -> { + worldStateKeyValueStorage + .updater() + .putStorageValueBySlotHash( + accountHash.get(), + getSlotHash(location, path), + Bytes32.leftPad(RLP.decodeValue(value))) + .commit(); + }); return Stream.empty(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/TrieNodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/TrieNodeDataRequest.java index 2761db14329..c9ea51ce0f4 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/TrieNodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/TrieNodeDataRequest.java @@ -17,7 +17,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.trie.Node; import org.hyperledger.besu.ethereum.trie.patricia.TrieNodeDecoder; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.List; import java.util.Objects; @@ -34,7 +34,8 @@ protected TrieNodeDataRequest( } @Override - public Stream getChildRequests(final WorldStateStorage worldStateStorage) { + public Stream getChildRequests( + final WorldStateStorageCoordinator worldStateStorageCoordinator) { if (getData() == null) { // If this node hasn't been downloaded yet, we can't return any child data return Stream.empty(); @@ -53,7 +54,10 @@ public Stream getChildRequests(final WorldStateStorage worldSta .map( value -> getRequestsFromTrieNodeValue( - worldStateStorage, node.getLocation(), node.getPath(), value)) + worldStateStorageCoordinator, + node.getLocation(), + node.getPath(), + value)) .orElseGet(Stream::empty); } }) @@ -68,7 +72,7 @@ protected abstract NodeDataRequest createChildNodeDataRequest( final Hash childHash, final Optional location); protected abstract Stream getRequestsFromTrieNodeValue( - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final Optional location, final Bytes path, final Bytes value); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStep.java index c24dbf6037d..132f22b92f0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStep.java @@ -18,9 +18,10 @@ import static org.hyperledger.besu.ethereum.eth.sync.StorageExceptionManager.errorCountAtThreshold; import static org.hyperledger.besu.ethereum.eth.sync.StorageExceptionManager.getRetryableErrorCounter; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.TrieNodeHealingRequest; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.exception.StorageException; @@ -38,7 +39,7 @@ public class LoadLocalDataStep { private static final Logger LOG = LoggerFactory.getLogger(LoadLocalDataStep.class); - private final WorldStateStorage worldStateStorage; + private final WorldStateStorageCoordinator worldStateStorageCoordinator; private final SnapWorldDownloadState downloadState; private final SnapSyncProcessState snapSyncState; @@ -46,12 +47,12 @@ public class LoadLocalDataStep { private final Counter existingNodeCounter; public LoadLocalDataStep( - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapWorldDownloadState downloadState, final SnapSyncConfiguration snapSyncConfiguration, final MetricsSystem metricsSystem, final SnapSyncProcessState snapSyncState) { - this.worldStateStorage = worldStateStorage; + this.worldStateStorageCoordinator = worldStateStorageCoordinator; this.downloadState = downloadState; this.snapSyncConfiguration = snapSyncConfiguration; existingNodeCounter = @@ -68,16 +69,22 @@ public Stream> loadLocalDataTrieNode( // check if node is already stored in the worldstate try { if (snapSyncState.hasPivotBlockHeader()) { - Optional existingData = request.getExistingData(downloadState, worldStateStorage); + Optional existingData = + request.getExistingData(downloadState, worldStateStorageCoordinator); if (existingData.isPresent()) { existingNodeCounter.inc(); request.setData(existingData.get()); request.setRequiresPersisting(false); - final WorldStateStorage.Updater updater = worldStateStorage.updater(); + final WorldStateKeyValueStorage.Updater updater = worldStateStorageCoordinator.updater(); request.persist( - worldStateStorage, updater, downloadState, snapSyncState, snapSyncConfiguration); + worldStateStorageCoordinator, + updater, + downloadState, + snapSyncState, + snapSyncConfiguration); updater.commit(); - downloadState.enqueueRequests(request.getRootStorageRequests(worldStateStorage)); + downloadState.enqueueRequests( + request.getRootStorageRequests(worldStateStorageCoordinator)); completedTasks.put(task); return Stream.empty(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java index df3696ccdf7..2e10aa80d7d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java @@ -18,10 +18,11 @@ import static org.hyperledger.besu.ethereum.eth.sync.StorageExceptionManager.errorCountAtThreshold; import static org.hyperledger.besu.ethereum.eth.sync.StorageExceptionManager.getRetryableErrorCounter; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.TrieNodeHealingRequest; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.services.tasks.Task; @@ -35,30 +36,31 @@ public class PersistDataStep { private static final Logger LOG = LoggerFactory.getLogger(PersistDataStep.class); private final SnapSyncProcessState snapSyncState; - private final WorldStateStorage worldStateStorage; + private final WorldStateStorageCoordinator worldStateStorageCoordinator; private final SnapWorldDownloadState downloadState; private final SnapSyncConfiguration snapSyncConfiguration; public PersistDataStep( final SnapSyncProcessState snapSyncState, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapWorldDownloadState downloadState, final SnapSyncConfiguration snapSyncConfiguration) { this.snapSyncState = snapSyncState; - this.worldStateStorage = worldStateStorage; + this.worldStateStorageCoordinator = worldStateStorageCoordinator; this.downloadState = downloadState; this.snapSyncConfiguration = snapSyncConfiguration; } public List> persist(final List> tasks) { try { - final WorldStateStorage.Updater updater = worldStateStorage.updater(); + final WorldStateKeyValueStorage.Updater updater = worldStateStorageCoordinator.updater(); for (Task task : tasks) { if (task.getData().isResponseReceived()) { // enqueue child requests final Stream childRequests = - task.getData().getChildRequests(downloadState, worldStateStorage, snapSyncState); + task.getData() + .getChildRequests(downloadState, worldStateStorageCoordinator, snapSyncState); if (!(task.getData() instanceof TrieNodeHealingRequest)) { enqueueChildren(childRequests); } else { @@ -73,7 +75,7 @@ public List> persist(final List> tas final int persistedNodes = task.getData() .persist( - worldStateStorage, + worldStateStorageCoordinator, updater, downloadState, snapSyncState, @@ -115,15 +117,21 @@ public List> persist(final List> tas */ public List> healFlatDatabase(final List> tasks) { final BonsaiWorldStateKeyValueStorage.Updater updater = - (BonsaiWorldStateKeyValueStorage.Updater) worldStateStorage.updater(); + (BonsaiWorldStateKeyValueStorage.Updater) worldStateStorageCoordinator.updater(); for (Task task : tasks) { // heal and/or persist task.getData() - .persist(worldStateStorage, updater, downloadState, snapSyncState, snapSyncConfiguration); + .persist( + worldStateStorageCoordinator, + updater, + downloadState, + snapSyncState, + snapSyncConfiguration); // enqueue child requests, these will be the right part of the ranges to complete if we have // not healed all the range enqueueChildren( - task.getData().getChildRequests(downloadState, worldStateStorage, snapSyncState)); + task.getData() + .getChildRequests(downloadState, worldStateStorageCoordinator, snapSyncState)); } updater.commit(); return tasks; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java index f78ffe3af9a..fdde0a5eea9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java @@ -31,7 +31,8 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.StorageFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.TrieNodeHealingRequest; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.tasks.Task; @@ -50,7 +51,7 @@ public class RequestDataStep { - private final WorldStateStorage worldStateStorage; + private final WorldStateStorageCoordinator worldStateStorageCoordinator; private final SnapSyncProcessState fastSyncState; private final SnapWorldDownloadState downloadState; private final SnapSyncConfiguration snapSyncConfiguration; @@ -60,18 +61,18 @@ public class RequestDataStep { public RequestDataStep( final EthContext ethContext, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapSyncProcessState fastSyncState, final SnapWorldDownloadState downloadState, final SnapSyncConfiguration snapSyncConfiguration, final MetricsSystem metricsSystem) { - this.worldStateStorage = worldStateStorage; + this.worldStateStorageCoordinator = worldStateStorageCoordinator; this.fastSyncState = fastSyncState; this.downloadState = downloadState; this.snapSyncConfiguration = snapSyncConfiguration; this.metricsSystem = metricsSystem; this.ethContext = ethContext; - this.worldStateProofProvider = new WorldStateProofProvider(worldStateStorage); + this.worldStateProofProvider = new WorldStateProofProvider(worldStateStorageCoordinator); } public CompletableFuture> requestAccount( @@ -230,12 +231,18 @@ public CompletableFuture> requestLocalFlatAccounts( final BlockHeader blockHeader = fastSyncState.getPivotBlockHeader().get(); // retrieve accounts from flat database - final TreeMap accounts = - (TreeMap) - worldStateStorage.streamFlatAccounts( - accountDataRequest.getStartKeyHash(), - accountDataRequest.getEndKeyHash(), - snapSyncConfiguration.getLocalFlatAccountCountToHealPerRequest()); + final TreeMap accounts = new TreeMap<>(); + + worldStateStorageCoordinator.applyOnMatchingFlatMode( + FlatDbMode.FULL, + onBonsai -> { + accounts.putAll( + onBonsai.streamFlatAccounts( + accountDataRequest.getStartKeyHash(), + accountDataRequest.getEndKeyHash(), + snapSyncConfiguration.getLocalFlatAccountCountToHealPerRequest())); + }); + final List proofs = new ArrayList<>(); if (!accounts.isEmpty()) { // generate range proof if accounts are present @@ -270,13 +277,18 @@ public CompletableFuture> requestLocalFlatStorages( storageDataRequest.setRootHash(blockHeader.getStateRoot()); // retrieve slots from flat database - final TreeMap slots = - (TreeMap) - worldStateStorage.streamFlatStorages( - storageDataRequest.getAccountHash(), - storageDataRequest.getStartKeyHash(), - storageDataRequest.getEndKeyHash(), - snapSyncConfiguration.getLocalFlatStorageCountToHealPerRequest()); + final TreeMap slots = new TreeMap<>(); + worldStateStorageCoordinator.applyOnMatchingFlatMode( + FlatDbMode.FULL, + onBonsai -> { + slots.putAll( + onBonsai.streamFlatStorages( + storageDataRequest.getAccountHash(), + storageDataRequest.getStartKeyHash(), + storageDataRequest.getEndKeyHash(), + snapSyncConfiguration.getLocalFlatStorageCountToHealPerRequest())); + }); + final List proofs = new ArrayList<>(); if (!slots.isEmpty()) { // generate range proof if slots are present diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java index 65fb117788e..44077ade5b1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapDownloaderFactory.java @@ -32,7 +32,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.trie.CompactEncoding; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; @@ -56,7 +56,7 @@ public static Optional> createSnapDownloader( final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SyncState syncState, final Clock clock) { @@ -105,7 +105,7 @@ public static Optional> createSnapDownloader( ethContext, snapContext, protocolContext, - worldStateStorage, + worldStateStorageCoordinator, snapTaskCollection, syncConfig.getSnapSyncConfiguration(), syncConfig.getWorldStateRequestParallelism(), @@ -117,14 +117,14 @@ public static Optional> createSnapDownloader( new SnapSyncDownloader( new FastSyncActions( syncConfig, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, syncState, pivotBlockSelector, metricsSystem), - worldStateStorage, + worldStateStorageCoordinator, snapWorldStateDownloader, fastSyncStateStorage, snapTaskCollection, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java index ac409513156..6f08ecb1aa1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncDownloader.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncStateStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.TaskCollection; import java.nio.file.Path; @@ -35,7 +35,7 @@ public class SnapSyncDownloader extends FastSyncDownloader { public SnapSyncDownloader( final FastSyncActions fastSyncActions, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final WorldStateDownloader worldStateDownloader, final FastSyncStateStorage fastSyncStateStorage, final TaskCollection taskCollection, @@ -43,7 +43,7 @@ public SnapSyncDownloader( final FastSyncState initialFastSyncState) { super( fastSyncActions, - worldStateStorage, + worldStateStorageCoordinator, worldStateDownloader, fastSyncStateStorage, taskCollection, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index dd7481cdd00..a92f6e20551 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -16,7 +16,10 @@ import static org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest.createAccountFlatHealingRangeRequest; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest.createAccountTrieNodeDataRequest; +import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.BlockAddedObserver; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; @@ -28,8 +31,9 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.AccountFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.StorageFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldDownloadState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.services.tasks.InMemoryTaskQueue; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; @@ -86,7 +90,7 @@ public class SnapWorldDownloadState extends WorldDownloadState private final SnapsyncMetricsManager metricsManager; public SnapWorldDownloadState( - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapSyncStatePersistenceManager snapContext, final Blockchain blockchain, final SnapSyncProcessState snapSyncState, @@ -96,7 +100,7 @@ public SnapWorldDownloadState( final SnapsyncMetricsManager metricsManager, final Clock clock) { super( - worldStateStorage, + worldStateStorageCoordinator, pendingRequests, maxRequestsWithoutProgress, minMillisBeforeStalling, @@ -190,16 +194,28 @@ else if (pivotBlockSelector.isBlockchainBehind()) { blockObserverId.ifPresent(blockchain::removeObserver); // If the flat database healing process is not in progress and the flat database mode is // FULL - if (!snapSyncState.isHealFlatDatabaseInProgress() - && worldStateStorage.getFlatDbMode().equals(FlatDbMode.FULL)) { - // Start the flat database healing process - startFlatDatabaseHeal(header); + if (!snapSyncState.isHealFlatDatabaseInProgress()) { + worldStateStorageCoordinator.applyOnMatchingFlatMode( + FlatDbMode.FULL, + bonsaiWorldStateStorageStrategy -> { + // Start the flat database healing process + startFlatDatabaseHeal(header); + }); } + // If the flat database healing process is in progress or the flat database mode is not FULL else { - final WorldStateStorage.Updater updater = worldStateStorage.updater(); - updater.saveWorldState(header.getHash(), header.getStateRoot(), rootNodeData); + final WorldStateKeyValueStorage.Updater updater = worldStateStorageCoordinator.updater(); + applyForStrategy( + updater, + onBonsai -> { + onBonsai.saveWorldState(header.getHash(), header.getStateRoot(), rootNodeData); + }, + onForest -> { + onForest.saveWorldState(header.getStateRoot(), rootNodeData); + }); updater.commit(); + // Notify that the snap sync has completed metricsManager.notifySnapSyncCompleted(); // Clear the snap context @@ -242,8 +258,14 @@ public synchronized void startTrieHeal() { /** Method to reload the healing process of the trie */ public synchronized void reloadTrieHeal() { // Clear the flat database and trie log from the world state storage if needed - worldStateStorage.clearFlatDatabase(); - worldStateStorage.clearTrieLog(); + worldStateStorageCoordinator.applyOnMatchingStrategy( + DataStorageFormat.BONSAI, + worldStateKeyValueStorage -> { + final BonsaiWorldStateKeyValueStorage strategy = + worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class); + strategy.clearFlatDatabase(); + strategy.clearTrieLog(); + }); // Clear pending trie node and code requests pendingTrieNodeRequests.clear(); pendingCodeRequests.clear(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java index 91bdd83a0f3..3825779749c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java @@ -29,7 +29,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; @@ -63,7 +63,7 @@ public class SnapWorldStateDownloader implements WorldStateDownloader { private final int maxOutstandingRequests; private final int maxNodeRequestsWithoutProgress; private final ProtocolContext protocolContext; - private final WorldStateStorage worldStateStorage; + private final WorldStateStorageCoordinator worldStateStorageCoordinator; private final AtomicReference downloadState = new AtomicReference<>(); @@ -71,7 +71,7 @@ public SnapWorldStateDownloader( final EthContext ethContext, final SnapSyncStatePersistenceManager snapContext, final ProtocolContext protocolContext, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final InMemoryTasksPriorityQueues snapTaskCollection, final SnapSyncConfiguration snapSyncConfiguration, final int maxOutstandingRequests, @@ -81,7 +81,7 @@ public SnapWorldStateDownloader( final MetricsSystem metricsSystem) { this.ethContext = ethContext; this.protocolContext = protocolContext; - this.worldStateStorage = worldStateStorage; + this.worldStateStorageCoordinator = worldStateStorageCoordinator; this.snapContext = snapContext; this.snapTaskCollection = snapTaskCollection; this.snapSyncConfiguration = snapSyncConfiguration; @@ -138,7 +138,7 @@ public CompletableFuture run( final SnapWorldDownloadState newDownloadState = new SnapWorldDownloadState( - worldStateStorage, + worldStateStorageCoordinator, snapContext, protocolContext.getBlockchain(), snapSyncState, @@ -167,19 +167,30 @@ public CompletableFuture run( }); } else if (!snapContext.getAccountsHealingList().isEmpty()) { // restart only the heal step snapSyncState.setHealTrieStatus(true); - worldStateStorage.clearFlatDatabase(); - worldStateStorage.clearTrieLog(); + worldStateStorageCoordinator.applyOnMatchingStrategy( + DataStorageFormat.BONSAI, + strategy -> { + BonsaiWorldStateKeyValueStorage onBonsai = (BonsaiWorldStateKeyValueStorage) strategy; + onBonsai.clearFlatDatabase(); + onBonsai.clearTrieLog(); + }); + newDownloadState.setAccountsHealingList(inconsistentAccounts); newDownloadState.enqueueRequest( SnapDataRequest.createAccountTrieNodeDataRequest( stateRoot, Bytes.EMPTY, snapContext.getAccountsHealingList())); } else { // start from scratch - worldStateStorage.clear(); + worldStateStorageCoordinator.clear(); // we have to upgrade to full flat db mode if we are in bonsai mode - if (worldStateStorage.getDataStorageFormat().equals(DataStorageFormat.BONSAI) - && snapSyncConfiguration.isFlatDbHealingEnabled()) { - ((BonsaiWorldStateKeyValueStorage) worldStateStorage).upgradeToFullFlatDbMode(); + if (snapSyncConfiguration.isFlatDbHealingEnabled()) { + worldStateStorageCoordinator.applyOnMatchingStrategy( + DataStorageFormat.BONSAI, + strategy -> { + BonsaiWorldStateKeyValueStorage onBonsai = + (BonsaiWorldStateKeyValueStorage) strategy; + onBonsai.upgradeToFullFlatDbMode(); + }); } ranges.forEach( (key, value) -> @@ -205,7 +216,7 @@ public CompletableFuture run( .dynamicPivotBlockSelector(dynamicPivotBlockManager) .loadLocalDataStep( new LoadLocalDataStep( - worldStateStorage, + worldStateStorageCoordinator, newDownloadState, snapSyncConfiguration, metricsSystem, @@ -213,14 +224,17 @@ public CompletableFuture run( .requestDataStep( new RequestDataStep( ethContext, - worldStateStorage, + worldStateStorageCoordinator, snapSyncState, newDownloadState, snapSyncConfiguration, metricsSystem)) .persistDataStep( new PersistDataStep( - snapSyncState, worldStateStorage, newDownloadState, snapSyncConfiguration)) + snapSyncState, + worldStateStorageCoordinator, + newDownloadState, + snapSyncConfiguration)) .completeTaskStep(maybeCompleteTask.get()) .downloadState(newDownloadState) .fastSyncState(snapSyncState) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java index 06181fd09f1..9d077bc3c19 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java @@ -20,8 +20,10 @@ import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.ACCOUNT_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager.Step.DOWNLOAD; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.StackTrie.FlatDatabaseUpdater.noop; +import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; @@ -33,8 +35,7 @@ import org.hyperledger.besu.ethereum.trie.NodeUpdater; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage.Updater; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.ArrayList; import java.util.List; @@ -42,6 +43,7 @@ import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; import com.google.common.annotations.VisibleForTesting; @@ -104,8 +106,8 @@ protected AccountRangeDataRequest( @Override protected int doPersist( - final WorldStateStorage worldStateStorage, - final Updater updater, + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final WorldStateKeyValueStorage.Updater updater, final SnapWorldDownloadState downloadState, final SnapSyncProcessState snapSyncState, final SnapSyncConfiguration snapSyncConfiguration) { @@ -120,19 +122,31 @@ protected int doPersist( final AtomicInteger nbNodesSaved = new AtomicInteger(); final NodeUpdater nodeUpdater = (location, hash, value) -> { - updater.putAccountStateTrieNode(location, hash, value); + applyForStrategy( + updater, + onBonsai -> { + onBonsai.putAccountStateTrieNode(location, hash, value); + }, + onForest -> { + onForest.putAccountStateTrieNode(hash, value); + }); nbNodesSaved.getAndIncrement(); }; - StackTrie.FlatDatabaseUpdater flatDatabaseUpdater = noop(); - if (worldStateStorage.getFlatDbMode().equals(FlatDbMode.FULL)) { - // we have a flat DB only with Bonsai - flatDatabaseUpdater = - (key, value) -> - ((BonsaiWorldStateKeyValueStorage.BonsaiUpdater) updater) - .putAccountInfoState(Hash.wrap(key), value); - } - stackTrie.commit(flatDatabaseUpdater, nodeUpdater); + final AtomicReference flatDatabaseUpdater = + new AtomicReference<>(noop()); + + // we have a flat DB only with Bonsai + worldStateStorageCoordinator.applyOnMatchingFlatMode( + FlatDbMode.FULL, + bonsaiWorldStateStorageStrategy -> { + flatDatabaseUpdater.set( + (key, value) -> + ((BonsaiWorldStateKeyValueStorage.Updater) updater) + .putAccountInfoState(Hash.wrap(key), value)); + }); + + stackTrie.commit(flatDatabaseUpdater.get(), nodeUpdater); downloadState.getMetricsManager().notifyAccountsDownloaded(stackTrie.getElementsCount().get()); @@ -162,7 +176,7 @@ public boolean isResponseReceived() { @Override public Stream getChildRequests( final SnapWorldDownloadState downloadState, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapSyncProcessState snapSyncState) { final List childRequests = new ArrayList<>(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/BytecodeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/BytecodeRequest.java index 5db5ec0211e..bb709d14966 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/BytecodeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/BytecodeRequest.java @@ -15,14 +15,15 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.BYTECODES; +import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; import static org.slf4j.LoggerFactory.getLogger; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage.Updater; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.stream.Stream; @@ -51,18 +52,26 @@ protected BytecodeRequest( @Override protected int doPersist( - final WorldStateStorage worldStateStorage, - final Updater updater, + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final WorldStateKeyValueStorage.Updater updater, final SnapWorldDownloadState downloadState, final SnapSyncProcessState snapSyncState, final SnapSyncConfiguration snapSyncConfiguration) { - updater.putCode(Hash.wrap(accountHash), code); + + applyForStrategy( + updater, + onBonsai -> { + onBonsai.putCode(Hash.wrap(accountHash), codeHash, code); + }, + onForest -> { + onForest.putCode(codeHash, code); + }); downloadState.getMetricsManager().notifyCodeDownloaded(); return possibleParent .map( trieNodeDataRequest -> trieNodeDataRequest.saveParent( - worldStateStorage, + worldStateStorageCoordinator, updater, downloadState, snapSyncState, @@ -79,7 +88,7 @@ public boolean isResponseReceived() { @Override public Stream getChildRequests( final SnapWorldDownloadState downloadState, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapSyncProcessState snapSyncState) { return Stream.empty(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java index 76d2d9fe596..4011a691f3c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java @@ -17,6 +17,7 @@ import static org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.NodeDataRequest.MAX_CHILDREN; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; @@ -27,7 +28,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.StorageTrieNodeHealingRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.TrieNodeHealingRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloaderException; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.TasksPriorityProvider; import java.util.HashSet; @@ -115,18 +116,18 @@ public static BytecodeRequest createBytecodeRequest( } public int persist( - final WorldStateStorage worldStateStorage, - final WorldStateStorage.Updater updater, + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final WorldStateKeyValueStorage.Updater updater, final SnapWorldDownloadState downloadState, final SnapSyncProcessState snapSyncState, final SnapSyncConfiguration snapSyncConfiguration) { return doPersist( - worldStateStorage, updater, downloadState, snapSyncState, snapSyncConfiguration); + worldStateStorageCoordinator, updater, downloadState, snapSyncState, snapSyncConfiguration); } protected abstract int doPersist( - final WorldStateStorage worldStateStorage, - final WorldStateStorage.Updater updater, + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final WorldStateKeyValueStorage.Updater updater, final SnapWorldDownloadState downloadState, final SnapSyncProcessState snapSyncState, final SnapSyncConfiguration snapSyncConfiguration); @@ -139,7 +140,7 @@ public boolean isExpired(final SnapSyncProcessState snapSyncState) { public abstract Stream getChildRequests( final SnapWorldDownloadState downloadState, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapSyncProcessState snapSyncState); public void registerParent(final TrieNodeHealingRequest parent) { @@ -156,14 +157,18 @@ protected int incrementChildren() { } protected int saveParent( - final WorldStateStorage worldStateStorage, - final WorldStateStorage.Updater updater, + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final WorldStateKeyValueStorage.Updater updater, final SnapWorldDownloadState downloadState, final SnapSyncProcessState snapSyncState, final SnapSyncConfiguration snapSyncConfiguration) { if (pendingChildren.decrementAndGet() == 0) { return persist( - worldStateStorage, updater, downloadState, snapSyncState, snapSyncConfiguration); + worldStateStorageCoordinator, + updater, + downloadState, + snapSyncState, + snapSyncConfiguration); } return 0; } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java index 14839f0ad6f..119d6b3e4d0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java @@ -20,8 +20,10 @@ import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.getRangeCount; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.STORAGE_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.StackTrie.FlatDatabaseUpdater.noop; +import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; @@ -32,14 +34,14 @@ import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.NodeUpdater; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage.Updater; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; import com.google.common.annotations.VisibleForTesting; @@ -86,8 +88,8 @@ protected StorageRangeDataRequest( @Override protected int doPersist( - final WorldStateStorage worldStateStorage, - final Updater updater, + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final WorldStateKeyValueStorage.Updater updater, final SnapWorldDownloadState downloadState, final SnapSyncProcessState snapSyncState, final SnapSyncConfiguration snapSyncConfiguration) { @@ -96,20 +98,31 @@ protected int doPersist( final AtomicInteger nbNodesSaved = new AtomicInteger(); final NodeUpdater nodeUpdater = (location, hash, value) -> { - updater.putAccountStorageTrieNode(accountHash, location, hash, value); + applyForStrategy( + updater, + onBonsai -> { + onBonsai.putAccountStorageTrieNode(accountHash, location, hash, value); + }, + onForest -> { + onForest.putAccountStorageTrieNode(hash, value); + }); }; - StackTrie.FlatDatabaseUpdater flatDatabaseUpdater = noop(); - if (worldStateStorage.getFlatDbMode().equals(FlatDbMode.FULL)) { - // we have a flat DB only with Bonsai - flatDatabaseUpdater = - (key, value) -> - ((BonsaiWorldStateKeyValueStorage.Updater) updater) - .putStorageValueBySlotHash( - accountHash, Hash.wrap(key), Bytes32.leftPad(RLP.decodeValue(value))); - } + final AtomicReference flatDatabaseUpdater = + new AtomicReference<>(noop()); + + // we have a flat DB only with Bonsai + worldStateStorageCoordinator.applyOnMatchingFlatMode( + FlatDbMode.FULL, + bonsaiWorldStateStorageStrategy -> { + flatDatabaseUpdater.set( + (key, value) -> + ((BonsaiWorldStateKeyValueStorage.Updater) updater) + .putStorageValueBySlotHash( + accountHash, Hash.wrap(key), Bytes32.leftPad(RLP.decodeValue(value)))); + }); - stackTrie.commit(flatDatabaseUpdater, nodeUpdater); + stackTrie.commit(flatDatabaseUpdater.get(), nodeUpdater); downloadState.getMetricsManager().notifySlotsDownloaded(stackTrie.getElementsCount().get()); @@ -153,7 +166,7 @@ public boolean isExpired(final SnapSyncProcessState snapSyncState) { @Override public Stream getChildRequests( final SnapWorldDownloadState downloadState, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapSyncProcessState snapSyncState) { final List childRequests = new ArrayList<>(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java index a40a4fb11a3..83d8852c8ce 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java @@ -19,6 +19,7 @@ import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager.Step.HEAL_FLAT; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType; @@ -34,7 +35,7 @@ import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.math.BigInteger; import java.util.ArrayList; @@ -75,7 +76,7 @@ public AccountFlatDatabaseHealingRangeRequest( @Override public Stream getChildRequests( final SnapWorldDownloadState downloadState, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapSyncProcessState snapSyncState) { final List childRequests = new ArrayList<>(); if (!existingAccounts.isEmpty()) { @@ -144,8 +145,8 @@ public void addLocalData( @Override protected int doPersist( - final WorldStateStorage worldStateStorage, - final WorldStateStorage.Updater updater, + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final WorldStateKeyValueStorage.Updater updater, final SnapWorldDownloadState downloadState, final SnapSyncProcessState snapSyncState, final SnapSyncConfiguration syncConfig) { @@ -157,7 +158,7 @@ protected int doPersist( final MerkleTrie accountTrie = new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, + worldStateStorageCoordinator::getAccountStateTrieNode, getRootHash(), Function.identity(), Function.identity()); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java index d9297217146..91ac16e9e66 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java @@ -15,9 +15,10 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest.createAccountTrieNodeDataRequest; +import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; @@ -26,9 +27,8 @@ import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.ArrayList; import java.util.HashSet; @@ -56,22 +56,30 @@ public AccountTrieNodeHealingRequest( @Override protected int doPersist( - final WorldStateStorage worldStateStorage, - final WorldStateStorage.Updater updater, + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final WorldStateKeyValueStorage.Updater updater, final SnapWorldDownloadState downloadState, final SnapSyncProcessState snapSyncState, final SnapSyncConfiguration snapSyncConfiguration) { if (isRoot()) { downloadState.setRootNodeData(data); } - updater.putAccountStateTrieNode(getLocation(), getNodeHash(), data); + applyForStrategy( + updater, + onBonsai -> { + onBonsai.putAccountStateTrieNode(getLocation(), getNodeHash(), data); + }, + onForest -> { + onForest.putAccountStateTrieNode(getNodeHash(), data); + }); return 1; } @Override public Optional getExistingData( - final SnapWorldDownloadState downloadState, final WorldStateStorage worldStateStorage) { - return worldStateStorage + final SnapWorldDownloadState downloadState, + final WorldStateStorageCoordinator worldStateStorageCoordinator) { + return worldStateStorageCoordinator .getAccountStateTrieNode(getLocation(), getNodeHash()) .filter(data -> !getLocation().isEmpty()); } @@ -93,11 +101,12 @@ private HashSet getSubLocation(final Bytes location) { } @Override - public Stream getRootStorageRequests(final WorldStateStorage worldStateStorage) { + public Stream getRootStorageRequests( + final WorldStateStorageCoordinator worldStateStorageCoordinator) { final List requests = new ArrayList<>(); final StoredMerklePatriciaTrie accountTrie = new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, + worldStateStorageCoordinator::getAccountStateTrieNode, Hash.hash(data), getLocation(), Function.identity(), @@ -137,7 +146,7 @@ public Stream getRootStorageRequests(final WorldStateStorage wo @Override protected Stream getRequestsFromTrieNodeValue( - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapWorldDownloadState downloadState, final Bytes location, final Bytes path, @@ -151,11 +160,10 @@ protected Stream getRequestsFromTrieNodeValue( Bytes32.wrap(CompactEncoding.pathToBytes(Bytes.concatenate(getLocation(), path)))); // update the flat db only for bonsai - if (!worldStateStorage.getFlatDbMode().equals(FlatDbMode.NO_FLATTENED)) { - ((BonsaiWorldStateKeyValueStorage.Updater) worldStateStorage.updater()) - .putAccountInfoState(accountHash, value) - .commit(); - } + worldStateStorageCoordinator.applyWhenFlatModeEnabled( + onBonsai -> { + onBonsai.updater().putAccountInfoState(accountHash, value).commit(); + }); // Add code, if appropriate if (!accountValue.getCodeHash().equals(Hash.EMPTY)) { @@ -164,7 +172,7 @@ protected Stream getRequestsFromTrieNodeValue( // Retrieve the storage root from the database, if available final Hash storageRootFoundInDb = - worldStateStorage + worldStateStorageCoordinator .getTrieNodeUnsafe(Bytes.concatenate(accountHash, Bytes.EMPTY)) .map(Hash::hash) .orElse(Hash.wrap(MerkleTrie.EMPTY_TRIE_NODE_HASH)); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java index a94424f023f..023e687f149 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java @@ -18,6 +18,7 @@ import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.STORAGE_RANGE; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; @@ -29,7 +30,7 @@ import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.math.BigInteger; import java.util.ArrayList; @@ -75,7 +76,7 @@ public StorageFlatDatabaseHealingRangeRequest( @Override public Stream getChildRequests( final SnapWorldDownloadState downloadState, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapSyncProcessState snapSyncState) { final List childRequests = new ArrayList<>(); if (!slots.isEmpty()) { @@ -132,8 +133,8 @@ public void addLocalData( @Override protected int doPersist( - final WorldStateStorage worldStateStorage, - final WorldStateStorage.Updater updater, + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final WorldStateKeyValueStorage.Updater updater, final SnapWorldDownloadState downloadState, final SnapSyncProcessState snapSyncState, final SnapSyncConfiguration snapSyncConfiguration) { @@ -147,7 +148,8 @@ protected int doPersist( final MerkleTrie storageTrie = new StoredMerklePatriciaTrie<>( (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash), + worldStateStorageCoordinator.getAccountStorageTrieNode( + accountHash, location, hash), storageRoot, Function.identity(), Function.identity()); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequest.java index ffcdb3e77b7..47757ef0fcf 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequest.java @@ -14,16 +14,16 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; +import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; + import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.trie.CompactEncoding; -import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage.Updater; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import java.util.List; import java.util.Optional; @@ -46,19 +46,27 @@ public StorageTrieNodeHealingRequest( @Override protected int doPersist( - final WorldStateStorage worldStateStorage, - final Updater updater, + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final WorldStateKeyValueStorage.Updater updater, final SnapWorldDownloadState downloadState, final SnapSyncProcessState snapSyncState, final SnapSyncConfiguration snapSyncConfiguration) { - updater.putAccountStorageTrieNode(getAccountHash(), getLocation(), getNodeHash(), data); + applyForStrategy( + updater, + onBonsai -> { + onBonsai.putAccountStorageTrieNode(getAccountHash(), getLocation(), getNodeHash(), data); + }, + onForest -> { + onForest.putAccountStorageTrieNode(getNodeHash(), data); + }); return 1; } @Override public Optional getExistingData( - final SnapWorldDownloadState downloadState, final WorldStateStorage worldStateStorage) { - return worldStateStorage.getAccountStorageTrieNode( + final SnapWorldDownloadState downloadState, + final WorldStateStorageCoordinator worldStateStorageCoordinator) { + return worldStateStorageCoordinator.getAccountStorageTrieNode( getAccountHash(), getLocation(), getNodeHash()); } @@ -70,17 +78,19 @@ protected SnapDataRequest createChildNodeDataRequest(final Hash childHash, final @Override protected Stream getRequestsFromTrieNodeValue( - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapWorldDownloadState downloadState, final Bytes location, final Bytes path, final Bytes value) { - if (!worldStateStorage.getFlatDbMode().equals(FlatDbMode.NO_FLATTENED)) { - ((BonsaiWorldStateKeyValueStorage.Updater) worldStateStorage.updater()) - .putStorageValueBySlotHash( - accountHash, getSlotHash(location, path), Bytes32.leftPad(RLP.decodeValue(value))) - .commit(); - } + worldStateStorageCoordinator.applyWhenFlatModeEnabled( + onBonsai -> { + onBonsai + .updater() + .putStorageValueBySlotHash( + accountHash, getSlotHash(location, path), Bytes32.leftPad(RLP.decodeValue(value))) + .commit(); + }); return Stream.empty(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/TrieNodeHealingRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/TrieNodeHealingRequest.java index ef7191a0167..27b37045675 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/TrieNodeHealingRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/TrieNodeHealingRequest.java @@ -18,13 +18,14 @@ import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.TRIE_NODE; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.trie.Node; import org.hyperledger.besu.ethereum.trie.patricia.TrieNodeDecoder; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.TasksPriorityProvider; import java.util.ArrayList; @@ -54,8 +55,8 @@ protected TrieNodeHealingRequest(final Hash nodeHash, final Hash rootHash, final @Override public int persist( - final WorldStateStorage worldStateStorage, - final WorldStateStorage.Updater updater, + final WorldStateStorageCoordinator worldStateStorageCoordinator, + final WorldStateKeyValueStorage.Updater updater, final SnapWorldDownloadState downloadState, final SnapSyncProcessState snapSyncState, final SnapSyncConfiguration snapSyncConfiguration) { @@ -68,13 +69,21 @@ public int persist( checkNotNull(data, "Must set data before node can be persisted."); saved = doPersist( - worldStateStorage, updater, downloadState, snapSyncState, snapSyncConfiguration); + worldStateStorageCoordinator, + updater, + downloadState, + snapSyncState, + snapSyncConfiguration); } if (possibleParent.isPresent()) { return possibleParent .get() .saveParent( - worldStateStorage, updater, downloadState, snapSyncState, snapSyncConfiguration) + worldStateStorageCoordinator, + updater, + downloadState, + snapSyncState, + snapSyncConfiguration) + saved; } return saved; @@ -83,7 +92,7 @@ public int persist( @Override public Stream getChildRequests( final SnapWorldDownloadState downloadState, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapSyncProcessState snapSyncState) { if (!isResponseReceived()) { // If this node hasn't been downloaded yet, we can't return any child data @@ -103,7 +112,7 @@ public Stream getChildRequests( .map( value -> getRequestsFromTrieNodeValue( - worldStateStorage, + worldStateStorageCoordinator, downloadState, node.getLocation().orElse(Bytes.EMPTY), node.getPath(), @@ -172,19 +181,21 @@ private boolean nodeIsHashReferencedDescendant(final Node node) { } public abstract Optional getExistingData( - final SnapWorldDownloadState downloadState, final WorldStateStorage worldStateStorage); + final SnapWorldDownloadState downloadState, + final WorldStateStorageCoordinator worldStateStorageCoordinator); public abstract List getTrieNodePath(); protected abstract SnapDataRequest createChildNodeDataRequest( final Hash childHash, final Bytes location); - public Stream getRootStorageRequests(final WorldStateStorage worldStateStorage) { + public Stream getRootStorageRequests( + final WorldStateStorageCoordinator worldStateStorageCoordinator) { return Stream.empty(); } protected abstract Stream getRequestsFromTrieNodeValue( - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final SnapWorldDownloadState downloadState, final Bytes location, final Bytes path, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadState.java index 8f53921f4dd..fd3ae682da9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadState.java @@ -17,7 +17,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.manager.task.EthTask; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; import org.hyperledger.besu.services.tasks.Task; import org.hyperledger.besu.services.tasks.TasksPriorityProvider; @@ -53,16 +53,16 @@ public abstract class WorldDownloadState private volatile long timestampOfLastProgress; protected Bytes rootNodeData; - protected final WorldStateStorage worldStateStorage; + protected final WorldStateStorageCoordinator worldStateStorageCoordinator; protected WorldStateDownloadProcess worldStateDownloadProcess; public WorldDownloadState( - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final InMemoryTasksPriorityQueues pendingRequests, final int maxRequestsWithoutProgress, final long minMillisBeforeStalling, final Clock clock) { - this.worldStateStorage = worldStateStorage; + this.worldStateStorageCoordinator = worldStateStorageCoordinator; this.minMillisBeforeStalling = minMillisBeforeStalling; this.timestampOfLastProgress = clock.millis(); this.downloadWasResumed = !pendingRequests.isEmpty(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index 62ef446503b..256daf538ee 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -37,7 +37,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.Optional; @@ -53,7 +53,8 @@ public class CheckPointSyncChainDownloaderTest { - private final WorldStateStorage worldStateStorage = mock(WorldStateStorage.class); + private final WorldStateStorageCoordinator worldStateStorageCoordinator = + mock(WorldStateStorageCoordinator.class); protected ProtocolSchedule protocolSchedule; protected EthProtocolManager ethProtocolManager; @@ -75,7 +76,7 @@ public Stream provideArguments(final ExtensionContext conte } public void setup(final DataStorageFormat storageFormat) { - when(worldStateStorage.isWorldStateAvailable(any(), any())).thenReturn(true); + when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); final BlockchainSetupUtil localBlockchainSetup = BlockchainSetupUtil.forTesting(storageFormat); localBlockchain = localBlockchainSetup.getBlockchain(); otherBlockchainSetup = BlockchainSetupUtil.forTesting(storageFormat); @@ -114,7 +115,7 @@ private ChainDownloader downloader( final SynchronizerConfiguration syncConfig, final long pivotBlockNumber) { return CheckpointSyncChainDownloader.create( syncConfig, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java index 1cb6521e00e..8d7b5a33379 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.FastDownloaderFactory; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.MetricsSystem; import java.io.File; @@ -59,7 +59,7 @@ public class FastDownloaderFactoryTest { @Mock private ProtocolContext protocolContext; @Mock private MetricsSystem metricsSystem; @Mock private EthContext ethContext; - @Mock private WorldStateStorage worldStateStorage; + @Mock private WorldStateStorageCoordinator worldStateStorageCoordinator; @Mock private SyncState syncState; @Mock private Clock clock; @Mock private Path dataDirectory; @@ -81,7 +81,7 @@ public void shouldThrowIfSyncModeChangedWhileFastSyncIncomplete() { protocolContext, metricsSystem, ethContext, - worldStateStorage, + worldStateStorageCoordinator, syncState, clock)) .isInstanceOf(IllegalStateException.class); @@ -102,7 +102,7 @@ public void shouldNotThrowIfSyncModeChangedWhileFastSyncComplete() { protocolContext, metricsSystem, ethContext, - worldStateStorage, + worldStateStorageCoordinator, syncState, clock); assertThat(result).isEmpty(); @@ -126,7 +126,7 @@ public void shouldNotThrowWhenFastSyncModeRequested() throws NoSuchFieldExceptio protocolContext, metricsSystem, ethContext, - worldStateStorage, + worldStateStorageCoordinator, syncState, clock); @@ -156,11 +156,11 @@ public void shouldClearWorldStateDuringFastSyncWhenStateQueDirectoryExists() thr protocolContext, metricsSystem, ethContext, - worldStateStorage, + worldStateStorageCoordinator, syncState, clock); - verify(worldStateStorage).clear(); + verify(worldStateStorageCoordinator).clear(); assertThat(Files.exists(stateQueueDir)).isFalse(); } @@ -188,7 +188,7 @@ public void shouldCrashWhenStateQueueIsNotDirectory() throws IOException { protocolContext, metricsSystem, ethContext, - worldStateStorage, + worldStateStorageCoordinator, syncState, clock)) .isInstanceOf(IllegalStateException.class); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java index a707228e92d..31529cb5b21 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java @@ -41,7 +41,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -64,7 +64,8 @@ public class FastSyncActionsTest { private final SynchronizerConfiguration.Builder syncConfigBuilder = new SynchronizerConfiguration.Builder().syncMode(SyncMode.FAST).fastSyncPivotDistance(1000); - private final WorldStateStorage worldStateStorage = mock(WorldStateStorage.class); + private final WorldStateStorageCoordinator worldStateStorageCoordinator = + mock(WorldStateStorageCoordinator.class); private final AtomicInteger timeoutCount = new AtomicInteger(0); private SynchronizerConfiguration syncConfig = syncConfigBuilder.build(); private FastSyncActions fastSyncActions; @@ -529,7 +530,7 @@ private FastSyncActions createFastSyncActions( final EthContext ethContext = ethProtocolManager.ethContext(); return new FastSyncActions( syncConfig, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java index d2c63a18a15..fde30bfa5bd 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java @@ -36,7 +36,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.concurrent.CompletableFuture; @@ -52,7 +52,8 @@ public class FastSyncChainDownloaderTest { - private final WorldStateStorage worldStateStorage = mock(WorldStateStorage.class); + private final WorldStateStorageCoordinator worldStateStorageCoordinator = + mock(WorldStateStorageCoordinator.class); protected ProtocolSchedule protocolSchedule; protected EthProtocolManager ethProtocolManager; @@ -73,7 +74,7 @@ public Stream provideArguments(final ExtensionContext conte } public void setup(final DataStorageFormat storageFormat) { - when(worldStateStorage.isWorldStateAvailable(any(), any())).thenReturn(true); + when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); final BlockchainSetupUtil localBlockchainSetup = BlockchainSetupUtil.forTesting(storageFormat); localBlockchain = localBlockchainSetup.getBlockchain(); otherBlockchainSetup = BlockchainSetupUtil.forTesting(storageFormat); @@ -100,7 +101,7 @@ private ChainDownloader downloader( final SynchronizerConfiguration syncConfig, final long pivotBlockNumber) { return FastSyncChainDownloader.create( syncConfig, - worldStateStorage, + worldStateStorageCoordinator, protocolSchedule, protocolContext, ethContext, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java index 632fed6ede1..dc9a7244c4f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java @@ -35,7 +35,7 @@ import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.TaskCollection; import java.nio.file.Path; @@ -54,7 +54,8 @@ public class FastSyncDownloaderTest { @SuppressWarnings("unchecked") private final FastSyncActions fastSyncActions = mock(FastSyncActions.class); - private final WorldStateStorage worldStateStorage = mock(WorldStateStorage.class); + private final WorldStateStorageCoordinator worldStateStorageCoordinator = + mock(WorldStateStorageCoordinator.class); private final WorldStateDownloader worldStateDownloader = mock(FastWorldStateDownloader.class); private final FastSyncStateStorage storage = mock(FastSyncStateStorage.class); @@ -69,7 +70,7 @@ public class FastSyncDownloaderTest { private final FastSyncDownloader downloader = new FastSyncDownloader<>( fastSyncActions, - worldStateStorage, + worldStateStorageCoordinator, worldStateDownloader, storage, taskCollection, @@ -78,8 +79,8 @@ public class FastSyncDownloaderTest { @BeforeEach public void setup() { - when(worldStateStorage.getDataStorageFormat()).thenReturn(DataStorageFormat.FOREST); - when(worldStateStorage.isWorldStateAvailable(any(), any())).thenReturn(true); + when(worldStateStorageCoordinator.getDataStorageFormat()).thenReturn(DataStorageFormat.FOREST); + when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); } @Test @@ -127,7 +128,7 @@ public void shouldResumeFastSync() { final FastSyncDownloader resumedDownloader = new FastSyncDownloader<>( fastSyncActions, - worldStateStorage, + worldStateStorageCoordinator, worldStateDownloader, storage, taskCollection, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java index b8531288276..09b8bca462d 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.verify; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; @@ -27,9 +28,9 @@ import org.hyperledger.besu.ethereum.eth.manager.task.EthTask; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; @@ -54,7 +55,9 @@ public class FastWorldDownloadStateTest { private static final int MAX_REQUESTS_WITHOUT_PROGRESS = 10; private static final long MIN_MILLIS_BEFORE_STALLING = 50_000; - private WorldStateStorage worldStateStorage; + private WorldStateKeyValueStorage worldStateKeyValueStorage; + + private WorldStateStorageCoordinator worldStateStorageCoordinator; private final BlockHeader header = new BlockHeaderTestFixture().stateRoot(ROOT_NODE_HASH).buildHeader(); @@ -78,15 +81,18 @@ public Stream provideArguments(final ExtensionContext conte public void setUp(final DataStorageFormat storageFormat) { if (storageFormat == DataStorageFormat.BONSAI) { - worldStateStorage = + worldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage( new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); } else { - worldStateStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); } + worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); + downloadState = new FastWorldDownloadState( - worldStateStorage, + worldStateStorageCoordinator, pendingRequests, MAX_REQUESTS_WITHOUT_PROGRESS, MIN_MILLIS_BEFORE_STALLING, @@ -115,7 +121,9 @@ public void shouldStoreRootNodeBeforeReturnedFutureCompletes( final CompletableFuture postFutureChecks = future.thenAccept( result -> - assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)) + assertThat( + worldStateStorageCoordinator.getAccountStateTrieNode( + Bytes.EMPTY, ROOT_NODE_HASH)) .contains(ROOT_NODE_DATA)); downloadState.checkCompletion(header); @@ -134,7 +142,8 @@ public void shouldNotCompleteWhenThereArePendingTasks(final DataStorageFormat st downloadState.checkCompletion(header); assertThat(future).isNotDone(); - assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)).isEmpty(); + assertThat(worldStateStorageCoordinator.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)) + .isEmpty(); assertThat(downloadState.isDownloading()).isTrue(); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java index 6d352abb7a8..d00fdf08972 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.verify; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.BlockDataGenerator.BlockOptions; @@ -46,20 +47,19 @@ import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; +import org.hyperledger.besu.ethereum.forest.ForestWorldStateArchive; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.Node; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.TrieNodeDecoder; -import org.hyperledger.besu.ethereum.worldstate.DefaultWorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage.Updater; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -178,8 +178,8 @@ void downloadEmptyWorldState() { final InMemoryTasksPriorityQueues taskCollection = new InMemoryTasksPriorityQueues<>(); - final WorldStateStorage localStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage localStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStateDownloader downloader = createDownloader(ethProtocolManager.ethContext(), localStorage, taskCollection); @@ -196,7 +196,7 @@ void downloadEmptyWorldState() { @Timeout(value = 60) void downloadAlreadyAvailableWorldState() { // Setup existing state - final DefaultWorldStateArchive worldStateArchive = createInMemoryWorldStateArchive(); + final ForestWorldStateArchive worldStateArchive = createInMemoryWorldStateArchive(); final MutableWorldState worldState = worldStateArchive.getMutable(); // Generate accounts and save corresponding state root @@ -218,7 +218,7 @@ void downloadAlreadyAvailableWorldState() { final WorldStateDownloader downloader = createDownloader( ethProtocolManager.ethContext(), - worldStateArchive.getWorldStateStorage(), + worldStateArchive.getWorldStateStorage().worldStateKeyValueStorage(), taskCollection); final FastSyncState fastSyncState = new FastSyncState(header); @@ -263,8 +263,8 @@ void canRecoverFromTimeouts() { final InMemoryTasksPriorityQueues taskCollection = new InMemoryTasksPriorityQueues<>(); - final WorldStateStorage localStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage localStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStateDownloader downloader = createDownloader(ethProtocolManager.ethContext(), localStorage, taskCollection); @@ -282,8 +282,10 @@ void canRecoverFromTimeouts() { // Check that all expected account data was downloaded final WorldStateArchive localWorldStateArchive = - new DefaultWorldStateArchive( - localStorage, createPreimageStorage(), EvmConfiguration.DEFAULT); + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(localStorage), + createPreimageStorage(), + EvmConfiguration.DEFAULT); final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get(); assertThat(result).isDone(); assertAccountsMatch(localWorldState, accounts); @@ -318,13 +320,13 @@ void doesNotRequestKnownCodeFromNetwork() { final InMemoryTasksPriorityQueues taskCollection = new InMemoryTasksPriorityQueues<>(); - final WorldStateStorage localStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage localStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); // Seed local storage with some contract values final Map knownCode = new HashMap<>(); accounts.subList(0, 5).forEach(a -> knownCode.put(a.getCodeHash(), a.getCode())); - final Updater localStorageUpdater = localStorage.updater(); + final ForestWorldStateKeyValueStorage.Updater localStorageUpdater = localStorage.updater(); knownCode.forEach((bytes32, code) -> localStorageUpdater.putCode(null, code)); localStorageUpdater.commit(); @@ -356,8 +358,10 @@ void doesNotRequestKnownCodeFromNetwork() { // Check that all expected account data was downloaded final WorldStateArchive localWorldStateArchive = - new DefaultWorldStateArchive( - localStorage, createPreimageStorage(), EvmConfiguration.DEFAULT); + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(localStorage), + createPreimageStorage(), + EvmConfiguration.DEFAULT); final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get(); assertThat(result).isDone(); assertAccountsMatch(localWorldState, accounts); @@ -403,8 +407,8 @@ private void testCancellation(final boolean shouldCancelFuture) { final InMemoryTasksPriorityQueues taskCollection = new InMemoryTasksPriorityQueues<>(); - final WorldStateStorage localStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage localStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStateDownloader downloader = createDownloader(ethProtocolManager.ethContext(), localStorage, taskCollection); @@ -449,19 +453,20 @@ private void testCancellation(final boolean shouldCancelFuture) { verify(taskCollection, never()).remove(); verify(taskCollection, never()).add(any(NodeDataRequest.class)); // Target world state should not be available - assertThat(localStorage.isWorldStateAvailable(header.getStateRoot(), header.getHash())) - .isFalse(); + assertThat(localStorage.isWorldStateAvailable(header.getStateRoot())).isFalse(); } @Test @Timeout(value = 60) void doesNotRequestKnownAccountTrieNodesFromNetwork() { // Setup "remote" state - final WorldStateStorage remoteStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage remoteStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStateArchive remoteWorldStateArchive = - new DefaultWorldStateArchive( - remoteStorage, createPreimageStorage(), EvmConfiguration.DEFAULT); + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(remoteStorage), + createPreimageStorage(), + EvmConfiguration.DEFAULT); final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable(); // Generate accounts and save corresponding state root @@ -480,8 +485,8 @@ void doesNotRequestKnownAccountTrieNodesFromNetwork() { final InMemoryTasksPriorityQueues taskCollection = new InMemoryTasksPriorityQueues<>(); - final WorldStateStorage localStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage localStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); // Seed local storage with some trie node values final Map allNodes = @@ -489,12 +494,12 @@ void doesNotRequestKnownAccountTrieNodesFromNetwork() { final Set knownNodes = new HashSet<>(); final Set unknownNodes = new HashSet<>(); assertThat(allNodes).isNotEmpty(); // Sanity check - final Updater localStorageUpdater = localStorage.updater(); + final ForestWorldStateKeyValueStorage.Updater localStorageUpdater = localStorage.updater(); final AtomicBoolean storeNode = new AtomicBoolean(true); allNodes.forEach( (nodeHash, node) -> { if (storeNode.get()) { - localStorageUpdater.putAccountStateTrieNode(null, nodeHash, node); + localStorageUpdater.putAccountStateTrieNode(nodeHash, node); knownNodes.add(nodeHash); } else { unknownNodes.add(nodeHash); @@ -533,8 +538,10 @@ void doesNotRequestKnownAccountTrieNodesFromNetwork() { // Check that all expected account data was downloaded final WorldStateArchive localWorldStateArchive = - new DefaultWorldStateArchive( - localStorage, createPreimageStorage(), EvmConfiguration.DEFAULT); + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(localStorage), + createPreimageStorage(), + EvmConfiguration.DEFAULT); final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get(); assertThat(result).isDone(); assertAccountsMatch(localWorldState, accounts); @@ -544,11 +551,13 @@ void doesNotRequestKnownAccountTrieNodesFromNetwork() { @Timeout(value = 60) void doesNotRequestKnownStorageTrieNodesFromNetwork() { // Setup "remote" state - final WorldStateStorage remoteStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage remoteStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStateArchive remoteWorldStateArchive = - new DefaultWorldStateArchive( - remoteStorage, createPreimageStorage(), EvmConfiguration.DEFAULT); + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(remoteStorage), + createPreimageStorage(), + EvmConfiguration.DEFAULT); final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable(); // Generate accounts and save corresponding state root @@ -567,13 +576,13 @@ void doesNotRequestKnownStorageTrieNodesFromNetwork() { final InMemoryTasksPriorityQueues taskCollection = new InMemoryTasksPriorityQueues<>(); - final WorldStateStorage localStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage localStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); // Seed local storage with some trie node values final List storageRootHashes = new StoredMerklePatriciaTrie<>( - remoteStorage::getNodeData, + (location, hash) -> remoteStorage.getNodeData(hash), remoteWorldState.rootHash(), Function.identity(), Function.identity()) @@ -590,13 +599,13 @@ void doesNotRequestKnownStorageTrieNodesFromNetwork() { collectTrieNodesToBeRequestedAfterRoot(remoteStorage, storageRootHash, 5)); } assertThat(allTrieNodes).isNotEmpty(); // Sanity check - final Updater localStorageUpdater = localStorage.updater(); + final ForestWorldStateKeyValueStorage.Updater localStorageUpdater = localStorage.updater(); boolean storeNode = true; for (final Map.Entry entry : allTrieNodes.entrySet()) { final Bytes32 hash = entry.getKey(); final Bytes data = entry.getValue(); if (storeNode) { - localStorageUpdater.putAccountStorageTrieNode(null, null, hash, data); + localStorageUpdater.putAccountStorageTrieNode(hash, data); knownNodes.add(hash); } else { unknownNodes.add(hash); @@ -622,7 +631,7 @@ void doesNotRequestKnownStorageTrieNodesFromNetwork() { respondUntilDone(peers, responder, result); // World state should be available by the time the result is complete - assertThat(localStorage.isWorldStateAvailable(stateRoot, header.getHash())).isTrue(); + assertThat(localStorage.isWorldStateAvailable(stateRoot)).isTrue(); // Check that unknown trie nodes were requested final List requestedHashes = @@ -638,8 +647,10 @@ void doesNotRequestKnownStorageTrieNodesFromNetwork() { // Check that all expected account data was downloaded final WorldStateArchive localWorldStateArchive = - new DefaultWorldStateArchive( - localStorage, createPreimageStorage(), EvmConfiguration.DEFAULT); + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(localStorage), + createPreimageStorage(), + EvmConfiguration.DEFAULT); final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get(); assertThat(result).isDone(); assertAccountsMatch(localWorldState, accounts); @@ -652,11 +663,13 @@ void stalledDownloader() { EthProtocolManagerTestUtil.create(new EthScheduler(1, 1, 1, 1, new NoOpMetricsSystem())); // Setup "remote" state - final WorldStateStorage remoteStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage remoteStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStateArchive remoteWorldStateArchive = - new DefaultWorldStateArchive( - remoteStorage, createPreimageStorage(), EvmConfiguration.DEFAULT); + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(remoteStorage), + createPreimageStorage(), + EvmConfiguration.DEFAULT); final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable(); // Generate accounts and save corresponding state root @@ -668,8 +681,8 @@ void stalledDownloader() { final InMemoryTasksPriorityQueues taskCollection = new InMemoryTasksPriorityQueues<>(); - final WorldStateStorage localStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage localStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateMaxRequestsWithoutProgress(10).build(); final WorldStateDownloader downloader = @@ -714,11 +727,13 @@ void stalledDownloader() { @Timeout(value = 60) void resumesFromNonEmptyQueue() { // Setup "remote" state - final WorldStateStorage remoteStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage remoteStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStateArchive remoteWorldStateArchive = - new DefaultWorldStateArchive( - remoteStorage, createPreimageStorage(), EvmConfiguration.DEFAULT); + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(remoteStorage), + createPreimageStorage(), + EvmConfiguration.DEFAULT); final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable(); // Generate accounts and save corresponding state root @@ -742,8 +757,8 @@ void resumesFromNonEmptyQueue() { verify(taskCollection, times(1)).add(argThat((r) -> r.getHash().equals(hash))); } - final WorldStateStorage localStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage localStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateMaxRequestsWithoutProgress(10).build(); final WorldStateDownloader downloader = @@ -762,7 +777,7 @@ void resumesFromNonEmptyQueue() { CompletableFuture result = downloader.run(null, new FastSyncState(header)); peer.respondWhileOtherThreadsWork(responder, () -> !result.isDone()); - assertThat(localStorage.isWorldStateAvailable(stateRoot, header.getHash())).isTrue(); + assertThat(localStorage.isWorldStateAvailable(stateRoot)).isTrue(); // Check that already enqueued trie nodes were requested final List requestedHashes = @@ -782,8 +797,10 @@ void resumesFromNonEmptyQueue() { // Check that all expected account data was downloaded assertThat(result).isDone(); final WorldStateArchive localWorldStateArchive = - new DefaultWorldStateArchive( - localStorage, createPreimageStorage(), EvmConfiguration.DEFAULT); + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(localStorage), + createPreimageStorage(), + EvmConfiguration.DEFAULT); final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get(); assertAccountsMatch(localWorldState, accounts); } @@ -800,10 +817,10 @@ void resumesFromNonEmptyQueue() { * @return A list of hash-node pairs */ private Map collectTrieNodesToBeRequestedAfterRoot( - final WorldStateStorage storage, final Bytes32 rootHash, final int maxNodes) { + final ForestWorldStateKeyValueStorage storage, final Bytes32 rootHash, final int maxNodes) { final Map trieNodes = new HashMap<>(); - TrieNodeDecoder.breadthFirstDecoder(storage::getNodeData, rootHash) + TrieNodeDecoder.breadthFirstDecoder((location, hash) -> storage.getNodeData(hash), rootHash) .filter(n -> !Objects.equals(n.getHash(), rootHash)) .filter(Node::isReferencedByHash) .limit(maxNodes) @@ -823,10 +840,10 @@ private Map collectTrieNodesToBeRequestedAfterRoot( * @return A list of node hashes */ private List getFirstSetOfChildNodeRequests( - final WorldStateStorage storage, final Bytes32 rootHash) { + final ForestWorldStateKeyValueStorage storage, final Bytes32 rootHash) { final List hashesToRequest = new ArrayList<>(); - final Bytes rootNodeRlp = storage.getNodeData(Bytes.EMPTY, rootHash).get(); + final Bytes rootNodeRlp = storage.getNodeData(rootHash).get(); TrieNodeDecoder.decodeNodes(Bytes.EMPTY, rootNodeRlp).stream() .filter(n -> !Objects.equals(n.getHash(), rootHash)) .filter(Node::isReferencedByHash) @@ -853,11 +870,13 @@ private void downloadAvailableWorldStateFromPeers( final int trailingPeerCount = 5; // Setup "remote" state - final WorldStateStorage remoteStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage remoteStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStateArchive remoteWorldStateArchive = - new DefaultWorldStateArchive( - remoteStorage, createPreimageStorage(), EvmConfiguration.DEFAULT); + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(remoteStorage), + createPreimageStorage(), + EvmConfiguration.DEFAULT); final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable(); // Generate accounts and save corresponding state root @@ -878,11 +897,13 @@ private void downloadAvailableWorldStateFromPeers( final InMemoryTasksPriorityQueues taskCollection = new InMemoryTasksPriorityQueues<>(); - final WorldStateStorage localStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage localStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStateArchive localWorldStateArchive = - new DefaultWorldStateArchive( - localStorage, createPreimageStorage(), EvmConfiguration.DEFAULT); + new ForestWorldStateArchive( + new WorldStateStorageCoordinator(localStorage), + createPreimageStorage(), + EvmConfiguration.DEFAULT); final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder() .worldStateHashCountPerRequest(hashesPerRequest) @@ -1010,7 +1031,7 @@ private void assertAccountsMatch( private WorldStateDownloader createDownloader( final EthContext context, - final WorldStateStorage storage, + final WorldStateKeyValueStorage storage, final InMemoryTasksPriorityQueues taskCollection) { return createDownloader( SynchronizerConfiguration.builder().build(), context, storage, taskCollection); @@ -1019,11 +1040,11 @@ private WorldStateDownloader createDownloader( private WorldStateDownloader createDownloader( final SynchronizerConfiguration config, final EthContext context, - final WorldStateStorage storage, + final WorldStateKeyValueStorage storage, final InMemoryTasksPriorityQueues taskCollection) { return new FastWorldStateDownloader( context, - storage, + new WorldStateStorageCoordinator(storage), taskCollection, config.getWorldStateHashCountPerRequest(), config.getWorldStateRequestParallelism(), diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java index 625b3f72eaa..b157f4aada4 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java @@ -22,7 +22,8 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.pipeline.Pipe; import org.hyperledger.besu.services.tasks.Task; @@ -38,8 +39,10 @@ public class LoadLocalDataStepTest { private static final Bytes DATA = Bytes.of(1, 2, 3); private static final Hash HASH = Hash.hash(DATA); - private final WorldStateStorage worldStateStorage = mock(WorldStateStorage.class); - private final WorldStateStorage.Updater updater = mock(WorldStateStorage.Updater.class); + private final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = + mock(BonsaiWorldStateKeyValueStorage.class); + private final BonsaiWorldStateKeyValueStorage.Updater updater = + mock(BonsaiWorldStateKeyValueStorage.Updater.class); private final CodeNodeDataRequest request = NodeDataRequest.createCodeRequest(HASH, Optional.empty()); @@ -48,7 +51,8 @@ public class LoadLocalDataStepTest { private final Pipe> completedTasks = new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); private final LoadLocalDataStep loadLocalDataStep = - new LoadLocalDataStep(worldStateStorage, new NoOpMetricsSystem()); + new LoadLocalDataStep( + new WorldStateStorageCoordinator(worldStateKeyValueStorage), new NoOpMetricsSystem()); @Test public void shouldReturnStreamWithUnchangedTaskWhenDataNotPresent() { @@ -61,8 +65,8 @@ public void shouldReturnStreamWithUnchangedTaskWhenDataNotPresent() { @Test public void shouldReturnEmptyStreamAndSendTaskToCompletedPipeWhenDataIsPresent() { - when(worldStateStorage.getCode(HASH, Hash.EMPTY)).thenReturn(Optional.of(DATA)); - when(worldStateStorage.updater()).thenReturn(updater); + when(worldStateKeyValueStorage.getCode(HASH, Hash.EMPTY)).thenReturn(Optional.of(DATA)); + when(worldStateKeyValueStorage.updater()).thenReturn(updater); final Stream> output = loadLocalDataStep.loadLocalData(task, completedTasks); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java index e857648f7fc..d6f3c98a137 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/PersistDataStepTest.java @@ -23,11 +23,11 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; -import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.patricia.SimpleMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.tasks.Task; import java.nio.charset.StandardCharsets; @@ -39,15 +39,17 @@ public class PersistDataStepTest { - private final WorldStateStorage worldStateStorage = - new InMemoryKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.FOREST); + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + private final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); private final FastWorldDownloadState downloadState = mock(FastWorldDownloadState.class); private final Bytes rootNodeData = Bytes.of(1, 1, 1, 1); private final BlockHeader blockHeader = new BlockHeaderTestFixture().stateRoot(Hash.hash(rootNodeData)).buildHeader(); - private final PersistDataStep persistDataStep = new PersistDataStep(worldStateStorage); + private final PersistDataStep persistDataStep = new PersistDataStep(worldStateStorageCoordinator); @Test public void shouldPersistDataWhenPresentWithoutChildren() { @@ -75,8 +77,8 @@ public void shouldSkipPersistingTasksWithNoData() { persistDataStep.persist(tasks, blockHeader, downloadState); assertThat(result).isSameAs(tasks); - assertThat(worldStateStorage.contains(withData.getData().getHash())).isTrue(); - assertThat(worldStateStorage.contains(withoutData.getData().getHash())).isFalse(); + assertThat(worldStateKeyValueStorage.contains(withData.getData().getHash())).isTrue(); + assertThat(worldStateKeyValueStorage.contains(withoutData.getData().getHash())).isFalse(); } @Test @@ -87,7 +89,7 @@ public void shouldStoreRootNodeDataInDownloadStateInsteadOfPersisting() { persistDataStep.persist(tasks, blockHeader, downloadState); assertThat(result).isSameAs(tasks); - assertThat(worldStateStorage.contains(rootNode.getData().getHash())).isFalse(); + assertThat(worldStateKeyValueStorage.contains(rootNode.getData().getHash())).isFalse(); verify(downloadState).setRootNodeData(rootNode.getData().getData()); } @@ -119,7 +121,7 @@ private StubTask createTaskWithoutData(final Bytes data) { private void assertDataPersisted(final List> tasks) { tasks.forEach( task -> - assertThat(worldStateStorage.getNodeData(null, task.getData().getHash())) + assertThat(worldStateKeyValueStorage.getNodeData(task.getData().getHash())) .isEqualTo(Optional.of(task.getData().getData()))); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java index ca39d0c86fe..be66c25812c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java @@ -35,7 +35,7 @@ import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.List; @@ -56,10 +56,12 @@ public class AccountHealingTrackingTest { private final List
accounts = List.of(Address.fromHexString("0xdeadbeef")); - private final WorldStateStorage worldStateStorage = + private final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage( new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); + private WorldStateStorageCoordinator worldStateStorageCoordinator; + private WorldStateProofProvider worldStateProofProvider; private MerkleTrie accountStateTrie; @@ -68,11 +70,12 @@ public class AccountHealingTrackingTest { @BeforeEach public void setup() { + worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); accountStateTrie = TrieGenerator.generateTrie( - worldStateStorage, + worldStateStorageCoordinator, accounts.stream().map(Address::addressHash).collect(Collectors.toList())); - worldStateProofProvider = new WorldStateProofProvider(worldStateStorage); + worldStateProofProvider = new WorldStateProofProvider(worldStateStorageCoordinator); } @Test @@ -87,7 +90,8 @@ void avoidMarkingAccountWhenStorageProofValid() { new StoredMerklePatriciaTrie<>( new StoredNodeFactory<>( (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash), + worldStateKeyValueStorage.getAccountStorageTrieNode( + accountHash, location, hash), Function.identity(), Function.identity()), stateTrieAccountValue.getStorageRoot()); @@ -118,7 +122,8 @@ void avoidMarkingAccountWhenStorageProofValid() { MAX_RANGE); storageRangeDataRequest.addResponse( snapWorldDownloadState, worldStateProofProvider, slots, new ArrayDeque<>(proofs)); - storageRangeDataRequest.getChildRequests(snapWorldDownloadState, worldStateStorage, null); + storageRangeDataRequest.getChildRequests( + snapWorldDownloadState, worldStateStorageCoordinator, null); verify(snapWorldDownloadState, never()).addAccountToHealingList(any(Bytes.class)); } @@ -130,7 +135,7 @@ void markAccountOnInvalidStorageProof() { final List proofs = List.of( - worldStateStorage + worldStateKeyValueStorage .getAccountStorageTrieNode( accountHash, Bytes.EMPTY, stateTrieAccountValue.getStorageRoot()) .get()); @@ -159,7 +164,8 @@ void markAccountOnPartialStorageRange() { new StoredMerklePatriciaTrie<>( new StoredNodeFactory<>( (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash), + worldStateKeyValueStorage.getAccountStorageTrieNode( + accountHash, location, hash), Function.identity(), Function.identity()), stateTrieAccountValue.getStorageRoot()); @@ -197,7 +203,8 @@ void markAccountOnPartialStorageRange() { verify(snapWorldDownloadState, never()).addAccountToHealingList(any(Bytes.class)); // should mark during the getchild request - storageRangeDataRequest.getChildRequests(snapWorldDownloadState, worldStateStorage, null); + storageRangeDataRequest.getChildRequests( + snapWorldDownloadState, worldStateStorageCoordinator, null); verify(snapWorldDownloadState).addAccountToHealingList(any(Bytes.class)); } @@ -212,7 +219,8 @@ void avoidMarkingAccountOnValidStorageTrieNodeDetection() { accountHash, Hash.wrap(accountStateTrie.getRootHash()), Bytes.EMPTY); - storageTrieNodeHealingRequest.getExistingData(snapWorldDownloadState, worldStateStorage); + storageTrieNodeHealingRequest.getExistingData( + snapWorldDownloadState, worldStateStorageCoordinator); verify(snapWorldDownloadState, never()).addAccountToHealingList(any(Bytes.class)); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStepTest.java index 7ecd284e98f..ad49c2144e3 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStepTest.java @@ -23,11 +23,12 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.AccountTrieNodeHealingRequest; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.pipeline.Pipe; import org.hyperledger.besu.services.tasks.Task; @@ -58,14 +59,16 @@ public class LoadLocalDataStepTest { private final SnapSyncProcessState snapSyncState = mock(SnapSyncProcessState.class); private final SnapWorldDownloadState downloadState = mock(SnapWorldDownloadState.class); - private final WorldStateStorage worldStateStorage = mock(WorldStateStorage.class); - private final WorldStateStorage.Updater updater = mock(WorldStateStorage.Updater.class); + private final WorldStateStorageCoordinator worldStateStorageCoordinator = + mock(WorldStateStorageCoordinator.class); + private final WorldStateKeyValueStorage.Updater updater = + mock(WorldStateKeyValueStorage.Updater.class); private final SnapSyncConfiguration snapSyncConfiguration = mock(SnapSyncConfiguration.class); private final LoadLocalDataStep loadLocalDataStep = new LoadLocalDataStep( - worldStateStorage, + worldStateStorageCoordinator, downloadState, snapSyncConfiguration, new NoOpMetricsSystem(), @@ -92,8 +95,10 @@ public void shouldReturnStreamWithSameRootHashTaskWhenDataArePresent() { task.getData().setRootHash(blockHeader.getStateRoot()); - when(worldStateStorage.getAccountStateTrieNode(any(), any())).thenReturn(Optional.of(DATA)); - when(worldStateStorage.updater()).thenReturn(mock(WorldStateStorage.Updater.class)); + when(worldStateStorageCoordinator.getAccountStateTrieNode(any(), any())) + .thenReturn(Optional.of(DATA)); + when(worldStateStorageCoordinator.updater()) + .thenReturn(mock(WorldStateKeyValueStorage.Updater.class)); final BlockHeader newBlockHeader = new BlockHeaderTestFixture().stateRoot(Hash.EMPTY).buildHeader(); @@ -107,8 +112,9 @@ public void shouldReturnStreamWithSameRootHashTaskWhenDataArePresent() { @Test public void shouldReturnEmptyStreamAndSendTaskToCompletedPipeWhenDataIsPresent() { - when(worldStateStorage.getAccountStateTrieNode(any(), any())).thenReturn(Optional.of(DATA)); - when(worldStateStorage.updater()).thenReturn(updater); + when(worldStateStorageCoordinator.getAccountStateTrieNode(any(), any())) + .thenReturn(Optional.of(DATA)); + when(worldStateStorageCoordinator.updater()).thenReturn(updater); final Stream> output = loadLocalDataStep.loadLocalDataTrieNode(task, completedTasks); @@ -122,7 +128,7 @@ public void shouldReturnEmptyStreamAndSendTaskToCompletedPipeWhenDataIsPresent() // Should not require persisting. request.persist( - worldStateStorage, updater, downloadState, snapSyncState, snapSyncConfiguration); + worldStateStorageCoordinator, updater, downloadState, snapSyncState, snapSyncConfiguration); verifyNoInteractions(updater); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java index 69047bc6e50..d1c38cb28bc 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java @@ -25,9 +25,10 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataRequest; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.Task; import java.util.List; @@ -38,15 +39,17 @@ public class PersistDataStepTest { - private final WorldStateStorage worldStateStorage = - new InMemoryKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.FOREST); + private final WorldStateStorageCoordinator worldStateStorageCoordinator = + new InMemoryKeyValueStorageProvider() + .createWorldStateStorageCoordinator(DataStorageFormat.FOREST); private final SnapSyncProcessState snapSyncState = mock(SnapSyncProcessState.class); private final SnapWorldDownloadState downloadState = mock(SnapWorldDownloadState.class); private final SnapSyncConfiguration snapSyncConfiguration = mock(SnapSyncConfiguration.class); private final PersistDataStep persistDataStep = - new PersistDataStep(snapSyncState, worldStateStorage, downloadState, snapSyncConfiguration); + new PersistDataStep( + snapSyncState, worldStateStorageCoordinator, downloadState, snapSyncConfiguration); @BeforeEach public void setUp() { @@ -69,7 +72,10 @@ public void shouldSkipPersistDataWhenNoData() { final List> result = persistDataStep.persist(tasks); assertThat(result).isSameAs(tasks); - assertThat(worldStateStorage.getNodeData(Bytes.EMPTY, tasks.get(0).getData().getRootHash())) + assertThat( + worldStateStorageCoordinator + .getStrategy(ForestWorldStateKeyValueStorage.class) + .getNodeData(tasks.get(0).getData().getRootHash())) .isEmpty(); } @@ -80,14 +86,17 @@ private void assertDataPersisted(final List> tasks) { final AccountRangeDataRequest data = (AccountRangeDataRequest) task.getData(); StoredMerklePatriciaTrie trie = new StoredMerklePatriciaTrie<>( - worldStateStorage::getAccountStateTrieNode, data.getRootHash(), b -> b, b -> b); + worldStateStorageCoordinator::getAccountStateTrieNode, + data.getRootHash(), + b -> b, + b -> b); data.getAccounts().forEach((key, value) -> assertThat(trie.get(key)).isPresent()); } else if (task.getData() instanceof StorageRangeDataRequest) { final StorageRangeDataRequest data = (StorageRangeDataRequest) task.getData(); final StoredMerklePatriciaTrie trie = new StoredMerklePatriciaTrie<>( (location, hash) -> - worldStateStorage.getAccountStorageTrieNode( + worldStateStorageCoordinator.getAccountStorageTrieNode( Hash.wrap(data.getAccountHash()), location, hash), data.getStorageRoot(), b -> b, @@ -96,7 +105,9 @@ private void assertDataPersisted(final List> tasks) { } else if (task.getData() instanceof BytecodeRequest) { final BytecodeRequest data = (BytecodeRequest) task.getData(); assertThat( - worldStateStorage.getCode(data.getCodeHash(), Hash.wrap(data.getAccountHash()))) + worldStateStorageCoordinator + .getStrategy(ForestWorldStateKeyValueStorage.class) + .getCode(data.getCodeHash())) .isPresent(); } else { fail("not expected message"); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java index 003db79a7e2..d733dfec6af 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java @@ -18,12 +18,12 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.TrieGenerator; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.HashMap; @@ -117,11 +117,14 @@ public void testGenerateRangesWithSize3() { @Test public void testFindNewBeginElement() { - final WorldStateStorage worldStateStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); final MerkleTrie accountStateTrie = - TrieGenerator.generateTrie(worldStateStorage, 15); + TrieGenerator.generateTrie(worldStateStorageCoordinator, 15); final RangeStorageEntriesCollector collector = RangeStorageEntriesCollector.createCollector( @@ -135,7 +138,7 @@ public void testFindNewBeginElement() { collector, visitor, root, Hash.ZERO)); final WorldStateProofProvider worldStateProofProvider = - new WorldStateProofProvider(worldStateStorage); + new WorldStateProofProvider(worldStateStorageCoordinator); // generate the proof final List proofs = @@ -156,11 +159,13 @@ public void testFindNewBeginElement() { @Test public void testFindNewBeginElementWhenNothingIsMissing() { - final WorldStateStorage worldStateStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); final MerkleTrie accountStateTrie = - TrieGenerator.generateTrie(worldStateStorage, 15); + TrieGenerator.generateTrie(worldStateStorageCoordinator, 15); final RangeStorageEntriesCollector collector = RangeStorageEntriesCollector.createCollector( @@ -174,7 +179,7 @@ public void testFindNewBeginElementWhenNothingIsMissing() { collector, visitor, root, Hash.ZERO)); final WorldStateProofProvider worldStateProofProvider = - new WorldStateProofProvider(worldStateStorage); + new WorldStateProofProvider(worldStateStorageCoordinator); // generate the proof final List proofs = diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index 22c2b73765d..4a58a1d247a 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.BlockAddedEvent; import org.hyperledger.besu.ethereum.chain.BlockAddedObserver; @@ -39,9 +40,9 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; @@ -71,7 +72,9 @@ public class SnapWorldDownloadStateTest { private static final int MAX_REQUESTS_WITHOUT_PROGRESS = 10; private static final long MIN_MILLIS_BEFORE_STALLING = 50_000; - private WorldStateStorage worldStateStorage; + private WorldStateKeyValueStorage worldStateKeyValueStorage; + + private WorldStateStorageCoordinator worldStateStorageCoordinator; private final BlockHeader header = new BlockHeaderTestFixture().stateRoot(ROOT_NODE_HASH).buildHeader(); private final InMemoryTasksPriorityQueues pendingRequests = @@ -106,15 +109,18 @@ public void setUp(final DataStorageFormat storageFormat) { when(metricsManager.getMetricsSystem()).thenReturn(new NoOpMetricsSystem()); if (storageFormat == DataStorageFormat.BONSAI) { - worldStateStorage = + worldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage( new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem()); } else { - worldStateStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); } + worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); + downloadState = new SnapWorldDownloadState( - worldStateStorage, + worldStateStorageCoordinator, snapContext, blockchain, snapSyncState, @@ -176,7 +182,9 @@ public void shouldStoreRootNodeBeforeReturnedFutureCompletes( final CompletableFuture postFutureChecks = future.thenAccept( result -> - assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)) + assertThat( + worldStateStorageCoordinator.getAccountStateTrieNode( + Bytes.EMPTY, ROOT_NODE_HASH)) .contains(ROOT_NODE_DATA)); downloadState.checkCompletion(header); @@ -201,7 +209,8 @@ public void shouldNotCompleteWhenThereAreAccountPendingTasks( downloadState.checkCompletion(header); assertThat(future).isNotDone(); - assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)).isEmpty(); + assertThat(worldStateStorageCoordinator.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)) + .isEmpty(); assertThat(downloadState.isDownloading()).isTrue(); } @@ -218,7 +227,8 @@ public void shouldNotCompleteWhenThereAreStoragePendingTasks( downloadState.checkCompletion(header); assertThat(future).isNotDone(); - assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)).isEmpty(); + assertThat(worldStateStorageCoordinator.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)) + .isEmpty(); assertThat(downloadState.isDownloading()).isTrue(); downloadState.pendingLargeStorageRequests.add( @@ -228,7 +238,8 @@ public void shouldNotCompleteWhenThereAreStoragePendingTasks( downloadState.checkCompletion(header); assertThat(future).isNotDone(); - assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)).isEmpty(); + assertThat(worldStateStorageCoordinator.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)) + .isEmpty(); assertThat(downloadState.isDownloading()).isTrue(); } @@ -245,7 +256,8 @@ public void shouldNotCompleteWhenThereAreTriePendingTasks( downloadState.checkCompletion(header); assertThat(future).isNotDone(); - assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)).isEmpty(); + assertThat(worldStateStorageCoordinator.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)) + .isEmpty(); assertThat(downloadState.isDownloading()).isTrue(); } @@ -263,7 +275,8 @@ public void shouldNotCompleteWhenThereAreFlatDBHealingPendingTasks( downloadState.checkCompletion(header); assertThat(future).isNotDone(); - assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)).isEmpty(); + assertThat(worldStateStorageCoordinator.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)) + .isEmpty(); assertThat(downloadState.isDownloading()).isTrue(); } @@ -341,7 +354,8 @@ public void shouldListeningBlockchainDuringHeal( verify(snapSyncState, atLeastOnce()).setHealTrieStatus(true); assertThat(future).isNotDone(); - assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)).isEmpty(); + assertThat(worldStateStorageCoordinator.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)) + .isEmpty(); assertThat(downloadState.isDownloading()).isTrue(); } @@ -450,13 +464,14 @@ public void shouldNotCompleteReturnedFutureWhenNoPendingTasksRemainAndFlatDBHeal setUp(storageFormat); Assumptions.assumeTrue(storageFormat == DataStorageFormat.BONSAI); Assumptions.assumeTrue(isFlatDbHealingEnabled); - ((BonsaiWorldStateKeyValueStorage) worldStateStorage).upgradeToFullFlatDbMode(); + ((BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage).upgradeToFullFlatDbMode(); when(snapSyncState.isHealTrieInProgress()).thenReturn(true); downloadState.checkCompletion(header); assertThat(future).isNotDone(); verify(snapSyncState).setHealFlatDatabaseInProgress(true); - assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)).isEmpty(); + assertThat(worldStateStorageCoordinator.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)) + .isEmpty(); assertThat(downloadState.isDownloading()).isTrue(); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java index 10a0cf3f84c..4d6ad60968a 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java @@ -16,12 +16,12 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.TrieGenerator; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.List; @@ -41,14 +41,16 @@ public void shouldNotSaveTheRootWhenIncomplete() { final int nbAccounts = 15; - final WorldStateStorage worldStateStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); - final WorldStateStorage recreatedWorldStateStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage recreatedWorldStateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final MerkleTrie accountStateTrie = - TrieGenerator.generateTrie(worldStateStorage, nbAccounts); + TrieGenerator.generateTrie(worldStateStorageCoordinator, nbAccounts); final StackTrie stackTrie = new StackTrie(Hash.wrap(accountStateTrie.getRootHash()), 0, 256, lastAccount); @@ -66,7 +68,7 @@ public void shouldNotSaveTheRootWhenIncomplete() { collector, visitor, root, lastAccount)); final WorldStateProofProvider worldStateProofProvider = - new WorldStateProofProvider(worldStateStorage); + new WorldStateProofProvider(worldStateStorageCoordinator); // generate the proof final List proofs = @@ -78,13 +80,12 @@ public void shouldNotSaveTheRootWhenIncomplete() { stackTrie.addElement(Bytes32.random(), proofs, accounts); - final WorldStateStorage.Updater updater = recreatedWorldStateStorage.updater(); - stackTrie.commit(updater::putAccountStateTrieNode); + final ForestWorldStateKeyValueStorage.Updater updater = recreatedWorldStateStorage.updater(); + stackTrie.commit(((location, hash, value) -> updater.putAccountStateTrieNode(hash, value))); updater.commit(); Assertions.assertThat( - recreatedWorldStateStorage.getAccountStateTrieNode( - Bytes.EMPTY, accountStateTrie.getRootHash())) + recreatedWorldStateStorage.getAccountStateTrieNode(accountStateTrie.getRootHash())) .isEmpty(); } @@ -93,14 +94,16 @@ public void shouldSaveTheRootWhenComplete() { final int nbAccounts = 15; - final WorldStateStorage worldStateStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); - final WorldStateStorage recreatedWorldStateStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final ForestWorldStateKeyValueStorage recreatedWorldStateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final MerkleTrie accountStateTrie = - TrieGenerator.generateTrie(worldStateStorage, nbAccounts); + TrieGenerator.generateTrie(worldStateStorageCoordinator, nbAccounts); final StackTrie stackTrie = new StackTrie(Hash.wrap(accountStateTrie.getRootHash()), 0, 256, lastAccount); @@ -119,7 +122,7 @@ public void shouldSaveTheRootWhenComplete() { collector, visitor, root, lastAccount)); final WorldStateProofProvider worldStateProofProvider = - new WorldStateProofProvider(worldStateStorage); + new WorldStateProofProvider(worldStateStorageCoordinator); // generate the proof final List proofs = @@ -131,13 +134,13 @@ public void shouldSaveTheRootWhenComplete() { stackTrie.addElement(Bytes32.random(), proofs, accounts); - final WorldStateStorage.Updater updater = recreatedWorldStateStorage.updater(); - stackTrie.commit(updater::putAccountStateTrieNode); + final ForestWorldStateKeyValueStorage.Updater updater = recreatedWorldStateStorage.updater(); + stackTrie.commit((location, hash, value) -> updater.putAccountStateTrieNode(hash, value)); updater.commit(); } Assertions.assertThat( - worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, accountStateTrie.getRootHash())) + worldStateKeyValueStorage.getAccountStateTrieNode(accountStateTrie.getRootHash())) .isPresent(); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java index 64bda37ae88..b3ee13d89b0 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java @@ -15,21 +15,21 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.AccountRangeDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataRequest; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import org.hyperledger.besu.services.tasks.Task; import java.util.List; @@ -43,13 +43,16 @@ public class TaskGenerator { public static List> createAccountRequest(final boolean withData) { - final WorldStateStorage worldStateStorage = - new InMemoryKeyValueStorageProvider().createWorldStateStorage(DataStorageFormat.FOREST); + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); final WorldStateProofProvider worldStateProofProvider = - new WorldStateProofProvider(worldStateStorage); + new WorldStateProofProvider(worldStateStorageCoordinator); - final MerkleTrie trie = TrieGenerator.generateTrie(worldStateStorage, 1); + final MerkleTrie trie = + TrieGenerator.generateTrie(worldStateStorageCoordinator, 1); final RangeStorageEntriesCollector collector = RangeStorageEntriesCollector.createCollector( Bytes32.ZERO, RangeManager.MAX_RANGE, 1, Integer.MAX_VALUE); @@ -77,14 +80,14 @@ public static List> createAccountRequest(final boolean wit final StorageRangeDataRequest storageRangeDataRequest = createStorageRangeDataRequest( worldStateProofProvider, - worldStateStorage, + worldStateStorageCoordinator, rootHash, accountHash, stateTrieAccountValue.getStorageRoot(), withData); final BytecodeRequest bytecodeRequest = createBytecodeDataRequest( - worldStateStorage, + worldStateKeyValueStorage, rootHash, accountHash, stateTrieAccountValue.getCodeHash(), @@ -98,7 +101,7 @@ public static List> createAccountRequest(final boolean wit private static StorageRangeDataRequest createStorageRangeDataRequest( final WorldStateProofProvider worldStateProofProvider, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateKeyValueStorage, final Hash rootHash, final Hash accountHash, final Bytes32 storageRoot, @@ -110,7 +113,7 @@ private static StorageRangeDataRequest createStorageRangeDataRequest( final StoredMerklePatriciaTrie storageTrie = new StoredMerklePatriciaTrie<>( (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(accountHash, location, hash), + worldStateKeyValueStorage.getAccountStorageTrieNode(accountHash, location, hash), storageRoot, b -> b, b -> b); @@ -134,7 +137,7 @@ private static StorageRangeDataRequest createStorageRangeDataRequest( } private static BytecodeRequest createBytecodeDataRequest( - final WorldStateStorage worldStateStorage, + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage, final Hash rootHash, final Hash accountHash, final Hash codeHash, @@ -142,7 +145,7 @@ private static BytecodeRequest createBytecodeDataRequest( final BytecodeRequest request = SnapDataRequest.createBytecodeRequest(accountHash, rootHash, codeHash); if (withData) { - request.setCode(worldStateStorage.getCode(codeHash, accountHash).get()); + request.setCode(worldStateKeyValueStorage.getCode(codeHash).get()); } return request; } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java index a95d4c34114..1977f19f7ed 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; @@ -24,14 +25,14 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapsyncMetricsManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -72,11 +73,14 @@ public void setup() { @Test public void shouldReturnChildRequests() { - final WorldStateStorage worldStateStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); - final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); + final WorldStateProofProvider proofProvider = + new WorldStateProofProvider(worldStateStorageCoordinator); final MerkleTrie accountStateTrie = - TrieGenerator.generateTrie(worldStateStorage, 15); + TrieGenerator.generateTrie(worldStateStorageCoordinator, 15); // Create a collector to gather account entries within a specific range final RangeStorageEntriesCollector collector = @@ -113,7 +117,9 @@ public void shouldReturnChildRequests() { // Verify that the start key hash of the snapDataRequest is greater than the last key in the // accounts TreeMap List childRequests = - request.getChildRequests(downloadState, worldStateStorage, snapSyncState).toList(); + request + .getChildRequests(downloadState, worldStateStorageCoordinator, snapSyncState) + .toList(); Assertions.assertThat(childRequests).hasSize(1); AccountFlatDatabaseHealingRangeRequest snapDataRequest = (AccountFlatDatabaseHealingRangeRequest) childRequests.get(0); @@ -127,7 +133,9 @@ public void shouldReturnChildRequests() { .map(CompactEncoding::bytesToPath) .collect(Collectors.toList()))); childRequests = - request.getChildRequests(downloadState, worldStateStorage, snapSyncState).toList(); + request + .getChildRequests(downloadState, worldStateStorageCoordinator, snapSyncState) + .toList(); Assertions.assertThat(childRequests).hasSizeGreaterThan(1); Assertions.assertThat(childRequests) .hasAtLeastOneElementOfType(AccountFlatDatabaseHealingRangeRequest.class); @@ -137,11 +145,14 @@ public void shouldReturnChildRequests() { @Test public void shouldNotReturnChildRequestsWhenNoMoreAccounts() { - final WorldStateStorage worldStateStorage = - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); - final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); + final WorldStateProofProvider proofProvider = + new WorldStateProofProvider(worldStateStorageCoordinator); final MerkleTrie accountStateTrie = - TrieGenerator.generateTrie(worldStateStorage, 15); + TrieGenerator.generateTrie(worldStateStorageCoordinator, 15); // Create a collector to gather account entries within a specific range final RangeStorageEntriesCollector collector = @@ -169,7 +180,7 @@ public void shouldNotReturnChildRequestsWhenNoMoreAccounts() { // Verify that no child requests are returned from the request final Stream childRequests = - request.getChildRequests(downloadState, worldStateStorage, snapSyncState); + request.getChildRequests(downloadState, worldStateStorageCoordinator, snapSyncState); Assertions.assertThat(childRequests).isEmpty(); } @@ -178,11 +189,14 @@ public void doNotPersistWhenProofIsValid() { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); - final WorldStateStorage worldStateStorage = + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); - final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); + final WorldStateProofProvider proofProvider = + new WorldStateProofProvider(worldStateStorageCoordinator); final MerkleTrie accountStateTrie = - TrieGenerator.generateTrie(worldStateStorage, 15); + TrieGenerator.generateTrie(worldStateStorageCoordinator, 15); // Create a collector to gather account entries within a specific range final RangeStorageEntriesCollector collector = RangeStorageEntriesCollector.createCollector( @@ -217,9 +231,9 @@ public void doNotPersistWhenProofIsValid() { // an ArrayDeque request.addLocalData(proofProvider, accounts, new ArrayDeque<>(proofs)); - WorldStateStorage.Updater updater = Mockito.spy(worldStateStorage.updater()); + WorldStateKeyValueStorage.Updater updater = Mockito.spy(worldStateKeyValueStorage.updater()); request.doPersist( - worldStateStorage, + worldStateStorageCoordinator, updater, downloadState, snapSyncState, @@ -232,11 +246,14 @@ public void doHealAndPersistWhenProofIsInvalid() { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); - final WorldStateStorage worldStateStorage = + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); - final WorldStateProofProvider proofProvider = new WorldStateProofProvider(worldStateStorage); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); + final WorldStateProofProvider proofProvider = + new WorldStateProofProvider(worldStateStorageCoordinator); final MerkleTrie accountStateTrie = - TrieGenerator.generateTrie(worldStateStorage, 15); + TrieGenerator.generateTrie(worldStateStorageCoordinator, 15); // Create a collector to gather account entries within a specific range final RangeStorageEntriesCollector collector = RangeStorageEntriesCollector.createCollector( @@ -286,9 +303,9 @@ public void doHealAndPersistWhenProofIsInvalid() { request.addLocalData(proofProvider, accounts, new ArrayDeque<>(proofs)); BonsaiWorldStateKeyValueStorage.Updater updater = - (BonsaiWorldStateKeyValueStorage.Updater) Mockito.spy(worldStateStorage.updater()); + Mockito.spy(worldStateKeyValueStorage.updater()); request.doPersist( - worldStateStorage, + worldStateStorageCoordinator, updater, downloadState, snapSyncState, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java index a281385dd5b..bafb6a2dfe5 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; @@ -34,7 +35,7 @@ import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import java.util.Iterator; @@ -69,7 +70,9 @@ class StorageFlatDatabaseHealingRangeRequestTest { Address.fromHexString("0xdeadbeeb")); private MerkleTrie trie; - private BonsaiWorldStateKeyValueStorage worldStateStorage; + + private WorldStateStorageCoordinator worldStateStorageCoordinator; + private BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; private WorldStateProofProvider proofProvider; private Hash account0Hash; private Hash account0StorageRoot; @@ -77,12 +80,13 @@ class StorageFlatDatabaseHealingRangeRequestTest { @BeforeEach public void setup() { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); - worldStateStorage = + worldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); - proofProvider = new WorldStateProofProvider(worldStateStorage); + worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); + proofProvider = new WorldStateProofProvider(worldStateStorageCoordinator); trie = TrieGenerator.generateTrie( - worldStateStorage, + worldStateStorageCoordinator, accounts.stream().map(Address::addressHash).collect(Collectors.toList())); account0Hash = accounts.get(0).addressHash(); account0StorageRoot = @@ -99,7 +103,7 @@ void shouldReturnChildRequests() { final StoredMerklePatriciaTrie storageTrie = new StoredMerklePatriciaTrie<>( (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(account0Hash, location, hash), + worldStateKeyValueStorage.getAccountStorageTrieNode(account0Hash, location, hash), account0StorageRoot, b -> b, b -> b); @@ -143,7 +147,9 @@ void shouldReturnChildRequests() { // Verify that the start key hash of the snapDataRequest is greater than the last key in the // slots TreeMap List childRequests = - request.getChildRequests(downloadState, worldStateStorage, snapSyncState).toList(); + request + .getChildRequests(downloadState, worldStateStorageCoordinator, snapSyncState) + .toList(); Assertions.assertThat(childRequests).hasSizeGreaterThan(1); StorageFlatDatabaseHealingRangeRequest snapDataRequest = (StorageFlatDatabaseHealingRangeRequest) childRequests.get(0); @@ -156,7 +162,7 @@ void shouldNotReturnChildRequestsWhenNoMoreSlots() { final StoredMerklePatriciaTrie storageTrie = new StoredMerklePatriciaTrie<>( (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(account0Hash, location, hash), + worldStateKeyValueStorage.getAccountStorageTrieNode(account0Hash, location, hash), account0StorageRoot, b -> b, b -> b); @@ -187,7 +193,7 @@ void shouldNotReturnChildRequestsWhenNoMoreSlots() { // Verify that no child requests are returned from the request final Stream childRequests = - request.getChildRequests(downloadState, worldStateStorage, snapSyncState); + request.getChildRequests(downloadState, worldStateStorageCoordinator, snapSyncState); Assertions.assertThat(childRequests).isEmpty(); } @@ -197,7 +203,7 @@ void doNotPersistWhenProofIsValid() { final StoredMerklePatriciaTrie storageTrie = new StoredMerklePatriciaTrie<>( (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(account0Hash, location, hash), + worldStateKeyValueStorage.getAccountStorageTrieNode(account0Hash, location, hash), account0StorageRoot, b -> b, b -> b); @@ -238,9 +244,9 @@ void doNotPersistWhenProofIsValid() { // Add local data to the request request.addLocalData(proofProvider, slots, new ArrayDeque<>(proofs)); - WorldStateStorage.Updater updater = Mockito.spy(worldStateStorage.updater()); + WorldStateKeyValueStorage.Updater updater = Mockito.spy(worldStateKeyValueStorage.updater()); request.doPersist( - worldStateStorage, + worldStateStorageCoordinator, updater, downloadState, snapSyncState, @@ -254,7 +260,7 @@ void doHealAndPersistWhenProofIsInvalid() { final StoredMerklePatriciaTrie storageTrie = new StoredMerklePatriciaTrie<>( (location, hash) -> - worldStateStorage.getAccountStorageTrieNode(account0Hash, location, hash), + worldStateKeyValueStorage.getAccountStorageTrieNode(account0Hash, location, hash), account0StorageRoot, b -> b, b -> b); @@ -309,9 +315,9 @@ void doHealAndPersistWhenProofIsInvalid() { request.addLocalData(proofProvider, slots, new ArrayDeque<>(proofs)); BonsaiWorldStateKeyValueStorage.Updater updater = - (BonsaiWorldStateKeyValueStorage.Updater) Mockito.spy(worldStateStorage.updater()); + Mockito.spy(worldStateKeyValueStorage.updater()); request.doPersist( - worldStateStorage, + worldStateStorageCoordinator, updater, downloadState, snapSyncState, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java index 447b37d14ce..822dfeb2cc5 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java @@ -20,13 +20,13 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -56,7 +56,7 @@ class StorageTrieNodeHealingRequestTest { Address.fromHexString("0xdeadbeea"), Address.fromHexString("0xdeadbeeb")); - private WorldStateStorage worldStateStorage; + private WorldStateStorageCoordinator worldStateStorageCoordinator; private Hash account0Hash; private Hash account0StorageRoot; @@ -70,15 +70,18 @@ public Stream provideArguments(final ExtensionContext conte public void setup(final DataStorageFormat storageFormat) { if (storageFormat.equals(DataStorageFormat.FOREST)) { - worldStateStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + worldStateStorageCoordinator = + new WorldStateStorageCoordinator( + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage())); } else { final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); - worldStateStorage = - new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()); + worldStateStorageCoordinator = + new WorldStateStorageCoordinator( + new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem())); } final MerkleTrie trie = TrieGenerator.generateTrie( - worldStateStorage, + worldStateStorageCoordinator, accounts.stream().map(Address::addressHash).collect(Collectors.toList())); account0Hash = accounts.get(0).addressHash(); @@ -99,7 +102,8 @@ void shouldDetectExistingData(final DataStorageFormat storageFormat) { new StorageTrieNodeHealingRequest( account0StorageRoot, account0Hash, Hash.EMPTY, Bytes.EMPTY); - Assertions.assertThat(request.getExistingData(downloadState, worldStateStorage)).isPresent(); + Assertions.assertThat(request.getExistingData(downloadState, worldStateStorageCoordinator)) + .isPresent(); } @ParameterizedTest @@ -109,6 +113,7 @@ void shouldDetectMissingData(final DataStorageFormat storageFormat) { final StorageTrieNodeHealingRequest request = new StorageTrieNodeHealingRequest(Hash.EMPTY, account0Hash, Hash.EMPTY, Bytes.EMPTY); - Assertions.assertThat(request.getExistingData(downloadState, worldStateStorage)).isEmpty(); + Assertions.assertThat(request.getExistingData(downloadState, worldStateStorageCoordinator)) + .isEmpty(); } } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BlockchainModule.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BlockchainModule.java index 0109d6c819a..ff461f1e830 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BlockchainModule.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BlockchainModule.java @@ -23,11 +23,11 @@ import org.hyperledger.besu.ethereum.chain.GenesisState; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -57,28 +57,33 @@ Blockchain provideBlockchain( @Singleton MutableWorldState getMutableWorldState( @Named("StateRoot") final Bytes32 stateRoot, - final WorldStateStorage worldStateStorage, + final WorldStateStorageCoordinator worldStateStorageCoordinator, final WorldStatePreimageStorage worldStatePreimageStorage, final GenesisState genesisState, @Named("KeyValueStorageName") final String keyValueStorageName, final EvmConfiguration evmConfiguration) { if ("memory".equals(keyValueStorageName)) { final MutableWorldState mutableWorldState = - new DefaultMutableWorldState( - worldStateStorage, worldStatePreimageStorage, evmConfiguration); + new ForestMutableWorldState( + worldStateStorageCoordinator.worldStateKeyValueStorage(), + worldStatePreimageStorage, + evmConfiguration); genesisState.writeStateTo(mutableWorldState); return mutableWorldState; } else { - return new DefaultMutableWorldState( - stateRoot, worldStateStorage, worldStatePreimageStorage, evmConfiguration); + return new ForestMutableWorldState( + stateRoot, + worldStateStorageCoordinator.worldStateKeyValueStorage(), + worldStatePreimageStorage, + evmConfiguration); } } @Provides @Singleton - WorldStateStorage provideWorldStateStorage( + WorldStateStorageCoordinator provideWorldStateStorage( @Named("worldState") final KeyValueStorage keyValueStorage) { - return new WorldStateKeyValueStorage(keyValueStorage); + return new WorldStateStorageCoordinator(new ForestWorldStateKeyValueStorage(keyValueStorage)); } @Provides diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index e56d5c858f8..e3ff35c03d1 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -50,19 +50,19 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState private final EvmConfiguration evmConfiguration; protected BonsaiReferenceTestWorldState( - final BonsaiReferenceTestWorldStateStorage worldStateStorage, + final BonsaiReferenceTestWorldStateStorage worldStateKeyValueStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, final CachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final BonsaiPreImageProxy preImageProxy, final EvmConfiguration evmConfiguration) { super( - worldStateStorage, + worldStateKeyValueStorage, cachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager, evmConfiguration); - this.refTestStorage = worldStateStorage; + this.refTestStorage = worldStateKeyValueStorage; this.preImageProxy = preImageProxy; this.evmConfiguration = evmConfiguration; setAccumulator( @@ -79,7 +79,8 @@ protected BonsaiReferenceTestWorldState( @Override public ReferenceTestWorldState copy() { - var layerCopy = new BonsaiReferenceTestWorldStateStorage(worldStateStorage, preImageProxy); + var layerCopy = + new BonsaiReferenceTestWorldStateStorage(worldStateKeyValueStorage, preImageProxy); return new BonsaiReferenceTestWorldState( layerCopy, cachedMerkleTrieLoader, @@ -117,7 +118,7 @@ public static BonsaiReferenceTestWorldState create( final BonsaiPreImageProxy preImageProxy = new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); - final BonsaiReferenceTestWorldStateStorage worldStateStorage = + final BonsaiReferenceTestWorldStateStorage worldStateKeyValueStorage = new BonsaiReferenceTestWorldStateStorage( new BonsaiWorldStateKeyValueStorage( new InMemoryKeyValueStorageProvider(), metricsSystem), @@ -128,7 +129,7 @@ public static BonsaiReferenceTestWorldState create( final BonsaiReferenceTestWorldState worldState = new BonsaiReferenceTestWorldState( - worldStateStorage, + worldStateKeyValueStorage, cachedMerkleTrieLoader, noOpCachedWorldStorageManager, trieLogManager, diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java index 45771578e14..e411962dfa4 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java @@ -15,9 +15,9 @@ package org.hyperledger.besu.ethereum.referencetests; import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.evm.worldstate.WorldUpdater; @@ -27,12 +27,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; -public class DefaultReferenceTestWorldState extends DefaultMutableWorldState +public class DefaultReferenceTestWorldState extends ForestMutableWorldState implements ReferenceTestWorldState { DefaultReferenceTestWorldState() { super( - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()), + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()), new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()), EvmConfiguration.DEFAULT); } diff --git a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java index 2286fe67cfe..66e13839989 100644 --- a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java +++ b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java @@ -47,6 +47,8 @@ import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolFactory; +import org.hyperledger.besu.ethereum.forest.ForestWorldStateArchive; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.mainnet.EpochCalculator; import org.hyperledger.besu.ethereum.mainnet.HeaderValidationMode; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; @@ -59,10 +61,9 @@ import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DefaultWorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -166,8 +167,9 @@ private boolean buildContext( mixHash = Optional.ofNullable(genesisState.getBlock().getHeader().getMixHashOrPrevRandao()); final WorldStateArchive worldStateArchive = - new DefaultWorldStateArchive( - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()), + new ForestWorldStateArchive( + new WorldStateStorageCoordinator( + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage())), new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()), EvmConfiguration.DEFAULT); final MutableWorldState worldState = worldStateArchive.getMutable(); diff --git a/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java b/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java index 140e60d9d24..494106f98fa 100644 --- a/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java +++ b/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java @@ -92,7 +92,7 @@ default void writeUInt256Scalar(final UInt256Value v) { /** * Writes a RLP "null", that is an empty value. * - *

This is a shortcut for {@code writeBytes(Bytes.EMPTY)}. + *

This is a shortcut for writeUInt256Scalar{@code writeBytes(Bytes.EMPTY)}. */ default void writeNull() { writeBytes(Bytes.EMPTY); diff --git a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/RangeStorageEntriesCollectorTest.java b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/RangeStorageEntriesCollectorTest.java index 6b931d041e5..09ddc1b2608 100644 --- a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/RangeStorageEntriesCollectorTest.java +++ b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/RangeStorageEntriesCollectorTest.java @@ -28,10 +28,11 @@ public class RangeStorageEntriesCollectorTest { @Test public void shouldRetrieveAllLeavesInRangeWhenStartFromZero() { - InMemoryKeyValueStorage worldStateStorage = new InMemoryKeyValueStorage(); + InMemoryKeyValueStorage worldStateKeyValueStorage = new InMemoryKeyValueStorage(); final MerkleTrie accountStateTrie = new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateStorage.get(hash.toArrayUnsafe()).map(Bytes::wrap), + (location, hash) -> + worldStateKeyValueStorage.get(hash.toArrayUnsafe()).map(Bytes::wrap), b -> b, b -> b); final List lists = @@ -47,10 +48,11 @@ public void shouldRetrieveAllLeavesInRangeWhenStartFromZero() { @Test public void shouldRetrieveAllLeavesInRangeWhenStartFromSpecificRange() { - InMemoryKeyValueStorage worldStateStorage = new InMemoryKeyValueStorage(); + InMemoryKeyValueStorage worldStateKeyValueStorage = new InMemoryKeyValueStorage(); final MerkleTrie accountStateTrie = new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateStorage.get(hash.toArrayUnsafe()).map(Bytes::wrap), + (location, hash) -> + worldStateKeyValueStorage.get(hash.toArrayUnsafe()).map(Bytes::wrap), b -> b, b -> b); final List lists = @@ -66,10 +68,11 @@ public void shouldRetrieveAllLeavesInRangeWhenStartFromSpecificRange() { @Test public void shouldExcludeLeavesNotInRange() { - InMemoryKeyValueStorage worldStateStorage = new InMemoryKeyValueStorage(); + InMemoryKeyValueStorage worldStateKeyValueStorage = new InMemoryKeyValueStorage(); final MerkleTrie accountStateTrie = new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateStorage.get(hash.toArrayUnsafe()).map(Bytes::wrap), + (location, hash) -> + worldStateKeyValueStorage.get(hash.toArrayUnsafe()).map(Bytes::wrap), b -> b, b -> b); final List lists = From 6d4a4bb5826adac6fb26c39fd1883d19b5bc63eb Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Tue, 28 Nov 2023 14:51:01 +0100 Subject: [PATCH 02/59] fix unit tests Signed-off-by: Karim TAAM --- .../WorldStateStorageCoordinator.java | 20 ++- .../sync/snapsync/SnapWorldDownloadState.java | 11 +- .../CheckPointSyncChainDownloaderTest.java | 31 +++- .../fastsync/FastDownloaderFactoryTest.java | 73 +++++++-- .../sync/fastsync/FastSyncActionsTest.java | 1 + .../fastsync/FastSyncChainDownloaderTest.java | 1 + .../sync/fastsync/FastSyncDownloaderTest.java | 147 ++++++++++++------ .../FastWorldStateDownloaderTest.java | 2 +- .../worldstate/LoadLocalDataStepTest.java | 7 + 9 files changed, 217 insertions(+), 76 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java index 041de8f8b2b..5c0e0b8e389 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java @@ -26,7 +26,12 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -public record WorldStateStorageCoordinator(WorldStateKeyValueStorage worldStateKeyValueStorage) { +public class WorldStateStorageCoordinator { + private final WorldStateKeyValueStorage worldStateKeyValueStorage; + + public WorldStateStorageCoordinator(final WorldStateKeyValueStorage worldStateKeyValueStorage) { + this.worldStateKeyValueStorage = worldStateKeyValueStorage; + } public DataStorageFormat getDataStorageFormat() { return worldStateKeyValueStorage.getDataStorageFormat(); @@ -63,6 +68,15 @@ public STRATEGY getStrategy( return (STRATEGY) worldStateKeyValueStorage; } + public boolean isMatchingFlatMode(final FlatDbMode flatDbMode) { + if (getDataStorageFormat().equals(DataStorageFormat.BONSAI)) { + final BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorageStrategy = + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage(); + return bonsaiWorldStateStorageStrategy.getFlatDbMode().equals(flatDbMode); + } + return false; + } + public void applyOnMatchingFlatMode( final FlatDbMode flatDbMode, final Consumer onStrategy) { applyOnMatchingStrategy( @@ -134,4 +148,8 @@ public WorldStateKeyValueStorage.Updater updater() { public void clear() { worldStateKeyValueStorage.clear(); } + + public WorldStateKeyValueStorage worldStateKeyValueStorage() { + return worldStateKeyValueStorage; + } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index a92f6e20551..28ff8943999 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -194,15 +194,10 @@ else if (pivotBlockSelector.isBlockchainBehind()) { blockObserverId.ifPresent(blockchain::removeObserver); // If the flat database healing process is not in progress and the flat database mode is // FULL - if (!snapSyncState.isHealFlatDatabaseInProgress()) { - worldStateStorageCoordinator.applyOnMatchingFlatMode( - FlatDbMode.FULL, - bonsaiWorldStateStorageStrategy -> { - // Start the flat database healing process - startFlatDatabaseHeal(header); - }); + if (!snapSyncState.isHealFlatDatabaseInProgress() + && worldStateStorageCoordinator.isMatchingFlatMode(FlatDbMode.FULL)) { + startFlatDatabaseHeal(header); } - // If the flat database healing process is in progress or the flat database mode is not FULL else { final WorldStateKeyValueStorage.Updater updater = worldStateStorageCoordinator.updater(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index 256daf538ee..8ac6463291f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -20,6 +20,8 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; @@ -35,6 +37,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.Checkpoint; import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.ImmutableCheckpoint; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; @@ -53,9 +56,6 @@ public class CheckPointSyncChainDownloaderTest { - private final WorldStateStorageCoordinator worldStateStorageCoordinator = - mock(WorldStateStorageCoordinator.class); - protected ProtocolSchedule protocolSchedule; protected EthProtocolManager ethProtocolManager; protected EthContext ethContext; @@ -67,6 +67,8 @@ public class CheckPointSyncChainDownloaderTest { protected Blockchain otherBlockchain; private Checkpoint checkpoint; + private WorldStateStorageCoordinator worldStateStorageCoordinator; + static class CheckPointSyncChainDownloaderTestArguments implements ArgumentsProvider { @Override public Stream provideArguments(final ExtensionContext context) { @@ -75,11 +77,26 @@ public Stream provideArguments(final ExtensionContext conte } } - public void setup(final DataStorageFormat storageFormat) { - when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); - final BlockchainSetupUtil localBlockchainSetup = BlockchainSetupUtil.forTesting(storageFormat); + public void setup(final DataStorageFormat dataStorageFormat) { + final WorldStateKeyValueStorage worldStateKeyValueStorage; + if (dataStorageFormat.equals(DataStorageFormat.BONSAI)) { + worldStateKeyValueStorage = mock(BonsaiWorldStateKeyValueStorage.class); + when(((BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage) + .isWorldStateAvailable(any(), any())) + .thenReturn(true); + } else { + worldStateKeyValueStorage = mock(ForestWorldStateKeyValueStorage.class); + when(((ForestWorldStateKeyValueStorage) worldStateKeyValueStorage) + .isWorldStateAvailable(any())) + .thenReturn(true); + } + when(worldStateKeyValueStorage.getDataStorageFormat()).thenReturn(dataStorageFormat); + worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); + + final BlockchainSetupUtil localBlockchainSetup = + BlockchainSetupUtil.forTesting(dataStorageFormat); localBlockchain = localBlockchainSetup.getBlockchain(); - otherBlockchainSetup = BlockchainSetupUtil.forTesting(storageFormat); + otherBlockchainSetup = BlockchainSetupUtil.forTesting(dataStorageFormat); otherBlockchain = otherBlockchainSetup.getBlockchain(); protocolSchedule = localBlockchainSetup.getProtocolSchedule(); protocolContext = localBlockchainSetup.getProtocolContext(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java index 8d7b5a33379..080e827574b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java @@ -23,6 +23,8 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.ethereum.ProtocolContext; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector; @@ -30,7 +32,9 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.FastDownloaderFactory; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -40,10 +44,16 @@ import java.nio.file.Path; import java.time.Clock; import java.util.Optional; +import java.util.stream.Stream; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -59,15 +69,36 @@ public class FastDownloaderFactoryTest { @Mock private ProtocolContext protocolContext; @Mock private MetricsSystem metricsSystem; @Mock private EthContext ethContext; - @Mock private WorldStateStorageCoordinator worldStateStorageCoordinator; @Mock private SyncState syncState; @Mock private Clock clock; @Mock private Path dataDirectory; @Mock private PivotBlockSelector pivotBlockSelector; + private WorldStateKeyValueStorage worldStateKeyValueStorage; + private WorldStateStorageCoordinator worldStateStorageCoordinator; - @SuppressWarnings("unchecked") - @Test - public void shouldThrowIfSyncModeChangedWhileFastSyncIncomplete() { + static class FastDownloaderFactoryTestArguments implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext context) { + return Stream.of( + Arguments.of(DataStorageFormat.BONSAI), Arguments.of(DataStorageFormat.FOREST)); + } + } + + public void setup(final DataStorageFormat dataStorageFormat) { + if (dataStorageFormat.equals(DataStorageFormat.BONSAI)) { + worldStateKeyValueStorage = mock(BonsaiWorldStateKeyValueStorage.class); + } else { + worldStateKeyValueStorage = mock(ForestWorldStateKeyValueStorage.class); + } + when(worldStateKeyValueStorage.getDataStorageFormat()).thenReturn(dataStorageFormat); + worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); + } + + @ParameterizedTest + @ArgumentsSource(FastDownloaderFactoryTestArguments.class) + public void shouldThrowIfSyncModeChangedWhileFastSyncIncomplete( + final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); initDataDirectory(true); when(syncConfig.getSyncMode()).thenReturn(SyncMode.FULL); @@ -88,8 +119,11 @@ public void shouldThrowIfSyncModeChangedWhileFastSyncIncomplete() { } @SuppressWarnings({"unchecked", "rawtypes"}) - @Test - public void shouldNotThrowIfSyncModeChangedWhileFastSyncComplete() { + @ParameterizedTest + @ArgumentsSource(FastDownloaderFactoryTestArguments.class) + public void shouldNotThrowIfSyncModeChangedWhileFastSyncComplete( + final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); initDataDirectory(false); when(syncConfig.getSyncMode()).thenReturn(SyncMode.FULL); @@ -109,8 +143,11 @@ public void shouldNotThrowIfSyncModeChangedWhileFastSyncComplete() { } @SuppressWarnings("unchecked") - @Test - public void shouldNotThrowWhenFastSyncModeRequested() throws NoSuchFieldException { + @ParameterizedTest + @ArgumentsSource(FastDownloaderFactoryTestArguments.class) + public void shouldNotThrowWhenFastSyncModeRequested(final DataStorageFormat dataStorageFormat) + throws NoSuchFieldException { + setup(dataStorageFormat); initDataDirectory(false); final MutableBlockchain mutableBlockchain = mock(MutableBlockchain.class); @@ -133,8 +170,12 @@ public void shouldNotThrowWhenFastSyncModeRequested() throws NoSuchFieldExceptio verify(mutableBlockchain).getChainHeadBlockNumber(); } - @Test - public void shouldClearWorldStateDuringFastSyncWhenStateQueDirectoryExists() throws IOException { + @ParameterizedTest + @ArgumentsSource(FastDownloaderFactoryTestArguments.class) + public void shouldClearWorldStateDuringFastSyncWhenStateQueDirectoryExists( + final DataStorageFormat dataStorageFormat) throws IOException { + Assumptions.assumeTrue(dataStorageFormat == DataStorageFormat.FOREST); + setup(dataStorageFormat); when(syncConfig.getSyncMode()).thenReturn(SyncMode.FAST); final MutableBlockchain mutableBlockchain = mock(MutableBlockchain.class); when(mutableBlockchain.getChainHeadBlockNumber()).thenReturn(0L); @@ -160,12 +201,16 @@ public void shouldClearWorldStateDuringFastSyncWhenStateQueDirectoryExists() thr syncState, clock); - verify(worldStateStorageCoordinator).clear(); + verify(worldStateKeyValueStorage).clear(); assertThat(Files.exists(stateQueueDir)).isFalse(); } - @Test - public void shouldCrashWhenStateQueueIsNotDirectory() throws IOException { + @ParameterizedTest + @ArgumentsSource(FastDownloaderFactoryTestArguments.class) + public void shouldCrashWhenStateQueueIsNotDirectory(final DataStorageFormat dataStorageFormat) + throws IOException { + Assumptions.assumeTrue(dataStorageFormat == DataStorageFormat.FOREST); + setup(dataStorageFormat); when(syncConfig.getSyncMode()).thenReturn(SyncMode.FAST); final MutableBlockchain mutableBlockchain = mock(MutableBlockchain.class); when(mutableBlockchain.getChainHeadBlockNumber()).thenReturn(0L); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java index 31529cb5b21..59ea933d58c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java @@ -86,6 +86,7 @@ public Stream provideArguments(final ExtensionContext conte } public void setUp(final DataStorageFormat storageFormat) { + when(worldStateStorageCoordinator.getDataStorageFormat()).thenReturn(storageFormat); blockchainSetupUtil = BlockchainSetupUtil.forTesting(storageFormat); blockchainSetupUtil.importAllBlocks(); blockchain = blockchainSetupUtil.getBlockchain(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java index fde30bfa5bd..a68695815f5 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncChainDownloaderTest.java @@ -74,6 +74,7 @@ public Stream provideArguments(final ExtensionContext conte } public void setup(final DataStorageFormat storageFormat) { + when(worldStateStorageCoordinator.getDataStorageFormat()).thenReturn(storageFormat); when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); final BlockchainSetupUtil localBlockchainSetup = BlockchainSetupUtil.forTesting(storageFormat); localBlockchain = localBlockchainSetup.getBlockchain(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java index dc9a7244c4f..88e3910db74 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java @@ -26,6 +26,8 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.eth.sync.ChainDownloader; @@ -34,6 +36,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.NodeDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.TaskCollection; @@ -44,19 +47,20 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import java.util.stream.Stream; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; public class FastSyncDownloaderTest { @SuppressWarnings("unchecked") private final FastSyncActions fastSyncActions = mock(FastSyncActions.class); - private final WorldStateStorageCoordinator worldStateStorageCoordinator = - mock(WorldStateStorageCoordinator.class); - private final WorldStateDownloader worldStateDownloader = mock(FastWorldStateDownloader.class); private final FastSyncStateStorage storage = mock(FastSyncStateStorage.class); @@ -66,25 +70,47 @@ public class FastSyncDownloaderTest { private final ChainDownloader chainDownloader = mock(ChainDownloader.class); private final Path fastSyncDataDirectory = null; + private WorldStateStorageCoordinator worldStateStorageCoordinator; + private FastSyncDownloader downloader; + + static class FastSyncDownloaderTestArguments implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext context) { + return Stream.of( + Arguments.of(DataStorageFormat.BONSAI), Arguments.of(DataStorageFormat.FOREST)); + } + } - private final FastSyncDownloader downloader = - new FastSyncDownloader<>( - fastSyncActions, - worldStateStorageCoordinator, - worldStateDownloader, - storage, - taskCollection, - fastSyncDataDirectory, - FastSyncState.EMPTY_SYNC_STATE); - - @BeforeEach - public void setup() { - when(worldStateStorageCoordinator.getDataStorageFormat()).thenReturn(DataStorageFormat.FOREST); - when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); + public void setup(final DataStorageFormat dataStorageFormat) { + final WorldStateKeyValueStorage worldStateKeyValueStorage; + if (dataStorageFormat.equals(DataStorageFormat.BONSAI)) { + worldStateKeyValueStorage = mock(BonsaiWorldStateKeyValueStorage.class); + when(((BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage) + .isWorldStateAvailable(any(), any())) + .thenReturn(true); + } else { + worldStateKeyValueStorage = mock(ForestWorldStateKeyValueStorage.class); + when(((ForestWorldStateKeyValueStorage) worldStateKeyValueStorage) + .isWorldStateAvailable(any())) + .thenReturn(true); + } + when(worldStateKeyValueStorage.getDataStorageFormat()).thenReturn(dataStorageFormat); + worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); + downloader = + new FastSyncDownloader<>( + fastSyncActions, + worldStateStorageCoordinator, + worldStateDownloader, + storage, + taskCollection, + fastSyncDataDirectory, + FastSyncState.EMPTY_SYNC_STATE); } - @Test - public void shouldCompleteFastSyncSuccessfully() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldCompleteFastSyncSuccessfully(final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); final FastSyncState selectPivotBlockState = new FastSyncState(50); final BlockHeader pivotBlockHeader = new BlockHeaderTestFixture().number(50).buildHeader(); final FastSyncState downloadPivotBlockHeaderState = new FastSyncState(pivotBlockHeader); @@ -112,8 +138,10 @@ public void shouldCompleteFastSyncSuccessfully() { assertThat(result).isCompletedWithValue(downloadPivotBlockHeaderState); } - @Test - public void shouldResumeFastSync() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldResumeFastSync(final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); final BlockHeader pivotBlockHeader = new BlockHeaderTestFixture().number(50).buildHeader(); final FastSyncState fastSyncState = new FastSyncState(pivotBlockHeader); final CompletableFuture complete = completedFuture(fastSyncState); @@ -148,8 +176,10 @@ public void shouldResumeFastSync() { assertThat(result).isCompletedWithValue(fastSyncState); } - @Test - public void shouldAbortIfSelectPivotBlockFails() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldAbortIfSelectPivotBlockFails(final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); when(fastSyncActions.selectPivotBlock(FastSyncState.EMPTY_SYNC_STATE)) .thenThrow(new FastSyncException(FastSyncError.UNEXPECTED_ERROR)); @@ -161,8 +191,10 @@ public void shouldAbortIfSelectPivotBlockFails() { verifyNoMoreInteractions(fastSyncActions); } - @Test - public void shouldAbortIfWorldStateDownloadFails() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldAbortIfWorldStateDownloadFails(final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); final CompletableFuture worldStateFuture = new CompletableFuture<>(); final CompletableFuture chainFuture = new CompletableFuture<>(); final FastSyncState selectPivotBlockState = new FastSyncState(50); @@ -199,8 +231,10 @@ public void shouldAbortIfWorldStateDownloadFails() { assertThat(chainFuture).isCancelled(); } - @Test - public void shouldAbortIfChainDownloadFails() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldAbortIfChainDownloadFails(final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); final CompletableFuture chainFuture = new CompletableFuture<>(); final CompletableFuture worldStateFuture = new CompletableFuture<>(); final FastSyncState selectPivotBlockState = new FastSyncState(50); @@ -235,8 +269,10 @@ public void shouldAbortIfChainDownloadFails() { assertThat(worldStateFuture).isCancelled(); } - @Test - public void shouldAbortIfStopped() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldAbortIfStopped(final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); final FastSyncState selectPivotBlockState = new FastSyncState(50); final BlockHeader pivotBlockHeader = new BlockHeaderTestFixture().number(50).buildHeader(); final FastSyncState downloadPivotBlockHeaderState = new FastSyncState(pivotBlockHeader); @@ -269,8 +305,11 @@ public void shouldAbortIfStopped() { verifyNoMoreInteractions(fastSyncActions, worldStateDownloader, storage); } - @Test - public void shouldNotConsiderFastSyncCompleteIfOnlyWorldStateDownloadIsComplete() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldNotConsiderFastSyncCompleteIfOnlyWorldStateDownloadIsComplete( + final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); final CompletableFuture chainFuture = new CompletableFuture<>(); final CompletableFuture worldStateFuture = new CompletableFuture<>(); final FastSyncState selectPivotBlockState = new FastSyncState(50); @@ -304,8 +343,11 @@ public void shouldNotConsiderFastSyncCompleteIfOnlyWorldStateDownloadIsComplete( assertThat(result).isNotDone(); } - @Test - public void shouldNotConsiderFastSyncCompleteIfOnlyChainDownloadIsComplete() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldNotConsiderFastSyncCompleteIfOnlyChainDownloadIsComplete( + final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); final CompletableFuture chainFuture = new CompletableFuture<>(); final CompletableFuture worldStateFuture = new CompletableFuture<>(); final FastSyncState selectPivotBlockState = new FastSyncState(50); @@ -340,8 +382,11 @@ public void shouldNotConsiderFastSyncCompleteIfOnlyChainDownloadIsComplete() { } @SuppressWarnings("unchecked") - @Test - public void shouldResetFastSyncStateAndRestartProcessIfWorldStateIsUnavailable() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldResetFastSyncStateAndRestartProcessIfWorldStateIsUnavailable( + final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); final CompletableFuture firstWorldStateFuture = new CompletableFuture<>(); final CompletableFuture secondWorldStateFuture = new CompletableFuture<>(); final CompletableFuture chainFuture = new CompletableFuture<>(); @@ -411,8 +456,11 @@ public void shouldResetFastSyncStateAndRestartProcessIfWorldStateIsUnavailable() } @SuppressWarnings("unchecked") - @Test - public void shouldResetFastSyncStateAndRestartProcessIfANonFastSyncExceptionOccurs() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldResetFastSyncStateAndRestartProcessIfANonFastSyncExceptionOccurs( + final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); final CompletableFuture firstWorldStateFuture = new CompletableFuture<>(); final CompletableFuture secondWorldStateFuture = new CompletableFuture<>(); final CompletableFuture chainFuture = new CompletableFuture<>(); @@ -485,14 +533,20 @@ public void shouldResetFastSyncStateAndRestartProcessIfANonFastSyncExceptionOccu assertThat(result).isCompletedWithValue(secondDownloadPivotBlockHeaderState); } - @Test - public void shouldNotHaveTrailingPeerRequirementsBeforePivotBlockSelected() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldNotHaveTrailingPeerRequirementsBeforePivotBlockSelected( + final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); downloader.start(); Assertions.assertThat(downloader.calculateTrailingPeerRequirements()).isEmpty(); } - @Test - public void shouldNotAllowPeersBeforePivotBlockOnceSelected() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldNotAllowPeersBeforePivotBlockOnceSelected( + final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); final FastSyncState selectPivotBlockState = new FastSyncState(50); final BlockHeader pivotBlockHeader = new BlockHeaderTestFixture().number(50).buildHeader(); final FastSyncState downloadPivotBlockHeaderState = new FastSyncState(pivotBlockHeader); @@ -513,8 +567,11 @@ public void shouldNotAllowPeersBeforePivotBlockOnceSelected() { .contains(new TrailingPeerRequirements(50, 0)); } - @Test - public void shouldNotHaveTrailingPeerRequirementsAfterDownloadCompletes() { + @ParameterizedTest + @ArgumentsSource(FastSyncDownloaderTestArguments.class) + public void shouldNotHaveTrailingPeerRequirementsAfterDownloadCompletes( + final DataStorageFormat dataStorageFormat) { + setup(dataStorageFormat); final FastSyncState selectPivotBlockState = new FastSyncState(50); final BlockHeader pivotBlockHeader = new BlockHeaderTestFixture().number(50).buildHeader(); final FastSyncState downloadPivotBlockHeaderState = new FastSyncState(pivotBlockHeader); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java index d00fdf08972..027b3317c79 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java @@ -218,7 +218,7 @@ void downloadAlreadyAvailableWorldState() { final WorldStateDownloader downloader = createDownloader( ethProtocolManager.ethContext(), - worldStateArchive.getWorldStateStorage().worldStateKeyValueStorage(), + worldStateArchive.getWorldStateStorage(), taskCollection); final FastSyncState fastSyncState = new FastSyncState(header); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java index b157f4aada4..c412a38f607 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.pipeline.Pipe; @@ -32,6 +33,7 @@ import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -54,6 +56,11 @@ public class LoadLocalDataStepTest { new LoadLocalDataStep( new WorldStateStorageCoordinator(worldStateKeyValueStorage), new NoOpMetricsSystem()); + @BeforeEach + public void setup() { + when(worldStateKeyValueStorage.getDataStorageFormat()).thenReturn(DataStorageFormat.BONSAI); + } + @Test public void shouldReturnStreamWithUnchangedTaskWhenDataNotPresent() { final Stream> output = From d8a411ea3c9a304853b9a85eb2d104952fb7db38 Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Tue, 28 Nov 2023 17:10:59 +0100 Subject: [PATCH 03/59] fix unit tests Signed-off-by: Karim TAAM --- .../controller/BesuControllerBuilderTest.java | 15 ++++++++++----- .../MergeBesuControllerBuilderTest.java | 14 +++++++++----- .../controller/QbftBesuControllerBuilderTest.java | 12 ++++++++---- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index 209a2284706..c52493eb267 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -30,7 +30,6 @@ import org.hyperledger.besu.cryptoservices.NodeKeyUtils; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.GasLimitCalculator; -import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; @@ -41,6 +40,7 @@ import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.forest.pruner.PrunerConfiguration; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; @@ -89,7 +89,6 @@ public class BesuControllerBuilderTest { @Mock Clock clock; @Mock StorageProvider storageProvider; @Mock GasLimitCalculator gasLimitCalculator; - @Mock WorldStateStorageCoordinator worldStateStorageCoordinator; @Mock WorldStateArchive worldStateArchive; @Mock BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorage; @Mock WorldStatePreimageStorage worldStatePreimageStorage; @@ -105,6 +104,12 @@ public class BesuControllerBuilderTest { @Before public void setup() { + + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + mock(ForestWorldStateKeyValueStorage.class); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); + when(genesisConfigFile.getParentHash()).thenReturn(Hash.ZERO.toHexString()); when(genesisConfigFile.getDifficulty()).thenReturn(Bytes.of(0).toHexString()); when(genesisConfigFile.getExtraData()).thenReturn(Bytes.EMPTY.toHexString()); @@ -134,11 +139,11 @@ public void setup() { .thenReturn(worldStateStorageCoordinator); when(storageProvider.createWorldStatePreimageStorage()).thenReturn(worldStatePreimageStorage); - when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); + when(worldStateKeyValueStorage.isWorldStateAvailable(any())).thenReturn(true); when(worldStatePreimageStorage.updater()) .thenReturn(mock(WorldStatePreimageStorage.Updater.class)); - when(worldStateStorageCoordinator.updater()) - .thenReturn(mock(WorldStateKeyValueStorage.Updater.class)); + when(worldStateKeyValueStorage.updater()) + .thenReturn(mock(ForestWorldStateKeyValueStorage.Updater.class)); besuControllerBuilder = spy(visitWithMockConfigs(new MainnetBesuControllerBuilder())); } diff --git a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java index c961aa0c323..efe98387fd4 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java @@ -31,7 +31,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.GasLimitCalculator; -import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.GenesisState; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -45,6 +44,7 @@ import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket; import org.hyperledger.besu.ethereum.mainnet.feemarket.LondonFeeMarket; @@ -99,7 +99,6 @@ public class MergeBesuControllerBuilderTest { @Mock Clock clock; @Mock StorageProvider storageProvider; @Mock GasLimitCalculator gasLimitCalculator; - @Mock WorldStateStorageCoordinator worldStateStorageCoordinator; @Mock WorldStatePreimageStorage worldStatePreimageStorage; BigInteger networkId = BigInteger.ONE; @@ -113,6 +112,11 @@ public class MergeBesuControllerBuilderTest { @Before public void setup() { + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + mock(ForestWorldStateKeyValueStorage.class); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); + when(genesisConfigFile.getParentHash()).thenReturn(Hash.ZERO.toHexString()); when(genesisConfigFile.getDifficulty()).thenReturn(Bytes.of(0).toHexString()); when(genesisConfigFile.getExtraData()).thenReturn(Bytes.EMPTY.toHexString()); @@ -144,11 +148,11 @@ public void setup() { .thenReturn(worldStateStorageCoordinator); when(storageProvider.createWorldStatePreimageStorage()).thenReturn(worldStatePreimageStorage); - when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); + when(worldStateKeyValueStorage.isWorldStateAvailable(any())).thenReturn(true); when(worldStatePreimageStorage.updater()) .thenReturn(mock(WorldStatePreimageStorage.Updater.class)); - when(worldStateStorageCoordinator.updater()) - .thenReturn(mock(WorldStateKeyValueStorage.Updater.class)); + when(worldStateKeyValueStorage.updater()) + .thenReturn(mock(ForestWorldStateKeyValueStorage.Updater.class)); when(miningParameters.getTargetGasLimit()).thenReturn(OptionalLong.empty()); besuControllerBuilder = visitWithMockConfigs(new MergeBesuControllerBuilder()); diff --git a/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java index 55b2db4159c..e7db7e64b97 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java @@ -36,13 +36,13 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.GasLimitCalculator; import org.hyperledger.besu.ethereum.ProtocolContext; -import org.hyperledger.besu.ethereum.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; @@ -85,7 +85,6 @@ public class QbftBesuControllerBuilderTest { @Mock private Clock clock; @Mock private StorageProvider storageProvider; @Mock private GasLimitCalculator gasLimitCalculator; - @Mock private WorldStateStorageCoordinator worldStateStorageCoordinator; @Mock private WorldStatePreimageStorage worldStatePreimageStorage; private static final BigInteger networkId = BigInteger.ONE; private static final NodeKey nodeKey = NodeKeyUtils.generate(); @@ -98,6 +97,11 @@ public class QbftBesuControllerBuilderTest { @Before public void setup() { // besu controller setup + final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + mock(ForestWorldStateKeyValueStorage.class); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateKeyValueStorage); + when(genesisConfigFile.getParentHash()).thenReturn(Hash.ZERO.toHexString()); when(genesisConfigFile.getDifficulty()).thenReturn(Bytes.of(0).toHexString()); when(genesisConfigFile.getExtraData()).thenReturn(Bytes.EMPTY.toHexString()); @@ -114,9 +118,9 @@ public void setup() { new MainnetBlockHeaderFunctions())); when(storageProvider.createWorldStateStorageCoordinator(DataStorageFormat.FOREST)) .thenReturn(worldStateStorageCoordinator); - when(worldStateStorageCoordinator.isWorldStateAvailable(any(), any())).thenReturn(true); + when(worldStateKeyValueStorage.isWorldStateAvailable(any())).thenReturn(true); when(worldStateStorageCoordinator.updater()) - .thenReturn(mock(WorldStateKeyValueStorage.Updater.class)); + .thenReturn(mock(ForestWorldStateKeyValueStorage.Updater.class)); when(worldStatePreimageStorage.updater()) .thenReturn(mock(WorldStatePreimageStorage.Updater.class)); when(storageProvider.createWorldStatePreimageStorage()).thenReturn(worldStatePreimageStorage); From e9558d3a3890ee77705c5996f704b41326924dc7 Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Wed, 29 Nov 2023 16:34:49 +0100 Subject: [PATCH 04/59] fix unit tests Signed-off-by: Karim TAAM --- .../cli/subcommands/operator/RestoreState.java | 2 +- ...estKeyValueStorageWorldStateStorageTest.java} | 16 ++++++++-------- .../worldstate/FastWorldStateDownloaderTest.java | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/{storage/keyvalue/KeyValueStorageWorldStateStorageTest.java => forest/ForestKeyValueStorageWorldStateStorageTest.java} (94%) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java index c8470f701f5..c9e4dc903d6 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java @@ -263,7 +263,7 @@ private void maybeCommitUpdater() { private void updateCode(final Bytes code) { maybeCommitUpdater(); - updater.putCode(null, code); + updater.putCode(code); } private void updateAccountState(final Bytes32 key, final Bytes value) { diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/forest/ForestKeyValueStorageWorldStateStorageTest.java similarity index 94% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/forest/ForestKeyValueStorageWorldStateStorageTest.java index ea6f9d5e3db..73fbdb17bfc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/forest/ForestKeyValueStorageWorldStateStorageTest.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.storage.keyvalue; +package org.hyperledger.besu.ethereum.forest; import static org.assertj.core.api.Assertions.assertThat; @@ -26,7 +26,7 @@ import org.apache.tuweni.units.bigints.UInt256; import org.junit.jupiter.api.Test; -public class KeyValueStorageWorldStateStorageTest { +public class ForestKeyValueStorageWorldStateStorageTest { @Test public void getCode_returnsEmpty() { @@ -64,7 +64,7 @@ public void getNodeData_returnsEmptyNode() { @Test public void getCode_saveAndGetSpecialValues() { final ForestWorldStateKeyValueStorage storage = emptyStorage(); - storage.updater().putCode(null, MerkleTrie.EMPTY_TRIE_NODE); + storage.updater().putCode(MerkleTrie.EMPTY_TRIE_NODE); storage.updater().putCode(Bytes.EMPTY).commit(); assertThat(storage.getCode(MerkleTrie.EMPTY_TRIE_NODE_HASH)) @@ -76,7 +76,7 @@ public void getCode_saveAndGetSpecialValues() { public void getCode_saveAndGetRegularValue() { final Bytes bytes = Bytes.fromHexString("0x123456"); final ForestWorldStateKeyValueStorage storage = emptyStorage(); - storage.updater().putCode(null, bytes).commit(); + storage.updater().putCode(bytes).commit(); assertThat(storage.getCode(Hash.hash(bytes))).contains(bytes); } @@ -162,10 +162,10 @@ public void reconcilesNonConflictingUpdaters() { final Updater updaterA = storage.updater(); final Updater updaterB = storage.updater(); - updaterA.putCode(null, bytesA); - updaterB.putCode(null, bytesA); - updaterB.putCode(null, bytesB); - updaterA.putCode(null, bytesC); + updaterA.putCode(bytesA); + updaterB.putCode(bytesA); + updaterB.putCode(bytesB); + updaterA.putCode(bytesC); updaterA.commit(); updaterB.commit(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java index 027b3317c79..5b81a246c2e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldStateDownloaderTest.java @@ -327,7 +327,7 @@ void doesNotRequestKnownCodeFromNetwork() { final Map knownCode = new HashMap<>(); accounts.subList(0, 5).forEach(a -> knownCode.put(a.getCodeHash(), a.getCode())); final ForestWorldStateKeyValueStorage.Updater localStorageUpdater = localStorage.updater(); - knownCode.forEach((bytes32, code) -> localStorageUpdater.putCode(null, code)); + knownCode.forEach((bytes32, code) -> localStorageUpdater.putCode(code)); localStorageUpdater.commit(); final WorldStateDownloader downloader = From 4a3ea8a2dcad5d03281a23d765e75f7947896e8a Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Tue, 19 Dec 2023 14:59:28 +0100 Subject: [PATCH 05/59] merge main Signed-off-by: Karim Taam --- .../org/hyperledger/besu/cli/BesuCommand.java | 1 - .../forest/ForestWorldStateArchive.java | 6 ++-- .../besu/ethereum/core/TransactionTest.java | 33 +++++++++++++++++-- .../core/TransactionTestCaseSpec.java | 5 ++- .../besu/ethereum/rlp/RLPRefTest.java | 33 ++++++++++++++++--- 5 files changed, 63 insertions(+), 15 deletions(-) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/{ => trie}/forest/ForestWorldStateArchive.java (95%) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 1001b247827..07bcabcaa22 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2905,7 +2905,6 @@ private TransactionPoolConfiguration buildTransactionPoolConfiguration() { + getMiningParameters().getMinTransactionGasPrice().toDecimalString() + ", since it cannot be greater than the value of min-gas-price"); txPoolConfBuilder.minGasPrice(getMiningParameters().getMinTransactionGasPrice()); - txPoolConfBuilder.minGasPrice(getMiningParameters().getMinTransactionGasPrice()); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/ForestWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/ForestWorldStateArchive.java similarity index 95% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/ForestWorldStateArchive.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/ForestWorldStateArchive.java index adbaf4c9fb5..d38976b644b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/forest/ForestWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/ForestWorldStateArchive.java @@ -13,17 +13,17 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.forest; +package org.hyperledger.besu.ethereum.trie.forest; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.forest.storage.ForestWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/core/TransactionTest.java b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/core/TransactionTest.java index d36a0b6a87d..b2dd41bbfa5 100644 --- a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/core/TransactionTest.java +++ b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/core/TransactionTest.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; import org.hyperledger.besu.evm.gascalculator.BerlinGasCalculator; import org.hyperledger.besu.evm.gascalculator.ByzantiumGasCalculator; +import org.hyperledger.besu.evm.gascalculator.CancunGasCalculator; import org.hyperledger.besu.evm.gascalculator.ConstantinopleGasCalculator; import org.hyperledger.besu.evm.gascalculator.FrontierGasCalculator; import org.hyperledger.besu.evm.gascalculator.GasCalculator; @@ -33,6 +34,7 @@ import org.hyperledger.besu.evm.gascalculator.IstanbulGasCalculator; import org.hyperledger.besu.evm.gascalculator.LondonGasCalculator; import org.hyperledger.besu.evm.gascalculator.PetersburgGasCalculator; +import org.hyperledger.besu.evm.gascalculator.ShanghaiGasCalculator; import org.hyperledger.besu.evm.gascalculator.SpuriousDragonGasCalculator; import org.hyperledger.besu.evm.gascalculator.TangerineWhistleGasCalculator; import org.hyperledger.besu.testutil.JsonTestParameters; @@ -54,7 +56,8 @@ private static TransactionValidator transactionValidator(final String name) { return REFERENCE_TEST_PROTOCOL_SCHEDULES .getByName(name) .getByBlockHeader(BlockHeaderBuilder.createDefault().buildBlockHeader()) - .getTransactionValidatorFactory().get(); + .getTransactionValidatorFactory() + .get(); } private static final String TEST_CONFIG_FILE_DIR_PATH = "TransactionTests/"; @@ -62,7 +65,9 @@ private static TransactionValidator transactionValidator(final String name) { public static Stream getTestParametersForConfig() { return JsonTestParameters.create(TransactionTestCaseSpec.class) .generator((name, fullPath, spec, collector) -> collector.add(name, fullPath, spec, true)) - .generate(TEST_CONFIG_FILE_DIR_PATH).stream().map(params -> Arguments.of(params[0], params[1])); + .generate(TEST_CONFIG_FILE_DIR_PATH) + .stream() + .map(params -> Arguments.of(params[0], params[1])); } @ParameterizedTest(name = "Name: {0}") @@ -125,8 +130,30 @@ public void london(final String name, final TransactionTestCaseSpec spec) { milestone(spec, name, "London", new LondonGasCalculator(), Optional.of(Wei.of(0))); } + @ParameterizedTest(name = "Name: {0}") + @MethodSource("getTestParametersForConfig") + public void merge(final String name, final TransactionTestCaseSpec spec) { + milestone(spec, name, "Merge", new LondonGasCalculator(), Optional.of(Wei.of(0))); + } + + @ParameterizedTest(name = "Name: {0}") + @MethodSource("getTestParametersForConfig") + public void shanghai(final String name, final TransactionTestCaseSpec spec) { + milestone(spec, name, "Shanghai", new ShanghaiGasCalculator(), Optional.of(Wei.of(0))); + } + + @ParameterizedTest(name = "Name: {0}") + @MethodSource("getTestParametersForConfig") + public void cancun(final String name, final TransactionTestCaseSpec spec) { + milestone(spec, name, "Cancun", new CancunGasCalculator(), Optional.of(Wei.of(0))); + } + public void milestone( - final TransactionTestCaseSpec spec, final String name, final String milestone, final GasCalculator gasCalculator, final Optional baseFee) { + final TransactionTestCaseSpec spec, + final String name, + final String milestone, + final GasCalculator gasCalculator, + final Optional baseFee) { final TransactionTestCaseSpec.Expectation expected = spec.expectation(milestone); diff --git a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/core/TransactionTestCaseSpec.java b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/core/TransactionTestCaseSpec.java index 7f9c3cb7c63..7ce7384e1ab 100644 --- a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/core/TransactionTestCaseSpec.java +++ b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/core/TransactionTestCaseSpec.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.Assumptions; /** A Transaction test case specification. */ @JsonIgnoreProperties({"_info"}) @@ -109,9 +110,7 @@ public Bytes getRlp() { public Expectation expectation(final String milestone) { final Expectation expectation = expectations.get(milestone); - if (expectation == null) { - throw new IllegalStateException("Expectation for milestone " + milestone + " not found"); - } + Assumptions.assumeFalse(expectation == null, () -> "No expectation for milestone " + milestone); return expectation; } diff --git a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/rlp/RLPRefTest.java b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/rlp/RLPRefTest.java index a4eff43c98d..b6d01dae83f 100644 --- a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/rlp/RLPRefTest.java +++ b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/rlp/RLPRefTest.java @@ -16,12 +16,13 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue; -import org.assertj.core.api.Assertions; import org.hyperledger.besu.ethereum.rlp.util.RLPTestUtil; import org.hyperledger.besu.testutil.JsonTestParameters; import java.util.stream.Stream; +import org.apache.tuweni.bytes.Bytes; +import org.assertj.core.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -29,23 +30,45 @@ /** The Ethereum reference RLP tests. */ public class RLPRefTest { - private static final String TEST_CONFIG_FILES = "RLPTests/rlptest.json"; + private static final String[] TEST_CONFIG_FILES = { + "RLPTests/rlptest.json", "RLPTests/invalidRLPTest.json" + }; + + private static final Bytes INVALID = Bytes.fromHexString("0x494e56414c4944"); public static Stream getTestParametersForConfig() { - return JsonTestParameters.create(RLPRefTestCaseSpec.class).generate(TEST_CONFIG_FILES).stream().map(params -> Arguments.of(params[0], params[1], params[2])); + return JsonTestParameters.create(RLPRefTestCaseSpec.class).generate(TEST_CONFIG_FILES).stream() + .map(params -> Arguments.of(params[0], params[1], params[2])); } @ParameterizedTest(name = "Name: {0}") @MethodSource("getTestParametersForConfig") public void encode(final String name, final RLPRefTestCaseSpec spec, final boolean runTest) { assumeTrue(runTest, "Test was blacklisted"); - Assertions.assertThat(RLPTestUtil.encode(spec.getIn())).isEqualTo(spec.getOut()); + if (!spec.getIn().equals(INVALID)) { + Assertions.assertThat(RLPTestUtil.encode(spec.getIn())).isEqualTo(spec.getOut()); + } } @ParameterizedTest(name = "Name: {0}") @MethodSource("getTestParametersForConfig") public void decode(final String name, final RLPRefTestCaseSpec spec, final boolean runTest) { assumeTrue(runTest, "Test was blacklisted"); - Assertions.assertThat(RLPTestUtil.decode(spec.getOut())).isEqualTo(spec.getIn()); + if (spec.getIn().equals(INVALID)) { + Assertions.assertThatThrownBy(() -> RLPTestUtil.decode(spec.getOut())) + .isInstanceOf(RLPException.class); + Assertions.assertThatThrownBy(() -> decode2(RLP.input(spec.getOut()))) + .isInstanceOf(RLPException.class); + } else { + Assertions.assertThat(RLPTestUtil.decode(spec.getOut())).isEqualTo(spec.getIn()); + } + } + + private static Object decode2(final RLPInput in) { + if (in.nextIsList()) { + return in.readList(RLPRefTest::decode2); + } else { + return in.readBytes(); + } } } From a368a353d80098e6d7970d9545a4097772bfcbe6 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Wed, 20 Dec 2023 10:33:55 +0100 Subject: [PATCH 06/59] fix tests Signed-off-by: Karim Taam --- .../storage/ForestKeyValueStorageWorldStateStorageTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java index 3148d6d974e..fb8750445fc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java @@ -63,8 +63,7 @@ public void getNodeData_returnsEmptyNode() { @Test public void getCode_saveAndGetSpecialValues() { final ForestWorldStateKeyValueStorage storage = emptyStorage(); - storage.updater().putCode(MerkleTrie.EMPTY_TRIE_NODE); - storage.updater().putCode(Bytes.EMPTY).commit(); + storage.updater().putCode(MerkleTrie.EMPTY_TRIE_NODE).putCode(Bytes.EMPTY).commit(); assertThat(storage.getCode(MerkleTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerkleTrie.EMPTY_TRIE_NODE); From 2ce37aea5e2268a051388098ff0c0bebc89163c4 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Wed, 20 Dec 2023 17:32:34 +0100 Subject: [PATCH 07/59] add diffbased for bonsai Signed-off-by: Karim Taam --- .../besu/components/BesuComponent.java | 8 +- .../controller/BesuControllerBuilder.java | 16 +- .../controller/BesuControllerBuilderTest.java | 8 +- .../internal/pojoadapter/AccountAdapter.java | 2 +- .../mainnet/AbstractBlockProcessor.java | 4 +- .../keyvalue/KeyValueStorageProvider.java | 2 +- .../{ => diffbased}/bonsai/BonsaiAccount.java | 187 ++------- .../bonsai/BonsaiWorldStateProvider.java | 39 +- .../cache/BonsaiCachedMerkleTrieLoader.java} | 10 +- .../BonsaiCachedMerkleTrieLoaderModule.java} | 8 +- .../BonsaiCachedWorldStorageManager.java | 60 +++ .../bonsai/storage/BonsaiPreImageProxy.java | 2 +- ...nsaiSnapshotWorldStateKeyValueStorage.java | 13 +- .../BonsaiWorldStateKeyValueStorage.java | 182 +-------- .../storage/BonsaiWorldStateLayerStorage.java | 13 +- .../bonsai/storage/flat/FlatDbStrategy.java | 2 +- .../storage/flat/FullFlatDbStrategy.java | 2 +- .../storage/flat/PartialFlatDbStrategy.java | 2 +- .../bonsai/trielog/TrieLogFactoryImpl.java | 17 +- .../bonsai/worldview/BonsaiWorldState.java | 359 +++--------------- .../BonsaiWorldStateUpdateAccumulator.java | 98 +++++ .../diffbased/common/DiffBasedAccount.java | 219 +++++++++++ .../common/DiffBasedValue.java} | 14 +- .../diffbased/common/StorageSubscriber.java | 11 + .../DiffBasedCachedWorldStorageManager.java} | 91 +++-- .../cache/DiffBasedCachedWorldView.java} | 22 +- ...BasedLayeredWorldStateKeyValueStorage.java | 21 + ...asedSnapshotWorldStateKeyValueStorage.java | 18 + .../DiffBasedWorldStateKeyValueStorage.java | 246 ++++++++++++ .../common}/trielog/TrieLogAddedEvent.java | 2 +- .../common}/trielog/TrieLogLayer.java | 50 +-- .../common}/trielog/TrieLogManager.java | 23 +- .../common}/trielog/TrieLogPruner.java | 18 +- .../common/worldview/DiffBasedWorldState.java | 352 +++++++++++++++++ .../common/worldview/DiffBasedWorldView.java} | 8 +- ...DiffBasedWorldStateUpdateAccumulator.java} | 305 +++++++-------- .../preload/AccountConsumingMap.java | 50 +++ .../accumulator/preload/Consumer.java | 22 ++ .../preload/StorageConsumingMap.java | 54 +++ .../worldstate/DataStorageFormat.java | 3 +- .../WorldStateStorageCoordinator.java | 2 +- .../core/InMemoryKeyValueStorageProvider.java | 12 +- .../besu/ethereum/core/TrieGenerator.java | 2 +- .../BlockImportExceptionHandlingTest.java | 6 +- .../trie/bonsai/AbstractIsolationTests.java | 9 +- .../bonsai/BonsaiWorldStateProviderTest.java | 37 +- .../bonsai/CachedMerkleTrieLoaderTest.java | 8 +- .../ethereum/trie/bonsai/LogRollingTests.java | 11 +- .../ethereum/trie/bonsai/RollingImport.java | 11 +- .../BonsaiWorldStateKeyValueStorageTest.java | 3 +- .../bonsai/trielog/TrieLogFactoryTests.java | 2 + .../bonsai/trielog/TrieLogLayerTests.java | 1 + .../bonsai/trielog/TrieLogManagerTests.java | 9 +- .../bonsai/trielog/TrieLogPrunerTest.java | 3 +- .../eth/sync/DefaultSynchronizer.java | 2 +- .../eth/sync/fastsync/FastSyncDownloader.java | 2 +- .../eth/sync/snapsync/PersistDataStep.java | 2 +- .../sync/snapsync/SnapWorldDownloadState.java | 2 +- .../snapsync/SnapWorldStateDownloader.java | 2 +- .../request/AccountRangeDataRequest.java | 2 +- .../request/StorageRangeDataRequest.java | 2 +- ...ccountFlatDatabaseHealingRangeRequest.java | 2 +- ...torageFlatDatabaseHealingRangeRequest.java | 2 +- .../CheckPointSyncChainDownloaderTest.java | 2 +- .../fastsync/FastDownloaderFactoryTest.java | 2 +- .../sync/fastsync/FastSyncDownloaderTest.java | 2 +- .../FastWorldDownloadStateTest.java | 2 +- .../worldstate/LoadLocalDataStepTest.java | 2 +- .../snapsync/AccountHealingTrackingTest.java | 2 +- .../snapsync/SnapWorldDownloadStateTest.java | 2 +- ...ntFlatDatabaseHealingRangeRequestTest.java | 2 +- ...geFlatDatabaseHealingRangeRequestTest.java | 2 +- .../StorageTrieNodeHealingRequestTest.java | 2 +- .../BonsaiReferenceTestUpdateAccumulator.java | 15 +- .../BonsaiReferenceTestWorldState.java | 40 +- .../BonsaiReferenceTestWorldStateStorage.java | 12 +- 76 files changed, 1716 insertions(+), 1068 deletions(-) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/BonsaiAccount.java (55%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/BonsaiWorldStateProvider.java (91%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedMerkleTrieLoader.java => diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java} (94%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedMerkleTrieLoaderModule.java => diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java} (77%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiPreImageProxy.java (97%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java (92%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiWorldStateKeyValueStorage.java (70%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiWorldStateLayerStorage.java (79%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/flat/FlatDbStrategy.java (99%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/flat/FullFlatDbStrategy.java (98%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/flat/PartialFlatDbStrategy.java (98%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/trielog/TrieLogFactoryImpl.java (92%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/worldview/BonsaiWorldState.java (57%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/BonsaiValue.java => diffbased/common/DiffBasedValue.java} (83%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedWorldStorageManager.java => diffbased/common/cache/DiffBasedCachedWorldStorageManager.java} (65%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedBonsaiWorldView.java => diffbased/common/cache/DiffBasedCachedWorldView.java} (71%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogAddedEvent.java (92%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogLayer.java (79%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogManager.java (88%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogPruner.java (91%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/worldview/BonsaiWorldView.java => diffbased/common/worldview/DiffBasedWorldView.java} (87%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java => diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java} (72%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java diff --git a/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java b/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java index f0093243a6c..65ba2d931d2 100644 --- a/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java +++ b/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java @@ -19,8 +19,8 @@ import org.hyperledger.besu.cli.BesuCommand; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; import org.hyperledger.besu.ethereum.eth.transactions.BlobCacheModule; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoaderModule; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoaderModule; import org.hyperledger.besu.metrics.MetricsSystemModule; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.services.BesuPluginContextImpl; @@ -37,7 +37,7 @@ modules = { BesuCommandModule.class, MetricsSystemModule.class, - CachedMerkleTrieLoaderModule.class, + BonsaiCachedMerkleTrieLoaderModule.class, BesuPluginContextModule.class, BlobCacheModule.class }) @@ -55,7 +55,7 @@ public interface BesuComponent { * * @return CachedMerkleTrieLoader */ - CachedMerkleTrieLoader getCachedMerkleTrieLoader(); + BonsaiCachedMerkleTrieLoader getCachedMerkleTrieLoader(); /** * a metrics system that is observable by a Prometheus or OTEL metrics collection subsystem diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 639d30c1da7..2140f5c53a5 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -80,10 +80,10 @@ import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.pruner.MarkSweepPruner; import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; @@ -611,10 +611,10 @@ public BesuController build() { dataDirectory.toString(), numberOfBlocksToCache); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = + final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader = besuComponent .map(BesuComponent::getCachedMerkleTrieLoader) - .orElseGet(() -> new CachedMerkleTrieLoader(metricsSystem)); + .orElseGet(() -> new BonsaiCachedMerkleTrieLoader(metricsSystem)); final WorldStateArchive worldStateArchive = createWorldStateArchive(worldStateStorageCoordinator, blockchain, cachedMerkleTrieLoader); @@ -1065,7 +1065,7 @@ private Optional createSnapProtocolManager( WorldStateArchive createWorldStateArchive( final WorldStateStorageCoordinator worldStateStorageCoordinator, final Blockchain blockchain, - final CachedMerkleTrieLoader cachedMerkleTrieLoader) { + final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader) { return switch (dataStorageConfiguration.getDataStorageFormat()) { case BONSAI -> { final GenesisConfigOptions genesisConfigOptions = configOptionsSupplier.get(); @@ -1099,6 +1099,8 @@ yield new BonsaiWorldStateProvider( yield new ForestWorldStateArchive( worldStateStorageCoordinator, preimageStorage, evmConfiguration); } + default -> throw new IllegalStateException( + "Unexpected value: " + dataStorageConfiguration.getDataStorageFormat()); }; } diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index fda1ffa6e80..e28b0379386 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -42,9 +42,9 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; @@ -174,7 +174,7 @@ public void shouldDisablePruningIfBonsaiIsEnabled() { .createWorldStateArchive( any(WorldStateStorageCoordinator.class), any(Blockchain.class), - any(CachedMerkleTrieLoader.class)); + any(BonsaiCachedMerkleTrieLoader.class)); doReturn(mockWorldState).when(worldStateArchive).getMutable(); when(storageProvider.createWorldStateStorageCoordinator(DataStorageFormat.BONSAI)) diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java index 13caadab6ab..faf42aafc8e 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java @@ -17,7 +17,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.AccountState; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index 8f1b01a50c1..4c5fb893fe1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -31,8 +31,8 @@ import org.hyperledger.besu.ethereum.privacy.storage.PrivateMetadataUpdater; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.vm.BlockHashLookup; import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup; import org.hyperledger.besu.evm.tracing.OperationTracer; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index e7dcffa2073..7b5dc7155ff 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -19,7 +19,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java similarity index 55% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java index d93f550cafa..ad52e57cd71 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java @@ -14,48 +14,34 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.evm.ModificationNotAllowedException; import org.hyperledger.besu.evm.account.AccountStorageEntry; -import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; -import java.util.HashMap; -import java.util.Map; import java.util.NavigableMap; import java.util.Objects; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; -public class BonsaiAccount implements MutableAccount, AccountValue { - private final BonsaiWorldView context; - private boolean immutable; - - private final Address address; - private final Hash addressHash; - private Hash codeHash; - private long nonce; - private Wei balance; +public class BonsaiAccount extends DiffBasedAccount { private Hash storageRoot; - private Bytes code; - - private final Map updatedStorage = new HashMap<>(); public BonsaiAccount( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final Hash addressHash, final long nonce, @@ -63,31 +49,24 @@ public BonsaiAccount( final Hash storageRoot, final Hash codeHash, final boolean mutable) { - this.context = context; - this.address = address; - this.addressHash = addressHash; - this.nonce = nonce; - this.balance = balance; + super(context, address, addressHash, nonce, balance, codeHash, !mutable); this.storageRoot = storageRoot; - this.codeHash = codeHash; - - this.immutable = !mutable; } public BonsaiAccount( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final AccountValue stateTrieAccount, final boolean mutable) { - this( + super( context, address, address.addressHash(), stateTrieAccount.getNonce(), stateTrieAccount.getBalance(), - stateTrieAccount.getStorageRoot(), stateTrieAccount.getCodeHash(), - mutable); + !mutable); + this.storageRoot = stateTrieAccount.getStorageRoot(); } public BonsaiAccount(final BonsaiAccount toCopy) { @@ -95,37 +74,35 @@ public BonsaiAccount(final BonsaiAccount toCopy) { } public BonsaiAccount( - final BonsaiAccount toCopy, final BonsaiWorldView context, final boolean mutable) { - this.context = context; - this.address = toCopy.address; - this.addressHash = toCopy.addressHash; - this.nonce = toCopy.nonce; - this.balance = toCopy.balance; + final BonsaiAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + super( + context, + toCopy.address, + toCopy.addressHash, + toCopy.nonce, + toCopy.balance, + toCopy.codeHash, + !mutable); this.storageRoot = toCopy.storageRoot; - this.codeHash = toCopy.codeHash; - this.code = toCopy.code; updatedStorage.putAll(toCopy.updatedStorage); - - this.immutable = !mutable; } public BonsaiAccount( - final BonsaiWorldView context, final UpdateTrackingAccount tracked) { - this.context = context; - this.address = tracked.getAddress(); - this.addressHash = tracked.getAddressHash(); - this.nonce = tracked.getNonce(); - this.balance = tracked.getBalance(); + final DiffBasedWorldView context, final UpdateTrackingAccount tracked) { + super( + context, + tracked.getAddress(), + tracked.getAddressHash(), + tracked.getNonce(), + tracked.getBalance(), + tracked.getCodeHash(), + false); this.storageRoot = Hash.EMPTY_TRIE_HASH; - this.codeHash = tracked.getCodeHash(); - this.code = tracked.getCode(); updatedStorage.putAll(tracked.getUpdatedStorage()); - - this.immutable = false; } public static BonsaiAccount fromRLP( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final Bytes encoded, final boolean mutable) @@ -144,88 +121,11 @@ public static BonsaiAccount fromRLP( context, address, address.addressHash(), nonce, balance, storageRoot, codeHash, mutable); } - @Override - public Address getAddress() { - return address; - } - - @Override - public Hash getAddressHash() { - return addressHash; - } - - @Override - public long getNonce() { - return nonce; - } - - @Override - public void setNonce(final long value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - nonce = value; - } - - @Override - public Wei getBalance() { - return balance; - } - - @Override - public void setBalance(final Wei value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - balance = value; - } - - @Override - public Bytes getCode() { - if (code == null) { - code = context.getCode(address, codeHash).orElse(Bytes.EMPTY); - } - return code; - } - - @Override - public void setCode(final Bytes code) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - this.code = code; - if (code == null || code.isEmpty()) { - this.codeHash = Hash.EMPTY; - } else { - this.codeHash = Hash.hash(code); - } - } - - @Override - public Hash getCodeHash() { - return codeHash; - } - - @Override - public UInt256 getStorageValue(final UInt256 key) { - return context.getStorageValue(address, key); - } - - @Override - public UInt256 getOriginalStorageValue(final UInt256 key) { - return context.getPriorStorageValue(address, key); - } - @Override public NavigableMap storageEntriesFrom( final Bytes32 startKeyHash, final int limit) { - return context.getWorldStateStorage().storageEntriesFrom(this.addressHash, startKeyHash, limit); - } - - public Bytes serializeAccount() { - final BytesValueRLPOutput out = new BytesValueRLPOutput(); - writeTo(out); - return out.encoded(); + return ((BonsaiWorldStateKeyValueStorage) context.getWorldStateStorage()) + .storageEntriesFrom(this.addressHash, startKeyHash, limit); } @Override @@ -240,24 +140,6 @@ public void writeTo(final RLPOutput out) { out.endList(); } - @Override - public void setStorageValue(final UInt256 key, final UInt256 value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - updatedStorage.put(key, value); - } - - @Override - public void clearStorage() { - updatedStorage.clear(); - } - - @Override - public Map getUpdatedStorage() { - return updatedStorage; - } - @Override public Hash getStorageRoot() { return storageRoot; @@ -270,11 +152,6 @@ public void setStorageRoot(final Hash storageRoot) { this.storageRoot = storageRoot; } - @Override - public void becomeImmutable() { - immutable = true; - } - @Override public String toString() { return "AccountState{" diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java similarity index 91% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java index 54b2a4db180..6ca43b7c28c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java @@ -14,9 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; - -import static org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager.RETAINED_LAYERS; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; @@ -27,13 +25,13 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; @@ -63,17 +61,17 @@ public class BonsaiWorldStateProvider implements WorldStateArchive { private final Blockchain blockchain; - private final CachedWorldStorageManager cachedWorldStorageManager; + private final BonsaiCachedWorldStorageManager cachedWorldStorageManager; private final TrieLogManager trieLogManager; private final BonsaiWorldState persistedState; private final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; - private final CachedMerkleTrieLoader cachedMerkleTrieLoader; + private final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader; public BonsaiWorldStateProvider( final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final Blockchain blockchain, final Optional maxLayersToLoad, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, + final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader, final ObservableMetricsSystem metricsSystem, final BesuContext pluginContext, final EvmConfiguration evmConfiguration, @@ -81,13 +79,13 @@ public BonsaiWorldStateProvider( this.worldStateKeyValueStorage = worldStateKeyValueStorage; this.cachedWorldStorageManager = - new CachedWorldStorageManager(this, worldStateKeyValueStorage, metricsSystem); + new BonsaiCachedWorldStorageManager(this, worldStateKeyValueStorage, metricsSystem); // TODO: de-dup constructors this.trieLogManager = new TrieLogManager( blockchain, worldStateKeyValueStorage, - maxLayersToLoad.orElse(RETAINED_LAYERS), + maxLayersToLoad.orElse(BonsaiCachedWorldStorageManager.RETAINED_LAYERS), pluginContext, trieLogPruner); this.blockchain = blockchain; @@ -103,11 +101,11 @@ public BonsaiWorldStateProvider( @VisibleForTesting BonsaiWorldStateProvider( - final CachedWorldStorageManager cachedWorldStorageManager, + final BonsaiCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final Blockchain blockchain, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, + final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader, final EvmConfiguration evmConfiguration) { this.cachedWorldStorageManager = cachedWorldStorageManager; this.trieLogManager = trieLogManager; @@ -165,7 +163,8 @@ public Optional getMutable( .or(() -> cachedWorldStorageManager.getHeadWorldState(blockchain::getBlockHeader)) .flatMap( bonsaiWorldState -> - rollMutableStateToBlockHash(bonsaiWorldState, blockHeader.getHash())) + rollMutableStateToBlockHash( + (BonsaiWorldState) bonsaiWorldState, blockHeader.getHash())) .map(MutableWorldState::freeze); } } @@ -274,7 +273,7 @@ Optional rollMutableStateToBlockHash( } } - public CachedMerkleTrieLoader getCachedMerkleTrieLoader() { + public BonsaiCachedMerkleTrieLoader getCachedMerkleTrieLoader() { return cachedMerkleTrieLoader; } @@ -347,7 +346,7 @@ public TrieLogManager getTrieLogManager() { return trieLogManager; } - public CachedWorldStorageManager getCachedWorldStorageManager() { + public BonsaiCachedWorldStorageManager getCachedWorldStorageManager() { return cachedWorldStorageManager; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java similarity index 94% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java index cbd1fc820d8..9f3251660dc 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java @@ -13,14 +13,15 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -37,8 +38,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -public class CachedMerkleTrieLoader - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { +public class BonsaiCachedMerkleTrieLoader implements StorageSubscriber { private static final int ACCOUNT_CACHE_SIZE = 100_000; private static final int STORAGE_CACHE_SIZE = 200_000; @@ -47,7 +47,7 @@ public class CachedMerkleTrieLoader private final Cache storageNodes = CacheBuilder.newBuilder().recordStats().maximumSize(STORAGE_CACHE_SIZE).build(); - public CachedMerkleTrieLoader(final ObservableMetricsSystem metricsSystem) { + public BonsaiCachedMerkleTrieLoader(final ObservableMetricsSystem metricsSystem) { CacheMetricsCollector cacheMetrics = new CacheMetricsCollector(); cacheMetrics.addCache("accountsNodes", accountNodes); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java similarity index 77% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java index 63c8051f253..04a8efc4514 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -21,11 +21,11 @@ import dagger.Provides; @Module -public class CachedMerkleTrieLoaderModule { +public class BonsaiCachedMerkleTrieLoaderModule { @Provides - CachedMerkleTrieLoader provideCachedMerkleTrieLoaderModule( + BonsaiCachedMerkleTrieLoader provideCachedMerkleTrieLoaderModule( final ObservableMetricsSystem metricsSystem) { - return new CachedMerkleTrieLoader(metricsSystem); + return new BonsaiCachedMerkleTrieLoader(metricsSystem); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java new file mode 100644 index 00000000000..17b32241eac --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java @@ -0,0 +1,60 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; + +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.metrics.ObservableMetricsSystem; + +public class BonsaiCachedWorldStorageManager extends DiffBasedCachedWorldStorageManager { + + public BonsaiCachedWorldStorageManager( + final BonsaiWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final ObservableMetricsSystem metricsSystem) { + super(archive, worldStateKeyValueStorage, metricsSystem); + } + + @Override + public DiffBasedWorldState createWorldState( + final BonsaiWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final EvmConfiguration evmConfiguration) { + return new BonsaiWorldState( + archive, (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, evmConfiguration); + } + + @Override + public DiffBasedWorldStateKeyValueStorage createLayeredKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { + return new BonsaiWorldStateLayerStorage( + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage); + } + + @Override + public DiffBasedWorldStateKeyValueStorage createSnapshotKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final ObservableMetricsSystem metricsSystem) { + return new BonsaiSnapshotWorldStateKeyValueStorage( + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, metricsSystem); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java similarity index 97% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java index 16cda3104f0..4bae322a855 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java similarity index 92% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index 8e256c39264..1a472460145 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -13,11 +13,12 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; @@ -33,7 +34,7 @@ import org.slf4j.LoggerFactory; public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKeyValueStorage - implements BonsaiStorageSubscriber { + implements DiffBasedSnapshotWorldStateKeyValueStorage, StorageSubscriber { protected final BonsaiWorldStateKeyValueStorage parentWorldStateStorage; private static final Logger LOG = @@ -60,9 +61,9 @@ public BonsaiSnapshotWorldStateKeyValueStorage( final ObservableMetricsSystem metricsSystem) { this( worldStateKeyValueStorage, - ((SnappableKeyValueStorage) worldStateKeyValueStorage.composedWorldStateStorage) + ((SnappableKeyValueStorage) worldStateKeyValueStorage.getComposedWorldStateStorage()) .takeSnapshot(), - worldStateKeyValueStorage.trieLogStorage, + worldStateKeyValueStorage.getTrieLogStorage(), metricsSystem); } @@ -215,7 +216,7 @@ public void onClearTrieLog() { protected synchronized void doClose() throws Exception { if (!isClosedGet()) { // alert any subscribers we are closing: - subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); + subscribers.forEach(StorageSubscriber::onCloseStorage); // close all of the SnappedKeyValueStorages: composedWorldStateStorage.close(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java similarity index 70% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 153c5fc54d0..72053fc9de5 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; @@ -24,9 +24,10 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FullFlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.PartialFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -37,57 +38,34 @@ import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; -import org.hyperledger.besu.util.Subscribers; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Map; import java.util.NavigableMap; import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; -import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorage, AutoCloseable { +public class BonsaiWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValueStorage + implements WorldStateKeyValueStorage { private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class); - - // 0x776f726c64526f6f74 - public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); - // 0x776f726c64426c6f636b48617368 - public static final byte[] WORLD_BLOCK_HASH_KEY = - "worldBlockHash".getBytes(StandardCharsets.UTF_8); - - // 0x666C61744462537461747573 public static final byte[] FLAT_DB_MODE = "flatDbStatus".getBytes(StandardCharsets.UTF_8); protected FlatDbMode flatDbMode; protected FlatDbStrategy flatDbStrategy; - protected final SegmentedKeyValueStorage composedWorldStateStorage; - protected final KeyValueStorage trieLogStorage; - - protected final ObservableMetricsSystem metricsSystem; - - private final AtomicBoolean shouldClose = new AtomicBoolean(false); - - protected final AtomicBoolean isClosed = new AtomicBoolean(false); - - protected final Subscribers subscribers = Subscribers.create(); - public BonsaiWorldStateKeyValueStorage( final StorageProvider provider, final ObservableMetricsSystem metricsSystem) { - this.composedWorldStateStorage = + super( provider.getStorageBySegmentIdentifiers( List.of( - ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)); - this.trieLogStorage = - provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); - this.metricsSystem = metricsSystem; + ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)), + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE), + metricsSystem); loadFlatDbStrategy(); } @@ -97,11 +75,9 @@ public BonsaiWorldStateKeyValueStorage( final SegmentedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage, final ObservableMetricsSystem metricsSystem) { + super(composedWorldStateStorage, trieLogStorage, metricsSystem); this.flatDbMode = flatDbMode; this.flatDbStrategy = flatDbStrategy; - this.composedWorldStateStorage = composedWorldStateStorage; - this.trieLogStorage = trieLogStorage; - this.metricsSystem = metricsSystem; } private void loadFlatDbStrategy() { @@ -131,6 +107,7 @@ public FlatDbMode deriveFlatDbStrategy() { return flatDbMode; } + @Override public FlatDbStrategy getFlatDbStrategy() { if (flatDbStrategy == null) { loadFlatDbStrategy(); @@ -143,6 +120,7 @@ public DataStorageFormat getDataStorageFormat() { return DataStorageFormat.BONSAI; } + @Override public FlatDbMode getFlatDbMode() { return flatDbMode; } @@ -193,31 +171,6 @@ public Optional getTrieNodeUnsafe(final Bytes key) { .map(Bytes::wrap); } - public Optional getTrieLog(final Hash blockHash) { - return trieLogStorage.get(blockHash.toArrayUnsafe()); - } - - public Stream streamTrieLogKeys(final int limit) { - return trieLogStorage.streamKeys().limit(limit); - } - - public Optional getStateTrieNode(final Bytes location) { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()) - .map(Bytes::wrap); - } - - public Optional getWorldStateRootHash() { - return composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).map(Bytes::wrap); - } - - public Optional getWorldStateBlockHash() { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY) - .map(Bytes32::wrap) - .map(Hash::wrap); - } - public Optional getStorageValueByStorageSlotKey( final Hash accountHash, final StorageSlotKey storageSlotKey) { return getStorageValueByStorageSlotKey( @@ -246,32 +199,11 @@ public Optional getStorageValueByStorageSlotKey( composedWorldStateStorage); } - public Map streamFlatAccounts( - final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return getFlatDbStrategy() - .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); - } - - public Map streamFlatStorages( - final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return getFlatDbStrategy() - .streamStorageFlatDatabase( - composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); - } - public NavigableMap storageEntriesFrom( final Hash addressHash, final Bytes32 startKeyHash, final int limit) { throw new RuntimeException("Bonsai Tries does not currently support enumerating storage"); } - public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY) - .map(Bytes32::wrap) - .map(hash -> hash.equals(rootHash) || trieLogStorage.containsKey(blockHash.toArrayUnsafe())) - .orElse(false); - } - public void upgradeToFullFlatDbMode() { final SegmentedKeyValueStorageTransaction transaction = composedWorldStateStorage.startTransaction(); @@ -293,23 +225,10 @@ public void downgradeToPartialFlatDbMode() { @Override public void clear() { - subscribers.forEach(BonsaiStorageSubscriber::onClearStorage); - getFlatDbStrategy().clearAll(composedWorldStateStorage); - composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE); - trieLogStorage.clear(); + super.clear(); loadFlatDbStrategy(); // force reload of flat db reader strategy } - public void clearTrieLog() { - subscribers.forEach(BonsaiStorageSubscriber::onClearTrieLog); - trieLogStorage.clear(); - } - - public void clearFlatDatabase() { - subscribers.forEach(BonsaiStorageSubscriber::onClearFlatDatabaseStorage); - getFlatDbStrategy().resetOnResync(composedWorldStateStorage); - } - @Override public Updater updater() { return new Updater( @@ -318,16 +237,7 @@ public Updater updater() { flatDbStrategy); } - public boolean pruneTrieLog(final Hash blockHash) { - try { - return trieLogStorage.tryDelete(blockHash.toArrayUnsafe()); - } catch (Exception e) { - LOG.error("Error pruning trie log for block hash {}", blockHash, e); - return false; - } - } - - public static class Updater implements WorldStateKeyValueStorage.Updater { + public static class Updater implements DiffBasedWorldStateKeyValueStorage.Updater { private final SegmentedKeyValueStorageTransaction composedWorldStateTransaction; private final KeyValueStorageTransaction trieLogStorageTransaction; @@ -377,6 +287,7 @@ public Updater putAccountInfoState(final Hash accountHash, final Bytes accountVa return this; } + @Override public Updater saveWorldState(final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { composedWorldStateTransaction.put( TRIE_BRANCH_STORAGE, Bytes.EMPTY.toArrayUnsafe(), node.toArrayUnsafe()); @@ -429,10 +340,12 @@ public synchronized void removeStorageValueBySlotHash( composedWorldStateTransaction, accountHash, slotHash); } + @Override public SegmentedKeyValueStorageTransaction getWorldStateTransaction() { return composedWorldStateTransaction; } + @Override public KeyValueStorageTransaction getTrieLogStorageTransaction() { return trieLogStorageTransaction; } @@ -444,65 +357,10 @@ public void commit() { composedWorldStateTransaction.commit(); } + @Override public void rollback() { composedWorldStateTransaction.rollback(); trieLogStorageTransaction.rollback(); } } - - @Override - public synchronized void close() throws Exception { - // when the storage clears, close - shouldClose.set(true); - tryClose(); - } - - public synchronized long subscribe(final BonsaiStorageSubscriber sub) { - if (isClosed.get()) { - throw new RuntimeException("Storage is marked to close or has already closed"); - } - return subscribers.subscribe(sub); - } - - public synchronized void unSubscribe(final long id) { - subscribers.unsubscribe(id); - try { - tryClose(); - } catch (Exception e) { - LOG.atWarn() - .setMessage("exception while trying to close : {}") - .addArgument(e::getMessage) - .log(); - } - } - - protected synchronized void tryClose() throws Exception { - if (shouldClose.get() && subscribers.getSubscriberCount() < 1) { - doClose(); - } - } - - protected synchronized void doClose() throws Exception { - if (!isClosed.get()) { - // alert any subscribers we are closing: - subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); - - // close all of the KeyValueStorages: - composedWorldStateStorage.close(); - trieLogStorage.close(); - - // set storage closed - isClosed.set(true); - } - } - - public interface BonsaiStorageSubscriber { - default void onClearStorage() {} - - default void onClearFlatDatabaseStorage() {} - - default void onClearTrieLog() {} - - default void onCloseStorage() {} - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java similarity index 79% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java index 0dd61b8d7a6..9369a0edd62 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java @@ -13,9 +13,10 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; @@ -23,14 +24,14 @@ import org.hyperledger.besu.services.kvstore.LayeredKeyValueStorage; public class BonsaiWorldStateLayerStorage extends BonsaiSnapshotWorldStateKeyValueStorage - implements BonsaiStorageSubscriber { + implements DiffBasedLayeredWorldStateKeyValueStorage, StorageSubscriber { public BonsaiWorldStateLayerStorage(final BonsaiWorldStateKeyValueStorage parent) { this( - new LayeredKeyValueStorage(parent.composedWorldStateStorage), - parent.trieLogStorage, + new LayeredKeyValueStorage(parent.getComposedWorldStateStorage()), + parent.getTrieLogStorage(), parent, - parent.metricsSystem); + parent.getMetricsSystem()); } public BonsaiWorldStateLayerStorage( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FlatDbStrategy.java similarity index 99% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FlatDbStrategy.java index 6c1c918172b..522b943a543 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java similarity index 98% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java index 1885069e3d4..ec3b6d6e1c4 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java similarity index 98% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java index 632f433f9b0..dd785badd64 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java similarity index 92% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java index 36a6510c91e..42eec39ae0d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; @@ -23,7 +23,8 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.plugin.data.BlockHeader; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; @@ -150,7 +151,7 @@ public static TrieLogLayer readFrom(final RLPInput input) { final TrieLogLayer newLayer = new TrieLogLayer(); input.enterList(); - newLayer.blockHash = Hash.wrap(input.readBytes32()); + newLayer.setBlockHash(Hash.wrap(input.readBytes32())); while (!input.isEndOfCurrentList()) { input.enterList(); @@ -164,7 +165,7 @@ public static TrieLogLayer readFrom(final RLPInput input) { final StateTrieAccountValue newValue = nullOrValue(input, StateTrieAccountValue::readFrom); final boolean isCleared = getOptionalIsCleared(input); input.leaveList(); - newLayer.accounts.put(address, new BonsaiValue<>(oldValue, newValue, isCleared)); + newLayer.getAccountChanges().put(address, new DiffBasedValue<>(oldValue, newValue, isCleared)); } if (input.nextIsNull()) { @@ -175,13 +176,13 @@ public static TrieLogLayer readFrom(final RLPInput input) { final Bytes newCode = nullOrValue(input, RLPInput::readBytes); final boolean isCleared = getOptionalIsCleared(input); input.leaveList(); - newLayer.code.put(address, new BonsaiValue<>(oldCode, newCode, isCleared)); + newLayer.getCodeChanges().put(address, new DiffBasedValue<>(oldCode, newCode, isCleared)); } if (input.nextIsNull()) { input.skipNext(); } else { - final Map> storageChanges = new TreeMap<>(); + final Map> storageChanges = new TreeMap<>(); input.enterList(); while (!input.isEndOfCurrentList()) { input.enterList(); @@ -190,11 +191,11 @@ public static TrieLogLayer readFrom(final RLPInput input) { final UInt256 oldValue = nullOrValue(input, RLPInput::readUInt256Scalar); final UInt256 newValue = nullOrValue(input, RLPInput::readUInt256Scalar); final boolean isCleared = getOptionalIsCleared(input); - storageChanges.put(storageSlotKey, new BonsaiValue<>(oldValue, newValue, isCleared)); + storageChanges.put(storageSlotKey, new DiffBasedValue<>(oldValue, newValue, isCleared)); input.leaveList(); } input.leaveList(); - newLayer.storage.put(address, storageChanges); + newLayer.getStorageChanges().put(address, storageChanges); } // TODO add trie nodes diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java similarity index 57% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java index 7e3736dc28c..f24f199f914 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java @@ -14,39 +14,33 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView.encodeTrieValue; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView.encodeTrieValue; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.NodeLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator.StorageConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.internal.EvmConfiguration; -import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import org.hyperledger.besu.plugin.services.exception.StorageException; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; @@ -60,24 +54,10 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class BonsaiWorldState - implements MutableWorldState, BonsaiWorldView, BonsaiStorageSubscriber { +public class BonsaiWorldState extends DiffBasedWorldState { - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldState.class); - - protected BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; - - protected final CachedMerkleTrieLoader cachedMerkleTrieLoader; - protected final CachedWorldStorageManager cachedWorldStorageManager; - protected final TrieLogManager trieLogManager; - private BonsaiWorldStateUpdateAccumulator accumulator; - - protected Hash worldStateRootHash; - Hash worldStateBlockHash; - private boolean isFrozen; + protected final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader; public BonsaiWorldState( final BonsaiWorldStateProvider archive, @@ -93,90 +73,33 @@ public BonsaiWorldState( protected BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final CachedWorldStorageManager cachedWorldStorageManager, + final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader, + final BonsaiCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final EvmConfiguration evmConfiguration) { - this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.worldStateRootHash = - Hash.wrap( - Bytes32.wrap( - worldStateKeyValueStorage.getWorldStateRootHash().orElse(getEmptyTrieHash()))); - this.worldStateBlockHash = - Hash.wrap( - Bytes32.wrap(worldStateKeyValueStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); - this.accumulator = + super(worldStateKeyValueStorage, cachedWorldStorageManager, trieLogManager); + this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; + this.setAccumulator( new BonsaiWorldStateUpdateAccumulator( this, (addr, value) -> cachedMerkleTrieLoader.preLoadAccount( - getWorldStateStorage(), worldStateRootHash, addr), + worldStateKeyValueStorage, worldStateRootHash, addr), (addr, value) -> - cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), - evmConfiguration); - this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; - this.cachedWorldStorageManager = cachedWorldStorageManager; - this.trieLogManager = trieLogManager; - } - - /** - * Having a protected method to override the accumulator solves the chicken-egg problem of needing - * a worldstate reference (this) when construction the Accumulator. - * - * @param accumulator accumulator to use. - */ - protected void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) { - this.accumulator = accumulator; - } - - /** - * Returns the world state block hash of this world state - * - * @return the world state block hash. - */ - public Hash getWorldStateBlockHash() { - return worldStateBlockHash; - } - - /** - * Returns the world state root hash of this world state - * - * @return the world state root hash. - */ - public Hash getWorldStateRootHash() { - return worldStateRootHash; - } - - @Override - public boolean isPersisted() { - return isPersisted(worldStateKeyValueStorage); - } - - private boolean isPersisted(final WorldStateKeyValueStorage worldStateKeyValueStorage) { - return !(worldStateKeyValueStorage instanceof BonsaiSnapshotWorldStateKeyValueStorage); - } - - @Override - public Optional getCode(@Nonnull final Address address, final Hash codeHash) { - return worldStateKeyValueStorage.getCode(codeHash, address.addressHash()); - } - - /** - * Reset the worldState to this block header - * - * @param blockHeader block to use - */ - public void resetWorldStateTo(final BlockHeader blockHeader) { - worldStateBlockHash = blockHeader.getBlockHash(); - worldStateRootHash = blockHeader.getStateRoot(); + cachedMerkleTrieLoader.preLoadStorageSlot(worldStateKeyValueStorage, addr, value), + evmConfiguration)); } @Override - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { - return worldStateKeyValueStorage; + protected Hash calculateRootHash( + final Optional maybeStateUpdater, + final DiffBasedWorldStateUpdateAccumulator worldStateUpdater) { + return internalCalculateRootHash( + maybeStateUpdater.map(BonsaiWorldStateKeyValueStorage.Updater.class::cast), + (BonsaiWorldStateUpdateAccumulator) worldStateUpdater); } - private Hash calculateRootHash( + private Hash internalCalculateRootHash( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { @@ -184,7 +107,7 @@ private Hash calculateRootHash( // This must be done before updating the accounts so // that we can get the storage state hash - Stream>>> + Stream>>> storageStream = worldStateUpdater.getStorageToUpdate().entrySet().stream(); if (maybeStateUpdater.isEmpty()) { storageStream = @@ -203,7 +126,7 @@ private Hash calculateRootHash( createTrie( (location, hash) -> cachedMerkleTrieLoader.getAccountStateTrieNode( - worldStateKeyValueStorage, location, hash), + getWorldStateStorage(), location, hash), worldStateRootHash); // for manicured tries and composting, collect branches here (not implemented) @@ -228,10 +151,10 @@ private void updateTheAccounts( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater, final StoredMerklePatriciaTrie accountTrie) { - for (final Map.Entry> accountUpdate : + for (final Map.Entry> accountUpdate : worldStateUpdater.getAccountsToUpdate().entrySet()) { final Bytes accountKey = accountUpdate.getKey(); - final BonsaiValue bonsaiValue = accountUpdate.getValue(); + final DiffBasedValue bonsaiValue = accountUpdate.getValue(); final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); try { if (updatedAccount == null) { @@ -260,7 +183,7 @@ private void updateCode( final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { maybeStateUpdater.ifPresent( bonsaiUpdater -> { - for (final Map.Entry> codeUpdate : + for (final Map.Entry> codeUpdate : worldStateUpdater.getCodeToUpdate().entrySet()) { final Bytes updatedCode = codeUpdate.getValue().getUpdated(); final Hash accountHash = codeUpdate.getKey().addressHash(); @@ -276,12 +199,12 @@ private void updateCode( private void updateAccountStorageState( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater, - final Map.Entry>> + final Map.Entry>> storageAccountUpdate) { final Address updatedAddress = storageAccountUpdate.getKey(); final Hash updatedAddressHash = updatedAddress.addressHash(); if (worldStateUpdater.getAccountsToUpdate().containsKey(updatedAddress)) { - final BonsaiValue accountValue = + final DiffBasedValue accountValue = worldStateUpdater.getAccountsToUpdate().get(updatedAddress); final BonsaiAccount accountOriginal = accountValue.getPrior(); final Hash storageRoot = @@ -293,11 +216,11 @@ private void updateAccountStorageState( createTrie( (location, key) -> cachedMerkleTrieLoader.getAccountStorageTrieNode( - worldStateKeyValueStorage, updatedAddressHash, location, key), + getWorldStateStorage(), updatedAddressHash, location, key), storageRoot); // for manicured tries and composting, collect branches here (not implemented) - for (final Map.Entry> storageUpdate : + for (final Map.Entry> storageUpdate : storageAccountUpdate.getValue().entrySet()) { final Hash slotHash = storageUpdate.getKey().getSlotHash(); final UInt256 updatedStorage = storageUpdate.getValue().getUpdated(); @@ -348,7 +271,7 @@ private void clearStorage( for (final Address address : worldStateUpdater.getStorageToClear()) { // because we are clearing persisted values we need the account root as persisted final BonsaiAccount oldAccount = - worldStateKeyValueStorage + getWorldStateStorage() .getAccount(address.addressHash()) .map( bytes -> BonsaiAccount.fromRLP(BonsaiWorldState.this, address, bytes, true)) @@ -389,165 +312,27 @@ private void clearStorage( } @Override - public void persist(final BlockHeader blockHeader) { - final Optional maybeBlockHeader = Optional.ofNullable(blockHeader); - LOG.atDebug() - .setMessage("Persist world state for block {}") - .addArgument(maybeBlockHeader) - .log(); - - final BonsaiWorldStateUpdateAccumulator localCopy = accumulator.copy(); - - boolean success = false; - - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = - worldStateKeyValueStorage.updater(); - Runnable saveTrieLog = () -> {}; - - try { - final Hash newWorldStateRootHash = - calculateRootHash(isFrozen ? Optional.empty() : Optional.of(stateUpdater), accumulator); - // if we are persisted with a block header, and the prior state is the parent - // then persist the TrieLog for that transition. - // If specified but not a direct descendant simply store the new block hash. - if (blockHeader != null) { - verifyWorldStateRoot(newWorldStateRootHash, blockHeader); - saveTrieLog = - () -> { - trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); - // not save a frozen state in the cache - if (!isFrozen) { - cachedWorldStorageManager.addCachedLayer(blockHeader, newWorldStateRootHash, this); - } - }; - - stateUpdater - .getWorldStateTransaction() - .put(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); - worldStateBlockHash = blockHeader.getHash(); - } else { - stateUpdater.getWorldStateTransaction().remove(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY); - worldStateBlockHash = null; - } - - stateUpdater - .getWorldStateTransaction() - .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); - worldStateRootHash = newWorldStateRootHash; - success = true; - } finally { - if (success) { - stateUpdater.commit(); - accumulator.reset(); - saveTrieLog.run(); - } else { - stateUpdater.rollback(); - accumulator.reset(); - } - } - } - - protected void verifyWorldStateRoot(final Hash calculatedStateRoot, final BlockHeader header) { - if (!calculatedStateRoot.equals(header.getStateRoot())) { - throw new RuntimeException( - "World State Root does not match expected value, header " - + header.getStateRoot().toHexString() - + " calculated " - + calculatedStateRoot.toHexString()); - } - } - - @Override - public WorldUpdater updater() { - return accumulator; - } - - @Override - public Hash rootHash() { - if (isFrozen && accumulator.isAccumulatorStateChanged()) { - worldStateRootHash = calculateRootHash(Optional.empty(), accumulator.copy()); - accumulator.resetAccumulatorStateChanged(); - } - return Hash.wrap(worldStateRootHash); - } - - static final KeyValueStorageTransaction noOpTx = - new KeyValueStorageTransaction() { - - @Override - public void put(final byte[] key, final byte[] value) { - // no-op - } - - @Override - public void remove(final byte[] key) { - // no-op - } - - @Override - public void commit() throws StorageException { - // no-op - } - - @Override - public void rollback() { - // no-op - } - }; - - static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = - new SegmentedKeyValueStorageTransaction() { - - @Override - public void put( - final SegmentIdentifier segmentIdentifier, final byte[] key, final byte[] value) { - // no-op - } - - @Override - public void remove(final SegmentIdentifier segmentIdentifier, final byte[] key) { - // no-op - } - - @Override - public void commit() throws StorageException { - // no-op - } - - @Override - public void rollback() { - // no-op - } - }; - - @Override - public Hash frontierRootHash() { - return calculateRootHash( - Optional.of( - new BonsaiWorldStateKeyValueStorage.Updater( - noOpSegmentedTx, noOpTx, worldStateKeyValueStorage.getFlatDbStrategy())), - accumulator.copy()); - } - - public Hash blockHash() { - return worldStateBlockHash; - } - - @Override - public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { - throw new RuntimeException("Bonsai Tries do not provide account streaming."); + public MutableWorldState freeze() { + this.isFrozen = true; + this.worldStateKeyValueStorage = new BonsaiWorldStateLayerStorage(getWorldStateStorage()); + return this; } @Override public Account get(final Address address) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getAccount(address.addressHash()) .map(bytes -> BonsaiAccount.fromRLP(accumulator, address, bytes, true)) .orElse(null); } + @Override + public Optional getCode(@Nonnull final Address address, final Hash codeHash) { + return worldStateKeyValueStorage.getCode(codeHash, address.addressHash()); + } + protected Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { - return worldStateKeyValueStorage.getAccountStateTrieNode(location, nodeHash); + return getWorldStateStorage().getAccountStateTrieNode(location, nodeHash); } protected void writeTrieNode( @@ -560,7 +345,7 @@ protected void writeTrieNode( protected Optional getStorageTrieNode( final Hash accountHash, final Bytes location, final Bytes32 nodeHash) { - return worldStateKeyValueStorage.getAccountStorageTrieNode(accountHash, location, nodeHash); + return getWorldStateStorage().getAccountStorageTrieNode(accountHash, location, nodeHash); } private void writeStorageTrieNode( @@ -581,7 +366,7 @@ public UInt256 getStorageValue(final Address address, final UInt256 storageKey) @Override public Optional getStorageValueByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getStorageValueByStorageSlotKey(address.addressHash(), storageSlotKey) .map(UInt256::fromBytes); } @@ -590,7 +375,7 @@ public Optional getStorageValueByStorageSlotKey( final Supplier> storageRootSupplier, final Address address, final StorageSlotKey storageSlotKey) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getStorageValueByStorageSlotKey(storageRootSupplier, address.addressHash(), storageSlotKey) .map(UInt256::fromBytes); } @@ -608,50 +393,18 @@ public Map getAllAccountStorage(final Address address, final Has return storageTrie.entriesFrom(Bytes32.ZERO, Integer.MAX_VALUE); } - @Override - public MutableWorldState freeze() { - this.isFrozen = true; - this.worldStateKeyValueStorage = new BonsaiWorldStateLayerStorage(worldStateKeyValueStorage); - return this; - } - private StoredMerklePatriciaTrie createTrie( final NodeLoader nodeLoader, final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( nodeLoader, rootHash, Function.identity(), Function.identity()); } - @Override - public void close() { - try { - if (!isPersisted()) { - this.worldStateKeyValueStorage.close(); - if (isFrozen) { - closeFrozenStorage(); - } - } - } catch (Exception e) { - // no op - } - } - - private void closeFrozenStorage() { - try { - final BonsaiWorldStateLayerStorage worldStateLayerStorage = - (BonsaiWorldStateLayerStorage) worldStateKeyValueStorage; - if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { - worldStateLayerStorage.getParentWorldStateStorage().close(); - } - } catch (Exception e) { - // no op - } - } - protected Hash hashAndSavePreImage(final Bytes value) { // by default do not save has preImages return Hash.hash(value); } + @Override protected Hash getEmptyTrieHash() { return Hash.EMPTY_TRIE_HASH; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java new file mode 100644 index 00000000000..0bb12ad7de8 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -0,0 +1,98 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview; + +import org.hyperledger.besu.datatypes.AccountValue; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; + +public class BonsaiWorldStateUpdateAccumulator + extends DiffBasedWorldStateUpdateAccumulator { + public BonsaiWorldStateUpdateAccumulator( + final DiffBasedWorldView world, + final Consumer> accountPreloader, + final Consumer storagePreloader, + final EvmConfiguration evmConfiguration) { + super(world, accountPreloader, storagePreloader, evmConfiguration); + } + + @Override + public DiffBasedWorldStateUpdateAccumulator copy() { + final BonsaiWorldStateUpdateAccumulator copy = + new BonsaiWorldStateUpdateAccumulator( + wrappedWorldView(), + getAccountPreloader(), + getStoragePreloader(), + getEvmConfiguration()); + copy.cloneFromUpdater(this); + return copy; + } + + @Override + protected BonsaiAccount copyAccount(final BonsaiAccount account) { + return new BonsaiAccount(account); + } + + @Override + protected BonsaiAccount copyAccount( + final BonsaiAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + return new BonsaiAccount(toCopy, context, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable) { + return new BonsaiAccount(context, address, stateTrieAccount, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash storageRoot, + final Hash codeHash, + final boolean mutable) { + return new BonsaiAccount( + context, address, addressHash, nonce, balance, storageRoot, codeHash, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, final UpdateTrackingAccount tracked) { + return new BonsaiAccount(context, tracked); + } + + @Override + protected void assertCloseEnoughForDiffing(final BonsaiAccount source, final AccountValue account, final String context) { + BonsaiAccount.assertCloseEnoughForDiffing(source,account,context); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java new file mode 100644 index 00000000000..54efcf0d9fb --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java @@ -0,0 +1,219 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.common; + +import org.hyperledger.besu.datatypes.AccountValue; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.evm.ModificationNotAllowedException; +import org.hyperledger.besu.evm.account.MutableAccount; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.Map; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; + +public abstract class DiffBasedAccount implements MutableAccount, AccountValue { + protected final DiffBasedWorldView context; + protected boolean immutable; + protected final Address address; + protected final Hash addressHash; + protected Hash codeHash; + protected long nonce; + protected Wei balance; + protected Bytes code; + + private final String stack; + protected final Map updatedStorage = new HashMap<>(); + + public DiffBasedAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash codeHash, + final boolean mutable) { + this.context = context; + this.address = address; + this.addressHash = addressHash; + this.nonce = nonce; + this.balance = balance; + this.codeHash = codeHash; + + this.immutable = mutable; + + StringWriter sw = new StringWriter(); + new Exception().printStackTrace(new PrintWriter(sw)); + stack = sw.toString(); + } + + public DiffBasedAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable) { + this( + context, + address, + address.addressHash(), + stateTrieAccount.getNonce(), + stateTrieAccount.getBalance(), + stateTrieAccount.getCodeHash(), + mutable); + } + + public DiffBasedAccount( + final DiffBasedAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + this.context = context; + this.address = toCopy.address; + this.addressHash = toCopy.addressHash; + this.nonce = toCopy.nonce; + this.balance = toCopy.balance; + this.codeHash = toCopy.codeHash; + this.code = toCopy.code; + updatedStorage.putAll(toCopy.updatedStorage); + + this.immutable = mutable; + + StringWriter sw = new StringWriter(); + new Exception().printStackTrace(new PrintWriter(sw)); + stack = sw.toString(); + } + + @Override + public Address getAddress() { + return address; + } + + @Override + public Hash getAddressHash() { + return addressHash; + } + + @Override + public long getNonce() { + return nonce; + } + + @Override + public void setNonce(final long value) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + nonce = value; + } + + @Override + public Wei getBalance() { + return balance; + } + + @Override + public void setBalance(final Wei value) { + if (immutable) { + System.out.println(stack); + throw new ModificationNotAllowedException(); + } + balance = value; + } + + @Override + public Bytes getCode() { + if (code == null) { + code = context.getCode(address, codeHash).orElse(Bytes.EMPTY); + } + return code; + } + + @Override + public void setCode(final Bytes code) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + this.code = code; + if (code == null || code.isEmpty()) { + this.codeHash = Hash.EMPTY; + } else { + this.codeHash = Hash.hash(code); + } + } + + @Override + public Hash getCodeHash() { + return codeHash; + } + + @Override + public UInt256 getStorageValue(final UInt256 key) { + return context.getStorageValue(address, key); + } + + @Override + public UInt256 getOriginalStorageValue(final UInt256 key) { + return context.getPriorStorageValue(address, key); + } + + public Bytes serializeAccount() { + final BytesValueRLPOutput out = new BytesValueRLPOutput(); + writeTo(out); + return out.encoded(); + } + + @Override + public void setStorageValue(final UInt256 key, final UInt256 value) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + updatedStorage.put(key, value); + } + + @Override + public void clearStorage() { + updatedStorage.clear(); + } + + @Override + public Map getUpdatedStorage() { + return updatedStorage; + } + + @Override + public void becomeImmutable() { + immutable = true; + } + + @Override + public String toString() { + return "AccountState{" + + "address=" + + address + + ", nonce=" + + nonce + + ", balance=" + + balance + + ", codeHash=" + + codeHash + + '}'; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java similarity index 83% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java index 5d14c882750..baf29c0749d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java @@ -14,25 +14,25 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.common; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; -public class BonsaiValue implements TrieLog.LogTuple { +public class DiffBasedValue implements TrieLog.LogTuple { private T prior; private T updated; private boolean cleared; - public BonsaiValue(final T prior, final T updated) { + public DiffBasedValue(final T prior, final T updated) { this.prior = prior; this.updated = updated; this.cleared = false; } - public BonsaiValue(final T prior, final T updated, final boolean cleared) { + public DiffBasedValue(final T prior, final T updated, final boolean cleared) { this.prior = prior; this.updated = updated; this.cleared = cleared; @@ -48,12 +48,12 @@ public T getUpdated() { return updated; } - public BonsaiValue setPrior(final T prior) { + public DiffBasedValue setPrior(final T prior) { this.prior = prior; return this; } - public BonsaiValue setUpdated(final T updated) { + public DiffBasedValue setUpdated(final T updated) { this.cleared = updated == null; this.updated = updated; return this; @@ -88,7 +88,7 @@ public boolean equals(final Object o) { if (o == null || getClass() != o.getClass()) { return false; } - BonsaiValue that = (BonsaiValue) o; + DiffBasedValue that = (DiffBasedValue) o; return new EqualsBuilder() .append(cleared, that.cleared) .append(prior, that.prior) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java new file mode 100644 index 00000000000..8ce83bca3b4 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java @@ -0,0 +1,11 @@ +package org.hyperledger.besu.ethereum.trie.diffbased.common; + +public interface StorageSubscriber { + default void onClearStorage() {} + + default void onClearFlatDatabaseStorage() {} + + default void onClearTrieLog() {} + + default void onCloseStorage() {} +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java similarity index 65% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java index 5728480562e..0f8021def81 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java @@ -12,15 +12,16 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.common.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -36,21 +37,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CachedWorldStorageManager - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { +public abstract class DiffBasedCachedWorldStorageManager implements StorageSubscriber { public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks - private static final Logger LOG = LoggerFactory.getLogger(CachedWorldStorageManager.class); + private static final Logger LOG = + LoggerFactory.getLogger(DiffBasedCachedWorldStorageManager.class); private final BonsaiWorldStateProvider archive; private final ObservableMetricsSystem metricsSystem; private final EvmConfiguration evmConfiguration; - private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; - private final Map cachedWorldStatesByHash; + private final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; + private final Map cachedWorldStatesByHash; - private CachedWorldStorageManager( + private DiffBasedCachedWorldStorageManager( final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, - final Map cachedWorldStatesByHash, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final Map cachedWorldStatesByHash, final ObservableMetricsSystem metricsSystem, final EvmConfiguration evmConfiguration) { worldStateKeyValueStorage.subscribe(this); @@ -61,9 +62,9 @@ private CachedWorldStorageManager( this.evmConfiguration = evmConfiguration; } - public CachedWorldStorageManager( + public DiffBasedCachedWorldStorageManager( final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final ObservableMetricsSystem metricsSystem) { this( archive, @@ -76,24 +77,23 @@ public CachedWorldStorageManager( public synchronized void addCachedLayer( final BlockHeader blockHeader, final Hash worldStateRootHash, - final BonsaiWorldState forWorldState) { - final Optional cachedBonsaiWorldView = + final DiffBasedWorldState forWorldState) { + final Optional cachedDiffBasedWorldView = Optional.ofNullable(this.cachedWorldStatesByHash.get(blockHeader.getBlockHash())); - if (cachedBonsaiWorldView.isPresent()) { + if (cachedDiffBasedWorldView.isPresent()) { // only replace if it is a layered storage if (forWorldState.isPersisted() - && cachedBonsaiWorldView.get().getWorldStateStorage() + && cachedDiffBasedWorldView.get().getWorldStateStorage() instanceof BonsaiWorldStateLayerStorage) { LOG.atDebug() .setMessage("updating layered world state for block {}, state root hash {}") .addArgument(blockHeader::toLogString) .addArgument(worldStateRootHash::toShortHexString) .log(); - cachedBonsaiWorldView + cachedDiffBasedWorldView .get() .updateWorldStateStorage( - new BonsaiSnapshotWorldStateKeyValueStorage( - forWorldState.getWorldStateStorage(), metricsSystem)); + createSnapshotKeyValueStorage(forWorldState.getWorldStateStorage(), metricsSystem)); } } else { LOG.atDebug() @@ -104,17 +104,18 @@ public synchronized void addCachedLayer( if (forWorldState.isPersisted()) { cachedWorldStatesByHash.put( blockHeader.getHash(), - new CachedBonsaiWorldView( + new DiffBasedCachedWorldView( blockHeader, - new BonsaiSnapshotWorldStateKeyValueStorage( + createSnapshotKeyValueStorage( forWorldState.getWorldStateStorage(), metricsSystem))); } else { // otherwise, add the layer to the cache cachedWorldStatesByHash.put( blockHeader.getHash(), - new CachedBonsaiWorldView( + new DiffBasedCachedWorldView( blockHeader, - ((BonsaiWorldStateLayerStorage) forWorldState.getWorldStateStorage()).clone())); + ((DiffBasedLayeredWorldStateKeyValueStorage) forWorldState.getWorldStateStorage()) + .clone())); } } scrubCachedLayers(blockHeader.getNumber()); @@ -134,15 +135,15 @@ private synchronized void scrubCachedLayers(final long newMaxHeight) { } } - public Optional getWorldState(final Hash blockHash) { + public Optional> getWorldState(final Hash blockHash) { if (cachedWorldStatesByHash.containsKey(blockHash)) { // return a new worldstate using worldstate storage and an isolated copy of the updater return Optional.ofNullable(cachedWorldStatesByHash.get(blockHash)) .map( cached -> - new BonsaiWorldState( + createWorldState( archive, - new BonsaiWorldStateLayerStorage(cached.getWorldStateStorage()), + createLayeredKeyValueStorage(cached.getWorldStateStorage()), evmConfiguration)); } LOG.atDebug() @@ -153,7 +154,7 @@ public Optional getWorldState(final Hash blockHash) { return Optional.empty(); } - public Optional getNearestWorldState(final BlockHeader blockHeader) { + public Optional> getNearestWorldState(final BlockHeader blockHeader) { LOG.atDebug() .setMessage("getting nearest worldstate for {}") .addArgument(blockHeader.toLogString()) @@ -161,7 +162,7 @@ public Optional getNearestWorldState(final BlockHeader blockHe return Optional.ofNullable( cachedWorldStatesByHash.get(blockHeader.getParentHash())) // search parent block - .map(CachedBonsaiWorldView::getWorldStateStorage) + .map(DiffBasedCachedWorldView::getWorldStateStorage) .or( () -> { // or else search the nearest state in the cache @@ -170,22 +171,22 @@ public Optional getNearestWorldState(final BlockHeader blockHe .addArgument(blockHeader.toLogString()) .log(); - final List cachedBonsaiWorldViews = + final List cachedDiffBasedWorldViews = new ArrayList<>(cachedWorldStatesByHash.values()); - return cachedBonsaiWorldViews.stream() + return cachedDiffBasedWorldViews.stream() .sorted( Comparator.comparingLong( view -> Math.abs(blockHeader.getNumber() - view.getBlockNumber()))) - .map(CachedBonsaiWorldView::getWorldStateStorage) + .map(DiffBasedCachedWorldView::getWorldStateStorage) .findFirst(); }) .map( storage -> - new BonsaiWorldState( // wrap the state in a layered worldstate - archive, new BonsaiWorldStateLayerStorage(storage), evmConfiguration)); + createWorldState( // wrap the state in a layered worldstate + archive, createLayeredKeyValueStorage(storage), evmConfiguration)); } - public Optional getHeadWorldState( + public Optional> getHeadWorldState( final Function> hashBlockHeaderFunction) { LOG.atDebug().setMessage("getting head worldstate").log(); @@ -199,7 +200,7 @@ public Optional getHeadWorldState( addCachedLayer( blockHeader, blockHeader.getStateRoot(), - new BonsaiWorldState(archive, rootWorldStateStorage, evmConfiguration)); + createWorldState(archive, rootWorldStateStorage, evmConfiguration)); return getWorldState(blockHeader.getHash()); }); } @@ -231,4 +232,16 @@ public void onClearTrieLog() { public void onCloseStorage() { this.cachedWorldStatesByHash.clear(); } + + public abstract DiffBasedWorldState createWorldState( + final BonsaiWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final EvmConfiguration evmConfiguration); + + public abstract DiffBasedWorldStateKeyValueStorage createLayeredKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage); + + public abstract DiffBasedWorldStateKeyValueStorage createSnapshotKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final ObservableMetricsSystem metricsSystem); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java similarity index 71% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java index 2b8fd69d114..951941f0ce1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java @@ -13,30 +13,30 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.common.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CachedBonsaiWorldView - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { - private BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; +public class DiffBasedCachedWorldView implements StorageSubscriber { + private DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; private final BlockHeader blockHeader; private long worldViewSubscriberId; - private static final Logger LOG = LoggerFactory.getLogger(CachedBonsaiWorldView.class); + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedCachedWorldView.class); - public CachedBonsaiWorldView( - final BlockHeader blockHeader, final BonsaiWorldStateKeyValueStorage worldView) { + public DiffBasedCachedWorldView( + final BlockHeader blockHeader, final DiffBasedWorldStateKeyValueStorage worldView) { this.blockHeader = blockHeader; this.worldStateKeyValueStorage = worldView; this.worldViewSubscriberId = worldStateKeyValueStorage.subscribe(this); } - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { return worldStateKeyValueStorage; } @@ -58,10 +58,10 @@ public synchronized void close() { } public synchronized void updateWorldStateStorage( - final BonsaiWorldStateKeyValueStorage newWorldStateStorage) { + final DiffBasedWorldStateKeyValueStorage newWorldStateStorage) { long newSubscriberId = newWorldStateStorage.subscribe(this); this.worldStateKeyValueStorage.unSubscribe(this.worldViewSubscriberId); - BonsaiWorldStateKeyValueStorage oldWorldStateStorage = this.worldStateKeyValueStorage; + final DiffBasedWorldStateKeyValueStorage oldWorldStateStorage = this.worldStateKeyValueStorage; this.worldStateKeyValueStorage = newWorldStateStorage; this.worldViewSubscriberId = newSubscriberId; try { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..a5b670a245b --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java @@ -0,0 +1,21 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; + +public interface DiffBasedLayeredWorldStateKeyValueStorage { + + DiffBasedWorldStateKeyValueStorage clone(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..2100243f7cf --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java @@ -0,0 +1,18 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; + +public interface DiffBasedSnapshotWorldStateKeyValueStorage {} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..9c5f0834824 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java @@ -0,0 +1,246 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.storage.StorageProvider; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; +import org.hyperledger.besu.metrics.ObservableMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; +import org.hyperledger.besu.util.Subscribers; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class DiffBasedWorldStateKeyValueStorage + implements WorldStateKeyValueStorage, AutoCloseable { + private static final Logger LOG = + LoggerFactory.getLogger(DiffBasedWorldStateKeyValueStorage.class); + + // 0x776f726c64526f6f74 + public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); + // 0x776f726c64426c6f636b48617368 + public static final byte[] WORLD_BLOCK_HASH_KEY = + "worldBlockHash".getBytes(StandardCharsets.UTF_8); + + private final AtomicBoolean shouldClose = new AtomicBoolean(false); + + protected final AtomicBoolean isClosed = new AtomicBoolean(false); + + protected final Subscribers subscribers = Subscribers.create(); + + protected final ObservableMetricsSystem metricsSystem; + + protected final SegmentedKeyValueStorage composedWorldStateStorage; + protected final KeyValueStorage trieLogStorage; + + public DiffBasedWorldStateKeyValueStorage( + final StorageProvider provider, final ObservableMetricsSystem metricsSystem) { + this.composedWorldStateStorage = + provider.getStorageBySegmentIdentifiers( + List.of( + ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)); + this.trieLogStorage = + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + this.metricsSystem = metricsSystem; + } + + public DiffBasedWorldStateKeyValueStorage( + final SegmentedKeyValueStorage composedWorldStateStorage, + final KeyValueStorage trieLogStorage, + final ObservableMetricsSystem metricsSystem) { + this.composedWorldStateStorage = composedWorldStateStorage; + this.trieLogStorage = trieLogStorage; + this.metricsSystem = metricsSystem; + } + + public abstract FlatDbMode getFlatDbMode(); + + public abstract FlatDbStrategy getFlatDbStrategy(); + + @Override + public abstract DataStorageFormat getDataStorageFormat(); + + public SegmentedKeyValueStorage getComposedWorldStateStorage() { + return composedWorldStateStorage; + } + + public KeyValueStorage getTrieLogStorage() { + return trieLogStorage; + } + + public Optional getTrieLog(final Hash blockHash) { + return trieLogStorage.get(blockHash.toArrayUnsafe()); + } + + public Stream streamTrieLogKeys(final int limit) { + return trieLogStorage.streamKeys().limit(limit); + } + + public Optional getStateTrieNode(final Bytes location) { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()) + .map(Bytes::wrap); + } + + public Optional getWorldStateRootHash() { + return composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).map(Bytes::wrap); + } + + public Optional getWorldStateBlockHash() { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY) + .map(Bytes32::wrap) + .map(Hash::wrap); + } + + public Map streamFlatAccounts( + final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { + return getFlatDbStrategy() + .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); + } + + public Map streamFlatStorages( + final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { + return getFlatDbStrategy() + .streamStorageFlatDatabase( + composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); + } + + public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY) + .map(Bytes32::wrap) + .map(hash -> hash.equals(rootHash) || trieLogStorage.containsKey(blockHash.toArrayUnsafe())) + .orElse(false); + } + + @Override + public void clear() { + subscribers.forEach(StorageSubscriber::onClearStorage); + getFlatDbStrategy().clearAll(composedWorldStateStorage); + composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE); + trieLogStorage.clear(); + } + + public void clearTrieLog() { + subscribers.forEach(StorageSubscriber::onClearTrieLog); + trieLogStorage.clear(); + } + + public void clearFlatDatabase() { + subscribers.forEach(StorageSubscriber::onClearFlatDatabaseStorage); + getFlatDbStrategy().resetOnResync(composedWorldStateStorage); + } + + @Override + public abstract Updater updater(); + + public boolean pruneTrieLog(final Hash blockHash) { + try { + return trieLogStorage.tryDelete(blockHash.toArrayUnsafe()); + } catch (Exception e) { + LOG.error("Error pruning trie log for block hash {}", blockHash, e); + return false; + } + } + + @Override + public synchronized void close() throws Exception { + // when the storage clears, close + shouldClose.set(true); + tryClose(); + } + + public synchronized long subscribe(final StorageSubscriber sub) { + if (isClosed.get()) { + throw new RuntimeException("Storage is marked to close or has already closed"); + } + return subscribers.subscribe(sub); + } + + public synchronized void unSubscribe(final long id) { + subscribers.unsubscribe(id); + try { + tryClose(); + } catch (Exception e) { + LOG.atWarn() + .setMessage("exception while trying to close : {}") + .addArgument(e::getMessage) + .log(); + } + } + + protected synchronized void tryClose() throws Exception { + if (shouldClose.get() && subscribers.getSubscriberCount() < 1) { + doClose(); + } + } + + public ObservableMetricsSystem getMetricsSystem() { + return metricsSystem; + } + + protected synchronized void doClose() throws Exception { + if (!isClosed.get()) { + // alert any subscribers we are closing: + subscribers.forEach(StorageSubscriber::onCloseStorage); + + // close all of the KeyValueStorages: + composedWorldStateStorage.close(); + trieLogStorage.close(); + + // set storage closed + isClosed.set(true); + } + } + + public interface Updater extends WorldStateKeyValueStorage.Updater { + + DiffBasedWorldStateKeyValueStorage.Updater saveWorldState( + final Bytes blockHash, final Bytes32 nodeHash, final Bytes node); + + SegmentedKeyValueStorageTransaction getWorldStateTransaction(); + + KeyValueStorageTransaction getTrieLogStorageTransaction(); + + @Override + void commit(); + + void rollback(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java similarity index 92% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java index 3e1387997b4..28edc5cc608 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java similarity index 79% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java index ff1e29c41b8..ff6ef226ef7 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import static com.google.common.base.Preconditions.checkState; @@ -22,7 +22,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.HashMap; @@ -49,21 +49,21 @@ public class TrieLogLayer implements TrieLog { protected Hash blockHash; protected Optional blockNumber = Optional.empty(); - Map> getAccounts() { + Map> getAccounts() { return accounts; } - Map> getCode() { + Map> getCode() { return code; } - Map>> getStorage() { + Map>> getStorage() { return storage; } - protected final Map> accounts; - protected final Map> code; - protected final Map>> storage; + protected final Map> accounts; + protected final Map> code; + protected final Map>> storage; protected boolean frozen = false; public TrieLogLayer() { @@ -104,14 +104,14 @@ public TrieLogLayer setBlockNumber(final long blockNumber) { public TrieLogLayer addAccountChange( final Address address, final AccountValue oldValue, final AccountValue newValue) { checkState(!frozen, "Layer is Frozen"); - accounts.put(address, new BonsaiValue<>(oldValue, newValue)); + accounts.put(address, new DiffBasedValue<>(oldValue, newValue)); return this; } public TrieLogLayer addCodeChange( final Address address, final Bytes oldValue, final Bytes newValue, final Hash blockHash) { checkState(!frozen, "Layer is Frozen"); - code.put(address, new BonsaiValue<>(oldValue, newValue, newValue == null)); + code.put(address, new DiffBasedValue<>(oldValue, newValue, newValue == null)); return this; } @@ -123,22 +123,22 @@ public TrieLogLayer addStorageChange( checkState(!frozen, "Layer is Frozen"); storage .computeIfAbsent(address, a -> new TreeMap<>()) - .put(slot, new BonsaiValue<>(oldValue, newValue)); + .put(slot, new DiffBasedValue<>(oldValue, newValue)); return this; } @Override - public Map> getAccountChanges() { + public Map> getAccountChanges() { return accounts; } @Override - public Map> getCodeChanges() { + public Map> getCodeChanges() { return code; } @Override - public Map>> getStorageChanges() { + public Map>> getStorageChanges() { return storage; } @@ -147,18 +147,18 @@ public boolean hasStorageChanges(final Address address) { } @Override - public Map> getStorageChanges(final Address address) { + public Map> getStorageChanges(final Address address) { return storage.getOrDefault(address, Map.of()); } @Override public Optional getPriorCode(final Address address) { - return Optional.ofNullable(code.get(address)).map(BonsaiValue::getPrior); + return Optional.ofNullable(code.get(address)).map(DiffBasedValue::getPrior); } @Override public Optional getCode(final Address address) { - return Optional.ofNullable(code.get(address)).map(BonsaiValue::getUpdated); + return Optional.ofNullable(code.get(address)).map(DiffBasedValue::getUpdated); } @Override @@ -166,7 +166,7 @@ public Optional getPriorStorageByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { return Optional.ofNullable(storage.get(address)) .map(i -> i.get(storageSlotKey)) - .map(BonsaiValue::getPrior); + .map(DiffBasedValue::getPrior); } @Override @@ -174,24 +174,24 @@ public Optional getStorageByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { return Optional.ofNullable(storage.get(address)) .map(i -> i.get(storageSlotKey)) - .map(BonsaiValue::getUpdated); + .map(DiffBasedValue::getUpdated); } @Override public Optional getPriorAccount(final Address address) { - return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getPrior); + return Optional.ofNullable(accounts.get(address)).map(DiffBasedValue::getPrior); } @Override public Optional getAccount(final Address address) { - return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getUpdated); + return Optional.ofNullable(accounts.get(address)).map(DiffBasedValue::getUpdated); } public String dump() { final StringBuilder sb = new StringBuilder(); sb.append("TrieLog{" + "blockHash=").append(blockHash).append(frozen).append('}'); sb.append("accounts\n"); - for (final Map.Entry> account : accounts.entrySet()) { + for (final Map.Entry> account : accounts.entrySet()) { sb.append(" : ").append(account.getKey()).append("\n"); if (Objects.equals(account.getValue().getPrior(), account.getValue().getUpdated())) { sb.append(" = ").append(account.getValue().getUpdated()).append("\n"); @@ -201,7 +201,7 @@ public String dump() { } } sb.append("code").append("\n"); - for (final Map.Entry> code : code.entrySet()) { + for (final Map.Entry> code : code.entrySet()) { sb.append(" : ").append(code.getKey()).append("\n"); if (Objects.equals(code.getValue().getPrior(), code.getValue().getUpdated())) { sb.append(" = ").append(code.getValue().getPrior()).append("\n"); @@ -211,10 +211,10 @@ public String dump() { } } sb.append("Storage").append("\n"); - for (final Map.Entry>> storage : + for (final Map.Entry>> storage : storage.entrySet()) { sb.append(" : ").append(storage.getKey()).append("\n"); - for (final Map.Entry> slot : + for (final Map.Entry> slot : storage.getValue().entrySet()) { final UInt256 originalValue = slot.getValue().getPrior(); final UInt256 updatedValue = slot.getValue().getUpdated(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java similarity index 88% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java index b546f208be1..1ac50435db3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java @@ -13,14 +13,15 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.services.TrieLogService; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; @@ -41,7 +42,7 @@ public class TrieLogManager { private static final Logger LOG = LoggerFactory.getLogger(TrieLogManager.class); public static final long LOG_RANGE_LIMIT = 1000; // restrict trielog range queries to 1k logs protected final Blockchain blockchain; - protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + protected final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; protected final long maxLayersToLoad; protected final Subscribers trieLogObservers = Subscribers.create(); @@ -51,7 +52,7 @@ public class TrieLogManager { public TrieLogManager( final Blockchain blockchain, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final long maxLayersToLoad, final BesuContext pluginContext, final TrieLogPruner trieLogPruner) { @@ -63,15 +64,15 @@ public TrieLogManager( } public synchronized void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, + final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // do not overwrite a trielog layer that already exists in the database. // if it's only in memory we need to save it // for example, in case of reorg we don't replace a trielog layer if (rootWorldStateStorage.getTrieLog(forBlockHeader.getHash()).isEmpty()) { - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater = forWorldState.getWorldStateStorage().updater(); boolean success = false; try { @@ -95,7 +96,7 @@ public synchronized void saveTrieLog( } private TrieLog prepareTrieLog( - final BlockHeader blockHeader, final BonsaiWorldStateUpdateAccumulator localUpdater) { + final BlockHeader blockHeader, final DiffBasedWorldStateUpdateAccumulator localUpdater) { LOG.atDebug() .setMessage("Adding layered world state for {}") .addArgument(blockHeader::toLogString) @@ -109,7 +110,7 @@ private void persistTrieLog( final BlockHeader blockHeader, final Hash worldStateRootHash, final TrieLog trieLog, - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater) { + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater) { LOG.atDebug() .setMessage("Persisting trie log for block hash {} and world state root {}") .addArgument(blockHeader::toLogString) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java similarity index 91% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java index 747a82e1621..c9478946183 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java @@ -13,13 +13,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import java.util.Comparator; import java.util.Optional; @@ -30,6 +29,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.TreeMultimap; import org.apache.tuweni.bytes.Bytes32; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +39,7 @@ public class TrieLogPruner { private final int pruningLimit; private final int loadingLimit; - private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + private final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; private final Blockchain blockchain; private final long numBlocksToRetain; private final boolean requireFinalizedBlock; @@ -48,7 +48,7 @@ public class TrieLogPruner { TreeMultimap.create(Comparator.reverseOrder(), Comparator.naturalOrder()); public TrieLogPruner( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final Blockchain blockchain, final long numBlocksToRetain, final int pruningLimit, @@ -91,7 +91,7 @@ private void preloadQueue() { } } - void addToPruneQueue(final long blockNumber, final Hash blockHash) { + public void addToPruneQueue(final long blockNumber, final Hash blockHash) { LOG.atTrace() .setMessage("adding trie log to queue for later pruning blockNumber {}; blockHash {}") .addArgument(blockNumber) @@ -100,7 +100,7 @@ void addToPruneQueue(final long blockNumber, final Hash blockHash) { trieLogBlocksAndForksByDescendingBlockNumber.put(blockNumber, blockHash); } - int pruneFromQueue() { + public int pruneFromQueue() { final long retainAboveThisBlock = blockchain.getChainHeadBlockNumber() - numBlocksToRetain; final Optional finalized = blockchain.getFinalized(); if (requireFinalizedBlock && finalized.isEmpty()) { @@ -168,7 +168,7 @@ public static TrieLogPruner noOpTrieLogPruner() { public static class NoOpTrieLogPruner extends TrieLogPruner { private NoOpTrieLogPruner( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final Blockchain blockchain, final long numBlocksToRetain, final int pruningLimit) { @@ -181,12 +181,12 @@ public void initialize() { } @Override - void addToPruneQueue(final long blockNumber, final Hash blockHash) { + public void addToPruneQueue(final long blockNumber, final Hash blockHash) { // no-op } @Override - int pruneFromQueue() { + public int pruneFromQueue() { // no-op return -1; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java new file mode 100644 index 00000000000..c4063e398dd --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java @@ -0,0 +1,352 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.util.Optional; +import java.util.stream.Stream; +import javax.annotation.Nonnull; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class DiffBasedWorldState< + WORLDSTATE_STORAGE extends DiffBasedWorldStateKeyValueStorage> + implements MutableWorldState, DiffBasedWorldView, StorageSubscriber { + + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedWorldState.class); + + protected WORLDSTATE_STORAGE worldStateKeyValueStorage; + protected final BonsaiCachedWorldStorageManager cachedWorldStorageManager; + protected final TrieLogManager trieLogManager; + protected BonsaiWorldStateUpdateAccumulator accumulator; + + protected Hash worldStateRootHash; + protected Hash worldStateBlockHash; + protected boolean isFrozen; + + protected DiffBasedWorldState( + final WORLDSTATE_STORAGE worldStateKeyValueStorage, + final BonsaiCachedWorldStorageManager cachedWorldStorageManager, + final TrieLogManager trieLogManager) { + this.worldStateKeyValueStorage = worldStateKeyValueStorage; + this.worldStateRootHash = + Hash.wrap( + Bytes32.wrap( + worldStateKeyValueStorage.getWorldStateRootHash().orElse(getEmptyTrieHash()))); + this.worldStateBlockHash = + Hash.wrap( + Bytes32.wrap(worldStateKeyValueStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); + this.cachedWorldStorageManager = cachedWorldStorageManager; + this.trieLogManager = trieLogManager; + } + + /** + * Having a protected method to override the accumulator solves the chicken-egg problem of needing + * a worldstate reference (this) when construction the Accumulator. + * + * @param accumulator accumulator to use. + */ + protected void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) { + this.accumulator = accumulator; + } + + /** + * Returns the world state block hash of this world state + * + * @return the world state block hash. + */ + public Hash getWorldStateBlockHash() { + return worldStateBlockHash; + } + + /** + * Returns the world state root hash of this world state + * + * @return the world state root hash. + */ + public Hash getWorldStateRootHash() { + return worldStateRootHash; + } + + @Override + public boolean isPersisted() { + return isPersisted(worldStateKeyValueStorage); + } + + private boolean isPersisted(final WorldStateKeyValueStorage worldStateKeyValueStorage) { + return !(worldStateKeyValueStorage instanceof BonsaiSnapshotWorldStateKeyValueStorage); + } + + /** + * Reset the worldState to this block header + * + * @param blockHeader block to use + */ + public void resetWorldStateTo(final BlockHeader blockHeader) { + worldStateBlockHash = blockHeader.getBlockHash(); + worldStateRootHash = blockHeader.getStateRoot(); + } + + @Override + public WORLDSTATE_STORAGE getWorldStateStorage() { + return worldStateKeyValueStorage; + } + + public BonsaiWorldStateUpdateAccumulator getAccumulator() { + return accumulator; + } + + @Override + public void persist(final BlockHeader blockHeader) { + final Optional maybeBlockHeader = Optional.ofNullable(blockHeader); + LOG.atDebug() + .setMessage("Persist world state for block {}") + .addArgument(maybeBlockHeader) + .log(); + + final DiffBasedWorldStateUpdateAccumulator localCopy = accumulator.copy(); + + boolean success = false; + + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater = + worldStateKeyValueStorage.updater(); + Runnable saveTrieLog = () -> {}; + + try { + final Hash newWorldStateRootHash = + calculateRootHash(isFrozen ? Optional.empty() : Optional.of(stateUpdater), accumulator); + // if we are persisted with a block header, and the prior state is the parent + // then persist the TrieLog for that transition. + // If specified but not a direct descendant simply store the new block hash. + if (blockHeader != null) { + verifyWorldStateRoot(newWorldStateRootHash, blockHeader); + saveTrieLog = + () -> { + trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); + // not save a frozen state in the cache + if (!isFrozen) { + cachedWorldStorageManager.addCachedLayer( + blockHeader, newWorldStateRootHash, (BonsaiWorldState) this); + } + }; + + stateUpdater + .getWorldStateTransaction() + .put( + TRIE_BRANCH_STORAGE, + BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY, + blockHeader.getHash().toArrayUnsafe()); + worldStateBlockHash = blockHeader.getHash(); + } else { + stateUpdater + .getWorldStateTransaction() + .remove(TRIE_BRANCH_STORAGE, BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY); + worldStateBlockHash = null; + } + + stateUpdater + .getWorldStateTransaction() + .put( + TRIE_BRANCH_STORAGE, + BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY, + newWorldStateRootHash.toArrayUnsafe()); + worldStateRootHash = newWorldStateRootHash; + success = true; + } finally { + if (success) { + stateUpdater.commit(); + accumulator.reset(); + saveTrieLog.run(); + } else { + stateUpdater.rollback(); + accumulator.reset(); + } + } + } + + protected void verifyWorldStateRoot(final Hash calculatedStateRoot, final BlockHeader header) { + if (!calculatedStateRoot.equals(header.getStateRoot())) { + throw new RuntimeException( + "World State Root does not match expected value, header " + + header.getStateRoot().toHexString() + + " calculated " + + calculatedStateRoot.toHexString()); + } + } + + @Override + public WorldUpdater updater() { + return accumulator; + } + + @Override + public Hash rootHash() { + if (isFrozen && accumulator.isAccumulatorStateChanged()) { + worldStateRootHash = calculateRootHash(Optional.empty(), accumulator.copy()); + accumulator.resetAccumulatorStateChanged(); + } + return Hash.wrap(worldStateRootHash); + } + + static final KeyValueStorageTransaction noOpTx = + new KeyValueStorageTransaction() { + + @Override + public void put(final byte[] key, final byte[] value) { + // no-op + } + + @Override + public void remove(final byte[] key) { + // no-op + } + + @Override + public void commit() throws StorageException { + // no-op + } + + @Override + public void rollback() { + // no-op + } + }; + + static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = + new SegmentedKeyValueStorageTransaction() { + + @Override + public void put( + final SegmentIdentifier segmentIdentifier, final byte[] key, final byte[] value) { + // no-op + } + + @Override + public void remove(final SegmentIdentifier segmentIdentifier, final byte[] key) { + // no-op + } + + @Override + public void commit() throws StorageException { + // no-op + } + + @Override + public void rollback() { + // no-op + } + }; + + @Override + public Hash frontierRootHash() { + return calculateRootHash( + Optional.of( + new BonsaiWorldStateKeyValueStorage.Updater( + noOpSegmentedTx, noOpTx, worldStateKeyValueStorage.getFlatDbStrategy())), + accumulator.copy()); + } + + public Hash blockHash() { + return worldStateBlockHash; + } + + @Override + public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { + throw new RuntimeException("Bonsai Tries do not provide account streaming."); + } + + @Override + public UInt256 getPriorStorageValue(final Address address, final UInt256 storageKey) { + return getStorageValue(address, storageKey); + } + + @Override + public void close() { + try { + if (!isPersisted()) { + this.worldStateKeyValueStorage.close(); + if (isFrozen) { + closeFrozenStorage(); + } + } + } catch (Exception e) { + // no op + } + } + + private void closeFrozenStorage() { + try { + final BonsaiWorldStateLayerStorage worldStateLayerStorage = + (BonsaiWorldStateLayerStorage) worldStateKeyValueStorage; + if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { + worldStateLayerStorage.getParentWorldStateStorage().close(); + } + } catch (Exception e) { + // no op + } + } + + @Override + public abstract MutableWorldState freeze(); + + @Override + public abstract Account get(final Address address); + + @Override + public abstract UInt256 getStorageValue(final Address address, final UInt256 storageKey); + + @Override + public abstract Optional getStorageValueByStorageSlotKey( + final Address address, final StorageSlotKey storageSlotKey); + + @Override + public abstract Optional getCode(@Nonnull final Address address, final Hash codeHash); + + protected abstract Hash calculateRootHash( + final Optional maybeStateUpdater, + final DiffBasedWorldStateUpdateAccumulator worldStateUpdater); + + protected abstract Hash getEmptyTrieHash(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java similarity index 87% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java index cd16e7dec8a..0009c1c98ed 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java @@ -14,13 +14,13 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.evm.worldstate.WorldView; @@ -31,7 +31,7 @@ import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -public interface BonsaiWorldView extends WorldView { +public interface DiffBasedWorldView extends WorldView { Optional getCode(Address address, final Hash codeHash); @@ -59,7 +59,7 @@ static Bytes encodeTrieValue(final Bytes bytes) { boolean isPersisted(); - BonsaiWorldStateKeyValueStorage getWorldStateStorage(); + DiffBasedWorldStateKeyValueStorage getWorldStateStorage(); WorldUpdater updater(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java similarity index 72% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java index 1c8cf159168..32c91366d3b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; @@ -23,9 +23,15 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.AccountConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -44,41 +50,39 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.function.Function; -import com.google.common.collect.ForwardingMap; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BonsaiWorldStateUpdateAccumulator - extends AbstractWorldUpdater - implements BonsaiWorldView, TrieLogAccumulator { +@SuppressWarnings("unchecked") +public abstract class DiffBasedWorldStateUpdateAccumulator + extends AbstractWorldUpdater + implements DiffBasedWorldView, TrieLogAccumulator { private static final Logger LOG = - LoggerFactory.getLogger(BonsaiWorldStateUpdateAccumulator.class); - private final Consumer> accountPreloader; + LoggerFactory.getLogger(DiffBasedWorldStateUpdateAccumulator.class); + private final Consumer> accountPreloader; private final Consumer storagePreloader; - private final AccountConsumingMap> accountsToUpdate; - private final Map> codeToUpdate = new ConcurrentHashMap<>(); + private final AccountConsumingMap> accountsToUpdate; + private final Map> codeToUpdate = new ConcurrentHashMap<>(); private final Set

storageToClear = Collections.synchronizedSet(new HashSet<>()); private final EvmConfiguration evmConfiguration; // storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to // enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the // alternative was to keep a giant pre-image cache of the entire trie. - private final Map>> + private final Map>> storageToUpdate = new ConcurrentHashMap<>(); private boolean isAccumulatorStateChanged; - public BonsaiWorldStateUpdateAccumulator( - final BonsaiWorldView world, - final Consumer> accountPreloader, + public DiffBasedWorldStateUpdateAccumulator( + final DiffBasedWorldView world, + final Consumer> accountPreloader, final Consumer storagePreloader, final EvmConfiguration evmConfiguration) { super(world, evmConfiguration); @@ -89,15 +93,7 @@ public BonsaiWorldStateUpdateAccumulator( this.evmConfiguration = evmConfiguration; } - public BonsaiWorldStateUpdateAccumulator copy() { - final BonsaiWorldStateUpdateAccumulator copy = - new BonsaiWorldStateUpdateAccumulator( - wrappedWorldView(), accountPreloader, storagePreloader, evmConfiguration); - copy.cloneFromUpdater(this); - return copy; - } - - void cloneFromUpdater(final BonsaiWorldStateUpdateAccumulator source) { + protected void cloneFromUpdater(final DiffBasedWorldStateUpdateAccumulator source) { accountsToUpdate.putAll(source.getAccountsToUpdate()); codeToUpdate.putAll(source.codeToUpdate); storageToClear.addAll(source.storageToClear); @@ -107,14 +103,25 @@ void cloneFromUpdater(final BonsaiWorldStateUpdateAccumulator source) { this.isAccumulatorStateChanged = true; } + protected Consumer> getAccountPreloader() { + return accountPreloader; + } + + protected Consumer getStoragePreloader() { + return storagePreloader; + } + + protected EvmConfiguration getEvmConfiguration() { + return evmConfiguration; + } + @Override public Account get(final Address address) { return super.get(address); } @Override - protected UpdateTrackingAccount track( - final UpdateTrackingAccount account) { + protected UpdateTrackingAccount track(final UpdateTrackingAccount account) { return super.track(account); } @@ -125,10 +132,10 @@ public MutableAccount getAccount(final Address address) { @Override public MutableAccount createAccount(final Address address, final long nonce, final Wei balance) { - BonsaiValue bonsaiValue = accountsToUpdate.get(address); + DiffBasedValue bonsaiValue = accountsToUpdate.get(address); if (bonsaiValue == null) { - bonsaiValue = new BonsaiValue<>(null, null); + bonsaiValue = new DiffBasedValue<>(null, null); accountsToUpdate.put(address, bonsaiValue); } else if (bonsaiValue.getUpdated() != null) { if (bonsaiValue.getUpdated().isEmpty()) { @@ -138,8 +145,8 @@ public MutableAccount createAccount(final Address address, final long nonce, fin } } - final BonsaiAccount newAccount = - new BonsaiAccount( + final ACCOUNT newAccount = + createAccount( this, address, hashAndSavePreImage(address), @@ -153,12 +160,12 @@ public MutableAccount createAccount(final Address address, final long nonce, fin } @Override - public Map> getAccountsToUpdate() { + public Map> getAccountsToUpdate() { return accountsToUpdate; } @Override - public Map> getCodeToUpdate() { + public Map> getCodeToUpdate() { return codeToUpdate; } @@ -167,40 +174,41 @@ public Set
getStorageToClear() { } @Override - public Map>> + public Map>> getStorageToUpdate() { return storageToUpdate; } @Override - protected BonsaiAccount getForMutation(final Address address) { - return loadAccount(address, BonsaiValue::getUpdated); + protected ACCOUNT getForMutation(final Address address) { + return loadAccount(address, DiffBasedValue::getUpdated); } - protected BonsaiAccount loadAccount( - final Address address, - final Function, BonsaiAccount> bonsaiAccountFunction) { + protected ACCOUNT loadAccount( + final Address address, final Function, ACCOUNT> accountFunction) { try { - final BonsaiValue bonsaiValue = accountsToUpdate.get(address); + final DiffBasedValue bonsaiValue = accountsToUpdate.get(address); if (bonsaiValue == null) { final Account account; - if (wrappedWorldView() - instanceof BonsaiWorldStateUpdateAccumulator bonsaiWorldStateUpdateAccumulator) { - account = bonsaiWorldStateUpdateAccumulator.loadAccount(address, bonsaiAccountFunction); + if (wrappedWorldView() instanceof DiffBasedWorldStateUpdateAccumulator) { + final DiffBasedWorldStateUpdateAccumulator worldStateUpdateAccumulator = + (DiffBasedWorldStateUpdateAccumulator) wrappedWorldView(); + account = worldStateUpdateAccumulator.loadAccount(address, accountFunction); } else { account = wrappedWorldView().get(address); } - if (account instanceof BonsaiAccount bonsaiAccount) { - BonsaiAccount mutableAccount = new BonsaiAccount(bonsaiAccount, this, true); - accountsToUpdate.put(address, new BonsaiValue<>(bonsaiAccount, mutableAccount)); + if (account instanceof DiffBasedAccount diffBasedAccount) { + ACCOUNT mutableAccount = copyAccount((ACCOUNT) diffBasedAccount, this, true); + accountsToUpdate.put( + address, new DiffBasedValue<>((ACCOUNT) diffBasedAccount, mutableAccount)); return mutableAccount; } else { // add the empty read in accountsToUpdate - accountsToUpdate.put(address, new BonsaiValue<>(null, null)); + accountsToUpdate.put(address, new DiffBasedValue<>(null, null)); return null; } } else { - return bonsaiAccountFunction.apply(bonsaiValue); + return accountFunction.apply(bonsaiValue); } } catch (MerkleTrieException e) { // need to throw to trigger the heal @@ -228,12 +236,12 @@ public void revert() { public void commit() { this.isAccumulatorStateChanged = true; for (final Address deletedAddress : getDeletedAccounts()) { - final BonsaiValue accountValue = + final DiffBasedValue accountValue = accountsToUpdate.computeIfAbsent( deletedAddress, - __ -> loadAccountFromParent(deletedAddress, new BonsaiValue<>(null, null, true))); + __ -> loadAccountFromParent(deletedAddress, new DiffBasedValue<>(null, null, true))); storageToClear.add(deletedAddress); - final BonsaiValue codeValue = codeToUpdate.get(deletedAddress); + final DiffBasedValue codeValue = codeToUpdate.get(deletedAddress); if (codeValue != null) { codeValue.setUpdated(null).setCleared(); } else { @@ -241,26 +249,27 @@ public void commit() { .getCode( deletedAddress, Optional.ofNullable(accountValue) - .map(BonsaiValue::getPrior) - .map(BonsaiAccount::getCodeHash) + .map(DiffBasedValue::getPrior) + .map(DiffBasedAccount::getCodeHash) .orElse(Hash.EMPTY)) .ifPresent( deletedCode -> - codeToUpdate.put(deletedAddress, new BonsaiValue<>(deletedCode, null, true))); + codeToUpdate.put( + deletedAddress, new DiffBasedValue<>(deletedCode, null, true))); } // mark all updated storage as to be cleared - final Map> deletedStorageUpdates = + final Map> deletedStorageUpdates = storageToUpdate.computeIfAbsent( deletedAddress, k -> new StorageConsumingMap<>( deletedAddress, new ConcurrentHashMap<>(), storagePreloader)); - final Iterator>> iter = + final Iterator>> iter = deletedStorageUpdates.entrySet().iterator(); while (iter.hasNext()) { - final Map.Entry> updateEntry = iter.next(); - final BonsaiValue updatedSlot = updateEntry.getValue(); + final Map.Entry> updateEntry = iter.next(); + final DiffBasedValue updatedSlot = updateEntry.getValue(); if (updatedSlot.getPrior() == null || updatedSlot.getPrior().isZero()) { iter.remove(); } else { @@ -268,7 +277,7 @@ public void commit() { } } - final BonsaiAccount originalValue = accountValue.getPrior(); + final ACCOUNT originalValue = accountValue.getPrior(); if (originalValue != null) { // Enumerate and delete addresses not updated wrappedWorldView() @@ -279,7 +288,8 @@ public void commit() { new StorageSlotKey(Hash.wrap(keyHash), Optional.empty()); if (!deletedStorageUpdates.containsKey(storageSlotKey)) { final UInt256 value = UInt256.fromBytes(RLP.decodeOne(entryValue)); - deletedStorageUpdates.put(storageSlotKey, new BonsaiValue<>(value, null, true)); + deletedStorageUpdates.put( + storageSlotKey, new DiffBasedValue<>(value, null, true)); } }); } @@ -293,11 +303,11 @@ public void commit() { .forEach( tracked -> { final Address updatedAddress = tracked.getAddress(); - final BonsaiAccount updatedAccount; - final BonsaiValue updatedAccountValue = + final ACCOUNT updatedAccount; + final DiffBasedValue updatedAccountValue = accountsToUpdate.get(updatedAddress); - final Map> pendingStorageUpdates = + final Map> pendingStorageUpdates = storageToUpdate.computeIfAbsent( updatedAddress, k -> @@ -310,12 +320,12 @@ public void commit() { } if (tracked.getWrappedAccount() == null) { - updatedAccount = new BonsaiAccount(this, tracked); + updatedAccount = createAccount(this, tracked); tracked.setWrappedAccount(updatedAccount); if (updatedAccountValue == null) { - accountsToUpdate.put(updatedAddress, new BonsaiValue<>(null, updatedAccount)); + accountsToUpdate.put(updatedAddress, new DiffBasedValue<>(null, updatedAccount)); codeToUpdate.put( - updatedAddress, new BonsaiValue<>(null, updatedAccount.getCode())); + updatedAddress, new DiffBasedValue<>(null, updatedAccount.getCode())); } else { updatedAccountValue.setUpdated(updatedAccount); } @@ -333,17 +343,17 @@ public void commit() { } if (tracked.codeWasUpdated()) { - final BonsaiValue pendingCode = + final DiffBasedValue pendingCode = codeToUpdate.computeIfAbsent( updatedAddress, addr -> - new BonsaiValue<>( + new DiffBasedValue<>( wrappedWorldView() .getCode( addr, Optional.ofNullable(updatedAccountValue) - .map(BonsaiValue::getPrior) - .map(BonsaiAccount::getCodeHash) + .map(DiffBasedValue::getPrior) + .map(DiffBasedAccount::getCodeHash) .orElse(Hash.EMPTY)) .orElse(null), null)); @@ -368,11 +378,11 @@ public void commit() { final StorageSlotKey slotKey = new StorageSlotKey(slotHash, Optional.of(keyUInt)); final UInt256 value = storageUpdate.getValue(); - final BonsaiValue pendingValue = pendingStorageUpdates.get(slotKey); + final DiffBasedValue pendingValue = pendingStorageUpdates.get(slotKey); if (pendingValue == null) { pendingStorageUpdates.put( slotKey, - new BonsaiValue<>( + new DiffBasedValue<>( updatedAccount.getOriginalStorageValue(keyUInt), value)); } else { pendingValue.setUpdated(value); @@ -393,7 +403,7 @@ public void commit() { @Override public Optional getCode(final Address address, final Hash codeHash) { - final BonsaiValue localCode = codeToUpdate.get(address); + final DiffBasedValue localCode = codeToUpdate.get(address); if (localCode == null) { final Optional code = wrappedWorldView().getCode(address, codeHash); if (code.isEmpty() && !codeHash.equals(Hash.EMPTY)) { @@ -416,10 +426,10 @@ public UInt256 getStorageValue(final Address address, final UInt256 slotKey) { @Override public Optional getStorageValueByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { - final Map> localAccountStorage = + final Map> localAccountStorage = storageToUpdate.get(address); if (localAccountStorage != null) { - final BonsaiValue value = localAccountStorage.get(storageSlotKey); + final DiffBasedValue value = localAccountStorage.get(storageSlotKey); if (value != null) { return Optional.ofNullable(value.getUpdated()); } @@ -429,8 +439,8 @@ public Optional getStorageValueByStorageSlotKey( (wrappedWorldView() instanceof BonsaiWorldState bonsaiWorldState) ? bonsaiWorldState.getStorageValueByStorageSlotKey( () -> - Optional.ofNullable(loadAccount(address, BonsaiValue::getPrior)) - .map(BonsaiAccount::getStorageRoot), + Optional.ofNullable(loadAccount(address, DiffBasedValue::getPrior)) + .map(DiffBasedAccount::getStorageRoot), address, storageSlotKey) : wrappedWorldView().getStorageValueByStorageSlotKey(address, storageSlotKey); @@ -439,7 +449,8 @@ public Optional getStorageValueByStorageSlotKey( address, key -> new StorageConsumingMap<>(address, new ConcurrentHashMap<>(), storagePreloader)) - .put(storageSlotKey, new BonsaiValue<>(valueUInt.orElse(null), valueUInt.orElse(null))); + .put( + storageSlotKey, new DiffBasedValue<>(valueUInt.orElse(null), valueUInt.orElse(null))); return valueUInt; } catch (MerkleTrieException e) { @@ -454,10 +465,10 @@ public UInt256 getPriorStorageValue(final Address address, final UInt256 storage // TODO maybe log the read into the trie layer? StorageSlotKey storageSlotKey = new StorageSlotKey(hashAndSavePreImage(storageKey), Optional.of(storageKey)); - final Map> localAccountStorage = + final Map> localAccountStorage = storageToUpdate.get(address); if (localAccountStorage != null) { - final BonsaiValue value = localAccountStorage.get(storageSlotKey); + final DiffBasedValue value = localAccountStorage.get(storageSlotKey); if (value != null) { if (value.isCleared()) { return UInt256.ZERO; @@ -481,7 +492,7 @@ public UInt256 getPriorStorageValue(final Address address, final UInt256 storage @Override public Map getAllAccountStorage(final Address address, final Hash rootHash) { final Map results = wrappedWorldView().getAllAccountStorage(address, rootHash); - final StorageConsumingMap> bonsaiValueStorage = + final StorageConsumingMap> bonsaiValueStorage = storageToUpdate.get(address); if (bonsaiValueStorage != null) { // hash the key to match the implied storage interface of hashed slotKey @@ -497,7 +508,7 @@ public boolean isPersisted() { } @Override - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { return wrappedWorldView().getWorldStateStorage(); } @@ -549,7 +560,7 @@ private void rollAccountChange( // non-change, a cached read. return; } - BonsaiValue accountValue = accountsToUpdate.get(address); + DiffBasedValue accountValue = accountsToUpdate.get(address); if (accountValue == null) { accountValue = loadAccountFromParent(address, accountValue); } @@ -557,7 +568,7 @@ private void rollAccountChange( if (expectedValue == null && replacementValue != null) { accountsToUpdate.put( address, - new BonsaiValue<>(null, new BonsaiAccount(this, address, replacementValue, true))); + new DiffBasedValue<>(null, createAccount(this, address, replacementValue, true))); } else { throw new IllegalStateException( String.format( @@ -571,10 +582,9 @@ private void rollAccountChange( "Expected to create account, but the account exists. Address=%s", address)); } } else { - BonsaiAccount.assertCloseEnoughForDiffing( - accountValue.getUpdated(), - expectedValue, - "Address=" + address + " Prior Value in Rolling Change"); + assertCloseEnoughForDiffing(accountValue.getUpdated(), + expectedValue, + "Address=" + address + " Prior Value in Rolling Change"); } if (replacementValue == null) { if (accountValue.getPrior() == null) { @@ -585,19 +595,18 @@ private void rollAccountChange( accountValue.setUpdated(null); } } else { - accountValue.setUpdated( - new BonsaiAccount(wrappedWorldView(), address, replacementValue, true)); + accountValue.setUpdated(createAccount(wrappedWorldView(), address, replacementValue, true)); } } } - private BonsaiValue loadAccountFromParent( - final Address address, final BonsaiValue defaultValue) { + private DiffBasedValue loadAccountFromParent( + final Address address, final DiffBasedValue defaultValue) { try { final Account parentAccount = wrappedWorldView().get(address); - if (parentAccount instanceof BonsaiAccount account) { - final BonsaiValue loadedAccountValue = - new BonsaiValue<>(new BonsaiAccount(account), account); + if (parentAccount instanceof DiffBasedAccount account) { + final DiffBasedValue loadedAccountValue = + new DiffBasedValue<>(copyAccount((ACCOUNT) account), ((ACCOUNT) account)); accountsToUpdate.put(address, loadedAccountValue); return loadedAccountValue; } else { @@ -616,7 +625,7 @@ private void rollCodeChange( // non-change, a cached read. return; } - BonsaiValue codeValue = codeToUpdate.get(address); + DiffBasedValue codeValue = codeToUpdate.get(address); if (codeValue == null) { final Bytes storedCode = wrappedWorldView() @@ -624,14 +633,14 @@ private void rollCodeChange( address, Optional.ofNullable(expectedCode).map(Hash::hash).orElse(Hash.EMPTY)) .orElse(Bytes.EMPTY); if (!storedCode.isEmpty()) { - codeValue = new BonsaiValue<>(storedCode, storedCode); + codeValue = new DiffBasedValue<>(storedCode, storedCode); codeToUpdate.put(address, codeValue); } } if (codeValue == null) { if ((expectedCode == null || expectedCode.isEmpty()) && replacementCode != null) { - codeToUpdate.put(address, new BonsaiValue<>(null, replacementCode)); + codeToUpdate.put(address, new DiffBasedValue<>(null, replacementCode)); } else { throw new IllegalStateException( String.format( @@ -659,10 +668,10 @@ private void rollCodeChange( } } - private Map> maybeCreateStorageMap( - final Map> storageMap, final Address address) { + private Map> maybeCreateStorageMap( + final Map> storageMap, final Address address) { if (storageMap == null) { - final StorageConsumingMap> newMap = + final StorageConsumingMap> newMap = new StorageConsumingMap<>(address, new ConcurrentHashMap<>(), storagePreloader); storageToUpdate.put(address, newMap); return newMap; @@ -684,13 +693,13 @@ private void rollStorageChange( // corner case on deletes, non-change return; } - final Map> storageMap = storageToUpdate.get(address); - BonsaiValue slotValue = storageMap == null ? null : storageMap.get(storageSlotKey); + final Map> storageMap = storageToUpdate.get(address); + DiffBasedValue slotValue = storageMap == null ? null : storageMap.get(storageSlotKey); if (slotValue == null) { final Optional storageValue = wrappedWorldView().getStorageValueByStorageSlotKey(address, storageSlotKey); if (storageValue.isPresent()) { - slotValue = new BonsaiValue<>(storageValue.get(), storageValue.get()); + slotValue = new DiffBasedValue<>(storageValue.get(), storageValue.get()); storageToUpdate .computeIfAbsent( address, @@ -702,7 +711,7 @@ private void rollStorageChange( if (slotValue == null) { if ((expectedValue == null || expectedValue.isZero()) && replacementValue != null) { maybeCreateStorageMap(storageMap, address) - .put(storageSlotKey, new BonsaiValue<>(null, replacementValue)); + .put(storageSlotKey, new DiffBasedValue<>(null, replacementValue)); } else { throw new IllegalStateException( String.format( @@ -729,7 +738,7 @@ private void rollStorageChange( existingSlotValue == null ? "null" : existingSlotValue.toShortHexString())); } if (replacementValue == null && slotValue.getPrior() == null) { - final Map> thisStorageUpdate = + final Map> thisStorageUpdate = maybeCreateStorageMap(storageMap, address); thisStorageUpdate.remove(storageSlotKey); if (thisStorageUpdate.isEmpty()) { @@ -767,69 +776,37 @@ public void reset() { deletedAccounts.clear(); } - public static class AccountConsumingMap extends ForwardingMap { - - private final ConcurrentMap accounts; - private final Consumer consumer; - - public AccountConsumingMap( - final ConcurrentMap accounts, final Consumer consumer) { - this.accounts = accounts; - this.consumer = consumer; - } - - @Override - public T put(@NotNull final Address address, @NotNull final T value) { - consumer.process(address, value); - return accounts.put(address, value); - } - - public Consumer getConsumer() { - return consumer; - } - - @Override - protected Map delegate() { - return accounts; - } + protected Hash hashAndSavePreImage(final Bytes bytes) { + // by default do not save hash preImages + return Hash.hash(bytes); } - public static class StorageConsumingMap extends ForwardingMap { + public abstract DiffBasedWorldStateUpdateAccumulator copy(); - private final Address address; + protected abstract ACCOUNT copyAccount(final ACCOUNT account); - private final ConcurrentMap storages; - private final Consumer consumer; + protected abstract ACCOUNT copyAccount( + final ACCOUNT toCopy, final DiffBasedWorldView context, final boolean mutable); - public StorageConsumingMap( - final Address address, final ConcurrentMap storages, final Consumer consumer) { - this.address = address; - this.storages = storages; - this.consumer = consumer; - } - - @Override - public T put(@NotNull final K slotKey, @NotNull final T value) { - consumer.process(address, slotKey); - return storages.put(slotKey, value); - } + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable); - public Consumer getConsumer() { - return consumer; - } + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash storageRoot, + final Hash codeHash, + final boolean mutable); - @Override - protected Map delegate() { - return storages; - } - } + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, final UpdateTrackingAccount tracked); - public interface Consumer { - void process(final Address address, T value); - } - protected Hash hashAndSavePreImage(final Bytes bytes) { - // by default do not save hash preImages - return Hash.hash(bytes); - } + protected abstract void assertCloseEnoughForDiffing(final ACCOUNT source, final AccountValue account, final String context); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java new file mode 100644 index 00000000000..8fe1ab55bbf --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java @@ -0,0 +1,50 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +import com.google.common.collect.ForwardingMap; +import org.jetbrains.annotations.NotNull; + +public class AccountConsumingMap extends ForwardingMap { + + private final ConcurrentMap accounts; + private final Consumer consumer; + + public AccountConsumingMap(final ConcurrentMap accounts, final Consumer consumer) { + this.accounts = accounts; + this.consumer = consumer; + } + + @Override + public T put(@NotNull final Address address, @NotNull final T value) { + consumer.process(address, value); + return accounts.put(address, value); + } + + public Consumer getConsumer() { + return consumer; + } + + @Override + protected Map delegate() { + return accounts; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java new file mode 100644 index 00000000000..4d48a9ed707 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java @@ -0,0 +1,22 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; + +public interface Consumer { + void process(final Address address, T value); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java new file mode 100644 index 00000000000..52970208cd1 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java @@ -0,0 +1,54 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +import com.google.common.collect.ForwardingMap; +import org.jetbrains.annotations.NotNull; + +public class StorageConsumingMap extends ForwardingMap { + + private final Address address; + + private final ConcurrentMap storages; + private final Consumer consumer; + + public StorageConsumingMap( + final Address address, final ConcurrentMap storages, final Consumer consumer) { + this.address = address; + this.storages = storages; + this.consumer = consumer; + } + + @Override + public T put(@NotNull final K slotKey, @NotNull final T value) { + consumer.process(address, slotKey); + return storages.put(slotKey, value); + } + + public Consumer getConsumer() { + return consumer; + } + + @Override + protected Map delegate() { + return storages; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java index b97f79006ae..c68258ba7b9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java @@ -18,7 +18,8 @@ public enum DataStorageFormat { FOREST(1), // Original format. Store all tries - BONSAI(2); // New format. Store one trie, and trie logs to roll forward and backward. + BONSAI(2), // New format. Store one trie, and trie logs to roll forward and backward. + VERKLE(3); private final int databaseVersion; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java index 9f38a49c617..f57161b88ff 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.worldstate; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import java.util.Optional; diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index f9030e67b99..46c941be536 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -25,10 +25,10 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; @@ -96,8 +96,8 @@ public static BonsaiWorldStateProvider createBonsaiInMemoryWorldStateArchive( final Blockchain blockchain, final EvmConfiguration evmConfiguration) { final InMemoryKeyValueStorageProvider inMemoryKeyValueStorageProvider = new InMemoryKeyValueStorageProvider(); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = - new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); return new BonsaiWorldStateProvider( (BonsaiWorldStateKeyValueStorage) inMemoryKeyValueStorageProvider.createWorldStateStorage(DataStorageFormat.BONSAI), diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java index c549c66eb93..a04592d3093 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java index de077b5e039..84e5ffcf0d8 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java @@ -41,9 +41,9 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java index 3252d2b7f19..5bf9b2cb3b4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java @@ -65,9 +65,10 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -153,7 +154,7 @@ public void createStorage() { (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, blockchain, Optional.of(16L), - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), new NoOpMetricsSystem(), null, EvmConfiguration.DEFAULT, diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java index 4c7ee1641df..1bb993b072e 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java @@ -18,8 +18,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.BLOCKCHAIN; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; @@ -36,14 +36,15 @@ import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; @@ -77,7 +78,7 @@ class BonsaiWorldStateProviderTest { @Mock SegmentedKeyValueStorageTransaction segmentedKeyValueStorageTransaction; BonsaiWorldStateProvider bonsaiWorldStateArchive; - @Mock CachedWorldStorageManager cachedWorldStorageManager; + @Mock BonsaiCachedWorldStorageManager cachedWorldStorageManager; @Mock TrieLogManager trieLogManager; @BeforeEach @@ -108,7 +109,7 @@ void testGetMutableReturnPersistedStateWhenNeeded() { trieLogManager, new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); assertThat(bonsaiWorldStateArchive.getMutable(chainHead, true)) @@ -122,7 +123,7 @@ void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() { new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), blockchain, Optional.of(512L), - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), new NoOpMetricsSystem(), null, EvmConfiguration.DEFAULT, @@ -143,7 +144,7 @@ void testGetMutableWhenLoadLessThanLimitLayersBack() { trieLogManager, new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(511).buildHeader(); @@ -175,7 +176,7 @@ void testGetMutableWithStorageInconsistencyRollbackTheState() { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -201,7 +202,7 @@ void testGetMutableWithStorageConsistencyNotRollbackTheState() { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -238,7 +239,7 @@ void testGetMutableWithStorageConsistencyToRollbackAndRollForwardTheState() { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); // initial persisted state hash key @@ -278,7 +279,7 @@ void testGetMutableWithRollbackNotOverrideTrieLogLayer() { trieLogManager, new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem()), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); // initial persisted state hash key diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java index 1a9c4f326ac..50829d2faaa 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java @@ -26,8 +26,8 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; @@ -46,7 +46,7 @@ class CachedMerkleTrieLoaderTest { - private CachedMerkleTrieLoader merkleTrieLoader; + private BonsaiCachedMerkleTrieLoader merkleTrieLoader; private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private final BonsaiWorldStateKeyValueStorage inMemoryWorldState = Mockito.spy(new BonsaiWorldStateKeyValueStorage(storageProvider, new NoOpMetricsSystem())); @@ -64,7 +64,7 @@ public void setup() { TrieGenerator.generateTrie( worldStateStorageCoordinator, accounts.stream().map(Address::addressHash).collect(Collectors.toList())); - merkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + merkleTrieLoader = new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); } @Test diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java index f2dad3bc6ea..6f90ba22482 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java @@ -29,11 +29,12 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.log.LogsBloomFilter; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java index 54d9afa270d..159c3505b58 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java @@ -25,11 +25,12 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java index e40c88fd938..ef70a2817b7 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java @@ -16,7 +16,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -35,6 +35,7 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.StorageEntriesCollector; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java index c74fc7e1805..ba61789b8bf 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java @@ -24,6 +24,8 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java index d7d49bace04..5faef343b98 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import java.util.Optional; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java index a0065e2cb95..bedfb8e7cb8 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java @@ -16,16 +16,17 @@ package org.hyperledger.besu.ethereum.trie.bonsai.trielog; import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner.noOpTrieLogPruner; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner.noOpTrieLogPruner; import static org.mockito.Mockito.spy; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java index af5acbf18cf..ac5aab1d837 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java @@ -26,7 +26,8 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import java.util.Optional; import java.util.stream.Stream; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index 6aedc7b9b7f..f412a0f7385 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -34,7 +34,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java index 7a66d41b34b..a3959d3418d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java @@ -21,7 +21,7 @@ import org.hyperledger.besu.ethereum.eth.sync.TrailingPeerRequirements; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.TaskCollection; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java index 264b29017d7..e9fb43ffeaf 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.TrieNodeHealingRequest; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.exception.StorageException; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index fbff9061c57..2366045668b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -29,7 +29,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.AccountFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.StorageFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldDownloadState; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java index 2166ed7b55e..04acc3176b9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java @@ -27,7 +27,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.AccountRangeDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java index 4977d3b9431..a4776ced624 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.trie.NodeUpdater; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java index 751d8e3ac2d..30142785edc 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.NodeUpdater; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java index 428ab47189c..410741095a0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java index ac554439e91..c827855637e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java @@ -27,7 +27,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index 44786688f24..54a969fbc28 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -36,7 +36,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.ImmutableCheckpoint; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java index 564147bb06e..9fcb7a2e730 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.FastDownloaderFactory; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java index 34beae396f4..19f57749e60 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java @@ -34,7 +34,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.NodeDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java index 7e4594da59e..ccd826a530b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java @@ -26,7 +26,7 @@ import org.hyperledger.besu.ethereum.eth.manager.task.EthTask; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java index 7c0d546402d..2aead3a557a 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java @@ -22,7 +22,7 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java index ad4504cbf0b..caced0e850b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index 8444ef7f4e7..0dc1a340446 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -38,7 +38,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java index 433b5485ed0..f903d2fc4c1 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java @@ -29,7 +29,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java index 90339efb032..0ba70bdbd3b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java index c85d4f85b2c..cf0a285c78e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java @@ -22,7 +22,7 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java index b56053f54af..1d28a5a86aa 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java @@ -16,11 +16,12 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.apache.tuweni.bytes.Bytes; @@ -29,8 +30,8 @@ public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdate private final BonsaiPreImageProxy preImageProxy; public BonsaiReferenceTestUpdateAccumulator( - final BonsaiWorldView world, - final Consumer> accountPreloader, + final DiffBasedWorldView world, + final Consumer> accountPreloader, final Consumer storagePreloader, final BonsaiPreImageProxy preImageProxy, final EvmConfiguration evmConfiguration) { diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 3114d696597..0d4e3259d45 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -18,15 +18,16 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogAddedEvent; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogAddedEvent; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -51,8 +52,8 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState protected BonsaiReferenceTestWorldState( final BonsaiReferenceTestWorldStateStorage worldStateKeyValueStorage, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final CachedWorldStorageManager cachedWorldStorageManager, + final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader, + final BonsaiCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final BonsaiPreImageProxy preImageProxy, final EvmConfiguration evmConfiguration) { @@ -113,7 +114,8 @@ public static BonsaiReferenceTestWorldState create( final Map accounts, final EvmConfiguration evmConfiguration) { final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(metricsSystem); + final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(metricsSystem); final TrieLogManager trieLogManager = new NoOpTrieLogManager(); final BonsaiPreImageProxy preImageProxy = new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); @@ -150,7 +152,7 @@ public Stream streamAccounts(final Bytes32 startKeyHash, fina return this.refTestStorage.streamAccounts(this, startKeyHash, limit); } - static class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { + static class NoOpCachedWorldStorageManager extends BonsaiCachedWorldStorageManager { public NoOpCachedWorldStorageManager() { super( @@ -165,7 +167,7 @@ public NoOpCachedWorldStorageManager() { public void addCachedLayer( final BlockHeader blockHeader, final Hash worldStateRootHash, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // reference test world states are not cached } @@ -175,17 +177,17 @@ public boolean containWorldStateStorage(final Hash blockHash) { } @Override - public Optional getWorldState(final Hash blockHash) { + public Optional> getWorldState(final Hash blockHash) { return Optional.empty(); } @Override - public Optional getNearestWorldState(final BlockHeader blockHeader) { + public Optional> getNearestWorldState(final BlockHeader blockHeader) { return Optional.empty(); } @Override - public Optional getHeadWorldState( + public Optional> getHeadWorldState( final Function> hashBlockHeaderFunction) { return Optional.empty(); } @@ -205,10 +207,10 @@ public NoOpTrieLogManager() { @SuppressWarnings({"UnsynchronizedOverridesSynchronized", "squid:S3551"}) @Override public void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, + final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // notify trie log added observers, synchronously TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java index 68057523a08..0993d4425be 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java @@ -16,11 +16,11 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.evm.worldstate.WorldState; @@ -64,7 +64,7 @@ public NavigableMap storageEntriesFrom( } public Stream streamAccounts( - final BonsaiWorldView context, final Bytes32 startKeyHash, final int limit) { + final DiffBasedWorldView context, final Bytes32 startKeyHash, final int limit) { return streamFlatAccounts(startKeyHash, UInt256.MAX_VALUE, limit) .entrySet() // map back to addresses using preImage provider: From 06cda0014120e90c26733ee1cf3711efae811338 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Thu, 21 Dec 2023 10:26:02 +0100 Subject: [PATCH 08/59] finish diffbased refactor Signed-off-by: Karim Taam --- .../bonsai/BonsaiWorldStateProvider.java | 286 ++-------------- .../BonsaiCachedWorldStorageManager.java | 2 +- .../bonsai/trielog/TrieLogFactoryImpl.java | 4 +- .../bonsai/worldview/BonsaiWorldState.java | 15 +- .../BonsaiWorldStateUpdateAccumulator.java | 5 +- .../common/DiffBasedWorldStateProvider.java | 310 ++++++++++++++++++ .../DiffBasedCachedWorldStorageManager.java | 10 +- .../common/trielog/TrieLogManager.java | 2 +- .../common/trielog/TrieLogPruner.java | 2 +- .../common/worldview/DiffBasedWorldState.java | 21 +- .../DiffBasedWorldStateUpdateAccumulator.java | 12 +- .../bonsai/AbstractIsolationTests.java | 3 +- .../bonsai/BonsaiSnapshotIsolationTests.java | 2 +- .../bonsai/BonsaiWorldStateProviderTest.java | 5 +- .../bonsai/CachedMerkleTrieLoaderTest.java | 2 +- .../bonsai/LogRollingTests.java | 5 +- .../{ => diffbased}/bonsai/RollingImport.java | 5 +- .../BonsaiWorldStateKeyValueStorageTest.java | 3 +- .../bonsai/trielog/TrieLogFactoryTests.java | 3 +- .../common}/trielog/TrieLogLayerTests.java | 3 +- .../common}/trielog/TrieLogManagerTests.java | 3 +- .../common}/trielog/TrieLogPrunerTest.java | 3 +- .../BonsaiReferenceTestWorldState.java | 18 +- 23 files changed, 393 insertions(+), 331 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/AbstractIsolationTests.java (98%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/BonsaiSnapshotIsolationTests.java (99%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/BonsaiWorldStateProviderTest.java (99%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/CachedMerkleTrieLoaderTest.java (99%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/LogRollingTests.java (99%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/RollingImport.java (97%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java (99%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/trielog/TrieLogFactoryTests.java (95%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogLayerTests.java (97%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogManagerTests.java (95%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogPrunerTest.java (98%) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java index 6ca43b7c28c..93f6428e045 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java @@ -19,52 +19,33 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.proof.WorldStateProof; -import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.evm.internal.EvmConfiguration; -import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.BesuContext; -import org.hyperledger.besu.plugin.services.trielogs.TrieLog; -import java.util.ArrayList; import java.util.HashSet; -import java.util.List; import java.util.Optional; import java.util.Set; import java.util.function.Function; import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BonsaiWorldStateProvider implements WorldStateArchive { +public class BonsaiWorldStateProvider extends DiffBasedWorldStateProvider { private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateProvider.class); - - private final Blockchain blockchain; - - private final BonsaiCachedWorldStorageManager cachedWorldStorageManager; - private final TrieLogManager trieLogManager; - private final BonsaiWorldState persistedState; - private final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; private final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader; public BonsaiWorldStateProvider( @@ -76,27 +57,11 @@ public BonsaiWorldStateProvider( final BesuContext pluginContext, final EvmConfiguration evmConfiguration, final TrieLogPruner trieLogPruner) { - - this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.cachedWorldStorageManager = - new BonsaiCachedWorldStorageManager(this, worldStateKeyValueStorage, metricsSystem); - // TODO: de-dup constructors - this.trieLogManager = - new TrieLogManager( - blockchain, - worldStateKeyValueStorage, - maxLayersToLoad.orElse(BonsaiCachedWorldStorageManager.RETAINED_LAYERS), - pluginContext, - trieLogPruner); - this.blockchain = blockchain; + super(worldStateKeyValueStorage, blockchain, maxLayersToLoad, pluginContext, trieLogPruner); this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; - this.persistedState = new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration); - blockchain - .getBlockHeader(persistedState.getWorldStateBlockHash()) - .ifPresent( - blockHeader -> - this.cachedWorldStorageManager.addCachedLayer( - blockHeader, persistedState.getWorldStateRootHash(), persistedState)); + provideCachedWorldStorageManager( + new BonsaiCachedWorldStorageManager(this, worldStateKeyValueStorage, metricsSystem)); + loadPersistedState(new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration)); } @VisibleForTesting @@ -107,181 +72,19 @@ public BonsaiWorldStateProvider( final Blockchain blockchain, final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader, final EvmConfiguration evmConfiguration) { - this.cachedWorldStorageManager = cachedWorldStorageManager; - this.trieLogManager = trieLogManager; - this.blockchain = blockchain; - this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.persistedState = new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration); + super(worldStateKeyValueStorage, blockchain, trieLogManager); this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; - blockchain - .getBlockHeader(persistedState.getWorldStateBlockHash()) - .ifPresent( - blockHeader -> - this.cachedWorldStorageManager.addCachedLayer( - blockHeader, persistedState.getWorldStateRootHash(), persistedState)); - } - - @Override - public Optional get(final Hash rootHash, final Hash blockHash) { - return cachedWorldStorageManager - .getWorldState(blockHash) - .or( - () -> { - if (blockHash.equals(persistedState.blockHash())) { - return Optional.of(persistedState); - } else { - return Optional.empty(); - } - }) - .map(WorldState.class::cast); - } - - @Override - public boolean isWorldStateAvailable(final Hash rootHash, final Hash blockHash) { - return cachedWorldStorageManager.containWorldStateStorage(blockHash) - || persistedState.blockHash().equals(blockHash) - || worldStateKeyValueStorage.isWorldStateAvailable(rootHash, blockHash); - } - - @Override - public Optional getMutable( - final BlockHeader blockHeader, final boolean shouldPersistState) { - if (shouldPersistState) { - return getMutable(blockHeader.getStateRoot(), blockHeader.getHash()); - } else { - final BlockHeader chainHeadBlockHeader = blockchain.getChainHeadHeader(); - if (chainHeadBlockHeader.getNumber() - blockHeader.getNumber() - >= trieLogManager.getMaxLayersToLoad()) { - LOG.warn( - "Exceeded the limit of back layers that can be loaded ({})", - trieLogManager.getMaxLayersToLoad()); - return Optional.empty(); - } - return cachedWorldStorageManager - .getWorldState(blockHeader.getHash()) - .or(() -> cachedWorldStorageManager.getNearestWorldState(blockHeader)) - .or(() -> cachedWorldStorageManager.getHeadWorldState(blockchain::getBlockHeader)) - .flatMap( - bonsaiWorldState -> - rollMutableStateToBlockHash( - (BonsaiWorldState) bonsaiWorldState, blockHeader.getHash())) - .map(MutableWorldState::freeze); - } - } - - @Override - public synchronized Optional getMutable( - final Hash rootHash, final Hash blockHash) { - return rollMutableStateToBlockHash(persistedState, blockHash); - } - - Optional rollMutableStateToBlockHash( - final BonsaiWorldState mutableState, final Hash blockHash) { - if (blockHash.equals(mutableState.blockHash())) { - return Optional.of(mutableState); - } else { - try { - - final Optional maybePersistedHeader = - blockchain.getBlockHeader(mutableState.blockHash()).map(BlockHeader.class::cast); - - final List rollBacks = new ArrayList<>(); - final List rollForwards = new ArrayList<>(); - if (maybePersistedHeader.isEmpty()) { - trieLogManager.getTrieLogLayer(mutableState.blockHash()).ifPresent(rollBacks::add); - } else { - BlockHeader targetHeader = blockchain.getBlockHeader(blockHash).get(); - BlockHeader persistedHeader = maybePersistedHeader.get(); - // roll back from persisted to even with target - Hash persistedBlockHash = persistedHeader.getBlockHash(); - while (persistedHeader.getNumber() > targetHeader.getNumber()) { - LOG.debug("Rollback {}", persistedBlockHash); - rollBacks.add(trieLogManager.getTrieLogLayer(persistedBlockHash).get()); - persistedHeader = blockchain.getBlockHeader(persistedHeader.getParentHash()).get(); - persistedBlockHash = persistedHeader.getBlockHash(); - } - // roll forward to target - Hash targetBlockHash = targetHeader.getBlockHash(); - while (persistedHeader.getNumber() < targetHeader.getNumber()) { - LOG.debug("Rollforward {}", targetBlockHash); - rollForwards.add(trieLogManager.getTrieLogLayer(targetBlockHash).get()); - targetHeader = blockchain.getBlockHeader(targetHeader.getParentHash()).get(); - targetBlockHash = targetHeader.getBlockHash(); - } - - // roll back in tandem until we hit a shared state - while (!persistedBlockHash.equals(targetBlockHash)) { - LOG.debug("Paired Rollback {}", persistedBlockHash); - LOG.debug("Paired Rollforward {}", targetBlockHash); - rollForwards.add(trieLogManager.getTrieLogLayer(targetBlockHash).get()); - targetHeader = blockchain.getBlockHeader(targetHeader.getParentHash()).get(); - - rollBacks.add(trieLogManager.getTrieLogLayer(persistedBlockHash).get()); - persistedHeader = blockchain.getBlockHeader(persistedHeader.getParentHash()).get(); - - targetBlockHash = targetHeader.getBlockHash(); - persistedBlockHash = persistedHeader.getBlockHash(); - } - } - - // attempt the state rolling - final BonsaiWorldStateUpdateAccumulator bonsaiUpdater = - (BonsaiWorldStateUpdateAccumulator) mutableState.updater(); - try { - for (final TrieLog rollBack : rollBacks) { - LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash()); - bonsaiUpdater.rollBack(rollBack); - } - for (int i = rollForwards.size() - 1; i >= 0; i--) { - final var forward = rollForwards.get(i); - LOG.debug("Attempting Rollforward of {}", rollForwards.get(i).getBlockHash()); - bonsaiUpdater.rollForward(forward); - } - bonsaiUpdater.commit(); - - mutableState.persist(blockchain.getBlockHeader(blockHash).get()); - - LOG.debug( - "Archive rolling finished, {} now at {}", - mutableState.getWorldStateStorage().getClass().getSimpleName(), - blockHash); - return Optional.of(mutableState); - } catch (final MerkleTrieException re) { - // need to throw to trigger the heal - throw re; - } catch (final Exception e) { - // if we fail we must clean up the updater - bonsaiUpdater.reset(); - LOG.debug( - "State rolling failed on " - + mutableState.getWorldStateStorage().getClass().getSimpleName() - + " for block hash " - + blockHash, - e); - - return Optional.empty(); - } - } catch (final RuntimeException re) { - LOG.info("Archive rolling failed for block hash " + blockHash, re); - if (re instanceof MerkleTrieException) { - // need to throw to trigger the heal - throw re; - } - throw new MerkleTrieException( - "invalid", Optional.of(Address.ZERO), Hash.EMPTY, Bytes.EMPTY); - } - } + provideCachedWorldStorageManager(cachedWorldStorageManager); + loadPersistedState(new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration)); } public BonsaiCachedMerkleTrieLoader getCachedMerkleTrieLoader() { return cachedMerkleTrieLoader; } - @Override - public MutableWorldState getMutable() { - return persistedState; + private BonsaiWorldStateKeyValueStorage getWorldStateKeyValueStorage() { + return (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage; } - /** * Prepares the state healing process for a given address and location. It prepares the state * healing, including retrieving data from storage, identifying invalid slots or nodes, removing @@ -290,12 +93,14 @@ public MutableWorldState getMutable() { */ public void prepareStateHealing(final Address address, final Bytes location) { final Set keysToDelete = new HashSet<>(); - final BonsaiWorldStateKeyValueStorage.Updater updater = worldStateKeyValueStorage.updater(); + final BonsaiWorldStateKeyValueStorage.Updater updater = + getWorldStateKeyValueStorage().updater(); final Hash accountHash = address.addressHash(); final StoredMerklePatriciaTrie accountTrie = new StoredMerklePatriciaTrie<>( (l, h) -> { - final Optional node = worldStateKeyValueStorage.getAccountStateTrieNode(l, h); + final Optional node = + getWorldStateKeyValueStorage().getAccountStateTrieNode(l, h); if (node.isPresent()) { keysToDelete.add(l); } @@ -315,8 +120,8 @@ public void prepareStateHealing(final Address address, final Bytes location) { new StoredMerklePatriciaTrie<>( (l, h) -> { Optional node = - worldStateKeyValueStorage.getAccountStorageTrieNode( - accountHash, l, h); + getWorldStateKeyValueStorage() + .getAccountStorageTrieNode(accountHash, l, h); if (node.isPresent()) { keysToDelete.add(Bytes.concatenate(accountHash, l)); } @@ -336,60 +141,9 @@ public void prepareStateHealing(final Address address, final Bytes location) { LOG.warn("Invalid node for account {} at location {}", address, location); // ignore } - keysToDelete.forEach(bytes -> updater.removeAccountStateTrieNode(bytes)); + keysToDelete.forEach(updater::removeAccountStateTrieNode); updater.commit(); - worldStateKeyValueStorage.downgradeToPartialFlatDbMode(); - } - - public TrieLogManager getTrieLogManager() { - return trieLogManager; - } - - public BonsaiCachedWorldStorageManager getCachedWorldStorageManager() { - return cachedWorldStorageManager; - } - - @Override - public void resetArchiveStateTo(final BlockHeader blockHeader) { - persistedState.resetWorldStateTo(blockHeader); - this.cachedWorldStorageManager.reset(); - this.cachedWorldStorageManager.addCachedLayer( - blockHeader, persistedState.getWorldStateRootHash(), persistedState); - } - - @Override - public Optional getAccountProof( - final BlockHeader blockHeader, - final Address accountAddress, - final List accountStorageKeys, - final Function, ? extends Optional> mapper) { - try (BonsaiWorldState ws = (BonsaiWorldState) getMutable(blockHeader, false).orElse(null)) { - if (ws != null) { - final WorldStateProofProvider worldStateProofProvider = - new WorldStateProofProvider( - new WorldStateStorageCoordinator(ws.getWorldStateStorage())); - return mapper.apply( - worldStateProofProvider.getAccountProof( - ws.getWorldStateRootHash(), accountAddress, accountStorageKeys)); - } - } catch (Exception ex) { - LOG.error("failed proof query for " + blockHeader.getBlockHash().toShortHexString(), ex); - } - return Optional.empty(); - } - - @Override - public Optional getNodeData(final Hash hash) { - return Optional.empty(); - } - - @Override - public void close() { - try { - worldStateKeyValueStorage.close(); - } catch (Exception e) { - // no op - } + getWorldStateKeyValueStorage().downgradeToPartialFlatDbMode(); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java index 17b32241eac..67ec99d1cc2 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java @@ -35,7 +35,7 @@ public BonsaiCachedWorldStorageManager( } @Override - public DiffBasedWorldState createWorldState( + public DiffBasedWorldState createWorldState( final BonsaiWorldStateProvider archive, final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final EvmConfiguration evmConfiguration) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java index 42eec39ae0d..462f052f0d6 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java @@ -165,7 +165,9 @@ public static TrieLogLayer readFrom(final RLPInput input) { final StateTrieAccountValue newValue = nullOrValue(input, StateTrieAccountValue::readFrom); final boolean isCleared = getOptionalIsCleared(input); input.leaveList(); - newLayer.getAccountChanges().put(address, new DiffBasedValue<>(oldValue, newValue, isCleared)); + newLayer + .getAccountChanges() + .put(address, new DiffBasedValue<>(oldValue, newValue, isCleared)); } if (input.nextIsNull()) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java index f24f199f914..accc47c7182 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java @@ -29,12 +29,12 @@ import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; @@ -55,7 +55,7 @@ import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -public class BonsaiWorldState extends DiffBasedWorldState { +public class BonsaiWorldState extends DiffBasedWorldState { protected final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader; @@ -74,7 +74,7 @@ public BonsaiWorldState( protected BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader, - final BonsaiCachedWorldStorageManager cachedWorldStorageManager, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final EvmConfiguration evmConfiguration) { super(worldStateKeyValueStorage, cachedWorldStorageManager, trieLogManager); @@ -90,6 +90,11 @@ protected BonsaiWorldState( evmConfiguration)); } + @Override + public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + return (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage; + } + @Override protected Hash calculateRootHash( final Optional maybeStateUpdater, @@ -328,7 +333,7 @@ public Account get(final Address address) { @Override public Optional getCode(@Nonnull final Address address, final Hash codeHash) { - return worldStateKeyValueStorage.getCode(codeHash, address.addressHash()); + return getWorldStateStorage().getCode(codeHash, address.addressHash()); } protected Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index 0bb12ad7de8..ffc64e09d4b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -92,7 +92,8 @@ protected BonsaiAccount createAccount( } @Override - protected void assertCloseEnoughForDiffing(final BonsaiAccount source, final AccountValue account, final String context) { - BonsaiAccount.assertCloseEnoughForDiffing(source,account,context); + protected void assertCloseEnoughForDiffing( + final BonsaiAccount source, final AccountValue account, final String context) { + BonsaiAccount.assertCloseEnoughForDiffing(source, account, context); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java new file mode 100644 index 00000000000..0c8ede719c5 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java @@ -0,0 +1,310 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.common; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.proof.WorldStateProof; +import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; +import org.hyperledger.besu.ethereum.trie.MerkleTrieException; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; +import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; +import org.hyperledger.besu.evm.worldstate.WorldState; +import org.hyperledger.besu.plugin.BesuContext; +import org.hyperledger.besu.plugin.services.trielogs.TrieLog; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DiffBasedWorldStateProvider implements WorldStateArchive { + + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedWorldStateProvider.class); + + protected final Blockchain blockchain; + + protected final TrieLogManager trieLogManager; + protected DiffBasedCachedWorldStorageManager cachedWorldStorageManager; + protected DiffBasedWorldState persistedState; + protected final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; + + public DiffBasedWorldStateProvider( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final Blockchain blockchain, + final Optional maxLayersToLoad, + final BesuContext pluginContext, + final TrieLogPruner trieLogPruner) { + + this.worldStateKeyValueStorage = worldStateKeyValueStorage; + // TODO: de-dup constructors + this.trieLogManager = + new TrieLogManager( + blockchain, + worldStateKeyValueStorage, + maxLayersToLoad.orElse(DiffBasedCachedWorldStorageManager.RETAINED_LAYERS), + pluginContext, + trieLogPruner); + this.blockchain = blockchain; + } + + public DiffBasedWorldStateProvider( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final Blockchain blockchain, + final TrieLogManager trieLogManager) { + + this.worldStateKeyValueStorage = worldStateKeyValueStorage; + // TODO: de-dup constructors + this.trieLogManager = trieLogManager; + this.blockchain = blockchain; + } + + protected void provideCachedWorldStorageManager( + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager) { + this.cachedWorldStorageManager = cachedWorldStorageManager; + } + + protected void loadPersistedState(final DiffBasedWorldState persistedState) { + this.persistedState = persistedState; + blockchain + .getBlockHeader(persistedState.getWorldStateBlockHash()) + .ifPresent( + blockHeader -> + this.cachedWorldStorageManager.addCachedLayer( + blockHeader, persistedState.getWorldStateRootHash(), persistedState)); + } + + @Override + public Optional get(final Hash rootHash, final Hash blockHash) { + return cachedWorldStorageManager + .getWorldState(blockHash) + .or( + () -> { + if (blockHash.equals(persistedState.blockHash())) { + return Optional.of(persistedState); + } else { + return Optional.empty(); + } + }) + .map(WorldState.class::cast); + } + + @Override + public boolean isWorldStateAvailable(final Hash rootHash, final Hash blockHash) { + return cachedWorldStorageManager.containWorldStateStorage(blockHash) + || persistedState.blockHash().equals(blockHash) + || worldStateKeyValueStorage.isWorldStateAvailable(rootHash, blockHash); + } + + @Override + public Optional getMutable( + final BlockHeader blockHeader, final boolean shouldPersistState) { + if (shouldPersistState) { + return getMutable(blockHeader.getStateRoot(), blockHeader.getHash()); + } else { + final BlockHeader chainHeadBlockHeader = blockchain.getChainHeadHeader(); + if (chainHeadBlockHeader.getNumber() - blockHeader.getNumber() + >= trieLogManager.getMaxLayersToLoad()) { + LOG.warn( + "Exceeded the limit of back layers that can be loaded ({})", + trieLogManager.getMaxLayersToLoad()); + return Optional.empty(); + } + return cachedWorldStorageManager + .getWorldState(blockHeader.getHash()) + .or(() -> cachedWorldStorageManager.getNearestWorldState(blockHeader)) + .or(() -> cachedWorldStorageManager.getHeadWorldState(blockchain::getBlockHeader)) + .flatMap(worldState -> rollMutableStateToBlockHash(worldState, blockHeader.getHash())) + .map(MutableWorldState::freeze); + } + } + + @Override + public synchronized Optional getMutable( + final Hash rootHash, final Hash blockHash) { + return rollMutableStateToBlockHash(persistedState, blockHash); + } + + Optional rollMutableStateToBlockHash( + final DiffBasedWorldState mutableState, final Hash blockHash) { + if (blockHash.equals(mutableState.blockHash())) { + return Optional.of(mutableState); + } else { + try { + + final Optional maybePersistedHeader = + blockchain.getBlockHeader(mutableState.blockHash()).map(BlockHeader.class::cast); + + final List rollBacks = new ArrayList<>(); + final List rollForwards = new ArrayList<>(); + if (maybePersistedHeader.isEmpty()) { + trieLogManager.getTrieLogLayer(mutableState.blockHash()).ifPresent(rollBacks::add); + } else { + BlockHeader targetHeader = blockchain.getBlockHeader(blockHash).get(); + BlockHeader persistedHeader = maybePersistedHeader.get(); + // roll back from persisted to even with target + Hash persistedBlockHash = persistedHeader.getBlockHash(); + while (persistedHeader.getNumber() > targetHeader.getNumber()) { + LOG.debug("Rollback {}", persistedBlockHash); + rollBacks.add(trieLogManager.getTrieLogLayer(persistedBlockHash).get()); + persistedHeader = blockchain.getBlockHeader(persistedHeader.getParentHash()).get(); + persistedBlockHash = persistedHeader.getBlockHash(); + } + // roll forward to target + Hash targetBlockHash = targetHeader.getBlockHash(); + while (persistedHeader.getNumber() < targetHeader.getNumber()) { + LOG.debug("Rollforward {}", targetBlockHash); + rollForwards.add(trieLogManager.getTrieLogLayer(targetBlockHash).get()); + targetHeader = blockchain.getBlockHeader(targetHeader.getParentHash()).get(); + targetBlockHash = targetHeader.getBlockHash(); + } + + // roll back in tandem until we hit a shared state + while (!persistedBlockHash.equals(targetBlockHash)) { + LOG.debug("Paired Rollback {}", persistedBlockHash); + LOG.debug("Paired Rollforward {}", targetBlockHash); + rollForwards.add(trieLogManager.getTrieLogLayer(targetBlockHash).get()); + targetHeader = blockchain.getBlockHeader(targetHeader.getParentHash()).get(); + + rollBacks.add(trieLogManager.getTrieLogLayer(persistedBlockHash).get()); + persistedHeader = blockchain.getBlockHeader(persistedHeader.getParentHash()).get(); + + targetBlockHash = targetHeader.getBlockHash(); + persistedBlockHash = persistedHeader.getBlockHash(); + } + } + + // attempt the state rolling + final DiffBasedWorldStateUpdateAccumulator diffBasedUpdater = + (DiffBasedWorldStateUpdateAccumulator) mutableState.updater(); + try { + for (final TrieLog rollBack : rollBacks) { + LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash()); + diffBasedUpdater.rollBack(rollBack); + } + for (int i = rollForwards.size() - 1; i >= 0; i--) { + final var forward = rollForwards.get(i); + LOG.debug("Attempting Rollforward of {}", rollForwards.get(i).getBlockHash()); + diffBasedUpdater.rollForward(forward); + } + diffBasedUpdater.commit(); + + mutableState.persist(blockchain.getBlockHeader(blockHash).get()); + + LOG.debug( + "Archive rolling finished, {} now at {}", + mutableState.getWorldStateStorage().getClass().getSimpleName(), + blockHash); + return Optional.of(mutableState); + } catch (final MerkleTrieException re) { + // need to throw to trigger the heal + throw re; + } catch (final Exception e) { + // if we fail we must clean up the updater + diffBasedUpdater.reset(); + LOG.debug( + "State rolling failed on " + + mutableState.getWorldStateStorage().getClass().getSimpleName() + + " for block hash " + + blockHash, + e); + + return Optional.empty(); + } + } catch (final RuntimeException re) { + LOG.info("Archive rolling failed for block hash " + blockHash, re); + if (re instanceof MerkleTrieException) { + // need to throw to trigger the heal + throw re; + } + throw new MerkleTrieException( + "invalid", Optional.of(Address.ZERO), Hash.EMPTY, Bytes.EMPTY); + } + } + } + + @Override + public MutableWorldState getMutable() { + return persistedState; + } + + public TrieLogManager getTrieLogManager() { + return trieLogManager; + } + + public DiffBasedCachedWorldStorageManager getCachedWorldStorageManager() { + return cachedWorldStorageManager; + } + + @Override + public void resetArchiveStateTo(final BlockHeader blockHeader) { + persistedState.resetWorldStateTo(blockHeader); + this.cachedWorldStorageManager.reset(); + this.cachedWorldStorageManager.addCachedLayer( + blockHeader, persistedState.getWorldStateRootHash(), persistedState); + } + + @Override + public Optional getAccountProof( + final BlockHeader blockHeader, + final Address accountAddress, + final List accountStorageKeys, + final Function, ? extends Optional> mapper) { + try (DiffBasedWorldState ws = + (DiffBasedWorldState) getMutable(blockHeader, false).orElse(null)) { + if (ws != null) { + final WorldStateProofProvider worldStateProofProvider = + new WorldStateProofProvider( + new WorldStateStorageCoordinator(ws.getWorldStateStorage())); + return mapper.apply( + worldStateProofProvider.getAccountProof( + ws.getWorldStateRootHash(), accountAddress, accountStorageKeys)); + } + } catch (Exception ex) { + LOG.error("failed proof query for " + blockHeader.getBlockHash().toShortHexString(), ex); + } + return Optional.empty(); + } + + @Override + public Optional getNodeData(final Hash hash) { + return Optional.empty(); + } + + @Override + public void close() { + try { + worldStateKeyValueStorage.close(); + } catch (Exception e) { + // no op + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java index 0f8021def81..d8499cf8f06 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java @@ -77,7 +77,7 @@ public DiffBasedCachedWorldStorageManager( public synchronized void addCachedLayer( final BlockHeader blockHeader, final Hash worldStateRootHash, - final DiffBasedWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { final Optional cachedDiffBasedWorldView = Optional.ofNullable(this.cachedWorldStatesByHash.get(blockHeader.getBlockHash())); if (cachedDiffBasedWorldView.isPresent()) { @@ -135,7 +135,7 @@ private synchronized void scrubCachedLayers(final long newMaxHeight) { } } - public Optional> getWorldState(final Hash blockHash) { + public Optional getWorldState(final Hash blockHash) { if (cachedWorldStatesByHash.containsKey(blockHash)) { // return a new worldstate using worldstate storage and an isolated copy of the updater return Optional.ofNullable(cachedWorldStatesByHash.get(blockHash)) @@ -154,7 +154,7 @@ public Optional> getWorldState(final Hash blockHash) { return Optional.empty(); } - public Optional> getNearestWorldState(final BlockHeader blockHeader) { + public Optional getNearestWorldState(final BlockHeader blockHeader) { LOG.atDebug() .setMessage("getting nearest worldstate for {}") .addArgument(blockHeader.toLogString()) @@ -186,7 +186,7 @@ public Optional> getNearestWorldState(final BlockHeader b archive, createLayeredKeyValueStorage(storage), evmConfiguration)); } - public Optional> getHeadWorldState( + public Optional getHeadWorldState( final Function> hashBlockHeaderFunction) { LOG.atDebug().setMessage("getting head worldstate").log(); @@ -233,7 +233,7 @@ public void onCloseStorage() { this.cachedWorldStatesByHash.clear(); } - public abstract DiffBasedWorldState createWorldState( + public abstract DiffBasedWorldState createWorldState( final BonsaiWorldStateProvider archive, final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final EvmConfiguration evmConfiguration); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java index 1ac50435db3..e5602727118 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java @@ -67,7 +67,7 @@ public synchronized void saveTrieLog( final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final DiffBasedWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // do not overwrite a trielog layer that already exists in the database. // if it's only in memory we need to save it // for example, in case of reorg we don't replace a trielog layer diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java index c9478946183..d3cc94fe82c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import java.util.Comparator; import java.util.Optional; @@ -29,7 +30,6 @@ import com.google.common.collect.Multimap; import com.google.common.collect.TreeMultimap; import org.apache.tuweni.bytes.Bytes32; -import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java index c4063e398dd..18412ac9722 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java @@ -23,15 +23,15 @@ import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.evm.account.Account; @@ -51,24 +51,23 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class DiffBasedWorldState< - WORLDSTATE_STORAGE extends DiffBasedWorldStateKeyValueStorage> +public abstract class DiffBasedWorldState implements MutableWorldState, DiffBasedWorldView, StorageSubscriber { private static final Logger LOG = LoggerFactory.getLogger(DiffBasedWorldState.class); - protected WORLDSTATE_STORAGE worldStateKeyValueStorage; - protected final BonsaiCachedWorldStorageManager cachedWorldStorageManager; + protected DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; + protected final DiffBasedCachedWorldStorageManager cachedWorldStorageManager; protected final TrieLogManager trieLogManager; - protected BonsaiWorldStateUpdateAccumulator accumulator; + protected DiffBasedWorldStateUpdateAccumulator accumulator; protected Hash worldStateRootHash; protected Hash worldStateBlockHash; protected boolean isFrozen; protected DiffBasedWorldState( - final WORLDSTATE_STORAGE worldStateKeyValueStorage, - final BonsaiCachedWorldStorageManager cachedWorldStorageManager, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager) { this.worldStateKeyValueStorage = worldStateKeyValueStorage; this.worldStateRootHash = @@ -130,11 +129,11 @@ public void resetWorldStateTo(final BlockHeader blockHeader) { } @Override - public WORLDSTATE_STORAGE getWorldStateStorage() { + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { return worldStateKeyValueStorage; } - public BonsaiWorldStateUpdateAccumulator getAccumulator() { + public DiffBasedWorldStateUpdateAccumulator getAccumulator() { return accumulator; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java index 32c91366d3b..662bc360f83 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java @@ -23,7 +23,6 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedAccount; import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; @@ -582,9 +581,10 @@ private void rollAccountChange( "Expected to create account, but the account exists. Address=%s", address)); } } else { - assertCloseEnoughForDiffing(accountValue.getUpdated(), - expectedValue, - "Address=" + address + " Prior Value in Rolling Change"); + assertCloseEnoughForDiffing( + accountValue.getUpdated(), + expectedValue, + "Address=" + address + " Prior Value in Rolling Change"); } if (replacementValue == null) { if (accountValue.getPrior() == null) { @@ -807,6 +807,6 @@ protected abstract ACCOUNT createAccount( protected abstract ACCOUNT createAccount( final DiffBasedWorldView context, final UpdateTrackingAccount tracked); - - protected abstract void assertCloseEnoughForDiffing(final ACCOUNT source, final AccountValue account, final String context); + protected abstract void assertCloseEnoughForDiffing( + final ACCOUNT source, final AccountValue account, final String context); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java similarity index 98% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java index 5bf9b2cb3b4..e419452664e 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryBlockchain; import static org.mockito.ArgumentMatchers.any; @@ -65,7 +65,6 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java similarity index 99% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java index ce983e10a9a..507047604bc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java similarity index 99% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java index 1bb993b072e..75f016e5557 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.BLOCKCHAIN; @@ -36,15 +36,14 @@ import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/CachedMerkleTrieLoaderTest.java similarity index 99% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/CachedMerkleTrieLoaderTest.java index 50829d2faaa..f6687fa6db0 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/CachedMerkleTrieLoaderTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java similarity index 99% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java index 6f90ba22482..494d26e8753 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -29,12 +29,11 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.log.LogsBloomFilter; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java similarity index 97% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java index 159c3505b58..126d359b56e 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; @@ -25,12 +25,11 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java similarity index 99% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java index ef70a2817b7..d4fe213746f 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; @@ -35,7 +35,6 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.StorageEntriesCollector; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java similarity index 95% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java index ba61789b8bf..186fbf64ae9 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; @@ -24,7 +24,6 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java similarity index 97% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java index 5faef343b98..d967ca1a1b4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java @@ -13,14 +13,13 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import java.util.Optional; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManagerTests.java similarity index 95% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManagerTests.java index bedfb8e7cb8..a8de3783dbf 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManagerTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner.noOpTrieLogPruner; @@ -24,7 +24,6 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.evm.internal.EvmConfiguration; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPrunerTest.java similarity index 98% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPrunerTest.java index ac5aab1d837..650f9d1afe0 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPrunerTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -27,7 +27,6 @@ import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import java.util.Optional; import java.util.stream.Stream; diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 0d4e3259d45..229f58efeaf 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -22,10 +22,11 @@ import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogAddedEvent; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -53,7 +54,7 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState protected BonsaiReferenceTestWorldState( final BonsaiReferenceTestWorldStateStorage worldStateKeyValueStorage, final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader, - final BonsaiCachedWorldStorageManager cachedWorldStorageManager, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final BonsaiPreImageProxy preImageProxy, final EvmConfiguration evmConfiguration) { @@ -80,8 +81,7 @@ protected BonsaiReferenceTestWorldState( @Override public ReferenceTestWorldState copy() { - var layerCopy = - new BonsaiReferenceTestWorldStateStorage(worldStateKeyValueStorage, preImageProxy); + var layerCopy = new BonsaiReferenceTestWorldStateStorage(getWorldStateStorage(), preImageProxy); return new BonsaiReferenceTestWorldState( layerCopy, cachedMerkleTrieLoader, @@ -167,7 +167,7 @@ public NoOpCachedWorldStorageManager() { public void addCachedLayer( final BlockHeader blockHeader, final Hash worldStateRootHash, - final DiffBasedWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // reference test world states are not cached } @@ -177,17 +177,17 @@ public boolean containWorldStateStorage(final Hash blockHash) { } @Override - public Optional> getWorldState(final Hash blockHash) { + public Optional getWorldState(final Hash blockHash) { return Optional.empty(); } @Override - public Optional> getNearestWorldState(final BlockHeader blockHeader) { + public Optional getNearestWorldState(final BlockHeader blockHeader) { return Optional.empty(); } @Override - public Optional> getHeadWorldState( + public Optional getHeadWorldState( final Function> hashBlockHeaderFunction) { return Optional.empty(); } @@ -210,7 +210,7 @@ public void saveTrieLog( final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final DiffBasedWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // notify trie log added observers, synchronously TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); From 9967e186d097ecd47aef08b4d1ffd98545d620a3 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Thu, 21 Dec 2023 11:56:51 +0100 Subject: [PATCH 09/59] add verkle Signed-off-by: Karim Taam --- .../controller/BesuControllerBuilder.java | 27 + build.gradle | 1 + ethereum/core/build.gradle | 1 + .../keyvalue/KeyValueStorageProvider.java | 3 + .../trie/diffbased/bonsai/BonsaiAccount.java | 19 +- .../BonsaiCachedWorldStorageManager.java | 9 +- .../BonsaiWorldStateUpdateAccumulator.java | 5 + .../diffbased/common/DiffBasedAccount.java | 14 - .../DiffBasedCachedWorldStorageManager.java | 10 +- .../common/trielog/TrieLogManager.java | 2 +- .../common/worldview/DiffBasedWorldState.java | 7 +- .../DiffBasedWorldStateUpdateAccumulator.java | 5 +- .../trie/diffbased/verkle/VerkleAccount.java | 179 + .../verkle/VerkleWorldStateProvider.java | 61 + .../VerkleCachedWorldStorageManager.java | 63 + ...erkleLayeredWorldStateKeyValueStorage.java | 52 + ...rkleSnapshotWorldStateKeyValueStorage.java | 201 + .../VerkleWorldStateKeyValueStorage.java | 223 + .../verkle/trielog/TrieLogFactoryImpl.java | 272 + .../verkle/worldview/VerkleWorldState.java | 361 + .../VerkleWorldStateUpdateAccumulator.java | 104 + .../worldstate/DataStorageFormat.java | 2 +- .../core/InMemoryKeyValueStorageProvider.java | 22 + .../diffbased/verkle/LogRollingTests.java | 584 ++ ethereum/verkletrie/build.gradle | 7 +- .../besu/ethereum/verkletrie/VerkleTrie.java | 61 + .../VerkleTrieKeyValueGenerator.java | 95 + gradle/verification-metadata.xml | 6883 ----------------- gradle/versions.gradle | 6 + 29 files changed, 2348 insertions(+), 6931 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/VerkleAccount.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/VerkleWorldStateProvider.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleCachedWorldStorageManager.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleLayeredWorldStateKeyValueStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleWorldStateKeyValueStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/trielog/TrieLogFactoryImpl.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java create mode 100644 ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleTrie.java create mode 100644 ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleTrieKeyValueGenerator.java delete mode 100644 gradle/verification-metadata.xml diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 2140f5c53a5..ad152c21743 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -84,6 +84,8 @@ import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.pruner.MarkSweepPruner; import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; @@ -1093,6 +1095,31 @@ yield new BonsaiWorldStateProvider( evmConfiguration, trieLogPruner); } + case VERKLE -> { + final GenesisConfigOptions genesisConfigOptions = configOptionsSupplier.get(); + final boolean isProofOfStake = + genesisConfigOptions.getTerminalTotalDifficulty().isPresent(); + final VerkleWorldStateKeyValueStorage worldStateKeyValueStorage = + worldStateStorageCoordinator.getStrategy(VerkleWorldStateKeyValueStorage.class); + final TrieLogPruner trieLogPruner = + dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningEnabled() + ? new TrieLogPruner( + worldStateKeyValueStorage, + blockchain, + dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold(), + dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit(), + isProofOfStake) + : TrieLogPruner.noOpTrieLogPruner(); + trieLogPruner.initialize(); + yield new VerkleWorldStateProvider( + worldStateKeyValueStorage, + blockchain, + Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), + metricsSystem, + besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), + evmConfiguration, + trieLogPruner); + } case FOREST -> { final WorldStatePreimageStorage preimageStorage = storageProvider.createWorldStatePreimageStorage(); diff --git a/build.gradle b/build.gradle index a00ea343db6..abd37b005d2 100644 --- a/build.gradle +++ b/build.gradle @@ -154,6 +154,7 @@ allprojects { content { includeGroupByRegex('com\\.splunk\\..*') } } mavenCentral() + mavenLocal() // ethereum execution spec tests fixtures. Exclusively for ethereum submodule to run ref tests def ethExecSpecTestsRepo = ivy { diff --git a/ethereum/core/build.gradle b/ethereum/core/build.gradle index b616413dd22..10fbee02f77 100644 --- a/ethereum/core/build.gradle +++ b/ethereum/core/build.gradle @@ -39,6 +39,7 @@ dependencies { implementation project(':enclave') implementation project(':ethereum:rlp') implementation project(':ethereum:trie') + implementation project(':ethereum:verkletrie') implementation project(':evm') implementation project(':metrics:core') implementation project(':plugin-api') diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index 7b5dc7155ff..3beb5d7300a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; @@ -80,6 +81,8 @@ public WorldStateKeyValueStorage createWorldStateStorage( final DataStorageFormat dataStorageFormat) { if (dataStorageFormat.equals(DataStorageFormat.BONSAI)) { return new BonsaiWorldStateKeyValueStorage(this, metricsSystem); + } else if (dataStorageFormat.equals(DataStorageFormat.VERKLE)) { + return new VerkleWorldStateKeyValueStorage(this, metricsSystem); } else { return new ForestWorldStateKeyValueStorage( getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.WORLD_STATE)); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java index ad52e57cd71..93f407ec05f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java @@ -58,14 +58,7 @@ public BonsaiAccount( final Address address, final AccountValue stateTrieAccount, final boolean mutable) { - super( - context, - address, - address.addressHash(), - stateTrieAccount.getNonce(), - stateTrieAccount.getBalance(), - stateTrieAccount.getCodeHash(), - !mutable); + super(context, address, stateTrieAccount, !mutable); this.storageRoot = stateTrieAccount.getStorageRoot(); } @@ -75,16 +68,8 @@ public BonsaiAccount(final BonsaiAccount toCopy) { public BonsaiAccount( final BonsaiAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { - super( - context, - toCopy.address, - toCopy.addressHash, - toCopy.nonce, - toCopy.balance, - toCopy.codeHash, - !mutable); + super(toCopy, context, !mutable); this.storageRoot = toCopy.storageRoot; - updatedStorage.putAll(toCopy.updatedStorage); } public BonsaiAccount( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java index 67ec99d1cc2..08f694ed497 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; @@ -28,7 +29,7 @@ public class BonsaiCachedWorldStorageManager extends DiffBasedCachedWorldStorageManager { public BonsaiCachedWorldStorageManager( - final BonsaiWorldStateProvider archive, + final DiffBasedWorldStateProvider archive, final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final ObservableMetricsSystem metricsSystem) { super(archive, worldStateKeyValueStorage, metricsSystem); @@ -36,11 +37,13 @@ public BonsaiCachedWorldStorageManager( @Override public DiffBasedWorldState createWorldState( - final BonsaiWorldStateProvider archive, + final DiffBasedWorldStateProvider archive, final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final EvmConfiguration evmConfiguration) { return new BonsaiWorldState( - archive, (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, evmConfiguration); + (BonsaiWorldStateProvider) archive, + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, + evmConfiguration); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index ffc64e09d4b..2146e5a64f4 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -96,4 +96,9 @@ protected void assertCloseEnoughForDiffing( final BonsaiAccount source, final AccountValue account, final String context) { BonsaiAccount.assertCloseEnoughForDiffing(source, account, context); } + + @Override + protected boolean shouldIgnoreIdenticalValuesDuringAccountRollingUpdate() { + return true; + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java index 54efcf0d9fb..d4557efde81 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java @@ -25,8 +25,6 @@ import org.hyperledger.besu.evm.ModificationNotAllowedException; import org.hyperledger.besu.evm.account.MutableAccount; -import java.io.PrintWriter; -import java.io.StringWriter; import java.util.HashMap; import java.util.Map; @@ -42,8 +40,6 @@ public abstract class DiffBasedAccount implements MutableAccount, AccountValue { protected long nonce; protected Wei balance; protected Bytes code; - - private final String stack; protected final Map updatedStorage = new HashMap<>(); public DiffBasedAccount( @@ -62,10 +58,6 @@ public DiffBasedAccount( this.codeHash = codeHash; this.immutable = mutable; - - StringWriter sw = new StringWriter(); - new Exception().printStackTrace(new PrintWriter(sw)); - stack = sw.toString(); } public DiffBasedAccount( @@ -93,12 +85,7 @@ public DiffBasedAccount( this.codeHash = toCopy.codeHash; this.code = toCopy.code; updatedStorage.putAll(toCopy.updatedStorage); - this.immutable = mutable; - - StringWriter sw = new StringWriter(); - new Exception().printStackTrace(new PrintWriter(sw)); - stack = sw.toString(); } @Override @@ -132,7 +119,6 @@ public Wei getBalance() { @Override public void setBalance(final Wei value) { if (immutable) { - System.out.println(stack); throw new ModificationNotAllowedException(); } balance = value; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java index d8499cf8f06..266aa298f15 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java @@ -16,8 +16,8 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; @@ -41,7 +41,7 @@ public abstract class DiffBasedCachedWorldStorageManager implements StorageSubsc public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks private static final Logger LOG = LoggerFactory.getLogger(DiffBasedCachedWorldStorageManager.class); - private final BonsaiWorldStateProvider archive; + private final DiffBasedWorldStateProvider archive; private final ObservableMetricsSystem metricsSystem; private final EvmConfiguration evmConfiguration; @@ -49,7 +49,7 @@ public abstract class DiffBasedCachedWorldStorageManager implements StorageSubsc private final Map cachedWorldStatesByHash; private DiffBasedCachedWorldStorageManager( - final BonsaiWorldStateProvider archive, + final DiffBasedWorldStateProvider archive, final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final Map cachedWorldStatesByHash, final ObservableMetricsSystem metricsSystem, @@ -63,7 +63,7 @@ private DiffBasedCachedWorldStorageManager( } public DiffBasedCachedWorldStorageManager( - final BonsaiWorldStateProvider archive, + final DiffBasedWorldStateProvider archive, final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final ObservableMetricsSystem metricsSystem) { this( @@ -234,7 +234,7 @@ public void onCloseStorage() { } public abstract DiffBasedWorldState createWorldState( - final BonsaiWorldStateProvider archive, + final DiffBasedWorldStateProvider archive, final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final EvmConfiguration evmConfiguration); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java index e5602727118..e8875be4e83 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java @@ -18,10 +18,10 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.services.TrieLogService; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java index 18412ac9722..06419f002e8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java @@ -26,8 +26,6 @@ import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; @@ -87,7 +85,7 @@ protected DiffBasedWorldState( * * @param accumulator accumulator to use. */ - protected void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) { + protected void setAccumulator(final DiffBasedWorldStateUpdateAccumulator accumulator) { this.accumulator = accumulator; } @@ -166,8 +164,7 @@ public void persist(final BlockHeader blockHeader) { trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); // not save a frozen state in the cache if (!isFrozen) { - cachedWorldStorageManager.addCachedLayer( - blockHeader, newWorldStateRootHash, (BonsaiWorldState) this); + cachedWorldStorageManager.addCachedLayer(blockHeader, newWorldStateRootHash, this); } }; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java index 662bc360f83..7c795320aff 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java @@ -555,7 +555,8 @@ private void rollAccountChange( final Address address, final AccountValue expectedValue, final AccountValue replacementValue) { - if (Objects.equals(expectedValue, replacementValue)) { + if (shouldIgnoreIdenticalValuesDuringAccountRollingUpdate() + && Objects.equals(expectedValue, replacementValue)) { // non-change, a cached read. return; } @@ -809,4 +810,6 @@ protected abstract ACCOUNT createAccount( protected abstract void assertCloseEnoughForDiffing( final ACCOUNT source, final AccountValue account, final String context); + + protected abstract boolean shouldIgnoreIdenticalValuesDuringAccountRollingUpdate(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/VerkleAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/VerkleAccount.java new file mode 100644 index 00000000000..7d38db9df8c --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/VerkleAccount.java @@ -0,0 +1,179 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle; + +import org.hyperledger.besu.datatypes.AccountValue; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.rlp.RLPException; +import org.hyperledger.besu.ethereum.rlp.RLPInput; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.evm.ModificationNotAllowedException; +import org.hyperledger.besu.evm.account.AccountStorageEntry; +import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; + +import java.util.NavigableMap; +import java.util.Objects; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +public class VerkleAccount extends DiffBasedAccount { + private Hash storageRoot; // TODO REMOVE AS USELESS + + public VerkleAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash storageRoot, + final Hash codeHash, + final boolean mutable) { + super(context, address, addressHash, nonce, balance, codeHash, !mutable); + this.storageRoot = storageRoot; + } + + public VerkleAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable) { + super(context, address, stateTrieAccount, !mutable); + this.storageRoot = stateTrieAccount.getStorageRoot(); + } + + public VerkleAccount(final VerkleAccount toCopy) { + this(toCopy, toCopy.context, false); + } + + public VerkleAccount( + final VerkleAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + super(toCopy, context, !mutable); + this.storageRoot = toCopy.storageRoot; + } + + public VerkleAccount( + final DiffBasedWorldView context, final UpdateTrackingAccount tracked) { + super( + context, + tracked.getAddress(), + tracked.getAddressHash(), + tracked.getNonce(), + tracked.getBalance(), + tracked.getCodeHash(), + false); + this.storageRoot = Hash.EMPTY_TRIE_HASH; + updatedStorage.putAll(tracked.getUpdatedStorage()); + } + + public static VerkleAccount fromRLP( + final DiffBasedWorldView context, + final Address address, + final Bytes encoded, + final boolean mutable) + throws RLPException { + final RLPInput in = RLP.input(encoded); + in.enterList(); + + final long nonce = in.readLongScalar(); + final Wei balance = Wei.of(in.readUInt256Scalar()); + final Hash storageRoot = Hash.wrap(in.readBytes32()); + final Hash codeHash = Hash.wrap(in.readBytes32()); + + in.leaveList(); + + return new VerkleAccount( + context, address, address.addressHash(), nonce, balance, storageRoot, codeHash, mutable); + } + + @Override + public NavigableMap storageEntriesFrom( + final Bytes32 startKeyHash, final int limit) { + return ((BonsaiWorldStateKeyValueStorage) context.getWorldStateStorage()) + .storageEntriesFrom(this.addressHash, startKeyHash, limit); + } + + @Override + public void writeTo(final RLPOutput out) { + out.startList(); + + out.writeLongScalar(nonce); + out.writeUInt256Scalar(balance); + out.writeBytes(storageRoot); + out.writeBytes(codeHash); + + out.endList(); + } + + @Override + public Hash getStorageRoot() { + return storageRoot; + } + + public void setStorageRoot(final Hash storageRoot) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + this.storageRoot = storageRoot; + } + + @Override + public String toString() { + return "AccountState{" + + "address=" + + address + + ", nonce=" + + nonce + + ", balance=" + + balance + + ", storageRoot=" + + storageRoot + + ", codeHash=" + + codeHash + + '}'; + } + + /** + * Throws an exception if the two accounts represent different stored states + * + * @param source The bonsai account to compare + * @param account The State Trie account to compare + * @param context a description to be added to the thrown exceptions + * @throws IllegalStateException if the stored values differ + */ + public static void assertCloseEnoughForDiffing( + final VerkleAccount source, final AccountValue account, final String context) { + if (source == null) { + throw new IllegalStateException(context + ": source is null but target isn't"); + } else { + if (source.nonce != account.getNonce()) { + throw new IllegalStateException(context + ": nonces differ"); + } + if (!Objects.equals(source.balance, account.getBalance())) { + throw new IllegalStateException(context + ": balances differ"); + } + if (!Objects.equals(source.storageRoot, account.getStorageRoot())) { + throw new IllegalStateException(context + ": Storage Roots differ"); + } + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/VerkleWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/VerkleWorldStateProvider.java new file mode 100644 index 00000000000..95e83692f1f --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/VerkleWorldStateProvider.java @@ -0,0 +1,61 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.verkle; + +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.VerkleCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.worldview.VerkleWorldState; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.metrics.ObservableMetricsSystem; +import org.hyperledger.besu.plugin.BesuContext; + +import java.util.Optional; + +import com.google.common.annotations.VisibleForTesting; + +public class VerkleWorldStateProvider extends DiffBasedWorldStateProvider { + + public VerkleWorldStateProvider( + final VerkleWorldStateKeyValueStorage worldStateKeyValueStorage, + final Blockchain blockchain, + final Optional maxLayersToLoad, + final ObservableMetricsSystem metricsSystem, + final BesuContext pluginContext, + final EvmConfiguration evmConfiguration, + final TrieLogPruner trieLogPruner) { + super(worldStateKeyValueStorage, blockchain, maxLayersToLoad, pluginContext, trieLogPruner); + provideCachedWorldStorageManager( + new VerkleCachedWorldStorageManager(this, worldStateKeyValueStorage, metricsSystem)); + loadPersistedState(new VerkleWorldState(this, worldStateKeyValueStorage, evmConfiguration)); + } + + @VisibleForTesting + VerkleWorldStateProvider( + final VerkleCachedWorldStorageManager cachedWorldStorageManager, + final TrieLogManager trieLogManager, + final VerkleWorldStateKeyValueStorage worldStateKeyValueStorage, + final Blockchain blockchain, + final EvmConfiguration evmConfiguration) { + super(worldStateKeyValueStorage, blockchain, trieLogManager); + provideCachedWorldStorageManager(cachedWorldStorageManager); + loadPersistedState(new VerkleWorldState(this, worldStateKeyValueStorage, evmConfiguration)); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleCachedWorldStorageManager.java new file mode 100644 index 00000000000..56d7922e7d9 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleCachedWorldStorageManager.java @@ -0,0 +1,63 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache; + +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleLayeredWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.worldview.VerkleWorldState; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.metrics.ObservableMetricsSystem; + +public class VerkleCachedWorldStorageManager extends DiffBasedCachedWorldStorageManager { + + public VerkleCachedWorldStorageManager( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final ObservableMetricsSystem metricsSystem) { + super(archive, worldStateKeyValueStorage, metricsSystem); + } + + @Override + public DiffBasedWorldState createWorldState( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final EvmConfiguration evmConfiguration) { + return new VerkleWorldState( + (VerkleWorldStateProvider) archive, + (VerkleWorldStateKeyValueStorage) worldStateKeyValueStorage, + evmConfiguration); + } + + @Override + public DiffBasedWorldStateKeyValueStorage createLayeredKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { + return new VerkleLayeredWorldStateKeyValueStorage( + (VerkleWorldStateKeyValueStorage) worldStateKeyValueStorage); + } + + @Override + public DiffBasedWorldStateKeyValueStorage createSnapshotKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final ObservableMetricsSystem metricsSystem) { + return new VerkleSnapshotWorldStateKeyValueStorage( + (VerkleWorldStateKeyValueStorage) worldStateKeyValueStorage, metricsSystem); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleLayeredWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleLayeredWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..3a098e2f3ae --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleLayeredWorldStateKeyValueStorage.java @@ -0,0 +1,52 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage; + +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; +import org.hyperledger.besu.metrics.ObservableMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; +import org.hyperledger.besu.services.kvstore.LayeredKeyValueStorage; + +public class VerkleLayeredWorldStateKeyValueStorage extends VerkleSnapshotWorldStateKeyValueStorage + implements DiffBasedLayeredWorldStateKeyValueStorage, StorageSubscriber { + + public VerkleLayeredWorldStateKeyValueStorage(final VerkleWorldStateKeyValueStorage parent) { + this( + new LayeredKeyValueStorage(parent.getComposedWorldStateStorage()), + parent.getTrieLogStorage(), + parent, + parent.getMetricsSystem()); + } + + public VerkleLayeredWorldStateKeyValueStorage( + final SnappedKeyValueStorage composedWorldStateStorage, + final KeyValueStorage trieLogStorage, + final VerkleWorldStateKeyValueStorage parent, + final ObservableMetricsSystem metricsSystem) { + super(parent, composedWorldStateStorage, trieLogStorage, metricsSystem); + } + + @Override + public VerkleLayeredWorldStateKeyValueStorage clone() { + return new VerkleLayeredWorldStateKeyValueStorage( + ((LayeredKeyValueStorage) composedWorldStateStorage).clone(), + trieLogStorage, + parentWorldStateStorage, + metricsSystem); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..bb797e79045 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java @@ -0,0 +1,201 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.metrics.ObservableMetricsSystem; +import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; + +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VerkleSnapshotWorldStateKeyValueStorage extends VerkleWorldStateKeyValueStorage + implements DiffBasedSnapshotWorldStateKeyValueStorage, StorageSubscriber { + + protected final VerkleWorldStateKeyValueStorage parentWorldStateStorage; + private static final Logger LOG = + LoggerFactory.getLogger(VerkleSnapshotWorldStateKeyValueStorage.class); + private final long subscribeParentId; + + public VerkleSnapshotWorldStateKeyValueStorage( + final VerkleWorldStateKeyValueStorage parentWorldStateStorage, + final SnappedKeyValueStorage segmentedWorldStateStorage, + final KeyValueStorage trieLogStorage, + final ObservableMetricsSystem metricsSystem) { + super(segmentedWorldStateStorage, trieLogStorage, metricsSystem); + this.parentWorldStateStorage = parentWorldStateStorage; + this.subscribeParentId = parentWorldStateStorage.subscribe(this); + } + + public VerkleSnapshotWorldStateKeyValueStorage( + final VerkleWorldStateKeyValueStorage worldStateKeyValueStorage, + final ObservableMetricsSystem metricsSystem) { + this( + worldStateKeyValueStorage, + ((SnappableKeyValueStorage) worldStateKeyValueStorage.getComposedWorldStateStorage()) + .takeSnapshot(), + worldStateKeyValueStorage.getTrieLogStorage(), + metricsSystem); + } + + private boolean isClosedGet() { + if (isClosed.get()) { + Throwable t = new Throwable("Attempting to access closed worldstate"); + LOG.warn(t.getMessage(), t); + } + return isClosed.get(); + } + + @Override + public Updater updater() { + return new Updater( + ((SnappedKeyValueStorage) composedWorldStateStorage).getSnapshotTransaction(), + trieLogStorage.startTransaction(), + flatDbStrategy); + } + + @Override + public Optional getAccount(final Hash accountHash) { + return isClosedGet() ? Optional.empty() : super.getAccount(accountHash); + } + + @Override + public Optional getCode(final Bytes32 codeHash, final Hash accountHash) { + return isClosedGet() ? Optional.empty() : super.getCode(codeHash, accountHash); + } + + @Override + public Optional getTrieLog(final Hash blockHash) { + return isClosedGet() ? Optional.empty() : super.getTrieLog(blockHash); + } + + @Override + public Optional getStateTrieNode(final Bytes location) { + return isClosedGet() ? Optional.empty() : super.getStateTrieNode(location); + } + + @Override + public Optional getWorldStateRootHash() { + return isClosedGet() ? Optional.empty() : super.getWorldStateRootHash(); + } + + @Override + public Optional getWorldStateBlockHash() { + return isClosedGet() ? Optional.empty() : super.getWorldStateBlockHash(); + } + + @Override + public Optional getStorageValueByStorageSlotKey( + final Hash accountHash, final StorageSlotKey storageSlotKey) { + return isClosedGet() + ? Optional.empty() + : super.getStorageValueByStorageSlotKey(accountHash, storageSlotKey); + } + + @Override + public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { + return !isClosedGet() && super.isWorldStateAvailable(rootHash, blockHash); + } + + @Override + public void clear() { + // snapshot storage does not implement clear + throw new StorageException("Snapshot storage does not implement clear"); + } + + @Override + public void clearFlatDatabase() { + // snapshot storage does not implement clear + throw new StorageException("Snapshot storage does not implement clear"); + } + + @Override + public void clearTrieLog() { + // snapshot storage does not implement clear + throw new StorageException("Snapshot storage does not implement clear"); + } + + @Override + public void onCloseStorage() { + try { + // when the parent storage clears, close regardless of subscribers + doClose(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void onClearStorage() { + try { + // when the parent storage clears, close regardless of subscribers + doClose(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void onClearFlatDatabaseStorage() { + // when the parent storage clears, close regardless of subscribers + try { + doClose(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void onClearTrieLog() { + // when the parent storage clears, close regardless of subscribers + try { + doClose(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + protected synchronized void doClose() throws Exception { + if (!isClosedGet()) { + // alert any subscribers we are closing: + subscribers.forEach(StorageSubscriber::onCloseStorage); + + // close all of the SnappedKeyValueStorages: + composedWorldStateStorage.close(); + + // unsubscribe the parent worldstate + parentWorldStateStorage.unSubscribe(subscribeParentId); + + // set storage closed + isClosed.set(true); + } + } + + public VerkleWorldStateKeyValueStorage getParentWorldStateStorage() { + return parentWorldStateStorage; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..08f08e86fed --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleWorldStateKeyValueStorage.java @@ -0,0 +1,223 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.storage.StorageProvider; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; +import org.hyperledger.besu.metrics.ObservableMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +public class VerkleWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValueStorage + implements WorldStateKeyValueStorage { + + protected FullFlatDbStrategy flatDbStrategy; + + public VerkleWorldStateKeyValueStorage( + final StorageProvider provider, final ObservableMetricsSystem metricsSystem) { + super( + provider.getStorageBySegmentIdentifiers( + List.of( + ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)), + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE), + metricsSystem); + this.flatDbStrategy = new FullFlatDbStrategy(metricsSystem); + } + + public VerkleWorldStateKeyValueStorage( + final SegmentedKeyValueStorage composedWorldStateStorage, + final KeyValueStorage trieLogStorage, + final ObservableMetricsSystem metricsSystem) { + super(composedWorldStateStorage, trieLogStorage, metricsSystem); + this.flatDbStrategy = new FullFlatDbStrategy(metricsSystem); + } + + @Override + public FlatDbStrategy getFlatDbStrategy() { + return flatDbStrategy; + } + + @Override + public DataStorageFormat getDataStorageFormat() { + return DataStorageFormat.VERKLE; + } + + @Override + public FlatDbMode getFlatDbMode() { + return FlatDbMode.FULL; + } + + public Optional getCode(final Bytes32 codeHash, final Hash accountHash) { + if (codeHash.equals(Hash.EMPTY)) { + return Optional.of(Bytes.EMPTY); + } else { + return getFlatDbStrategy().getFlatCode(codeHash, accountHash, composedWorldStateStorage); + } + } + + public Optional getAccount(final Hash accountHash) { + return getFlatDbStrategy().getFlatAccount(null, null, accountHash, composedWorldStateStorage); + } + + public Optional getStorageValueByStorageSlotKey( + final Hash accountHash, final StorageSlotKey storageSlotKey) { + return getFlatDbStrategy() + .getFlatStorageValueByStorageSlotKey( + null, null, null, accountHash, storageSlotKey, composedWorldStateStorage); + } + + @Override + public void clear() { + super.clear(); + } + + @Override + public Updater updater() { + return new Updater( + composedWorldStateStorage.startTransaction(), + trieLogStorage.startTransaction(), + flatDbStrategy); + } + + public static class Updater implements DiffBasedWorldStateKeyValueStorage.Updater { + + private final SegmentedKeyValueStorageTransaction composedWorldStateTransaction; + private final KeyValueStorageTransaction trieLogStorageTransaction; + private final FlatDbStrategy flatDbStrategy; + + public Updater( + final SegmentedKeyValueStorageTransaction composedWorldStateTransaction, + final KeyValueStorageTransaction trieLogStorageTransaction, + final FlatDbStrategy flatDbStrategy) { + + this.composedWorldStateTransaction = composedWorldStateTransaction; + this.trieLogStorageTransaction = trieLogStorageTransaction; + this.flatDbStrategy = flatDbStrategy; + } + + public Updater removeCode(final Hash accountHash) { + flatDbStrategy.removeFlatCode(composedWorldStateTransaction, accountHash); + return this; + } + + public Updater putCode(final Hash accountHash, final Bytes code) { + // Skip the hash calculation for empty code + final Hash codeHash = code.size() == 0 ? Hash.EMPTY : Hash.hash(code); + return putCode(accountHash, codeHash, code); + } + + public Updater putCode(final Hash accountHash, final Bytes32 codeHash, final Bytes code) { + if (code.size() == 0) { + // Don't save empty values + return this; + } + flatDbStrategy.putFlatCode(composedWorldStateTransaction, accountHash, codeHash, code); + return this; + } + + public Updater removeAccountInfoState(final Hash accountHash) { + flatDbStrategy.removeFlatAccount(composedWorldStateTransaction, accountHash); + return this; + } + + public Updater putAccountInfoState(final Hash accountHash, final Bytes accountValue) { + if (accountValue.size() == 0) { + // Don't save empty values + return this; + } + flatDbStrategy.putFlatAccount(composedWorldStateTransaction, accountHash, accountValue); + return this; + } + + @Override + public Updater saveWorldState(final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { + composedWorldStateTransaction.put( + TRIE_BRANCH_STORAGE, Bytes.EMPTY.toArrayUnsafe(), node.toArrayUnsafe()); + composedWorldStateTransaction.put( + TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, nodeHash.toArrayUnsafe()); + composedWorldStateTransaction.put( + TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY, blockHash.toArrayUnsafe()); + return this; + } + + public Updater putStateTrieNode(final Bytes location, final Bytes node) { + composedWorldStateTransaction.put( + TRIE_BRANCH_STORAGE, location.toArrayUnsafe(), node.toArrayUnsafe()); + return this; + } + + public Updater removeStateTrieNode(final Bytes location) { + composedWorldStateTransaction.remove(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()); + return this; + } + + public synchronized Updater putStorageValueBySlotHash( + final Hash accountHash, final Hash slotHash, final Bytes storage) { + flatDbStrategy.putFlatAccountStorageValueByStorageSlotHash( + composedWorldStateTransaction, accountHash, slotHash, storage); + return this; + } + + public synchronized void removeStorageValueBySlotHash( + final Hash accountHash, final Hash slotHash) { + flatDbStrategy.removeFlatAccountStorageValueByStorageSlotHash( + composedWorldStateTransaction, accountHash, slotHash); + } + + @Override + public SegmentedKeyValueStorageTransaction getWorldStateTransaction() { + return composedWorldStateTransaction; + } + + @Override + public KeyValueStorageTransaction getTrieLogStorageTransaction() { + return trieLogStorageTransaction; + } + + @Override + public void commit() { + // write the log ahead, then the worldstate + trieLogStorageTransaction.commit(); + composedWorldStateTransaction.commit(); + } + + @Override + public void rollback() { + composedWorldStateTransaction.rollback(); + trieLogStorageTransaction.rollback(); + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/trielog/TrieLogFactoryImpl.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/trielog/TrieLogFactoryImpl.java new file mode 100644 index 00000000000..07c4b84ccbf --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/trielog/TrieLogFactoryImpl.java @@ -0,0 +1,272 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.trielog; + +import org.hyperledger.besu.datatypes.AccountValue; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLPInput; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.plugin.data.BlockHeader; +import org.hyperledger.besu.plugin.services.trielogs.TrieLog; +import org.hyperledger.besu.plugin.services.trielogs.TrieLogAccumulator; +import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.BiConsumer; +import java.util.function.Function; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; + +public class TrieLogFactoryImpl implements TrieLogFactory { + @Override + public TrieLogLayer create(final TrieLogAccumulator accumulator, final BlockHeader blockHeader) { + TrieLogLayer layer = new TrieLogLayer(); + layer.setBlockHash(blockHeader.getBlockHash()); + layer.setBlockNumber(blockHeader.getNumber()); + for (final var updatedAccount : accumulator.getAccountsToUpdate().entrySet()) { + final var bonsaiValue = updatedAccount.getValue(); + final var oldAccountValue = bonsaiValue.getPrior(); + final var newAccountValue = bonsaiValue.getUpdated(); + if (oldAccountValue == null && newAccountValue == null) { + // by default do not persist empty reads of accounts to the trie log + continue; + } + layer.addAccountChange(updatedAccount.getKey(), oldAccountValue, newAccountValue); + } + + for (final var updatedCode : accumulator.getCodeToUpdate().entrySet()) { + layer.addCodeChange( + updatedCode.getKey(), + updatedCode.getValue().getPrior(), + updatedCode.getValue().getUpdated(), + blockHeader.getBlockHash()); + } + + for (final var updatesStorage : accumulator.getStorageToUpdate().entrySet()) { + final Address address = updatesStorage.getKey(); + for (final var slotUpdate : updatesStorage.getValue().entrySet()) { + var val = slotUpdate.getValue(); + + if (val.getPrior() == null && val.getUpdated() == null) { + // by default do not persist empty reads to the trie log + continue; + } + + System.out.println(val.getPrior() + " " + val.getUpdated()); + layer.addStorageChange(address, slotUpdate.getKey(), val.getPrior(), val.getUpdated()); + } + } + return layer; + } + + @Override + public byte[] serialize(final TrieLog layer) { + final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); + writeTo(layer, rlpLog); + return rlpLog.encoded().toArrayUnsafe(); + } + + public static void writeTo(final TrieLog layer, final RLPOutput output) { + layer.freeze(); + + final Set
addresses = new TreeSet<>(); + addresses.addAll(layer.getAccountChanges().keySet()); + addresses.addAll(layer.getCodeChanges().keySet()); + addresses.addAll(layer.getStorageChanges().keySet()); + + output.startList(); // container + output.writeBytes(layer.getBlockHash()); + + for (final Address address : addresses) { + output.startList(); // this change + output.writeBytes(address); + + final TrieLog.LogTuple accountChange = layer.getAccountChanges().get(address); + if (accountChange == null || accountChange.isUnchanged()) { + output.writeNull(); + } else { + writeRlp(accountChange, output, (o, sta) -> sta.writeTo(o)); + } + + final TrieLog.LogTuple codeChange = layer.getCodeChanges().get(address); + if (codeChange == null || codeChange.isUnchanged()) { + output.writeNull(); + } else { + writeRlp(codeChange, output, RLPOutput::writeBytes); + } + + final Map> storageChanges = + layer.getStorageChanges().get(address); + if (storageChanges == null) { + output.writeNull(); + } else { + output.startList(); + for (final Map.Entry> storageChangeEntry : + storageChanges.entrySet()) { + output.startList(); + // do not write slotKey, it is not used in mainnet bonsai trielogs + StorageSlotKey storageSlotKey = storageChangeEntry.getKey(); + output.writeBytes(storageSlotKey.getSlotHash()); + writeInnerRlp(storageChangeEntry.getValue(), output, RLPOutput::writeBytes); + if (storageSlotKey.getSlotKey().isPresent()) { + output.writeUInt256Scalar(storageSlotKey.getSlotKey().get()); + } + output.endList(); + } + output.endList(); + } + + output.endList(); // this change + } + output.endList(); // container + } + + @Override + public TrieLogLayer deserialize(final byte[] bytes) { + return readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false)); + } + + public static TrieLogLayer readFrom(final RLPInput input) { + final TrieLogLayer newLayer = new TrieLogLayer(); + + input.enterList(); + newLayer.setBlockHash(Hash.wrap(input.readBytes32())); + + while (!input.isEndOfCurrentList()) { + input.enterList(); + final Address address = Address.readFrom(input); + + if (input.nextIsNull()) { + input.skipNext(); + } else { + input.enterList(); + final StateTrieAccountValue oldValue = nullOrValue(input, StateTrieAccountValue::readFrom); + final StateTrieAccountValue newValue = nullOrValue(input, StateTrieAccountValue::readFrom); + final boolean isCleared = getOptionalIsCleared(input); + input.leaveList(); + newLayer + .getAccountChanges() + .put(address, new DiffBasedValue<>(oldValue, newValue, isCleared)); + } + + if (input.nextIsNull()) { + input.skipNext(); + } else { + input.enterList(); + final Bytes oldCode = nullOrValue(input, RLPInput::readBytes); + final Bytes newCode = nullOrValue(input, RLPInput::readBytes); + final boolean isCleared = getOptionalIsCleared(input); + input.leaveList(); + newLayer.getCodeChanges().put(address, new DiffBasedValue<>(oldCode, newCode, isCleared)); + } + + if (input.nextIsNull()) { + input.skipNext(); + } else { + final Map> storageChanges = new TreeMap<>(); + input.enterList(); + while (!input.isEndOfCurrentList()) { + int storageElementlistSize = input.enterList(); + final Hash slotHash = Hash.wrap(input.readBytes32()); + final UInt256 oldValue = + nullOrValue(input, rlpInput -> UInt256.fromBytes(rlpInput.readBytes())); + final UInt256 newValue = + nullOrValue(input, rlpInput -> UInt256.fromBytes(rlpInput.readBytes())); + final boolean isCleared = getOptionalIsCleared(input); + final Optional slotKey = + Optional.of(storageElementlistSize) + .filter(listSize -> listSize == 5) + .map(__ -> input.readUInt256Scalar()) + .or(Optional::empty); + + final StorageSlotKey storageSlotKey = new StorageSlotKey(slotHash, slotKey); + storageChanges.put(storageSlotKey, new DiffBasedValue<>(oldValue, newValue, isCleared)); + input.leaveList(); + } + input.leaveList(); + newLayer.getStorageChanges().put(address, storageChanges); + } + + // TODO add trie nodes + + // lenient leave list for forward compatible additions. + input.leaveListLenient(); + } + input.leaveListLenient(); + newLayer.freeze(); + + return newLayer; + } + + protected static T nullOrValue(final RLPInput input, final Function reader) { + if (input.nextIsNull()) { + input.skipNext(); + return null; + } else { + return reader.apply(input); + } + } + + protected static boolean getOptionalIsCleared(final RLPInput input) { + return Optional.of(input.isEndOfCurrentList()) + .filter(isEnd -> !isEnd) // isCleared is optional + .map(__ -> nullOrValue(input, RLPInput::readInt)) + .filter(i -> i == 1) + .isPresent(); + } + + public static void writeRlp( + final TrieLog.LogTuple value, + final RLPOutput output, + final BiConsumer writer) { + output.startList(); + writeInnerRlp(value, output, writer); + output.endList(); + } + + public static void writeInnerRlp( + final TrieLog.LogTuple value, + final RLPOutput output, + final BiConsumer writer) { + if (value.getPrior() == null) { + output.writeNull(); + } else { + writer.accept(output, value.getPrior()); + } + if (value.getUpdated() == null) { + output.writeNull(); + } else { + writer.accept(output, value.getUpdated()); + } + if (!value.isCleared()) { + output.writeNull(); + } else { + output.writeInt(1); + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java new file mode 100644 index 00000000000..384bdf4cf29 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java @@ -0,0 +1,361 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.worldview; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.trie.NodeLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleLayeredWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.verkletrie.VerkleTrie; +import org.hyperledger.besu.ethereum.verkletrie.VerkleTrieKeyValueGenerator; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; +import javax.annotation.Nonnull; + +import kotlin.Pair; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VerkleWorldState extends DiffBasedWorldState { + + private static final Logger LOG = LoggerFactory.getLogger(VerkleWorldState.class); + + private final VerkleTrieKeyValueGenerator verkleTrieKeyValueGenerator = + new VerkleTrieKeyValueGenerator(); + + public VerkleWorldState( + final VerkleWorldStateProvider archive, + final VerkleWorldStateKeyValueStorage worldStateKeyValueStorage, + final EvmConfiguration evmConfiguration) { + this( + worldStateKeyValueStorage, + archive.getCachedWorldStorageManager(), + archive.getTrieLogManager(), + evmConfiguration); + } + + protected VerkleWorldState( + final VerkleWorldStateKeyValueStorage worldStateKeyValueStorage, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, + final TrieLogManager trieLogManager, + final EvmConfiguration evmConfiguration) { + super(worldStateKeyValueStorage, cachedWorldStorageManager, trieLogManager); + this.setAccumulator( + new VerkleWorldStateUpdateAccumulator( + this, (addr, value) -> {}, (addr, value) -> {}, evmConfiguration)); + } + + @Override + public VerkleWorldStateKeyValueStorage getWorldStateStorage() { + return (VerkleWorldStateKeyValueStorage) worldStateKeyValueStorage; + } + + @Override + protected Hash calculateRootHash( + final Optional maybeStateUpdater, + final DiffBasedWorldStateUpdateAccumulator worldStateUpdater) { + return internalCalculateRootHash( + maybeStateUpdater.map(VerkleWorldStateKeyValueStorage.Updater.class::cast), + (VerkleWorldStateUpdateAccumulator) worldStateUpdater); + } + + protected Hash internalCalculateRootHash( + final Optional maybeStateUpdater, + final VerkleWorldStateUpdateAccumulator worldStateUpdater) { + + final VerkleTrie stateTrie = + createTrie( + (location, hash) -> worldStateKeyValueStorage.getStateTrieNode(location), + worldStateRootHash); + // clearStorage(maybeStateUpdater, worldStateUpdater); + + Stream>>> + storageStream = worldStateUpdater.getStorageToUpdate().entrySet().stream(); + if (maybeStateUpdater.isEmpty()) { + storageStream = + storageStream + .parallel(); // if we are not updating the state updater we can use parallel stream + } + storageStream.forEach( + addressMapEntry -> + updateAccountStorageState( + stateTrie, maybeStateUpdater, worldStateUpdater, addressMapEntry)); + + // Third update the code. This has the side effect of ensuring a code hash is calculated. + updateCode(stateTrie, maybeStateUpdater, worldStateUpdater); + + // for manicured tries and composting, collect branches here (not implemented) + updateTheAccounts(maybeStateUpdater, worldStateUpdater, stateTrie); + + LOG.info("start commit "); + maybeStateUpdater.ifPresent( + bonsaiUpdater -> + stateTrie.commit( + (location, hash, value) -> { + writeTrieNode( + TRIE_BRANCH_STORAGE, + bonsaiUpdater.getWorldStateTransaction(), + location, + value); + })); + + LOG.info("end commit "); + LOG.info(stateTrie.toDotTree()); + final Bytes32 rootHash = stateTrie.getRootHash(); + + LOG.info("end commit "); + return Hash.wrap(rootHash); + } + + private void updateTheAccounts( + final Optional maybeStateUpdater, + final VerkleWorldStateUpdateAccumulator worldStateUpdater, + final VerkleTrie stateTrie) { + for (final Map.Entry> accountUpdate : + worldStateUpdater.getAccountsToUpdate().entrySet()) { + final Address accountKey = accountUpdate.getKey(); + final DiffBasedValue bonsaiValue = accountUpdate.getValue(); + final VerkleAccount priorAccount = bonsaiValue.getPrior(); + final VerkleAccount updatedAccount = bonsaiValue.getUpdated(); + if (updatedAccount == null) { + final Hash addressHash = hashAndSavePreImage(accountKey); + verkleTrieKeyValueGenerator + .generateKeysForAccount(accountKey) + .forEach( + bytes -> { + System.out.println("remove " + bytes); + stateTrie.remove(bytes); + }); + maybeStateUpdater.ifPresent( + bonsaiUpdater -> bonsaiUpdater.removeAccountInfoState(addressHash)); + } else { + final Bytes priorValue = priorAccount == null ? null : priorAccount.serializeAccount(); + final Bytes accountValue = updatedAccount.serializeAccount(); + if (!accountValue.equals(priorValue)) { + verkleTrieKeyValueGenerator + .generateKeyValuesForAccount( + accountKey, + updatedAccount.getNonce(), + updatedAccount.getBalance(), + updatedAccount.getCodeHash()) + .forEach( + (bytes, bytes2) -> { + System.out.println("add " + bytes + " " + bytes2); + stateTrie.put(bytes, bytes2); + }); + maybeStateUpdater.ifPresent( + bonsaiUpdater -> + bonsaiUpdater.putAccountInfoState(hashAndSavePreImage(accountKey), accountValue)); + } + } + } + } + + private void updateCode( + final VerkleTrie stateTrie, + final Optional maybeStateUpdater, + final VerkleWorldStateUpdateAccumulator worldStateUpdater) { + maybeStateUpdater.ifPresent( + bonsaiUpdater -> { + for (final Map.Entry> codeUpdate : + worldStateUpdater.getCodeToUpdate().entrySet()) { + final Bytes previousCode = codeUpdate.getValue().getPrior(); + final Bytes updatedCode = codeUpdate.getValue().getUpdated(); + final Address address = codeUpdate.getKey(); + final Hash accountHash = address.addressHash(); + if (updatedCode == null) { + verkleTrieKeyValueGenerator + .generateKeysForCode(address, previousCode) + .forEach( + bytes -> { + System.out.println("remove code " + bytes); + stateTrie.remove(bytes); + }); + bonsaiUpdater.removeCode(accountHash); + } else { + if (updatedCode.isEmpty()) { + final Hash codeHash = updatedCode.size() == 0 ? Hash.EMPTY : Hash.hash(updatedCode); + verkleTrieKeyValueGenerator + .generateKeyValuesForCode(address, codeHash, updatedCode) + .forEach( + (bytes, bytes2) -> { + // System.out.println("add code " + bytes + " " + bytes2); + stateTrie.put(bytes, bytes2); + }); + bonsaiUpdater.removeCode(accountHash); + } else { + final Hash codeHash = updatedCode.size() == 0 ? Hash.EMPTY : Hash.hash(updatedCode); + verkleTrieKeyValueGenerator + .generateKeyValuesForCode(address, codeHash, updatedCode) + .forEach( + (bytes, bytes2) -> { + System.out.println("add code " + bytes + " " + bytes2); + stateTrie.put(bytes, bytes2); + }); + bonsaiUpdater.putCode(accountHash, null, updatedCode); + } + } + } + }); + } + + private void updateAccountStorageState( + final VerkleTrie stateTrie, + final Optional maybeStateUpdater, + final VerkleWorldStateUpdateAccumulator worldStateUpdater, + final Map.Entry>> + storageAccountUpdate) { + final Address updatedAddress = storageAccountUpdate.getKey(); + final Hash updatedAddressHash = updatedAddress.addressHash(); + if (worldStateUpdater.getAccountsToUpdate().containsKey(updatedAddress)) { + + // for manicured tries and composting, collect branches here (not implemented) + for (final Map.Entry> storageUpdate : + storageAccountUpdate.getValue().entrySet()) { + final Hash slotHash = storageUpdate.getKey().getSlotHash(); + final UInt256 updatedStorage = storageUpdate.getValue().getUpdated(); + if (updatedStorage == null) { + verkleTrieKeyValueGenerator + .generateKeysForStorage(updatedAddress, storageUpdate.getKey()) + .forEach( + bytes -> { + System.out.println("remove storage" + bytes); + stateTrie.remove(bytes); + }); + maybeStateUpdater.ifPresent( + diffBasedUpdater -> + diffBasedUpdater.removeStorageValueBySlotHash(updatedAddressHash, slotHash)); + } else { + final Pair storage = + verkleTrieKeyValueGenerator.generateKeyValuesForStorage( + updatedAddress, storageUpdate.getKey(), updatedStorage); + System.out.println("add storage " + storage.getFirst() + " " + storage.getSecond()); + stateTrie + .put(storage.getFirst(), storage.getSecond()) + .ifPresentOrElse( + bytes -> { + System.out.println("found old key " + bytes); + storageUpdate.getValue().setPrior(UInt256.fromBytes(bytes)); + }, + () -> { + storageUpdate.getValue().setPrior(null); + }); + if (updatedStorage.equals(UInt256.ZERO)) { + maybeStateUpdater.ifPresent( + bonsaiUpdater -> + bonsaiUpdater.removeStorageValueBySlotHash(updatedAddressHash, slotHash)); + } else { + maybeStateUpdater.ifPresent( + bonsaiUpdater -> + bonsaiUpdater.putStorageValueBySlotHash( + updatedAddressHash, slotHash, updatedStorage)); + } + } + } + } + } + + @Override + public MutableWorldState freeze() { + this.isFrozen = true; + this.worldStateKeyValueStorage = + new VerkleLayeredWorldStateKeyValueStorage(getWorldStateStorage()); + return this; + } + + @Override + public Account get(final Address address) { + return getWorldStateStorage() + .getAccount(address.addressHash()) + .map(bytes -> VerkleAccount.fromRLP(accumulator, address, bytes, true)) + .orElse(null); + } + + @Override + public Optional getCode(@Nonnull final Address address, final Hash codeHash) { + return getWorldStateStorage().getCode(codeHash, address.addressHash()); + } + + protected void writeTrieNode( + final SegmentIdentifier segmentId, + final SegmentedKeyValueStorageTransaction tx, + final Bytes location, + final Bytes value) { + tx.put(segmentId, location.toArrayUnsafe(), value.toArrayUnsafe()); + } + + @Override + public UInt256 getStorageValue(final Address address, final UInt256 storageKey) { + return getStorageValueByStorageSlotKey(address, new StorageSlotKey(storageKey)) + .orElse(UInt256.ZERO); + } + + @Override + public Optional getStorageValueByStorageSlotKey( + final Address address, final StorageSlotKey storageSlotKey) { + return getWorldStateStorage() + .getStorageValueByStorageSlotKey(address.addressHash(), storageSlotKey) + .map(UInt256::fromBytes); + } + + @Override + public UInt256 getPriorStorageValue(final Address address, final UInt256 storageKey) { + return getStorageValue(address, storageKey); + } + + @Override + public Map getAllAccountStorage(final Address address, final Hash rootHash) { + throw new UnsupportedOperationException("getAllAccountStorage not yet available for verkle"); + } + + private VerkleTrie createTrie(final NodeLoader nodeLoader, final Bytes32 rootHash) { + return new VerkleTrie(nodeLoader, rootHash); + } + + protected Hash hashAndSavePreImage(final Bytes value) { + // by default do not save has preImages + return Hash.hash(value); + } + + @Override + protected Hash getEmptyTrieHash() { + return Hash.wrap(Bytes32.ZERO); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java new file mode 100644 index 00000000000..60806078436 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java @@ -0,0 +1,104 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.worldview; + +import org.hyperledger.besu.datatypes.AccountValue; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; + +public class VerkleWorldStateUpdateAccumulator + extends DiffBasedWorldStateUpdateAccumulator { + public VerkleWorldStateUpdateAccumulator( + final DiffBasedWorldView world, + final Consumer> accountPreloader, + final Consumer storagePreloader, + final EvmConfiguration evmConfiguration) { + super(world, accountPreloader, storagePreloader, evmConfiguration); + } + + @Override + public DiffBasedWorldStateUpdateAccumulator copy() { + final VerkleWorldStateUpdateAccumulator copy = + new VerkleWorldStateUpdateAccumulator( + wrappedWorldView(), + getAccountPreloader(), + getStoragePreloader(), + getEvmConfiguration()); + copy.cloneFromUpdater(this); + return copy; + } + + @Override + protected VerkleAccount copyAccount(final VerkleAccount account) { + return new VerkleAccount(account); + } + + @Override + protected VerkleAccount copyAccount( + final VerkleAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + return new VerkleAccount(toCopy, context, mutable); + } + + @Override + protected VerkleAccount createAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable) { + return new VerkleAccount(context, address, stateTrieAccount, mutable); + } + + @Override + protected VerkleAccount createAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash storageRoot, + final Hash codeHash, + final boolean mutable) { + return new VerkleAccount( + context, address, addressHash, nonce, balance, storageRoot, codeHash, mutable); + } + + @Override + protected VerkleAccount createAccount( + final DiffBasedWorldView context, final UpdateTrackingAccount tracked) { + return new VerkleAccount(context, tracked); + } + + @Override + protected void assertCloseEnoughForDiffing( + final VerkleAccount source, final AccountValue account, final String context) { + VerkleAccount.assertCloseEnoughForDiffing(source, account, context); + } + + @Override + protected boolean shouldIgnoreIdenticalValuesDuringAccountRollingUpdate() { + return false; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java index c68258ba7b9..baaae11f830 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java @@ -19,7 +19,7 @@ public enum DataStorageFormat { FOREST(1), // Original format. Store all tries BONSAI(2), // New format. Store one trie, and trie logs to roll forward and backward. - VERKLE(3); + VERKLE(2); // TODO MOVE TO 3 private final int databaseVersion; diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 46c941be536..5cf31e0209b 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -29,6 +29,8 @@ import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; @@ -110,6 +112,26 @@ public static BonsaiWorldStateProvider createBonsaiInMemoryWorldStateArchive( TrieLogPruner.noOpTrieLogPruner()); } + public static VerkleWorldStateProvider createVerkleInMemoryWorldStateArchive( + final Blockchain blockchain) { + return createVerkleInMemoryWorldStateArchive(blockchain, EvmConfiguration.DEFAULT); + } + + public static VerkleWorldStateProvider createVerkleInMemoryWorldStateArchive( + final Blockchain blockchain, final EvmConfiguration evmConfiguration) { + final InMemoryKeyValueStorageProvider inMemoryKeyValueStorageProvider = + new InMemoryKeyValueStorageProvider(); + return new VerkleWorldStateProvider( + (VerkleWorldStateKeyValueStorage) + inMemoryKeyValueStorageProvider.createWorldStateStorage(DataStorageFormat.VERKLE), + blockchain, + Optional.empty(), + new NoOpMetricsSystem(), + null, + evmConfiguration, + TrieLogPruner.noOpTrieLogPruner()); + } + public static MutableWorldState createInMemoryWorldState() { final InMemoryKeyValueStorageProvider provider = new InMemoryKeyValueStorageProvider(); return new ForestMutableWorldState( diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java new file mode 100644 index 00000000000..fb3653bc0f3 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java @@ -0,0 +1,584 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.verkle; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.core.Difficulty; +import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.worldview.VerkleWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.worldview.VerkleWorldStateUpdateAccumulator; +import org.hyperledger.besu.evm.account.MutableAccount; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.evm.log.LogsBloomFilter; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; + +import java.util.Optional; +import java.util.stream.Collectors; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@SuppressWarnings("unused") +@ExtendWith(MockitoExtension.class) +class LogRollingTests { + + private VerkleWorldStateProvider archive; + + private static final BlockHeaderTestFixture blockHeaderTestFixture = new BlockHeaderTestFixture(); + private InMemoryKeyValueStorageProvider provider; + private KeyValueStorage accountStorage; + private KeyValueStorage codeStorage; + private KeyValueStorage storageStorage; + private KeyValueStorage trieBranchStorage; + private KeyValueStorage trieLogStorage; + + private InMemoryKeyValueStorageProvider secondProvider; + private VerkleWorldStateProvider secondArchive; + private KeyValueStorage secondAccountStorage; + private KeyValueStorage secondCodeStorage; + private KeyValueStorage secondStorageStorage; + private KeyValueStorage secondTrieBranchStorage; + private KeyValueStorage secondTrieLogStorage; + private final Blockchain blockchain = mock(Blockchain.class); + + private static final Address addressOne = + Address.fromHexString("0x1111111111111111111111111111111111111111"); + + private static final BlockHeader headerOne = + new BlockHeader( + Hash.ZERO, + Hash.EMPTY_LIST_HASH, + Address.ZERO, + Hash.fromHexString("0x3869378cd87434ffd04c4e187312d69d1430dc62e575c4b4b061ac625b88ec08"), + Hash.EMPTY_TRIE_HASH, + Hash.EMPTY_LIST_HASH, + LogsBloomFilter.builder().build(), + Difficulty.ONE, + 1, + 0, + 0, + 0, + Bytes.EMPTY, + Wei.ZERO, + Hash.ZERO, + 0, + null, + null, // blobGasUSed + null, + null, + null, + new MainnetBlockHeaderFunctions()); + private static final BlockHeader headerTwo = + new BlockHeader( + headerOne.getHash(), + Hash.EMPTY_LIST_HASH, + Address.ZERO, + Hash.fromHexString("0x3e7c057b149069fadbb2bd2c752184cb5c7a9c736d27682c9e557ceda8ede10e"), + Hash.EMPTY_TRIE_HASH, + Hash.EMPTY_LIST_HASH, + LogsBloomFilter.builder().build(), + Difficulty.ONE, + 2, + 0, + 0, + 0, + Bytes.EMPTY, + Wei.ZERO, + Hash.ZERO, + 0, + null, + null, // blobGasUsed + null, + null, + null, + new MainnetBlockHeaderFunctions()); + private static final BlockHeader headerThree = + new BlockHeader( + headerOne.getHash(), + Hash.EMPTY_LIST_HASH, + Address.ZERO, + Hash.fromHexString("0xec5d7bd6bd7ce01e58bb389475767350852e2ce2bb72b8cd9c9b55d118c14e07"), + Hash.EMPTY_TRIE_HASH, + Hash.EMPTY_LIST_HASH, + LogsBloomFilter.builder().build(), + Difficulty.ONE, + 3, + 0, + 0, + 0, + Bytes.EMPTY, + Wei.ZERO, + Hash.ZERO, + 0, + null, + null, // blobGasUsed + null, + null, + null, + new MainnetBlockHeaderFunctions()); + + private static final BlockHeader headerFour = + new BlockHeader( + headerOne.getHash(), + Hash.EMPTY_LIST_HASH, + Address.ZERO, + Hash.fromHexString("0x3869378cd87434ffd04c4e187312d69d1430dc62e575c4b4b061ac625b88ec08"), + Hash.EMPTY_TRIE_HASH, + Hash.EMPTY_LIST_HASH, + LogsBloomFilter.builder().build(), + Difficulty.ONE, + 3, + 0, + 0, + 0, + Bytes.EMPTY, + Wei.ZERO, + Hash.ZERO, + 0, + null, + null, // blobGasUsed + null, + null, + null, + new MainnetBlockHeaderFunctions()); + + @BeforeEach + void createStorage() { + provider = new InMemoryKeyValueStorageProvider(); + archive = InMemoryKeyValueStorageProvider.createVerkleInMemoryWorldStateArchive(blockchain); + accountStorage = + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); + codeStorage = provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE); + storageStorage = + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE); + trieBranchStorage = + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE); + trieLogStorage = + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + + secondProvider = new InMemoryKeyValueStorageProvider(); + secondArchive = + InMemoryKeyValueStorageProvider.createVerkleInMemoryWorldStateArchive(blockchain); + secondAccountStorage = + secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); + secondCodeStorage = + secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE); + secondStorageStorage = + secondProvider.getStorageBySegmentIdentifier( + KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE); + secondTrieBranchStorage = + secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE); + secondTrieLogStorage = + secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + } + + @Test + void rollForwardComparedWithTestnet() { // TODO change the name + + final VerkleWorldState worldState = + new VerkleWorldState( + archive, + new VerkleWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + EvmConfiguration.DEFAULT); + final WorldUpdater updater = worldState.updater(); + + final MutableAccount contract = + updater.createAccount( + Address.fromHexString("0x2a97e18168654393a573599759104efdfec6d8bd"), 1, Wei.ZERO); + contract.setCode( + Bytes.fromHexString( + "608060405234801561000f575f80fd5b5060043610610034575f3560e01c80632e64cec1146100385780636057361d14610056575b5f80fd5b610040610072565b60405161004d919061029a565b60405180910390f35b610070600480360381019061006b91906102e1565b61019c565b005b5f8060405161008090610275565b604051809103905ff080158015610099573d5f803e3d5ffd5b5090505f60015f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16632e64cec16040518163ffffffff1660e01b8152600401602060405180830381865afa158015610107573d5f803e3d5ffd5b505050506040513d601f19601f8201168201806040525081019061012b9190610320565b90508173ffffffffffffffffffffffffffffffffffffffff16636057361d826040518263ffffffff1660e01b8152600401610166919061029a565b5f604051808303815f87803b15801561017d575f80fd5b505af115801561018f573d5f803e3d5ffd5b505050505f549250505090565b805f819055505f6040516101af90610275565b604051809103905ff0801580156101c8573d5f803e3d5ffd5b5090508073ffffffffffffffffffffffffffffffffffffffff16636057361d836040518263ffffffff1660e01b8152600401610204919061029a565b5f604051808303815f87803b15801561021b575f80fd5b505af115801561022d573d5f803e3d5ffd5b505050508060015f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055505050565b6101e38061034c83390190565b5f819050919050565b61029481610282565b82525050565b5f6020820190506102ad5f83018461028b565b92915050565b5f80fd5b6102c081610282565b81146102ca575f80fd5b50565b5f813590506102db816102b7565b92915050565b5f602082840312156102f6576102f56102b3565b5b5f610303848285016102cd565b91505092915050565b5f8151905061031a816102b7565b92915050565b5f60208284031215610335576103346102b3565b5b5f6103428482850161030c565b9150509291505056fe608060405234801561000f575f80fd5b506101c68061001d5f395ff3fe60806040526004361061003e575f3560e01c80632711432d146100425780632e64cec11461006c5780636057361d14610096578063d64c8ca4146100be575b5f80fd5b34801561004d575f80fd5b506100566100c8565b604051610063919061011e565b60405180910390f35b348015610077575f80fd5b506100806100d1565b60405161008d919061011e565b60405180910390f35b3480156100a1575f80fd5b506100bc60048036038101906100b79190610165565b6100d9565b005b6100c66100e9565b005b5f600154905090565b5f8054905090565b805f819055508060018190555050565b5f3390508073ffffffffffffffffffffffffffffffffffffffff16ff5b5f819050919050565b61011881610106565b82525050565b5f6020820190506101315f83018461010f565b92915050565b5f80fd5b61014481610106565b811461014e575f80fd5b50565b5f8135905061015f8161013b565b92915050565b5f6020828403121561017a57610179610137565b5b5f61018784828501610151565b9150509291505056fea2646970667358221220dc349a9524617af5742ac60346440c0d09b175e4d9c4d95e378a9652cb9acbb064736f6c63430008160033a264697066735822122079744fe4f745783dffcec2415a6b99b8b7b340bcf4a768d5563f00d2ec1f916b64736f6c63430008160033")); + contract.setStorageValue(UInt256.ZERO, UInt256.fromHexString("0x0c")); + final MutableAccount mutableAccount = + updater.createAccount( + Address.fromHexString("0xb247faa497c752519917402cd79414727222f792"), + 2, + Wei.fromHexString("56cdce8421269edc4")); + updater.commit(); + worldState.persist(null); + } + + @Test + void simpleRollForwardTest() { + + final VerkleWorldState worldState = + new VerkleWorldState( + archive, + new VerkleWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + EvmConfiguration.DEFAULT); + final WorldUpdater updater = worldState.updater(); + + final MutableAccount mutableAccount = updater.createAccount(addressOne, 1, Wei.of(1L)); + mutableAccount.setCode(Bytes.of(0, 1, 2)); + mutableAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + updater.commit(); + worldState.persist(headerOne); + + final VerkleWorldState secondWorldState = + new VerkleWorldState( + secondArchive, + new VerkleWorldStateKeyValueStorage(secondProvider, new NoOpMetricsSystem()), + EvmConfiguration.DEFAULT); + final VerkleWorldStateUpdateAccumulator secondUpdater = + (VerkleWorldStateUpdateAccumulator) secondWorldState.updater(); + + final Optional value = trieLogStorage.get(headerOne.getHash().toArrayUnsafe()); + + final TrieLogLayer layer = + TrieLogFactoryImpl.readFrom(new BytesValueRLPInput(Bytes.wrap(value.get()), false)); + + secondUpdater.rollForward(layer); + secondUpdater.commit(); + secondWorldState.persist(null); + + assertKeyValueStorageEqual(accountStorage, secondAccountStorage); + assertKeyValueStorageEqual(codeStorage, secondCodeStorage); + assertKeyValueStorageEqual(storageStorage, secondStorageStorage); + final KeyValueStorageTransaction tx = trieBranchStorage.startTransaction(); + tx.remove(VerkleWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY); + tx.commit(); + assertKeyValueStorageEqual(trieBranchStorage, secondTrieBranchStorage); + // trie logs won't be the same, we shouldn't generate logs on rolls. + assertKeyValueSubset(trieLogStorage, secondTrieLogStorage); + assertThat(secondWorldState.rootHash()).isEqualByComparingTo(worldState.rootHash()); + } + + @Test + void rollForwardTwice() { + final VerkleWorldState worldState = + new VerkleWorldState( + archive, + new VerkleWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + EvmConfiguration.DEFAULT); + + final WorldUpdater updater = worldState.updater(); + final MutableAccount mutableAccount = updater.createAccount(addressOne, 1, Wei.of(1L)); + mutableAccount.setCode(Bytes.of(0, 1, 2)); + mutableAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + updater.commit(); + + worldState.persist(headerOne); + + final WorldUpdater updater2 = worldState.updater(); + final MutableAccount mutableAccount2 = updater2.getAccount(addressOne); + mutableAccount2.setStorageValue(UInt256.ONE, UInt256.valueOf(2)); + updater2.commit(); + + worldState.persist(headerTwo); + + final VerkleWorldState secondWorldState = + new VerkleWorldState( + secondArchive, + new VerkleWorldStateKeyValueStorage(secondProvider, new NoOpMetricsSystem()), + EvmConfiguration.DEFAULT); + final VerkleWorldStateUpdateAccumulator secondUpdater = + (VerkleWorldStateUpdateAccumulator) secondWorldState.updater(); + + final TrieLogLayer layerOne = getTrieLogLayer(trieLogStorage, headerOne.getHash()); + secondUpdater.rollForward(layerOne); + secondUpdater.commit(); + secondWorldState.persist(null); + + final TrieLogLayer layerTwo = getTrieLogLayer(trieLogStorage, headerTwo.getHash()); + secondUpdater.rollForward(layerTwo); + secondUpdater.commit(); + secondWorldState.persist(null); + + assertKeyValueStorageEqual(accountStorage, secondAccountStorage); + assertKeyValueStorageEqual(codeStorage, secondCodeStorage); + assertKeyValueStorageEqual(storageStorage, secondStorageStorage); + final KeyValueStorageTransaction tx = trieBranchStorage.startTransaction(); + tx.remove(VerkleWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY); + tx.commit(); + assertKeyValueStorageEqual(trieBranchStorage, secondTrieBranchStorage); + // trie logs won't be the same, we shouldn't generate logs on rolls. + assertKeyValueSubset(trieLogStorage, secondTrieLogStorage); + assertThat(secondWorldState.rootHash()).isEqualByComparingTo(worldState.rootHash()); + } + + @Test + void rollBackOnce() { + final VerkleWorldState worldState = + new VerkleWorldState( + archive, + new VerkleWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + EvmConfiguration.DEFAULT); + + final WorldUpdater updater = worldState.updater(); + final MutableAccount mutableAccount = updater.createAccount(addressOne, 1, Wei.of(1L)); + mutableAccount.setCode(Bytes.of(0, 1, 2)); + mutableAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + updater.commit(); + + worldState.persist(headerOne); + + final WorldUpdater updater2 = worldState.updater(); + final MutableAccount mutableAccount2 = updater2.getAccount(addressOne); + mutableAccount2.setStorageValue(UInt256.ONE, UInt256.valueOf(2)); + updater2.commit(); + + worldState.persist(headerTwo); + final VerkleWorldStateUpdateAccumulator firstRollbackUpdater = + (VerkleWorldStateUpdateAccumulator) worldState.updater(); + + final TrieLogLayer layerTwo = getTrieLogLayer(trieLogStorage, headerTwo.getHash()); + firstRollbackUpdater.rollBack(layerTwo); + + worldState.persist(headerOne); + + final VerkleWorldState secondWorldState = + new VerkleWorldState( + secondArchive, + new VerkleWorldStateKeyValueStorage(secondProvider, new NoOpMetricsSystem()), + EvmConfiguration.DEFAULT); + + final WorldUpdater secondUpdater = secondWorldState.updater(); + final MutableAccount secondMutableAccount = + secondUpdater.createAccount(addressOne, 1, Wei.of(1L)); + secondMutableAccount.setCode(Bytes.of(0, 1, 2)); + secondMutableAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + secondUpdater.commit(); + + secondWorldState.persist(null); + + assertKeyValueStorageEqual(accountStorage, secondAccountStorage); + assertKeyValueStorageEqual(codeStorage, secondCodeStorage); + assertKeyValueStorageEqual(storageStorage, secondStorageStorage); + final KeyValueStorageTransaction tx = trieBranchStorage.startTransaction(); + tx.remove(VerkleWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY); + tx.commit(); + assertKeyValueStorageEqual(trieBranchStorage, secondTrieBranchStorage); + // trie logs won't be the same, we don't delete the roll back log + assertKeyValueSubset(trieLogStorage, secondTrieLogStorage); + assertThat(secondWorldState.rootHash()).isEqualByComparingTo(worldState.rootHash()); + } + + @Test + void rollBackTwice() { + final VerkleWorldState worldState = + new VerkleWorldState( + archive, + new VerkleWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + EvmConfiguration.DEFAULT); + + final WorldUpdater updater = worldState.updater(); + final MutableAccount mutableAccount = updater.createAccount(addressOne, 1, Wei.of(1L)); + mutableAccount.setCode(Bytes.of(0, 1, 2)); + mutableAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + updater.commit(); + + worldState.persist(headerOne); + final TrieLogLayer layerOne = getTrieLogLayer(trieLogStorage, headerOne.getHash()); + + final WorldUpdater updater2 = worldState.updater(); + final MutableAccount mutableAccount2 = updater2.getAccount(addressOne); + mutableAccount2.setStorageValue(UInt256.ONE, UInt256.valueOf(2)); + updater2.commit(); + + worldState.persist(headerTwo); + final VerkleWorldStateUpdateAccumulator firstRollbackUpdater = + (VerkleWorldStateUpdateAccumulator) worldState.updater(); + + final TrieLogLayer layerTwo = getTrieLogLayer(trieLogStorage, headerTwo.getHash()); + firstRollbackUpdater.rollBack(layerTwo); + + worldState.persist(headerOne); + + final VerkleWorldStateUpdateAccumulator secondRollbackUpdater = + (VerkleWorldStateUpdateAccumulator) worldState.updater(); + secondRollbackUpdater.rollBack(layerOne); + + worldState.persist(null); + + assertThat(worldState.rootHash()).isEqualTo(Bytes32.ZERO); + } + + @Test + void rollBackFourTimes() { + final VerkleWorldState worldState = + new VerkleWorldState( + archive, + new VerkleWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + EvmConfiguration.DEFAULT); + + final WorldUpdater updater = worldState.updater(); + final MutableAccount mutableAccount = updater.createAccount(addressOne, 1, Wei.of(1L)); + mutableAccount.setCode(Bytes.of(0, 1, 2)); + mutableAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + updater.commit(); + + worldState.persist(headerOne); + final TrieLogLayer layerOne = getTrieLogLayer(trieLogStorage, headerOne.getHash()); + + final WorldUpdater updater2 = worldState.updater(); + final MutableAccount mutableAccount2 = updater2.getAccount(addressOne); + mutableAccount2.setStorageValue(UInt256.ONE, UInt256.valueOf(2)); + updater2.commit(); + + worldState.persist(headerTwo); + final TrieLogLayer layerTwo = getTrieLogLayer(trieLogStorage, headerTwo.getHash()); + + final WorldUpdater updater3 = worldState.updater(); + final MutableAccount mutableAccount3 = updater3.getAccount(addressOne); + mutableAccount3.setStorageValue(UInt256.ONE, UInt256.valueOf(0)); + updater3.commit(); + + worldState.persist(headerThree); + final TrieLogLayer layerThree = getTrieLogLayer(trieLogStorage, headerThree.getHash()); + + final WorldUpdater updater4 = worldState.updater(); + final MutableAccount mutableAccount4 = updater4.getAccount(addressOne); + mutableAccount4.setStorageValue(UInt256.ONE, UInt256.valueOf(1)); + updater4.commit(); + + worldState.persist(headerFour); + final TrieLogLayer layerFour = getTrieLogLayer(trieLogStorage, headerFour.getHash()); + + final VerkleWorldStateUpdateAccumulator firstRollbackUpdater = + (VerkleWorldStateUpdateAccumulator) worldState.updater(); + + firstRollbackUpdater.rollBack(layerFour); + + System.out.println(layerFour.dump()); + worldState.persist(headerThree); + + final VerkleWorldStateUpdateAccumulator secondRollbackUpdater = + (VerkleWorldStateUpdateAccumulator) worldState.updater(); + secondRollbackUpdater.rollBack(layerThree); + + worldState.persist(headerTwo); + + final VerkleWorldStateUpdateAccumulator thirdRollbackUpdater = + (VerkleWorldStateUpdateAccumulator) worldState.updater(); + thirdRollbackUpdater.rollBack(layerTwo); + + worldState.persist(headerOne); + + final VerkleWorldStateUpdateAccumulator fourRollbackUpdater = + (VerkleWorldStateUpdateAccumulator) worldState.updater(); + fourRollbackUpdater.rollBack(layerOne); + + worldState.persist(null); + + assertThat(worldState.rootHash()).isEqualTo(Bytes32.ZERO); + } + + @Test + void rollingWithRemovedStorageValue() { + final VerkleWorldState worldState = + new VerkleWorldState( + archive, + new VerkleWorldStateKeyValueStorage(provider, new NoOpMetricsSystem()), + EvmConfiguration.DEFAULT); + + final WorldUpdater updater = worldState.updater(); + final MutableAccount mutableAccount = updater.createAccount(addressOne, 1, Wei.of(1L)); + mutableAccount.setCode(Bytes.of(0, 1, 2)); + mutableAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + updater.commit(); + + worldState.persist(headerOne); + + /*final WorldUpdater updater2 = worldState.updater(); + final MutableAccount mutableAccount2 = updater2.getAccount(addressOne); + mutableAccount2.setStorageValue(UInt256.ONE, UInt256.ZERO); + updater2.commit(); + + blockHeaderTestFixture.stateRoot(Hash.fromHexString("0x1879f69465e8ef937ce1f13cb5b328437239a2764982cea5e337cd5d217a2866")); + blockHeaderTestFixture.number(2); + final BlockHeader blockHeaderTwo = blockHeaderTestFixture.buildHeader(); + worldState.persist(blockHeaderTwo); + + final VerkleWorldStateUpdateAccumulator firstRollbackUpdater = + (VerkleWorldStateUpdateAccumulator) worldState.updater(); + + final TrieLogLayer layerTwo = getTrieLogLayer(trieLogStorage, blockHeaderTwo.getBlockHash()); + firstRollbackUpdater.rollBack(layerTwo); + System.out.println("rollback"); + + worldState.persist(null); + assertThat(worldState.rootHash()).isEqualTo(blockHeaderOne.getStateRoot());*/ + + final VerkleWorldStateUpdateAccumulator secondRollbackUpdater = + (VerkleWorldStateUpdateAccumulator) worldState.updater(); + + final TrieLogLayer layerOne = getTrieLogLayer(trieLogStorage, headerOne.getBlockHash()); + secondRollbackUpdater.rollBack(layerOne); + + worldState.persist(null); + assertThat(worldState.rootHash()).isEqualTo(Bytes32.ZERO); + } + + private TrieLogLayer getTrieLogLayer(final KeyValueStorage storage, final Bytes key) { + return storage + .get(key.toArrayUnsafe()) + .map(bytes -> TrieLogFactoryImpl.readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false))) + .get(); + } + + private static void assertKeyValueStorageEqual( + final KeyValueStorage first, final KeyValueStorage second) { + final var firstKeys = + first.getAllKeysThat(k -> true).stream().map(Bytes::wrap).collect(Collectors.toSet()); + final var secondKeys = + second.getAllKeysThat(k -> true).stream().map(Bytes::wrap).collect(Collectors.toSet()); + + assertThat(secondKeys).isEqualTo(firstKeys); + for (final Bytes key : firstKeys) { + assertThat(Bytes.wrap(second.get(key.toArrayUnsafe()).get())) + .isEqualByComparingTo(Bytes.wrap(first.get(key.toArrayUnsafe()).get())); + } + } + + private static void assertKeyValueSubset( + final KeyValueStorage largerSet, final KeyValueStorage smallerSet) { + final var largerKeys = + largerSet.getAllKeysThat(k -> true).stream().map(Bytes::wrap).collect(Collectors.toSet()); + final var smallerKeys = + smallerSet.getAllKeysThat(k -> true).stream().map(Bytes::wrap).collect(Collectors.toSet()); + + assertThat(largerKeys).containsAll(smallerKeys); + for (final Bytes key : largerKeys) { + if (smallerKeys.contains(key)) { + assertThat(Bytes.wrap(largerSet.get(key.toArrayUnsafe()).get())) + .isEqualByComparingTo(Bytes.wrap(smallerSet.get(key.toArrayUnsafe()).get())); + } + } + } +} diff --git a/ethereum/verkletrie/build.gradle b/ethereum/verkletrie/build.gradle index 166b02efe17..f5ff5b9d99f 100644 --- a/ethereum/verkletrie/build.gradle +++ b/ethereum/verkletrie/build.gradle @@ -31,6 +31,7 @@ dependencies { implementation project(':crypto:algorithms') implementation project(':ethereum:rlp') implementation project(':ethereum:trie') + implementation project(':datatypes') implementation "org.immutables:value-annotations" implementation 'com.google.guava:guava' @@ -38,7 +39,11 @@ dependencies { implementation 'io.tmio:tuweni-bytes' implementation 'io.tmio:tuweni-units' implementation 'org.bouncycastle:bcprov-jdk18on' - implementation 'org.hyperledger.besu:ipa-multipoint' + + implementation ('org.hyperledger.besu:besu-verkle-trie') { + exclude group: 'org.hyperledger.besu.internal' + exclude group: 'org.apache.tuweni' + } annotationProcessor "org.immutables:value" diff --git a/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleTrie.java b/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleTrie.java new file mode 100644 index 00000000000..d747e083c0a --- /dev/null +++ b/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleTrie.java @@ -0,0 +1,61 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.verkletrie; + +import org.hyperledger.besu.ethereum.trie.NodeLoader; +import org.hyperledger.besu.ethereum.trie.NodeUpdater; +import org.hyperledger.besu.ethereum.trie.verkle.StoredVerkleTrie; +import org.hyperledger.besu.ethereum.trie.verkle.factory.StoredNodeFactory; + +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; + +public class VerkleTrie { + + private final org.hyperledger.besu.ethereum.trie.verkle.StoredVerkleTrie verkleTrie; + + private final StoredNodeFactory nodeFactory; + + public VerkleTrie(final NodeLoader nodeLoader, final Bytes32 rootHash) { + nodeFactory = new StoredNodeFactory<>(nodeLoader, value -> value); + verkleTrie = new StoredVerkleTrie<>(nodeFactory); + } + + public Optional get(final Bytes key) { + return verkleTrie.get(key); + } + + public Optional put(final Bytes key, final Bytes value) { + return verkleTrie.put(key, Bytes32.leftPad(value)); + } + + public void remove(final Bytes key) { + verkleTrie.remove(Bytes32.wrap(key)); + } + + public Bytes32 getRootHash() { + return verkleTrie.getRootHash(); + } + + public void commit(final NodeUpdater nodeUpdater) { + verkleTrie.commit(nodeUpdater); + } + + public String toDotTree() { + return verkleTrie.toDotTree(false); + } +} diff --git a/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleTrieKeyValueGenerator.java b/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleTrieKeyValueGenerator.java new file mode 100644 index 00000000000..7180df1bd65 --- /dev/null +++ b/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleTrieKeyValueGenerator.java @@ -0,0 +1,95 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.verkletrie; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import kotlin.Pair; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +public class VerkleTrieKeyValueGenerator { + + final TrieKeyAdapter trieKeyAdapter = new TrieKeyAdapter(new PedersenHasher()); + + public Map generateKeyValuesForAccount( + final Address address, final long nonce, final Wei balance, final Hash codeHash) { + final Map keyValues = new HashMap<>(); + keyValues.put(trieKeyAdapter.versionKey(address), Bytes32.ZERO); + keyValues.put(trieKeyAdapter.balanceKey(address), toLittleEndian(balance)); + keyValues.put(trieKeyAdapter.nonceKey(address), toLittleEndian(UInt256.valueOf(nonce))); + keyValues.put(trieKeyAdapter.codeKeccakKey(address), codeHash); + return keyValues; + } + + public List generateKeysForAccount(final Address address) { + final List keys = new ArrayList<>(); + keys.add(trieKeyAdapter.versionKey(address)); + keys.add(trieKeyAdapter.balanceKey(address)); + keys.add(trieKeyAdapter.nonceKey(address)); + return keys; + } + + public Map generateKeyValuesForCode( + final Address address, final Bytes32 keccakCodeHash, final Bytes code) { + final Map keyValues = new HashMap<>(); + keyValues.put( + trieKeyAdapter.codeSizeKey(address), toLittleEndian(UInt256.valueOf(code.size()))); + List codeChunks = trieKeyAdapter.chunkifyCode(code); + for (int i = 0; i < codeChunks.size(); i++) { + // System.out.println("add code " + trieKeyAdapter.codeChunkKey(address, UInt256.valueOf(i)) + + // " " + codeChunks.get(i)); + keyValues.put(trieKeyAdapter.codeChunkKey(address, UInt256.valueOf(i)), codeChunks.get(i)); + } + return keyValues; + } + + public List generateKeysForCode(final Address address, final Bytes code) { + final List keys = new ArrayList<>(); + keys.add(trieKeyAdapter.codeKeccakKey(address)); + keys.add(trieKeyAdapter.codeSizeKey(address)); + List codeChunks = trieKeyAdapter.chunkifyCode(code); + for (int i = 0; i < codeChunks.size(); i++) { + keys.add(trieKeyAdapter.codeChunkKey(address, UInt256.valueOf(i))); + } + return keys; + } + + public Pair generateKeyValuesForStorage( + final Address address, final StorageSlotKey storageKey, final Bytes value) { + return new Pair<>( + trieKeyAdapter.storageKey(address, storageKey.getSlotKey().orElseThrow()), value); + } + + public List generateKeysForStorage( + final Address address, final StorageSlotKey storageKey) { + return List.of(trieKeyAdapter.storageKey(address, storageKey.getSlotKey().orElseThrow())); + } + + private static Bytes toLittleEndian(final Bytes originalValue) { + return originalValue.reverse(); + } +} diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml deleted file mode 100644 index 8265a2810c4..00000000000 --- a/gradle/verification-metadata.xml +++ /dev/null @@ -1,6883 +0,0 @@ - - - - true - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/gradle/versions.gradle b/gradle/versions.gradle index f239370ab7a..d647fa52d71 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -170,6 +170,12 @@ dependencyManagement { entry 'blake2bf' } + dependencySet(group: 'org.hyperledger.besu', version: '0.8.3-SNAPSHOT') { + entry 'ipa-multipoint' + } + + dependency 'org.hyperledger.besu:besu-verkle-trie:0.0.1-SNAPSHOT-KT5' + dependencySet(group: 'org.immutables', version: '2.9.3') { entry 'value-annotations' entry 'value' From 3851483f4141d0fb136abd23ca165c29bb24871a Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Wed, 10 Jan 2024 11:50:56 +0100 Subject: [PATCH 10/59] update besu native Signed-off-by: Karim Taam --- gradle/versions.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/versions.gradle b/gradle/versions.gradle index d647fa52d71..4a8b72146e3 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -170,7 +170,7 @@ dependencyManagement { entry 'blake2bf' } - dependencySet(group: 'org.hyperledger.besu', version: '0.8.3-SNAPSHOT') { + dependencySet(group: 'org.hyperledger.besu', version: '0.8.4-SNAPSHOT') { entry 'ipa-multipoint' } From 41fc466cd328c0147815561dd86cedac6b470488 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Wed, 10 Jan 2024 15:01:16 +0100 Subject: [PATCH 11/59] fix review Signed-off-by: Karim Taam --- .../besu/cli/subcommands/operator/BackupState.java | 4 ++-- .../ethereum/trie/bonsai/worldview/BonsaiWorldState.java | 2 +- .../besu/ethereum/trie/forest/pruner/Pruner.java | 4 ++-- .../ethereum/worldstate/WorldStateKeyValueStorage.java | 3 +-- .../worldstate/WorldStateStorageCoordinator.java | 2 +- .../ethereum/eth/sync/snapsync/LoadLocalDataStep.java | 3 +-- .../request/heal/AccountTrieNodeHealingRequest.java | 1 - .../request/heal/StorageTrieNodeHealingRequest.java | 1 - .../snapsync/request/heal/TrieNodeHealingRequest.java | 1 - .../eth/sync/snapsync/AccountHealingTrackingTest.java | 3 +-- .../request/heal/StorageTrieNodeHealingRequestTest.java | 9 ++------- .../org/hyperledger/besu/ethereum/rlp/RLPOutput.java | 2 +- 12 files changed, 12 insertions(+), 23 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/BackupState.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/BackupState.java index e56ccd960c7..62621328b46 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/BackupState.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/BackupState.java @@ -81,7 +81,7 @@ public void run() { final BesuController besuController = createBesuController(); final MutableBlockchain blockchain = besuController.getProtocolContext().getBlockchain(); - final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = + final ForestWorldStateKeyValueStorage forestWorldStateKeyValueStorage = ((ForestWorldStateArchive) besuController.getProtocolContext().getWorldStateArchive()) .getWorldStateStorage(); final EthScheduler scheduler = new EthScheduler(1, 1, 1, 1, new NoOpMetricsSystem()); @@ -93,7 +93,7 @@ public void run() { blockchain, backupDir.toPath(), scheduler, - worldStateKeyValueStorage); + forestWorldStateKeyValueStorage); final BackupStatus status = backup.requestBackup(targetBlock, compress, Optional.empty()); final double refValue = Math.pow(2, 256) / 100.0d; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java index e4e00b31b13..b1b650f4374 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java @@ -557,7 +557,7 @@ protected Optional getAccountStateTrieNode(final Bytes location, final By return worldStateKeyValueStorage.getAccountStateTrieNode(location, nodeHash); } - protected void writeTrieNode( + private void writeTrieNode( final SegmentIdentifier segmentId, final SegmentedKeyValueStorageTransaction tx, final Bytes location, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/Pruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/Pruner.java index 2898bfdaeec..78d5aa853d5 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/Pruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/Pruner.java @@ -179,11 +179,11 @@ private void execute(final Runnable action) { } } - public PruningPhase getPruningPhase() { + PruningPhase getPruningPhase() { return pruningPhase.get(); } - public enum PruningPhase { + enum PruningPhase { IDLE, MARK_BLOCK_CONFIRMATIONS_AWAITING, MARKING, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateKeyValueStorage.java index 5ff6a218ca8..4e3d6ceee6b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateKeyValueStorage.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,7 +11,6 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * */ package org.hyperledger.besu.ethereum.worldstate; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java index 9f38a49c617..b3abe347f3b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStep.java index 231b4c84bdf..394023aba06 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStep.java @@ -69,8 +69,7 @@ public Stream> loadLocalDataTrieNode( // check if node is already stored in the worldstate try { if (snapSyncState.hasPivotBlockHeader()) { - Optional existingData = - request.getExistingData(downloadState, worldStateStorageCoordinator); + Optional existingData = request.getExistingData(worldStateStorageCoordinator); if (existingData.isPresent()) { existingNodeCounter.inc(); request.setData(existingData.get()); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java index e698ca58b26..e2f3353f20e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java @@ -77,7 +77,6 @@ protected int doPersist( @Override public Optional getExistingData( - final SnapWorldDownloadState downloadState, final WorldStateStorageCoordinator worldStateStorageCoordinator) { return worldStateStorageCoordinator .getAccountStateTrieNode(getLocation(), getNodeHash()) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequest.java index 5cf151c51ec..d1cea17684a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequest.java @@ -64,7 +64,6 @@ protected int doPersist( @Override public Optional getExistingData( - final SnapWorldDownloadState downloadState, final WorldStateStorageCoordinator worldStateStorageCoordinator) { return worldStateStorageCoordinator.getAccountStorageTrieNode( getAccountHash(), getLocation(), getNodeHash()); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/TrieNodeHealingRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/TrieNodeHealingRequest.java index e0d92382bf8..e114ebb426a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/TrieNodeHealingRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/TrieNodeHealingRequest.java @@ -181,7 +181,6 @@ private boolean nodeIsHashReferencedDescendant(final Node node) { } public abstract Optional getExistingData( - final SnapWorldDownloadState downloadState, final WorldStateStorageCoordinator worldStateStorageCoordinator); public abstract List getTrieNodePath(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java index ad4504cbf0b..5cda25ee51a 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java @@ -219,8 +219,7 @@ void avoidMarkingAccountOnValidStorageTrieNodeDetection() { accountHash, Hash.wrap(accountStateTrie.getRootHash()), Bytes.EMPTY); - storageTrieNodeHealingRequest.getExistingData( - snapWorldDownloadState, worldStateStorageCoordinator); + storageTrieNodeHealingRequest.getExistingData(worldStateStorageCoordinator); verify(snapWorldDownloadState, never()).addAccountToHealingList(any(Bytes.class)); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java index c85d4f85b2c..1c4948ab908 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java @@ -18,7 +18,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; @@ -42,13 +41,11 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) class StorageTrieNodeHealingRequestTest { - @Mock private SnapWorldDownloadState downloadState; final List
accounts = List.of( Address.fromHexString("0xdeadbeef"), @@ -102,8 +99,7 @@ void shouldDetectExistingData(final DataStorageFormat storageFormat) { new StorageTrieNodeHealingRequest( account0StorageRoot, account0Hash, Hash.EMPTY, Bytes.EMPTY); - Assertions.assertThat(request.getExistingData(downloadState, worldStateStorageCoordinator)) - .isPresent(); + Assertions.assertThat(request.getExistingData(worldStateStorageCoordinator)).isPresent(); } @ParameterizedTest @@ -113,7 +109,6 @@ void shouldDetectMissingData(final DataStorageFormat storageFormat) { final StorageTrieNodeHealingRequest request = new StorageTrieNodeHealingRequest(Hash.EMPTY, account0Hash, Hash.EMPTY, Bytes.EMPTY); - Assertions.assertThat(request.getExistingData(downloadState, worldStateStorageCoordinator)) - .isEmpty(); + Assertions.assertThat(request.getExistingData(worldStateStorageCoordinator)).isEmpty(); } } diff --git a/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java b/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java index 494106f98fa..140e60d9d24 100644 --- a/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java +++ b/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java @@ -92,7 +92,7 @@ default void writeUInt256Scalar(final UInt256Value v) { /** * Writes a RLP "null", that is an empty value. * - *

This is a shortcut for writeUInt256Scalar{@code writeBytes(Bytes.EMPTY)}. + *

This is a shortcut for {@code writeBytes(Bytes.EMPTY)}. */ default void writeNull() { writeBytes(Bytes.EMPTY); From 99a47f312190fd922ef595225b2d8d4d46bd4e34 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Mon, 15 Jan 2024 09:39:06 +0100 Subject: [PATCH 12/59] fix state root generation Signed-off-by: Karim Taam --- .../controller/BesuControllerBuilder.java | 4 +- .../besu/ethereum/chain/GenesisState.java | 103 ++++++++++++++++-- .../BonsaiNoOpCachedWorldStorageManager.java | 66 +++++++++++ .../bonsai/worldview/BonsaiWorldState.java | 2 +- .../common/trielog/NoOpTrieLogManager.java | 52 +++++++++ .../VerkleNoOpCachedWorldStorageManager.java | 68 ++++++++++++ .../verkle/worldview/VerkleWorldState.java | 4 +- 7 files changed, 284 insertions(+), 15 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiNoOpCachedWorldStorageManager.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index ad152c21743..6e6bf9fb6e1 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -593,7 +593,9 @@ public BesuController build() { prepForBuild(); final ProtocolSchedule protocolSchedule = createProtocolSchedule(); - final GenesisState genesisState = GenesisState.fromConfig(genesisConfig, protocolSchedule); + final GenesisState genesisState = + GenesisState.fromConfig( + dataStorageConfiguration.getDataStorageFormat(), genesisConfig, protocolSchedule); final VariablesStorage variablesStorage = storageProvider.createVariablesStorage(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java index 87d45be8184..17075f288ce 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java @@ -32,13 +32,24 @@ import org.hyperledger.besu.ethereum.core.Withdrawal; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiNoOpCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.NoOpTrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.VerkleNoOpCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.worldview.VerkleWorldState; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.log.LogsBloomFilter; import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.math.BigInteger; @@ -55,6 +66,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; +import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; public final class GenesisState { @@ -66,6 +78,21 @@ private GenesisState(final Block block, final List genesisAccoun this.genesisAccounts = genesisAccounts; } + /** + * Construct a {@link GenesisState} from a JSON string. + * + * @param dataStorageFormat A {@link DataStorageFormat} describing the storage format to use + * @param json A JSON string describing the genesis block + * @param protocolSchedule A protocol Schedule associated with + * @return A new {@link GenesisState}. + */ + public static GenesisState fromJson( + final DataStorageFormat dataStorageFormat, + final String json, + final ProtocolSchedule protocolSchedule) { + return fromConfig(dataStorageFormat, GenesisConfigFile.fromConfig(json), protocolSchedule); + } + /** * Construct a {@link GenesisState} from a JSON string. * @@ -77,20 +104,35 @@ public static GenesisState fromJson(final String json, final ProtocolSchedule pr return fromConfig(GenesisConfigFile.fromConfig(json), protocolSchedule); } + /** + * Construct a {@link GenesisState} from a JSON object with FOREST as a default data storage format. + * @param config A {@link GenesisConfigFile} describing the genesis block. + * @param protocolSchedule A protocol Schedule associated with + */ + public static GenesisState fromConfig( + final GenesisConfigFile config, final ProtocolSchedule protocolSchedule) { + return fromConfig(DataStorageFormat.FOREST, config, protocolSchedule); + } /** * Construct a {@link GenesisState} from a JSON object. * + * @param dataStorageFormat A {@link DataStorageFormat} describing the storage format to use * @param config A {@link GenesisConfigFile} describing the genesis block. * @param protocolSchedule A protocol Schedule associated with * @return A new {@link GenesisState}. */ public static GenesisState fromConfig( - final GenesisConfigFile config, final ProtocolSchedule protocolSchedule) { + final DataStorageFormat dataStorageFormat, + final GenesisConfigFile config, + final ProtocolSchedule protocolSchedule) { final List genesisAccounts = parseAllocations(config).toList(); final Block block = - new Block( - buildHeader(config, calculateGenesisStateHash(genesisAccounts), protocolSchedule), - buildBody(config)); + new Block( + buildHeader( + config, + calculateGenesisStateHash(dataStorageFormat, genesisAccounts), + protocolSchedule), + buildBody(config)); return new GenesisState(block, genesisAccounts); } @@ -133,17 +175,56 @@ private static void writeAccountsTo( target.persist(rootHeader); } - private static Hash calculateGenesisStateHash(final List genesisAccounts) { - final ForestWorldStateKeyValueStorage stateStorage = - new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); - final WorldStatePreimageKeyValueStorage preimageStorage = - new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); - final MutableWorldState worldState = - new ForestMutableWorldState(stateStorage, preimageStorage, EvmConfiguration.DEFAULT); + private static Hash calculateGenesisStateHash( + final DataStorageFormat dataStorageFormat, final List genesisAccounts) { + final MutableWorldState worldState = loadWorldState(dataStorageFormat); writeAccountsTo(worldState, genesisAccounts, null); return worldState.rootHash(); } + private static MutableWorldState loadWorldState(final DataStorageFormat dataStorageFormat) { + switch (dataStorageFormat) { + case BONSAI -> { + final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); + final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = + new BonsaiWorldStateKeyValueStorage( + new KeyValueStorageProvider( + segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), + new InMemoryKeyValueStorage(), + new NoOpMetricsSystem()), + new NoOpMetricsSystem()); + return new BonsaiWorldState( + bonsaiWorldStateKeyValueStorage, + cachedMerkleTrieLoader, + new BonsaiNoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), + new NoOpTrieLogManager(), + EvmConfiguration.DEFAULT); + } + case VERKLE -> { + final VerkleWorldStateKeyValueStorage verkleWorldStateKeyValueStorage = + new VerkleWorldStateKeyValueStorage( + new KeyValueStorageProvider( + segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), + new InMemoryKeyValueStorage(), + new NoOpMetricsSystem()), + new NoOpMetricsSystem()); + return new VerkleWorldState( + verkleWorldStateKeyValueStorage, + new VerkleNoOpCachedWorldStorageManager(verkleWorldStateKeyValueStorage), + new NoOpTrieLogManager(), + EvmConfiguration.DEFAULT); + } + default -> { + final ForestWorldStateKeyValueStorage stateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final WorldStatePreimageKeyValueStorage preimageStorage = + new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); + return new ForestMutableWorldState(stateStorage, preimageStorage, EvmConfiguration.DEFAULT); + } + } + } + private static BlockHeader buildHeader( final GenesisConfigFile genesis, final Hash genesisRootHash, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiNoOpCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiNoOpCachedWorldStorageManager.java new file mode 100644 index 00000000000..3c9f082bd80 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiNoOpCachedWorldStorageManager.java @@ -0,0 +1,66 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; + +import java.util.Optional; +import java.util.function.Function; + +public class BonsaiNoOpCachedWorldStorageManager extends BonsaiCachedWorldStorageManager { + + public BonsaiNoOpCachedWorldStorageManager( + final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage) { + super(null, bonsaiWorldStateKeyValueStorage, new NoOpMetricsSystem()); + } + + @Override + public synchronized void addCachedLayer( + final BlockHeader blockHeader, + final Hash worldStateRootHash, + final DiffBasedWorldState forWorldState) { + // no cache + } + + @Override + public boolean containWorldStateStorage(final Hash blockHash) { + return false; + } + + @Override + public Optional getWorldState(final Hash blockHash) { + return Optional.empty(); + } + + @Override + public Optional getNearestWorldState(final BlockHeader blockHeader) { + return Optional.empty(); + } + + @Override + public Optional getHeadWorldState( + final Function> hashBlockHeaderFunction) { + return Optional.empty(); + } + + @Override + public void reset() { + // world states are not re-used + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java index accc47c7182..1b35363361a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java @@ -71,7 +71,7 @@ public BonsaiWorldState( evmConfiguration); } - protected BonsaiWorldState( + public BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader, final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java new file mode 100644 index 00000000000..593ea89e5ba --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java @@ -0,0 +1,52 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.plugin.services.trielogs.TrieLog; + +import java.util.Optional; + +public class NoOpTrieLogManager extends TrieLogManager { + + public NoOpTrieLogManager() { + super(null, null, 0, null, TrieLogPruner.noOpTrieLogPruner()); + } + + @SuppressWarnings({"UnsynchronizedOverridesSynchronized", "squid:S3551"}) + @Override + public void saveTrieLog( + final DiffBasedWorldStateUpdateAccumulator localUpdater, + final Hash forWorldStateRootHash, + final BlockHeader forBlockHeader, + final DiffBasedWorldState forWorldState) { + // notify trie log added observers, synchronously + TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); + trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); + } + + @Override + public long getMaxLayersToLoad() { + return 0; + } + + @Override + public Optional getTrieLogLayer(final Hash blockHash) { + return Optional.empty(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java new file mode 100644 index 00000000000..9df56a9e54e --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java @@ -0,0 +1,68 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; + +import java.util.Optional; +import java.util.function.Function; + +public class VerkleNoOpCachedWorldStorageManager extends VerkleCachedWorldStorageManager { + + public VerkleNoOpCachedWorldStorageManager( + final VerkleWorldStateKeyValueStorage verkleWorldStateKeyValueStorage) { + super(null, verkleWorldStateKeyValueStorage, new NoOpMetricsSystem()); + } + + @Override + public synchronized void addCachedLayer( + final BlockHeader blockHeader, + final Hash worldStateRootHash, + final DiffBasedWorldState forWorldState) { + // no cache + } + + @Override + public boolean containWorldStateStorage(final Hash blockHash) { + return false; + } + + @Override + public Optional getWorldState(final Hash blockHash) { + return Optional.empty(); + } + + @Override + public Optional getNearestWorldState(final BlockHeader blockHeader) { + return Optional.empty(); + } + + @Override + public Optional getHeadWorldState( + final Function> hashBlockHeaderFunction) { + return Optional.empty(); + } + + @Override + public void reset() { + // world states are not re-used + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java index 384bdf4cf29..b8b5b804db7 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java @@ -71,7 +71,7 @@ public VerkleWorldState( evmConfiguration); } - protected VerkleWorldState( + public VerkleWorldState( final VerkleWorldStateKeyValueStorage worldStateKeyValueStorage, final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, @@ -137,7 +137,7 @@ protected Hash internalCalculateRootHash( })); LOG.info("end commit "); - LOG.info(stateTrie.toDotTree()); + //LOG.info(stateTrie.toDotTree()); final Bytes32 rootHash = stateTrie.getRootHash(); LOG.info("end commit "); From 33e3a9ecf01dd1bf048b072086ede1dc5d290396 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Thu, 25 Jan 2024 12:16:21 +0100 Subject: [PATCH 13/59] fix storage Signed-off-by: Karim Taam --- .../controller/BesuControllerBuilder.java | 4 +- .../besu/ethereum/chain/GenesisState.java | 86 ++++++++++--------- ...nsaiSnapshotWorldStateKeyValueStorage.java | 1 + .../bonsai/worldview/BonsaiWorldState.java | 9 ++ ...asedSnapshotWorldStateKeyValueStorage.java | 5 +- .../common/trielog/NoOpTrieLogManager.java | 8 +- .../common/worldview/DiffBasedWorldState.java | 32 +++---- .../VerkleNoOpCachedWorldStorageManager.java | 2 - ...rkleSnapshotWorldStateKeyValueStorage.java | 1 + .../verkle/worldview/VerkleWorldState.java | 11 ++- 10 files changed, 87 insertions(+), 72 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 6e6bf9fb6e1..bf9ed73d895 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -594,8 +594,8 @@ public BesuController build() { final ProtocolSchedule protocolSchedule = createProtocolSchedule(); final GenesisState genesisState = - GenesisState.fromConfig( - dataStorageConfiguration.getDataStorageFormat(), genesisConfig, protocolSchedule); + GenesisState.fromConfig( + dataStorageConfiguration.getDataStorageFormat(), genesisConfig, protocolSchedule); final VariablesStorage variablesStorage = storageProvider.createVariablesStorage(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java index 17075f288ce..42268e9369d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java @@ -37,8 +37,8 @@ import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiNoOpCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.VerkleNoOpCachedWorldStorageManager; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.worldview.VerkleWorldState; @@ -51,6 +51,7 @@ import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; import java.math.BigInteger; import java.util.HashMap; @@ -66,7 +67,6 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -import org.hyperledger.besu.services.kvstore.SegmentedInMemoryKeyValueStorage; public final class GenesisState { @@ -87,9 +87,9 @@ private GenesisState(final Block block, final List genesisAccoun * @return A new {@link GenesisState}. */ public static GenesisState fromJson( - final DataStorageFormat dataStorageFormat, - final String json, - final ProtocolSchedule protocolSchedule) { + final DataStorageFormat dataStorageFormat, + final String json, + final ProtocolSchedule protocolSchedule) { return fromConfig(dataStorageFormat, GenesisConfigFile.fromConfig(json), protocolSchedule); } @@ -105,13 +105,15 @@ public static GenesisState fromJson(final String json, final ProtocolSchedule pr } /** - * Construct a {@link GenesisState} from a JSON object with FOREST as a default data storage format. + * Construct a {@link GenesisState} from a JSON object with FOREST as a default data storage + * format. + * * @param config A {@link GenesisConfigFile} describing the genesis block. * @param protocolSchedule A protocol Schedule associated with */ public static GenesisState fromConfig( - final GenesisConfigFile config, final ProtocolSchedule protocolSchedule) { - return fromConfig(DataStorageFormat.FOREST, config, protocolSchedule); + final GenesisConfigFile config, final ProtocolSchedule protocolSchedule) { + return fromConfig(DataStorageFormat.FOREST, config, protocolSchedule); } /** * Construct a {@link GenesisState} from a JSON object. @@ -122,17 +124,17 @@ public static GenesisState fromConfig( * @return A new {@link GenesisState}. */ public static GenesisState fromConfig( - final DataStorageFormat dataStorageFormat, - final GenesisConfigFile config, - final ProtocolSchedule protocolSchedule) { + final DataStorageFormat dataStorageFormat, + final GenesisConfigFile config, + final ProtocolSchedule protocolSchedule) { final List genesisAccounts = parseAllocations(config).toList(); final Block block = - new Block( - buildHeader( - config, - calculateGenesisStateHash(dataStorageFormat, genesisAccounts), - protocolSchedule), - buildBody(config)); + new Block( + buildHeader( + config, + calculateGenesisStateHash(dataStorageFormat, genesisAccounts), + protocolSchedule), + buildBody(config)); return new GenesisState(block, genesisAccounts); } @@ -176,7 +178,7 @@ private static void writeAccountsTo( } private static Hash calculateGenesisStateHash( - final DataStorageFormat dataStorageFormat, final List genesisAccounts) { + final DataStorageFormat dataStorageFormat, final List genesisAccounts) { final MutableWorldState worldState = loadWorldState(dataStorageFormat); writeAccountsTo(worldState, genesisAccounts, null); return worldState.rootHash(); @@ -186,40 +188,40 @@ private static MutableWorldState loadWorldState(final DataStorageFormat dataStor switch (dataStorageFormat) { case BONSAI -> { final BonsaiCachedMerkleTrieLoader cachedMerkleTrieLoader = - new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = - new BonsaiWorldStateKeyValueStorage( - new KeyValueStorageProvider( - segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), - new InMemoryKeyValueStorage(), - new NoOpMetricsSystem()), - new NoOpMetricsSystem()); + new BonsaiWorldStateKeyValueStorage( + new KeyValueStorageProvider( + segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), + new InMemoryKeyValueStorage(), + new NoOpMetricsSystem()), + new NoOpMetricsSystem()); return new BonsaiWorldState( - bonsaiWorldStateKeyValueStorage, - cachedMerkleTrieLoader, - new BonsaiNoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), - new NoOpTrieLogManager(), - EvmConfiguration.DEFAULT); + bonsaiWorldStateKeyValueStorage, + cachedMerkleTrieLoader, + new BonsaiNoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), + new NoOpTrieLogManager(), + EvmConfiguration.DEFAULT); } case VERKLE -> { final VerkleWorldStateKeyValueStorage verkleWorldStateKeyValueStorage = - new VerkleWorldStateKeyValueStorage( - new KeyValueStorageProvider( - segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), - new InMemoryKeyValueStorage(), - new NoOpMetricsSystem()), - new NoOpMetricsSystem()); + new VerkleWorldStateKeyValueStorage( + new KeyValueStorageProvider( + segmentIdentifiers -> new SegmentedInMemoryKeyValueStorage(), + new InMemoryKeyValueStorage(), + new NoOpMetricsSystem()), + new NoOpMetricsSystem()); return new VerkleWorldState( - verkleWorldStateKeyValueStorage, - new VerkleNoOpCachedWorldStorageManager(verkleWorldStateKeyValueStorage), - new NoOpTrieLogManager(), - EvmConfiguration.DEFAULT); + verkleWorldStateKeyValueStorage, + new VerkleNoOpCachedWorldStorageManager(verkleWorldStateKeyValueStorage), + new NoOpTrieLogManager(), + EvmConfiguration.DEFAULT); } default -> { final ForestWorldStateKeyValueStorage stateStorage = - new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); final WorldStatePreimageKeyValueStorage preimageStorage = - new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); + new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); return new ForestMutableWorldState(stateStorage, preimageStorage, EvmConfiguration.DEFAULT); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index 1a472460145..af5ab27cde3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -229,6 +229,7 @@ protected synchronized void doClose() throws Exception { } } + @Override public BonsaiWorldStateKeyValueStorage getParentWorldStateStorage() { return parentWorldStateStorage; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java index 1b35363361a..cedd8898450 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java @@ -409,6 +409,15 @@ protected Hash hashAndSavePreImage(final Bytes value) { return Hash.hash(value); } + @Override + public Hash frontierRootHash() { + return calculateRootHash( + Optional.of( + new BonsaiWorldStateKeyValueStorage.Updater( + noOpSegmentedTx, noOpTx, worldStateKeyValueStorage.getFlatDbStrategy())), + accumulator.copy()); + } + @Override protected Hash getEmptyTrieHash() { return Hash.EMPTY_TRIE_HASH; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java index 2100243f7cf..9c1a691cf31 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java @@ -15,4 +15,7 @@ */ package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; -public interface DiffBasedSnapshotWorldStateKeyValueStorage {} +public interface DiffBasedSnapshotWorldStateKeyValueStorage { + + DiffBasedWorldStateKeyValueStorage getParentWorldStateStorage(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java index 593ea89e5ba..11836f33442 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java @@ -31,10 +31,10 @@ public NoOpTrieLogManager() { @SuppressWarnings({"UnsynchronizedOverridesSynchronized", "squid:S3551"}) @Override public void saveTrieLog( - final DiffBasedWorldStateUpdateAccumulator localUpdater, - final Hash forWorldStateRootHash, - final BlockHeader forBlockHeader, - final DiffBasedWorldState forWorldState) { + final DiffBasedWorldStateUpdateAccumulator localUpdater, + final Hash forWorldStateRootHash, + final BlockHeader forBlockHeader, + final DiffBasedWorldState forWorldState) { // notify trie log added observers, synchronously TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java index 06419f002e8..8805f98a8c3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java @@ -23,11 +23,9 @@ import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; @@ -113,7 +111,7 @@ public boolean isPersisted() { } private boolean isPersisted(final WorldStateKeyValueStorage worldStateKeyValueStorage) { - return !(worldStateKeyValueStorage instanceof BonsaiSnapshotWorldStateKeyValueStorage); + return !(worldStateKeyValueStorage instanceof DiffBasedSnapshotWorldStateKeyValueStorage); } /** @@ -172,13 +170,13 @@ public void persist(final BlockHeader blockHeader) { .getWorldStateTransaction() .put( TRIE_BRANCH_STORAGE, - BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY, + DiffBasedWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); worldStateBlockHash = blockHeader.getHash(); } else { stateUpdater .getWorldStateTransaction() - .remove(TRIE_BRANCH_STORAGE, BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY); + .remove(TRIE_BRANCH_STORAGE, DiffBasedWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY); worldStateBlockHash = null; } @@ -186,7 +184,7 @@ public void persist(final BlockHeader blockHeader) { .getWorldStateTransaction() .put( TRIE_BRANCH_STORAGE, - BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY, + DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); worldStateRootHash = newWorldStateRootHash; success = true; @@ -226,7 +224,7 @@ public Hash rootHash() { return Hash.wrap(worldStateRootHash); } - static final KeyValueStorageTransaction noOpTx = + protected static final KeyValueStorageTransaction noOpTx = new KeyValueStorageTransaction() { @Override @@ -250,7 +248,7 @@ public void rollback() { } }; - static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = + protected static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = new SegmentedKeyValueStorageTransaction() { @Override @@ -275,15 +273,6 @@ public void rollback() { } }; - @Override - public Hash frontierRootHash() { - return calculateRootHash( - Optional.of( - new BonsaiWorldStateKeyValueStorage.Updater( - noOpSegmentedTx, noOpTx, worldStateKeyValueStorage.getFlatDbStrategy())), - accumulator.copy()); - } - public Hash blockHash() { return worldStateBlockHash; } @@ -314,8 +303,8 @@ public void close() { private void closeFrozenStorage() { try { - final BonsaiWorldStateLayerStorage worldStateLayerStorage = - (BonsaiWorldStateLayerStorage) worldStateKeyValueStorage; + final DiffBasedSnapshotWorldStateKeyValueStorage worldStateLayerStorage = + (DiffBasedSnapshotWorldStateKeyValueStorage) worldStateKeyValueStorage; if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { worldStateLayerStorage.getParentWorldStateStorage().close(); } @@ -324,6 +313,9 @@ private void closeFrozenStorage() { } } + @Override + public abstract Hash frontierRootHash(); + @Override public abstract MutableWorldState freeze(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java index 9df56a9e54e..fe9f8e5472b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/VerkleNoOpCachedWorldStorageManager.java @@ -16,8 +16,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java index bb797e79045..ada211afa94 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/storage/VerkleSnapshotWorldStateKeyValueStorage.java @@ -195,6 +195,7 @@ protected synchronized void doClose() throws Exception { } } + @Override public VerkleWorldStateKeyValueStorage getParentWorldStateStorage() { return parentWorldStateStorage; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java index b8b5b804db7..0d5269ecd9e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java @@ -137,7 +137,7 @@ protected Hash internalCalculateRootHash( })); LOG.info("end commit "); - //LOG.info(stateTrie.toDotTree()); + // LOG.info(stateTrie.toDotTree()); final Bytes32 rootHash = stateTrie.getRootHash(); LOG.info("end commit "); @@ -354,6 +354,15 @@ protected Hash hashAndSavePreImage(final Bytes value) { return Hash.hash(value); } + @Override + public Hash frontierRootHash() { + return calculateRootHash( + Optional.of( + new VerkleWorldStateKeyValueStorage.Updater( + noOpSegmentedTx, noOpTx, worldStateKeyValueStorage.getFlatDbStrategy())), + accumulator.copy()); + } + @Override protected Hash getEmptyTrieHash() { return Hash.wrap(Bytes32.ZERO); From d0ed27774495fb280227878191d8bf3e61993af0 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Thu, 25 Jan 2024 14:28:17 +0100 Subject: [PATCH 14/59] fix build Signed-off-by: Karim Taam --- .../ethereum/referencetests/BonsaiReferenceTestWorldState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 88ae0bf8623..73f8908be65 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -187,7 +187,7 @@ private void generateTrieLogFromState( private BonsaiWorldState createBonsaiWorldState(final boolean isFrozen) { BonsaiWorldState bonsaiWorldState = new BonsaiWorldState( - new BonsaiWorldStateLayerStorage(worldStateStorage), + new BonsaiWorldStateLayerStorage(worldStateKeyValueStorage), cachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager, From 28ca47c4b4d9f0ad070d50c016b0bcfbe986f114 Mon Sep 17 00:00:00 2001 From: Karim Taam Date: Tue, 9 Apr 2024 17:02:15 +0200 Subject: [PATCH 15/59] verkle implementation Signed-off-by: Karim Taam --- .circleci/config.yml | 3 - .github/issue_template.md | 2 +- .github/pull_request_template.md | 21 +- .github/workflows/acceptance-tests.yml | 89 +- .github/workflows/artifacts.yml | 76 +- .github/workflows/codeql.yml | 27 +- .github/workflows/docker.yml | 80 +- .github/workflows/integration-tests.yml | 69 +- .github/workflows/nightly.yml | 121 -- .github/workflows/parallel-unit-tests.yml | 49 - .github/workflows/pr-checklist-on-open.yml | 22 - .github/workflows/pre-review.yml | 102 +- .github/workflows/reference-tests.yml | 154 +- .github/workflows/release.yml | 23 +- .github/workflows/sonarcloud.yml | 14 +- CHANGELOG.md | 110 +- .../acceptance/dsl/AcceptanceTestBase.java | 9 +- .../besu/tests/acceptance/dsl/BlockUtils.java | 1 + .../tests/acceptance/dsl/node/BesuNode.java | 29 +- .../dsl/node/ProcessBesuNodeRunner.java | 28 +- .../dsl/node/ThreadBesuNodeRunner.java | 140 +- .../configuration/BesuNodeConfiguration.java | 8 + .../BesuNodeConfigurationBuilder.java | 10 + .../node/configuration/BesuNodeFactory.java | 20 +- .../NodeConfigurationFactory.java | 8 +- .../acceptance/dsl/privacy/PrivacyNode.java | 8 +- .../acceptance/dsl/pubsub/WebSocket.java | 2 +- .../DeploySmartContractTransaction.java | 3 +- .../dsl/transaction/NodeRequests.java | 9 +- acceptance-tests/tests/build.gradle | 46 +- .../tests/contracts/SimpleStorage.sol | 2 +- .../bft/BftMiningAcceptanceTest.java | 42 + .../clique/CliqueMiningAcceptanceTest.java | 180 +- .../EthSendRawTransactionAcceptanceTest.java | 12 + .../ExecutionEngineEip6110AcceptanceTest.java | 48 - ...ContractPermissioningV2AcceptanceTest.java | 14 +- .../plugins/PermissioningPluginTest.java | 27 +- .../privacy/PrivacyClusterAcceptanceTest.java | 9 +- .../NewPendingTransactionAcceptanceTest.java | 10 + .../jsonrpc/engine/eip6110/genesis.json | 77 - .../test-cases/01_cancun_prepare_payload.json | 34 - .../test-cases/02_cancun_getPayloadV3.json | 44 - .../test-cases/03_cancun_newPayloadV3.json | 40 - .../04_cancun_forkchoiceUpdatedV3.json | 28 - .../05_eip6110_forkchoiceUpdatedV3.json | 34 - .../06_eip6110_getPayloadV6110.json | 45 - .../07_eip6110_send_raw_transaction.json | 14 - ...invalid_null_deposits_execute_payload.json | 41 - .../09_eip6110_newPayloadV6110.json | 45 - .../10_eip6110_forkchoiceUpdatedV3.json | 34 - .../11_eip6110_getPayloadV6110.json | 45 - besu/build.gradle | 2 + .../org/hyperledger/besu/RunnerBuilder.java | 7 +- .../org/hyperledger/besu/cli/BesuCommand.java | 385 ++-- .../cli/ConfigurationOverviewBuilder.java | 16 + .../besu/cli/DefaultCommandValues.java | 2 - .../besu/cli/config/NetworkName.java | 16 +- .../besu/cli/config/ProfileName.java | 19 +- .../converter/MetricCategoryConverter.java | 3 +- .../besu/cli/options/MiningOptions.java | 26 +- .../cli/options/TransactionPoolOptions.java | 27 + .../options/stable/DataStorageOptions.java | 38 +- .../options/stable/LoggingLevelOption.java | 5 +- .../options/unstable/NetworkingOptions.java | 6 +- .../options/unstable/SynchronizerOptions.java | 61 +- .../operator/GenerateBlockchainConfig.java | 13 +- .../storage/RocksDbSubCommand.java | 106 +- .../storage/RocksDbUsageHelper.java | 105 -- .../storage/StorageSubCommand.java | 16 +- .../subcommands/storage/TrieLogHelper.java | 21 +- .../storage/TrieLogSubCommand.java | 88 +- .../controller/BesuControllerBuilder.java | 170 +- .../CliqueBesuControllerBuilder.java | 29 +- ...onsensusScheduleBesuControllerBuilder.java | 25 +- .../controller/IbftBesuControllerBuilder.java | 19 +- .../MainnetBesuControllerBuilder.java | 7 +- .../MergeBesuControllerBuilder.java | 7 +- .../controller/QbftBesuControllerBuilder.java | 19 +- .../TransitionBesuControllerBuilder.java | 27 +- .../besu/services/BesuConfigurationImpl.java | 67 +- .../besu/services/BlockchainServiceImpl.java | 51 +- ...PluginTransactionValidatorServiceImpl.java | 35 - .../besu/services/RpcEndpointServiceImpl.java | 9 +- .../TransactionSelectionServiceImpl.java | 13 +- besu/src/main/resources/log4j2.xml | 6 + .../org/hyperledger/besu/cli/launcher.json | 6 - .../besu/ForkIdsNetworkConfigTest.java | 15 +- .../org/hyperledger/besu/PrivacyTest.java | 19 +- .../java/org/hyperledger/besu/RunnerTest.java | 32 +- .../chainimport/JsonBlockImporterTest.java | 3 +- .../hyperledger/besu/cli/BesuCommandTest.java | 1547 ++--------------- .../cli/CascadingDefaultProviderTest.java | 32 +- .../besu/cli/CommandTestAbstract.java | 39 +- .../cli/options/AbstractCLIOptionsTest.java | 11 +- .../besu/cli/options/MiningOptionsTest.java | 23 +- .../cli/options/NetworkingOptionsTest.java | 1 - .../options/TransactionPoolOptionsTest.java | 30 + .../stable/DataStorageOptionsTest.java | 42 +- .../storage/StorageSubCommandTest.java | 25 +- .../storage/TrieLogHelperTest.java | 10 +- .../controller/BesuControllerBuilderTest.java | 219 --- .../besu/controller/BesuControllerTest.java | 8 +- .../MergeBesuControllerBuilderTest.java | 8 +- .../QbftBesuControllerBuilderTest.java | 157 +- .../besu/services/BesuEventsImplTest.java | 15 +- .../besu/services/TraceServiceImplTest.java | 2 +- .../src/test/resources/everything_config.toml | 9 +- build.gradle | 329 ++-- config/build.gradle | 2 + .../org/hyperledger/besu/config/BftFork.java | 6 +- .../besu/config/CliqueConfigOptions.java | 59 +- .../besu/config/GenesisConfigOptions.java | 8 +- .../besu/config/JsonBftConfigOptions.java | 3 +- .../besu/config/JsonGenesisConfigOptions.java | 16 +- .../org/hyperledger/besu/config/JsonUtil.java | 3 +- .../besu/config/StubGenesisConfigOptions.java | 21 +- .../besu/config/TransitionsConfigOptions.java | 29 +- config/src/main/resources/holesky.json | 1 + config/src/main/resources/mainnet.json | 1 + .../besu/config/CliqueConfigOptionsTest.java | 4 +- .../besu/config/GenesisConfigOptionsTest.java | 26 +- .../clique/CliqueProtocolSchedule.java | 64 +- .../blockcreation/CliqueBlockMiner.java | 12 +- .../blockcreation/CliqueBlockScheduler.java | 16 +- .../blockcreation/CliqueMinerExecutor.java | 12 +- .../CliqueDifficultyCalculatorTest.java | 4 +- .../clique/CliqueProtocolScheduleTest.java | 53 +- .../clique/NodeCanProduceNextBlockTest.java | 23 +- .../blockcreation/CliqueBlockCreatorTest.java | 12 +- .../blockcreation/CliqueBlockMinerTest.java | 24 +- .../CliqueBlockSchedulerTest.java | 73 +- .../CliqueMinerExecutorTest.java | 21 +- .../CliqueDifficultyValidationRuleTest.java | 4 +- .../CliqueExtraDataValidationRuleTest.java | 4 +- .../CombinedProtocolScheduleFactory.java | 14 +- .../common/MigratingProtocolContext.java | 16 +- .../bft/BaseBftProtocolScheduleBuilder.java | 12 +- .../common/bft/BftForksScheduleFactory.java | 82 - .../common/bft/BftProtocolSchedule.java | 20 +- .../bft/blockcreation/BftBlockCreator.java | 23 + .../bft/statemachine/BftFinalState.java | 6 - .../common/bft/inttest/NetworkLayout.java | 9 +- .../CombinedProtocolScheduleFactoryTest.java | 93 +- .../common/MigratingProtocolContextTest.java | 4 +- .../BaseBftProtocolScheduleBuilderTest.java | 6 +- .../bft/BftForksScheduleFactoryTest.java | 103 -- .../BftCoinbaseValidationRuleTest.java | 8 +- .../BftCommitSealsValidationRuleTest.java | 17 +- .../BftValidatorsValidationRuleTest.java | 14 +- .../BftVanityDataValidationRuleTest.java | 8 +- .../ibft/support/TestContextBuilder.java | 13 +- .../tests/round/IbftRoundIntegrationTest.java | 14 +- .../ibft/IbftForksSchedulesFactory.java | 4 +- .../ibft/IbftProtocolScheduleBuilder.java | 22 +- .../statemachine/IbftBlockHeightManager.java | 28 + .../IbftBlockHeightManagerFactory.java | 7 + .../ibft/statemachine/IbftRound.java | 15 +- .../ibft/statemachine/IbftRoundFactory.java | 5 +- .../ibft/validation/MessageValidator.java | 13 +- .../validation/MessageValidatorFactory.java | 36 +- ...ockHeaderValidationRulesetFactoryTest.java | 3 +- .../ibft/IbftProtocolScheduleTest.java | 11 +- .../blockcreation/BftBlockCreatorTest.java | 131 +- .../IbftBlockHeightManagerTest.java | 37 +- .../ibft/statemachine/IbftRoundTest.java | 44 +- .../ibft/validation/MessageValidatorTest.java | 16 +- .../besu/consensus/merge/MergeContext.java | 23 - .../merge/MergeProtocolSchedule.java | 26 +- .../consensus/merge/PostMergeContext.java | 32 - .../consensus/merge/TransitionContext.java | 5 - .../merge/TransitionProtocolSchedule.java | 13 +- .../merge/blockcreation/MergeCoordinator.java | 68 +- .../blockcreation/MergeMiningCoordinator.java | 16 +- .../blockcreation/TransitionCoordinator.java | 16 +- .../merge/MergeProtocolScheduleTest.java | 24 +- .../blockcreation/MergeCoordinatorTest.java | 21 +- .../MergeGenesisConfigHelper.java | 8 +- .../merge/blockcreation/MergeReorgTest.java | 3 +- .../qbft/support/TestContextBuilder.java | 34 +- .../qbft/test/ValidatorContractTest.java | 96 + .../test/round/QbftRoundIntegrationTest.java | 15 +- .../qbft/QbftForksSchedulesFactory.java | 4 +- .../qbft/QbftProtocolScheduleBuilder.java | 32 +- .../blockcreation/PkiQbftBlockCreator.java | 44 +- .../QbftBlockCreatorFactory.java | 6 +- .../statemachine/QbftBlockHeightManager.java | 28 + .../QbftBlockHeightManagerFactory.java | 7 + .../qbft/statemachine/QbftRound.java | 13 +- .../qbft/statemachine/QbftRoundFactory.java | 3 +- .../validation/MessageValidatorFactory.java | 44 +- .../validation/ProposalPayloadValidator.java | 4 + .../qbft/validation/ProposalValidator.java | 11 +- .../RoundChangeMessageValidator.java | 15 +- .../ValidatorContractController.java | 2 +- ...ockHeaderValidationRulesetFactoryTest.java | 3 +- .../qbft/QbftProtocolScheduleTest.java | 10 +- .../PkiQbftBlockCreatorTest.java | 31 +- .../QbftValidatorsValidationRuleTest.java | 9 +- .../qbft/messagewrappers/ProposalTest.java | 7 +- .../QbftBlockHeightManagerTest.java | 40 +- .../qbft/statemachine/QbftRoundTest.java | 47 +- .../ProposalPayloadValidatorTest.java | 7 +- .../validation/ProposalValidatorTest.java | 13 +- .../RoundChangeMessageValidatorTest.java | 123 +- .../crypto/SignatureAlgorithmFactory.java | 6 +- .../besu/crypto/SignatureAlgorithmType.java | 8 +- .../besu/crypto/SECP256R1Test.java | 2 +- .../hyperledger/besu/datatypes/Address.java | 2 +- .../org/hyperledger/besu/datatypes/Hash.java | 3 + .../besu/datatypes/StorageSlotKey.java | 4 +- .../org/hyperledger/besu/datatypes/Wei.java | 3 +- docker/graalvm/Dockerfile | 49 - docker/openj9-jdk-17/Dockerfile | 72 - docker/openjdk-17-debug/Dockerfile | 63 - docker/openjdk-17/Dockerfile | 53 - docker/openjdk-latest/Dockerfile | 55 - docker/test.sh | 6 +- .../hyperledger/besu/enclave/TlsHelpers.java | 3 +- errorprone-checks/README.md | 9 - errorprone-checks/build.gradle | 68 - .../errorpronechecks/BannedMethod.java | 60 - .../DoNotCreateSecureRandomDirectly.java | 59 - .../DoNotInvokeMessageDigestDirectly.java | 44 - .../DoNotReturnNullOptionals.java | 69 - ...ntalCliOptionMustBeCorrectlyDisplayed.java | 76 - .../MethodInputParametersMustBeFinal.java | 131 -- .../PrivateStaticFinalLoggers.java | 70 - .../errorpronechecks/BannedMethodTest.java | 39 - .../DoNotCreateSecureRandomDirectlyTest.java | 40 - .../DoNotInvokeMessageDigestDirectlyTest.java | 40 - .../DoNotReturnNullOptionalsTest.java | 40 - ...CliOptionMustBeCorrectlyDisplayedTest.java | 45 - .../MethodInputParametersMustBeFinalTest.java | 54 - .../PrivateStaticFinalLoggersTest.java | 40 - .../BannedMethodNegativeCases.java | 29 - .../BannedMethodPositiveCases.java | 50 - ...eateSecureRandomDirectlyNegativeCases.java | 34 - ...eateSecureRandomDirectlyPositiveCases.java | 41 - ...okeMessageDigestDirectlyNegativeCases.java | 26 - ...okeMessageDigestDirectlyPositiveCases.java | 26 - ...DoNotReturnNullOptionalsNegativeCases.java | 36 - ...DoNotReturnNullOptionalsPositiveCases.java | 35 - ...MustBeCorrectlyDisplayedNegativeCases.java | 50 - ...MustBeCorrectlyDisplayedPositiveCases.java | 41 - ...tersMustBeFinalInterfaceNegativeCases.java | 40 - ...tersMustBeFinalInterfacePositiveCases.java | 24 - ...putParametersMustBeFinalNegativeCases.java | 63 - ...putParametersMustBeFinalPositiveCases.java | 41 - ...rivateStaticFinalLoggersNegativeCases.java | 25 - ...rivateStaticFinalLoggersPositiveCases.java | 25 - .../api/jsonrpc/BlockchainImporter.java | 6 +- .../api/jsonrpc/JsonRpcResponseKey.java | 3 +- .../api/jsonrpc/JsonRpcResponseUtils.java | 3 + .../jsonrpc/JsonRpcTestMethodsFactory.java | 3 +- ...ckByNumberLatestDesyncIntegrationTest.java | 3 +- .../fork/frontier/EthCallIntegrationTest.java | 53 + .../EthCreateAccessListIntegrationTest.java | 16 +- .../EthEstimateGasIntegrationTest.java | 27 +- .../EthGetFilterChangesIntegrationTest.java | 3 +- .../fork/london/EthCallIntegrationTest.java | 18 + .../london/EthEstimateGasIntegrationTest.java | 12 +- .../EthGetFilterChangesIntegrationTest.java | 3 +- .../api/graphql/GraphQLHttpService.java | 26 +- .../api/graphql/internal/Scalars.java | 7 +- .../pojoadapter/TransactionAdapter.java | 2 +- .../api/jsonrpc/EngineJsonRpcService.java | 25 +- .../api/jsonrpc/JsonRpcErrorConverter.java | 6 +- .../api/jsonrpc/JsonRpcHttpService.java | 25 +- .../besu/ethereum/api/jsonrpc/RpcMethod.java | 6 +- .../internal/methods/AbstractEstimateGas.java | 11 +- .../internal/methods/AbstractTraceByHash.java | 25 +- .../internal/methods/AbstractTraceCall.java | 15 +- .../internal/methods/DebugAccountAt.java | 2 +- .../internal/methods/DebugGetBadBlocks.java | 19 +- .../internal/methods/DebugGetRawReceipts.java | 2 +- .../methods/DebugResyncWorldstate.java | 19 +- .../DebugStandardTraceBadBlockToFile.java | 15 +- .../internal/methods/DebugTraceBlock.java | 5 +- .../methods/DebugTraceBlockByHash.java | 5 +- .../methods/DebugTraceBlockByNumber.java | 16 +- .../internal/methods/DebugTraceCall.java | 4 +- .../methods/DebugTraceTransaction.java | 2 +- .../api/jsonrpc/internal/methods/EthCall.java | 7 +- .../internal/methods/EthEstimateGas.java | 2 +- .../internal/methods/EthFeeHistory.java | 73 +- .../methods/EthSendRawTransaction.java | 6 +- .../internal/methods/PluginJsonRpcMethod.java | 11 +- .../jsonrpc/internal/methods/TraceBlock.java | 2 +- .../jsonrpc/internal/methods/TraceCall.java | 4 +- .../internal/methods/TraceCallMany.java | 11 +- .../jsonrpc/internal/methods/TraceFilter.java | 6 +- .../internal/methods/TraceRawTransaction.java | 5 +- .../methods/TraceReplayBlockTransactions.java | 4 +- .../TxPoolBesuPendingTransactions.java | 8 +- .../AbstractEngineForkchoiceUpdated.java | 43 +- .../engine/AbstractEngineGetPayload.java | 9 +- .../engine/AbstractEngineNewPayload.java | 60 +- .../engine/EngineExchangeCapabilities.java | 1 - .../engine/EngineForkchoiceUpdatedV1.java | 2 +- .../engine/EngineForkchoiceUpdatedV2.java | 5 +- .../engine/EngineForkchoiceUpdatedV3.java | 15 +- .../methods/engine/EngineGetPayloadV6110.java | 87 - .../methods/engine/EngineNewPayloadV6110.java | 89 - .../internal/parameters/BlockParameter.java | 3 +- .../parameters/BlockParameterOrBlockHash.java | 3 +- .../parameters/EnginePayloadParameter.java | 20 +- .../parameters/JsonCallParameter.java | 11 +- .../parameters/PendingTransactionsParams.java | 3 +- .../internal/processor/BlockReplay.java | 16 +- .../internal/response/JsonRpcError.java | 31 +- .../internal/response/RpcErrorType.java | 30 +- .../internal/results/BlockResultFactory.java | 5 +- .../results/EngineGetPayloadResultV6110.java | 230 --- .../jsonrpc/internal/results/FeeHistory.java | 12 + .../results/TransactionCompleteResult.java | 2 +- .../results/TransactionPendingResult.java | 2 +- .../tracing/flat/FlatTraceGenerator.java | 20 +- .../results/tracing/vm/VmTraceGenerator.java | 3 +- .../jsonrpc/methods/DebugJsonRpcMethods.java | 8 +- .../jsonrpc/methods/EthJsonRpcMethods.java | 4 +- .../ExecutionEngineJsonRpcMethods.java | 20 +- .../methods/JsonRpcMethodsFactory.java | 3 +- .../jsonrpc/methods/TraceJsonRpcMethods.java | 6 +- .../jsonrpc/websocket/WebSocketService.java | 44 +- .../PendingTransactionDetailResult.java | 2 +- .../api/query/StateBackupService.java | 2 +- .../cache/TransactionLogBloomCacher.java | 2 +- .../AbstractEthGraphQLHttpServiceTest.java | 5 +- .../api/graphql/scalar/Bytes32ScalarTest.java | 13 + .../AbstractJsonRpcHttpServiceTest.java | 2 +- .../JsonRpcHttpServiceHostAllowlistTest.java | 5 +- .../jsonrpc/JsonRpcHttpServiceLoginTest.java | 4 +- .../api/jsonrpc/JsonRpcHttpServiceTest.java | 3 +- .../jsonrpc/JsonRpcHttpServiceTestBase.java | 10 +- .../JsonRpcHttpServiceTlsClientAuthTest.java | 5 +- ...RpcHttpServiceTlsMisconfigurationTest.java | 5 +- .../jsonrpc/JsonRpcHttpServiceTlsTest.java | 5 +- .../api/jsonrpc/JsonRpcTestHelper.java | 20 +- .../jsonrpc/RpcErrorTypeConverterTest.java | 8 +- .../DebugTraceJsonRpcHttpBySpecTest.java | 2 +- .../EthByzantiumJsonRpcHttpBySpecTest.java | 2 +- .../bonsai/TraceJsonRpcHttpBySpecTest.java | 2 +- .../DebugTraceJsonRpcHttpBySpecTest.java | 2 +- .../EthByzantiumJsonRpcHttpBySpecTest.java | 2 +- .../forest/TraceJsonRpcHttpBySpecTest.java | 2 +- .../filter/EthJsonRpcHttpServiceTest.java | 2 +- .../methods/DebugGetBadBlockTest.java | 32 +- .../DebugStandardTraceBadBlockToFileTest.java | 21 +- .../methods/DebugTraceBlockByHashTest.java | 1 + .../methods/DebugTraceBlockByNumberTest.java | 1 + .../internal/methods/DebugTraceBlockTest.java | 1 + .../methods/DebugTraceTransactionTest.java | 2 + .../jsonrpc/internal/methods/EthCallTest.java | 10 +- .../methods/EthCreateAccessListTest.java | 8 +- .../internal/methods/EthEstimateGasTest.java | 29 +- .../internal/methods/EthFeeHistoryTest.java | 102 +- .../internal/methods/EthGasPriceTest.java | 2 + .../methods/EthGetTransactionReceiptTest.java | 4 +- .../TxPoolBesuPendingTransactionsTest.java | 32 +- .../AbstractEngineForkchoiceUpdatedTest.java | 4 +- .../engine/AbstractEngineGetPayloadTest.java | 2 + .../engine/AbstractEngineNewPayloadTest.java | 85 +- .../engine/AbstractScheduledApiTest.java | 7 +- ...neExchangeTransitionConfigurationTest.java | 1 + .../engine/EngineForkchoiceUpdatedV2Test.java | 6 + .../EngineGetPayloadBodiesByHashV1Test.java | 3 + .../EngineGetPayloadBodiesByRangeV1Test.java | 7 + .../engine/EngineGetPayloadV3Test.java | 1 + .../engine/EngineGetPayloadV6110Test.java | 177 -- .../engine/EngineNewPayloadV2Test.java | 25 +- .../engine/EngineNewPayloadV3Test.java | 18 +- .../engine/EngineNewPayloadV6110Test.java | 204 --- .../privacy/methods/priv/PrivCallTest.java | 6 + .../processor/TransactionTracerTest.java | 8 +- .../websocket/WebSocketHostAllowlistTest.java | 83 +- .../websocket/WebSocketServiceLoginTest.java | 6 +- ...ewBlockHeadersSubscriptionServiceTest.java | 3 +- .../query/BlockchainQueriesLogCacheTest.java | 1 + .../cache/TransactionLogBloomCacherTest.java | 2 + .../api/graphql/eth_getTransaction_type2.json | 6 +- .../debug/trace-call/debug_traceCall_all.json | 2 +- .../trace-call/debug_traceCall_complete.json | 2 +- .../debug_traceCall_disableMemory.json | 2 +- .../debug_traceCall_disableStack.json | 2 +- .../debug_traceCall_disableStorage.json | 2 +- ...call_invalidWithInputAndDataAttribute.json | 25 - .../eth_feeHistory_fieldNamesWithReward.json | 2 + ...th_feeHistory_fieldNamesWithoutReward.json | 4 +- ...th_getBlockByNumber_complete_shanghai.json | 2 +- .../trace-call/trace_call_14_1_trace.json | 2 +- .../trace_callMany_14_trace.json | 2 +- .../trace_rawTransaction_14_1_trace.json | 2 +- ethereum/blockcreation/build.gradle | 1 + .../blockcreation/AbstractBlockCreator.java | 48 +- .../ethereum/blockcreation/BlockCreator.java | 2 + .../blockcreation/DefaultBlockScheduler.java | 17 +- .../txselection/BlockTransactionSelector.java | 66 +- .../AllAcceptingTransactionSelector.java | 55 - .../AbstractBlockCreatorTest.java | 8 +- .../AbstractBlockTransactionSelectorTest.java | 287 ++- .../blockcreation/BlockMinerTest.java | 10 +- ...FeeMarketBlockTransactionSelectorTest.java | 13 +- ...FeeMarketBlockTransactionSelectorTest.java | 33 +- .../blockcreation/PoWBlockCreatorTest.java | 20 +- .../blockcreation/PoWMinerExecutorTest.java | 5 +- ethereum/core/build.gradle | 4 + .../forest/pruner/PrunerIntegrationTest.java | 267 --- .../vm/TraceTransactionIntegrationTest.java | 4 +- .../besu/ethereum/MainnetBlockValidator.java | 64 +- .../besu/ethereum/ProtocolContext.java | 27 +- .../besu/ethereum/chain/BadBlockManager.java | 39 +- .../besu/ethereum/chain/GenesisState.java | 36 +- .../hyperledger/besu/ethereum/core/Block.java | 7 +- .../besu/ethereum/core/BlockBody.java | 32 +- .../besu/ethereum/core/BlockHeader.java | 52 +- .../ethereum/core/BlockHeaderBuilder.java | 27 +- .../besu/ethereum/core/MiningParameters.java | 44 +- .../ethereum/core/SealableBlockHeader.java | 23 +- .../besu/ethereum/core/Transaction.java | 36 +- .../ethereum/core/TransactionReceipt.java | 43 +- .../besu/ethereum/core/feemarket/BaseFee.java | 48 - .../feemarket/TransactionPriceCalculator.java | 34 - .../besu/ethereum/debug/TraceFrame.java | 7 + .../FixedDifficultyProtocolSchedule.java | 35 +- .../besu/ethereum/forkid/ForkIdManager.java | 3 +- .../mainnet/AbstractBlockProcessor.java | 36 +- .../AbstractGasLimitSpecification.java | 7 + .../besu/ethereum/mainnet/BodyValidation.java | 20 +- .../mainnet/ClassicProtocolSpecs.java | 8 +- .../mainnet/DefaultProtocolSchedule.java | 10 - .../mainnet/MainnetProtocolSchedule.java | 63 +- .../mainnet/MainnetProtocolSpecFactory.java | 33 +- .../mainnet/MainnetProtocolSpecs.java | 137 +- .../mainnet/MainnetTransactionProcessor.java | 43 +- .../mainnet/MainnetTransactionValidator.java | 26 +- .../PermissionTransactionValidator.java | 3 +- .../mainnet/ProtocolScheduleBuilder.java | 34 +- .../besu/ethereum/mainnet/ProtocolSpec.java | 40 +- .../ethereum/mainnet/ProtocolSpecBuilder.java | 30 +- .../mainnet/TransactionValidationParams.java | 2 +- .../mainnet/TransactionValidator.java | 1 + .../mainnet/WithdrawalsProcessor.java | 8 +- .../ethereum/mainnet/feemarket/FeeMarket.java | 4 + .../mainnet/feemarket/LondonFeeMarket.java | 2 +- .../mainnet/feemarket/ZeroBaseFeeMarket.java | 16 +- .../BlobGasValidationRule.java | 10 +- .../privacy/PluginPrivacyController.java | 6 +- .../privacy/PrivateTransactionReceipt.java | 14 +- .../TransactionProcessingResult.java | 32 +- .../proof/WorldStateProofProvider.java | 31 +- .../ethereum/storage/StorageProvider.java | 4 +- .../keyvalue/KeyValueSegmentIdentifier.java | 45 +- ...ueStoragePrefixedKeyBlockchainStorage.java | 18 +- .../keyvalue/KeyValueStorageProvider.java | 14 +- .../ethereum/transaction/CallParameter.java | 112 +- .../transaction/TransactionInvalidReason.java | 4 +- .../transaction/TransactionSimulator.java | 55 +- .../TransactionSimulatorResult.java | 52 +- .../common/GenesisWorldStateProvider.java | 42 +- .../trie/diffbased/bonsai/BonsaiAccount.java | 25 +- .../bonsai/BonsaiWorldStateProvider.java | 25 +- .../BonsaiCachedWorldStorageManager.java | 2 +- .../BonsaiNoOpCachedWorldStorageManager.java | 66 - .../NoOpBonsaiCachedWorldStorageManager.java | 2 +- ...nsaiSnapshotWorldStateKeyValueStorage.java | 2 +- .../BonsaiWorldStateKeyValueStorage.java | 20 +- .../bonsai/storage/flat/FlatDbStrategy.java | 235 --- .../storage/flat/FullFlatDbStrategy.java | 7 +- .../storage/flat/PartialFlatDbStrategy.java | 7 +- .../bonsai/worldview/BonsaiWorldState.java | 41 +- .../BonsaiWorldStateUpdateAccumulator.java | 23 +- .../diffbased/common/DiffBasedAccount.java | 61 +- .../trie/diffbased/common/DiffBasedValue.java | 5 +- .../common/DiffBasedWorldStateProvider.java | 23 +- .../diffbased/common/StorageSubscriber.java | 14 + .../DiffBasedCachedWorldStorageManager.java | 59 +- ...BasedLayeredWorldStateKeyValueStorage.java | 4 +- .../DiffBasedWorldStateKeyValueStorage.java | 30 +- .../storage/flat/FlatDbStrategyProvider.java | 49 +- .../common/trielog/NoOpTrieLogManager.java | 3 +- .../common/trielog/TrieLogManager.java | 13 +- .../common/worldview/DiffBasedWorldState.java | 25 +- .../common/worldview/DiffBasedWorldView.java | 3 +- .../DiffBasedWorldStateUpdateAccumulator.java | 48 +- .../preload/AccountConsumingMap.java | 6 +- .../accumulator/preload/Consumer.java | 3 +- .../preload/StorageConsumingMap.java | 6 +- .../trie/diffbased/verkle/VerkleAccount.java | 6 +- .../verkle/VerkleWorldStateProvider.java | 10 +- .../VerkleCachedWorldStorageManager.java | 1 - .../VerkleNoOpCachedWorldStorageManager.java | 3 +- ...erkleLayeredWorldStateKeyValueStorage.java | 1 - ...rkleSnapshotWorldStateKeyValueStorage.java | 3 +- .../VerkleWorldStateKeyValueStorage.java | 22 +- .../verkle/worldview/VerkleWorldState.java | 276 +-- .../VerkleWorldStateUpdateAccumulator.java | 14 + .../trie/forest/pruner/MarkSweepPruner.java | 317 ---- .../ethereum/trie/forest/pruner/Pruner.java | 199 --- .../forest/pruner/PrunerConfiguration.java | 42 - .../ForestWorldStateKeyValueStorage.java | 4 +- .../ethereum/vm/DebugOperationTracer.java | 34 +- .../worldstate/DataStorageConfiguration.java | 27 + .../worldstate/DataStorageFormat.java | 33 - .../worldstate/WorldStateArchive.java | 3 - .../worldstate/WorldStateKeyValueStorage.java | 2 + .../WorldStateStorageCoordinator.java | 6 + .../ethereum/core/BlockDataGenerator.java | 25 +- .../ethereum/core/BlockHeaderTestFixture.java | 7 + .../ethereum/core/BlockchainSetupUtil.java | 11 +- .../core/ExecutionContextTestFixture.java | 12 +- .../core/InMemoryKeyValueStorageProvider.java | 28 +- .../ethereum/core/NonBesuBlockHeader.java | 10 + .../core/ProtocolScheduleFixture.java | 8 +- .../BlockImportExceptionHandlingTest.java | 4 +- .../ethereum/MainnetBlockValidatorTest.java | 497 ++++-- .../ethereum/chain/ChainDataPrunerTest.java | 14 +- .../ethereum/chain/DefaultBlockchainTest.java | 3 +- .../besu/ethereum/chain/GenesisStateTest.java | 49 +- .../besu/ethereum/core/LogTest.java | 31 +- .../ethereum/core/TransactionReceiptTest.java | 37 +- .../fixed/FixedProtocolScheduleTest.java | 7 +- .../mainnet/DefaultProtocolScheduleTest.java | 6 +- .../besu/ethereum/mainnet/EthHashTest.java | 50 +- .../mainnet/MainnetProtocolScheduleTest.java | 20 +- .../MainnetTransactionProcessorTest.java | 9 +- .../MainnetTransactionValidatorTest.java | 95 +- .../PermissionTransactionValidatorTest.java | 3 - .../mainnet/ProtocolScheduleBuilderTest.java | 10 +- .../ethereum/mainnet/RefundSstoreGasTest.java | 10 +- .../TargetingGasLimitCalculatorTest.java | 6 + .../ethereum/mainnet/ValidationTestUtils.java | 7 +- .../mainnet/WithdrawalsProcessorTest.java | 37 +- ...BlockHeaderGasPriceValidationRuleTest.java | 25 + ...oragePrefixedKeyBlockchainStorageTest.java | 8 +- .../transaction/TransactionSimulatorTest.java | 103 ++ .../bonsai/AbstractIsolationTests.java | 53 +- .../BonsaiCachedMerkleTrieLoaderTest.java | 6 +- .../bonsai/BonsaiSnapshotIsolationTests.java | 8 +- .../bonsai/BonsaiWorldStateProviderTest.java | 31 +- .../diffbased/bonsai/LogRollingTests.java | 22 +- .../trie/diffbased/bonsai/RollingImport.java | 2 +- .../BonsaiWorldStateKeyValueStorageTest.java | 95 +- .../bonsai/trielog/TrieLogFactoryTests.java | 2 +- .../bonsai/trielog/TrieLogLayerTests.java | 136 -- .../bonsai/trielog/TrieLogManagerTests.java | 5 +- .../BonsaiWorldStateKeyValueStorageTest.java | 487 ------ .../flat/FlatDbStrategyProviderTest.java | 98 +- .../common/trielog/TrieLogManagerTests.java | 74 - .../common/trielog/TrieLogPrunerTest.java | 318 ---- .../diffbased/verkle/LogRollingTests.java | 18 +- .../forest/pruner/MarkSweepPrunerTest.java | 293 ---- .../trie/forest/pruner/PrunerTest.java | 210 --- ...tKeyValueStorageWorldStateStorageTest.java | 4 +- .../ethereum/vm/DebugOperationTracerTest.java | 57 +- .../WorldStateDownloaderBenchmark.java | 6 +- .../eth/EthProtocolConfiguration.java | 2 +- .../besu/ethereum/eth/manager/EthPeer.java | 34 +- .../besu/ethereum/eth/manager/EthPeers.java | 36 +- .../eth/manager/EthProtocolManager.java | 16 +- .../ethereum/eth/manager/EthScheduler.java | 4 + .../besu/ethereum/eth/manager/EthServer.java | 2 +- .../ethereum/eth/manager/MergePeerFilter.java | 4 +- .../ethereum/eth/manager/PeerReputation.java | 6 +- .../eth/manager/snap/SnapProtocolManager.java | 12 +- .../ethereum/eth/manager/snap/SnapServer.java | 630 ++++++- .../task/AbstractGetHeadersFromPeerTask.java | 4 +- .../AbstractRetryingSwitchingPeerTask.java | 2 +- ...dGetPooledTransactionsFromPeerFetcher.java | 4 +- .../manager/task/GetBodiesFromPeerTask.java | 27 +- .../task/GetHeadersFromPeerByHashTask.java | 2 +- .../manager/task/GetNodeDataFromPeerTask.java | 2 +- .../GetPooledTransactionsFromPeerTask.java | 2 +- .../manager/task/GetReceiptsFromPeerTask.java | 2 +- .../eth/messages/ReceiptsMessage.java | 2 +- .../messages/snap/AccountRangeMessage.java | 3 +- .../messages/snap/GetAccountRangeMessage.java | 3 + .../messages/snap/GetStorageRangeMessage.java | 2 +- .../messages/snap/StorageRangeMessage.java | 9 +- .../AbstractPeerBlockValidator.java | 24 +- .../peervalidation/DaoForkPeerValidator.java | 9 + .../eth/sync/BlockPropagationManager.java | 9 +- .../ethereum/eth/sync/ChainHeadTracker.java | 2 +- .../eth/sync/DefaultSynchronizer.java | 24 +- .../besu/ethereum/eth/sync/SyncMode.java | 20 +- .../eth/sync/SynchronizerConfiguration.java | 2 +- .../eth/sync/TrailingPeerLimiter.java | 2 +- .../backwardsync/BackwardSyncAlgorithm.java | 66 +- .../backwardsync/BackwardSyncContext.java | 17 +- .../sync/backwardsync/BackwardSyncStep.java | 3 +- .../FastSyncDownloadPipelineFactory.java | 19 +- .../eth/sync/fastsync/FastSyncDownloader.java | 2 +- .../eth/sync/fastsync/ImportBlocksStep.java | 5 +- .../sync/fastsync/PivotBlockConfirmer.java | 3 +- .../eth/sync/fastsync/SyncTargetManager.java | 2 +- .../sync/fullsync/FullImportBlockStep.java | 2 +- .../sync/fullsync/FullSyncTargetManager.java | 2 +- .../range/RangeHeadersValidationStep.java | 3 +- .../eth/sync/snapsync/RangeManager.java | 155 -- .../eth/sync/snapsync/RequestDataStep.java | 45 +- .../sync/snapsync/SnapSyncConfiguration.java | 7 + .../sync/snapsync/SnapSyncMetricsManager.java | 1 + .../sync/snapsync/SnapWorldDownloadState.java | 10 +- .../snapsync/SnapWorldStateDownloader.java | 7 +- .../ethereum/eth/sync/snapsync/StackTrie.java | 77 +- .../SnapSyncStatePersistenceManager.java | 3 +- .../request/AccountRangeDataRequest.java | 18 +- .../snapsync/request/BytecodeRequest.java | 2 +- .../snapsync/request/SnapDataRequest.java | 6 +- .../request/StorageRangeDataRequest.java | 44 +- ...ccountFlatDatabaseHealingRangeRequest.java | 13 +- .../heal/AccountTrieNodeHealingRequest.java | 7 +- ...torageFlatDatabaseHealingRangeRequest.java | 11 +- .../eth/sync/tasks/CompleteBlocksTask.java | 5 +- .../tasks/DownloadHeaderSequenceTask.java | 118 +- .../eth/sync/tasks/PersistBlockTask.java | 4 +- .../exceptions/InvalidBlockException.java | 20 +- ...wPooledTransactionHashesMessageSender.java | 2 +- .../transactions/PeerTransactionTracker.java | 22 + .../eth/transactions/PendingTransaction.java | 10 +- .../transactions/TransactionBroadcaster.java | 37 +- .../eth/transactions/TransactionPool.java | 67 +- .../TransactionPoolConfiguration.java | 23 + .../transactions/TransactionPoolFactory.java | 49 +- .../TransactionPoolReplacementHandler.java | 5 +- ...TransactionReplacementByFeeMarketRule.java | 46 +- .../AbstractPrioritizedTransactions.java | 9 +- .../AbstractSequentialTransactionsLayer.java | 3 +- .../layered/AbstractTransactionsLayer.java | 4 +- .../BaseFeePrioritizedTransactions.java | 40 +- .../GasPricePrioritizedTransactions.java | 14 +- .../layered/LayeredPendingTransactions.java | 50 +- .../layered/ReadyTransactions.java | 6 +- .../layered/SparseTransactions.java | 3 +- .../layered/TransactionsLayer.java | 3 +- .../AbstractPendingTransactionsSorter.java | 25 +- .../eth/manager/EthProtocolManagerTest.java | 24 +- .../manager/EthProtocolManagerTestUtil.java | 6 +- .../ethereum/eth/manager/EthServerTest.java | 2 +- .../eth/manager/PeerReputationTest.java | 42 +- .../ethtaskutils/AbstractMessageTaskTest.java | 26 +- .../RetryingSwitchingPeerMessageTaskTest.java | 4 + .../task/GetBodiesFromPeerTaskTest.java | 40 +- .../task/SnapProtocolManagerTestUtil.java | 61 - .../eth/messages/BlockBodiesMessageTest.java | 8 +- .../eth/messages/BlockHeadersMessageTest.java | 6 +- .../eth/messages/MessageWrapperTest.java | 4 + .../snap/GetAccountRangeMessageTest.java | 2 +- .../snap/GetStorageRangeMessageTest.java | 2 +- .../snap/StorageRangeMessageTest.java | 3 +- .../DaoForkPeerValidatorTest.java | 24 + .../RequiredBlocksPeerValidatorTest.java | 22 + .../AbstractBlockPropagationManagerTest.java | 10 +- .../BonsaiBlockPropagationManagerTest.java | 2 +- .../eth/sync/ChainHeadTrackerTest.java | 10 +- .../eth/sync/DownloadHeadersStepTest.java | 2 +- .../ForestBlockPropagationManagerTest.java | 2 +- .../eth/sync/PipelineChainDownloaderTest.java | 2 +- .../eth/sync/RangeHeadersFetcherTest.java | 2 +- .../eth/sync/TrailingPeerLimiterTest.java | 2 +- .../backwardsync/BackwardSyncAlgSpec.java | 2 +- .../backwardsync/BackwardSyncContextTest.java | 75 +- .../backwardsync/BackwardSyncStepTest.java | 5 +- .../backwardsync/ChainForTestCreator.java | 6 + .../backwardsync/ForwardSyncStepTest.java | 5 +- .../CheckPointBlockImportStepTest.java | 3 +- .../CheckPointSyncChainDownloaderTest.java | 2 +- .../fastsync/DownloadReceiptsStepTest.java | 2 +- .../fastsync/FastDownloaderFactoryTest.java | 2 +- .../sync/fastsync/FastSyncActionsTest.java | 2 +- .../fastsync/FastSyncChainDownloaderTest.java | 2 +- .../sync/fastsync/FastSyncDownloaderTest.java | 2 +- .../fastsync/PivotBlockConfirmerTest.java | 2 +- .../fastsync/PivotBlockRetrieverTest.java | 2 +- .../FastWorldDownloadStateTest.java | 4 +- .../worldstate/LoadLocalDataStepTest.java | 2 +- .../fullsync/FullSyncChainDownloaderTest.java | 2 +- ...DownloaderTotalTerminalDifficultyTest.java | 2 +- .../sync/fullsync/FullSyncDownloaderTest.java | 2 +- .../fullsync/FullSyncTargetManagerTest.java | 6 +- .../snapsync/AccountHealingTrackingTest.java | 4 +- .../sync/snapsync/PersistDataStepTest.java | 10 +- .../eth/sync/snapsync/RangeManagerTest.java | 1 + .../snapsync/SnapWorldDownloadStateTest.java | 5 +- .../eth/sync/snapsync/StackTrieTest.java | 126 ++ .../eth/sync/snapsync/TaskGenerator.java | 19 +- ...ntFlatDatabaseHealingRangeRequestTest.java | 10 +- ...geFlatDatabaseHealingRangeRequestTest.java | 6 +- .../StorageTrieNodeHealingRequestTest.java | 5 +- .../sync/tasks/CompleteBlocksTaskTest.java | 4 + ...neCommonAncestorTaskParameterizedTest.java | 4 +- .../DetermineCommonAncestorTaskTest.java | 4 +- .../tasks/DownloadHeaderSequenceTaskTest.java | 275 ++- .../eth/sync/tasks/PersistBlockTaskTest.java | 2 +- .../AbstractTransactionPoolTest.java | 80 +- .../AbstractTransactionReplacementTest.java | 18 + ...ledTransactionHashesMessageSenderTest.java | 9 +- ...ingTransactionEstimatedMemorySizeTest.java | 4 +- .../ethereum/eth/transactions/TestNode.java | 15 +- .../TransactionBroadcasterTest.java | 58 +- .../TransactionPoolFactoryTest.java | 148 +- ...sactionReplacementByFeeMarketRuleTest.java | 90 +- .../TransactionReplacementRulesTest.java | 76 +- ...stractPrioritizedTransactionsTestBase.java | 45 +- .../BaseFeePrioritizedTransactionsTest.java | 60 +- .../layered/BaseTransactionPoolTest.java | 3 +- .../GasPricePrioritizedTransactionsTest.java | 18 +- .../LayeredPendingTransactionsTest.java | 25 +- .../LayeredTransactionPoolBaseFeeTest.java | 4 +- .../LayeredTransactionPoolGasPriceTest.java | 8 +- .../eth/transactions/layered/LayersTest.java | 7 +- .../eth/transactions/layered/ReplayTest.java | 7 +- ethereum/evmtool/build.gradle | 3 +- .../besu/evmtool/DataStoreModule.java | 5 +- .../besu/evmtool/EvmToolCommand.java | 82 +- .../evmtool/EvmToolCommandOptionsModule.java | 4 +- .../evmtool/MainnetGenesisFileModule.java | 9 +- .../evmtool/benchmarks/BenchmarkExecutor.java | 3 +- .../state-test/all-trace-flags-disabled.json | 2 +- .../evmtool/state-test/all-trace-flags.json | 2 +- .../besu/evmtool/state-test/no-memory.json | 2 +- .../besu/evmtool/trace/badcode.json | 6 +- .../besu/evmtool/trace/coinbase-cold.json | 46 +- .../besu/evmtool/trace/coinbase-warm.json | 46 +- .../besu/evmtool/trace/revert.json | 12 +- .../p2p/config/NetworkingConfiguration.java | 11 - .../p2p/discovery/PeerDiscoveryAgent.java | 6 +- .../p2p/discovery/PeerDiscoveryStatus.java | 4 +- .../discovery/VertxPeerDiscoveryAgent.java | 14 +- .../p2p/network/DefaultP2PNetwork.java | 1 - .../besu/ethereum/p2p/peers/EnodeURLImpl.java | 3 +- .../besu/ethereum/p2p/rlpx/RlpxAgent.java | 4 +- .../p2p/rlpx/connections/netty/DeFramer.java | 3 +- .../ethereum/p2p/rlpx/wire/Capability.java | 2 +- .../besu/ethereum/p2p/rlpx/wire/PeerInfo.java | 2 +- .../rlpx/wire/messages/DisconnectMessage.java | 32 +- .../PeerDiscoveryTimestampsTest.java | 31 +- .../internal/PeerDiscoveryControllerTest.java | 4 +- .../NetworkingServiceLifecycleTest.java | 4 +- .../MockConnectionInitializer.java | 3 +- .../rlpx/connections/netty/DeFramerTest.java | 4 +- .../wire/messages/DisconnectMessageTest.java | 10 + ...eSmartContractPermissioningController.java | 2 +- ...martContractV2PermissioningController.java | 2 +- ...nSmartContractPermissioningController.java | 2 +- ethereum/referencetests/build.gradle | 3 +- .../BlockchainReferenceTestCaseSpec.java | 12 +- .../BonsaiReferenceTestWorldState.java | 22 +- .../ReferenceTestBlockchain.java | 1 + .../referencetests/ReferenceTestEnv.java | 5 +- .../ReferenceTestProtocolSchedules.java | 9 +- .../besu/ethereum/core/TransactionTest.java | 2 +- .../mainnet/DifficultyCalculatorTests.java | 37 +- .../NoRewardProtocolScheduleWrapper.java | 12 +- .../ethereum/retesteth/RetestethContext.java | 19 +- .../besu/ethereum/trie/CompactEncoding.java | 32 + .../trie/InnerNodeDiscoveryManager.java | 41 +- .../besu/ethereum/trie/SnapPutVisitor.java | 52 - .../ethereum/trie/patricia/BranchNode.java | 2 +- .../ethereum/trie/SnapPutVisitorTest.java | 129 -- .../besu/ethereum/verkletrie/VerkleTrie.java | 8 +- .../VerkleTrieKeyValueGenerator.java | 95 - evm/build.gradle | 1 + .../org/hyperledger/besu/evm/MainnetEVMs.java | 1 + .../besu/evm/frame/MessageFrame.java | 32 +- .../gascalculator/BerlinGasCalculator.java | 43 +- .../ConstantinopleGasCalculator.java | 4 +- .../gascalculator/FrontierGasCalculator.java | 52 +- .../besu/evm/gascalculator/GasCalculator.java | 84 +- .../gascalculator/IstanbulGasCalculator.java | 10 +- .../PetersburgGasCalculator.java | 7 +- .../gascalculator/PragueGasCalculator.java | 10 + .../gascalculator/ShanghaiGasCalculator.java | 205 ++- .../SpuriousDragonGasCalculator.java | 9 +- .../TangerineWhistleGasCalculator.java | 15 +- .../hyperledger/besu/evm/internal/Words.java | 27 + .../org/hyperledger/besu/evm/log/Log.java | 66 +- .../evm/operation/AbstractCallOperation.java | 48 +- .../operation/AbstractCreateOperation.java | 2 +- .../besu/evm/operation/BalanceOperation.java | 11 +- .../evm/operation/BlockHashOperation.java | 59 +- .../besu/evm/operation/CallCodeOperation.java | 28 - .../besu/evm/operation/CallOperation.java | 30 +- .../evm/operation/DelegateCallOperation.java | 28 - .../evm/operation/ExtCodeHashOperation.java | 11 +- .../besu/evm/operation/ReturnOperation.java | 5 +- .../besu/evm/operation/RevertOperation.java | 5 +- .../besu/evm/operation/SLoadOperation.java | 30 +- .../besu/evm/operation/SStoreOperation.java | 4 +- .../evm/operation/SelfDestructOperation.java | 4 +- .../evm/operation/StaticCallOperation.java | 23 - .../KZGPointEvalPrecompiledContract.java | 8 +- .../MainnetPrecompiledContracts.java | 2 + .../processor/ContractCreationProcessor.java | 47 +- .../besu/evm/tracing/StandardJsonTracer.java | 7 +- .../besu/evm/worldstate/WorldUpdater.java | 7 - .../besu/evm/StandardJsonTracerTest.java | 12 +- .../besu/evm/fluent/EVMExecutorTest.java | 4 +- .../evm/operations/Create2OperationTest.java | 4 +- .../evm/operations/CreateOperationTest.java | 4 +- .../ContractCreationProcessorTest.java | 22 +- .../tracing/ExtendedOperationTracerTest.java | 3 +- gradle.properties | 4 +- gradle/versions.gradle | 21 +- .../opentelemetry/DebugMetricReader.java | 6 +- .../prometheus/MetricsHttpService.java | 24 +- .../besu/metrics/rocksdb/RocksDBStats.java | 5 +- .../besu/nat/docker/HostBasedIpDetector.java | 1 + plugin-api/build.gradle | 2 +- .../besu/plugin/data/BlockBody.java | 8 + .../besu/plugin/data/BlockHeader.java | 16 + .../data/TransactionProcessingResult.java | 7 + .../plugin/services/BesuConfiguration.java | 28 +- .../plugin/services/BlockchainService.java | 24 + .../PluginTransactionValidatorService.java | 39 - .../services/TransactionSelectionService.java | 11 +- .../exception/PluginRpcEndpointException.java | 60 +- .../services/storage/SegmentIdentifier.java | 8 +- .../PluginTransactionSelector.java | 18 + .../PluginTransactionValidator.java | 35 - .../PluginTransactionValidatorFactory.java | 30 - plugins/rocksdb/build.gradle | 1 + .../RocksDBKeyValuePrivacyStorageFactory.java | 157 +- .../RocksDBKeyValueStorageFactory.java | 316 ++-- .../configuration/DatabaseMetadata.java | 251 ++- ...ksDBKeyValuePrivacyStorageFactoryTest.java | 132 +- .../RocksDBKeyValueStorageFactoryTest.java | 206 ++- .../configuration/DatabaseMetadataTest.java | 55 +- .../RocksDBColumnarKeyValueStorageTest.java | 97 +- .../kvstore/InMemoryKeyValueStorage.java | 10 +- .../kvstore/LayeredKeyValueStorage.java | 119 +- .../SegmentedInMemoryKeyValueStorage.java | 52 +- settings.gradle | 5 +- .../kvstore/AbstractKeyValueStorageTest.java | 507 +++--- .../besu/testutil/JsonTestParameters.java | 8 +- .../hyperledger/besu/util/NetworkUtility.java | 34 +- .../hyperledger/besu/util/Subscribers.java | 4 +- .../besu/util/NetworkUtilityTest.java | 12 +- .../besu/util/log/LogUtilTest.java | 1 + 838 files changed, 13031 insertions(+), 14455 deletions(-) delete mode 100644 .github/workflows/nightly.yml delete mode 100644 .github/workflows/parallel-unit-tests.yml delete mode 100644 .github/workflows/pr-checklist-on-open.yml delete mode 100644 acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineEip6110AcceptanceTest.java delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/genesis.json delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/01_cancun_prepare_payload.json delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/02_cancun_getPayloadV3.json delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/03_cancun_newPayloadV3.json delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/04_cancun_forkchoiceUpdatedV3.json delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/05_eip6110_forkchoiceUpdatedV3.json delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/06_eip6110_getPayloadV6110.json delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/07_eip6110_send_raw_transaction.json delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/08_eip6110_invalid_null_deposits_execute_payload.json delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/09_eip6110_newPayloadV6110.json delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/10_eip6110_forkchoiceUpdatedV3.json delete mode 100644 acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/11_eip6110_getPayloadV6110.json delete mode 100644 besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java delete mode 100644 besu/src/main/java/org/hyperledger/besu/services/PluginTransactionValidatorServiceImpl.java delete mode 100644 besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java delete mode 100644 consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BftForksScheduleFactory.java delete mode 100644 consensus/common/src/test/java/org/hyperledger/besu/consensus/common/bft/BftForksScheduleFactoryTest.java delete mode 100644 docker/graalvm/Dockerfile delete mode 100644 docker/openj9-jdk-17/Dockerfile delete mode 100644 docker/openjdk-17-debug/Dockerfile delete mode 100644 docker/openjdk-17/Dockerfile delete mode 100644 docker/openjdk-latest/Dockerfile delete mode 100644 errorprone-checks/README.md delete mode 100644 errorprone-checks/build.gradle delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/BannedMethod.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectly.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectly.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionals.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayed.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinal.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggers.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/BannedMethodTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersTest.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfaceNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfacePositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersPositiveCases.java delete mode 100644 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV6110.java delete mode 100644 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV6110.java delete mode 100644 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/EngineGetPayloadResultV6110.java delete mode 100644 ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV6110Test.java delete mode 100644 ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV6110Test.java delete mode 100644 ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_call_invalidWithInputAndDataAttribute.json delete mode 100644 ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/selectors/AllAcceptingTransactionSelector.java delete mode 100644 ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerIntegrationTest.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/feemarket/BaseFee.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiNoOpCachedWorldStorageManager.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FlatDbStrategy.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/MarkSweepPruner.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/Pruner.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerConfiguration.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogLayerTests.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/BonsaiWorldStateKeyValueStorageTest.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManagerTests.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPrunerTest.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/MarkSweepPrunerTest.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java delete mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManager.java delete mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/SnapProtocolManagerTestUtil.java delete mode 100644 ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitor.java delete mode 100644 ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitorTest.java delete mode 100644 ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/VerkleTrieKeyValueGenerator.java delete mode 100644 plugin-api/src/main/java/org/hyperledger/besu/plugin/services/PluginTransactionValidatorService.java delete mode 100644 plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txvalidator/PluginTransactionValidator.java delete mode 100644 plugin-api/src/main/java/org/hyperledger/besu/plugin/services/txvalidator/PluginTransactionValidatorFactory.java diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e84ab22aaf..a964f4d3be8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -500,6 +500,3 @@ workflows: jobs: - assemble - dockerScan - - acceptanceTestsPrivacy: - requires: - - assemble diff --git a/.github/issue_template.md b/.github/issue_template.md index cfae3400266..54acf4c1968 100644 --- a/.github/issue_template.md +++ b/.github/issue_template.md @@ -3,7 +3,7 @@ - + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 11bac0fb923..7e8cde29e7b 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,8 +1,21 @@ - - - ## PR description ## Fixed Issue(s) - \ No newline at end of file + + + +### Thanks for sending a pull request! Have you done the following? + +- [ ] Checked out our [contribution guidelines](https://github.com/hyperledger/besu/blob/main/CONTRIBUTING.md)? +- [ ] Considered documentation and added the `doc-change-required` label to this PR [if updates are required](https://wiki.hyperledger.org/display/BESU/Documentation). +- [ ] Considered the changelog and included an [update if required](https://wiki.hyperledger.org/display/BESU/Changelog). +- [ ] For database changes (e.g. KeyValueSegmentIdentifier) considered compatibility and performed forwards and backwards compatibility tests + +### Locally, you can run these tests to catch failures early: + +- [ ] unit tests: `./gradlew build` +- [ ] acceptance tests: `./gradlew acceptanceTest` +- [ ] integration tests: `./gradlew integrationTest` +- [ ] reference tests: `./gradlew ethereum:referenceTests:referenceTests` + diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index b94218e6087..74ba75c2f91 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -1,72 +1,42 @@ name: acceptance-tests on: + workflow_dispatch: pull_request: - pull_request_review: - types: [submitted] + branches: + - main + - release-* + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" total-runners: 16 jobs: - shouldRun: - name: checks to ensure we should run - # necessary because there is no single PR approved event, need to check all comments/approvals/denials - runs-on: ubuntu-22.04 - outputs: - shouldRun: ${{steps.shouldRun.outputs.result}} - steps: - - name: required check - id: shouldRun - uses: actions/github-script@v7.0.1 - env: - # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main - RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - with: - script: | - const { RELEVANT_SHA } = process.env; - const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: RELEVANT_SHA, - }); - const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); - const alreadyRun = acceptanceTested && acceptanceTested.find(({ state }) => state === 'success') > 0; - const { data: reviews } = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - }); - const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); - - console.log("tests should be run = %j", shouldRun); - console.log("alreadyRun = %j", alreadyRun); - console.log("approvingReviews = %j", approvingReviews.length); - - return shouldRun; acceptanceTestEthereum: runs-on: ubuntu-22.04 name: "Acceptance Runner" - needs: shouldRun permissions: statuses: write checks: write - if: ${{ needs.shouldRun.outputs.shouldRun == 'true'}} strategy: fail-fast: true matrix: runner_index: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] steps: - name: Checkout Repo - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Java - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 17 - name: get acceptance test report - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d with: branch: main name_is_regexp: true @@ -74,10 +44,12 @@ jobs: path: tmp/junit-xml-reports-downloaded if_no_artifact_found: true - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: Split tests id: split-tests - uses: r7kamura/split-tests-by-timings@v0 + uses: r7kamura/split-tests-by-timings@9322bd292d9423e2bc5a65bec548901801341e3f with: reports: tmp/junit-xml-reports-downloaded glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java' @@ -90,25 +62,30 @@ jobs: #then drop file extension, then insert --tests option between each. run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g' > gradleArgs.txt - name: run acceptance tests - run: ./gradlew acceptanceTest `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew acceptanceTestNotPrivacy `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: cleanup tempfiles run: rm testList.txt gradleArgs.txt - name: Upload Acceptance Test Results - uses: actions/upload-artifact@v3.1.0 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 with: name: acceptance-node-${{matrix.runner_index}}-test-results - path: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' - - name: Publish Test Report - uses: mikepenz/action-junit-report@v4 - if: (success() || failure()) # always run even if the build step fails - with: - report_paths: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' - acceptance-tests: + path: 'acceptance-tests/tests/build/test-results/**/TEST-*.xml' + accepttests-passed: + name: "accepttests-passed" runs-on: ubuntu-22.04 needs: [ acceptanceTestEthereum ] permissions: checks: write statuses: write + if: always() steps: - - name: consolidation - run: echo "consolidating statuses" + # Fail if any `needs` job was not a success. + # Along with `if: always()`, this allows this job to act as a single required status check for the entire workflow. + - name: Fail on workflow error + run: exit 1 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + }} \ No newline at end of file diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 3b33f43bce1..26beeed9cd0 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -1,10 +1,12 @@ -name: artifacts +name: release artifacts on: release: types: - prereleased +env: + GRADLE_OPTS: "-Dorg.gradle.parallel=true -Dorg.gradle.caching=true" jobs: artifacts: @@ -13,17 +15,19 @@ jobs: contents: write steps: - name: checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Set up JDK 17 - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: 'temurin' java-version: '17' - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 - - name: assemble distributions + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true + - name: assemble release run: - ./gradlew -Prelease.releaseVersion=${{github.ref_name}} assemble -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + ./gradlew -Prelease.releaseVersion=${{github.event.release.name}} -Pversion=${{github.event.release.name}} assemble - name: hashes id: hashes run: | @@ -31,46 +35,68 @@ jobs: echo "zipSha=$(shasum -a 256 besu*.zip)" >> $GITHUB_OUTPUT echo "tarSha=$(shasum -a 256 besu*.tar.gz)" >> $GITHUB_OUTPUT - name: upload tarball - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 with: path: 'build/distributions/besu*.tar.gz' - name: besu-${{ github.ref_name }}.tar.gz + name: besu-${{ github.event.release.name }}.tar.gz + compression-level: 0 - name: upload zipfile - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 with: path: 'build/distributions/besu*.zip' - name: besu-${{ github.ref_name }}.zip - - name: Upload Release assets - uses: softprops/action-gh-release@v1 - with: - append_body: true - files: | - build/distributions/besu*.tar.gz - build/distributions/besu*.zip - body: | - ${{steps.hashes.outputs.tarSha}} - ${{steps.hashes.outputs.zipSha}} + name: besu-${{ github.event.release.name }}.zip + compression-level: 0 + testWindows: runs-on: windows-2022 needs: artifacts timeout-minutes: 10 - if: ${{ github.actor != 'dependabot[bot]' }} steps: - name: Set up Java - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: adopt java-version: 17 - name: Download zip - uses: actions/download-artifact@v3 + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe with: - name: besu-${{ github.ref_name }}.zip + pattern: besu-*.zip + merge-multiple: true - name: test Besu run: | + dir unzip besu-*.zip -d besu-tmp cd besu-tmp mv besu-* ../besu cd .. besu\bin\besu.bat --help besu\bin\besu.bat --version - + publish: + runs-on: ubuntu-22.04 + needs: testWindows + steps: + - name: Download archives + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe + with: + pattern: besu-* + merge-multiple: true + path: 'build/distributions' + - name: Upload Release assets + uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 + with: + append_body: true + files: | + build/distributions/besu*.tar.gz + build/distributions/besu*.zip + body: | + ${{steps.hashes.outputs.tarSha}} + ${{steps.hashes.outputs.zipSha}} + arifactoryPublish: + runs-on: ubuntu-22.04 + needs: artifacts + steps: + - name: Artifactory Publish + env: + ARTIFACTORY_USER: ${{ secrets.BESU_ARTIFACTORY_USER }} + ARTIFACTORY_KEY: ${{ secrets.BESU_ARTIFACTORY_TOKEN }} + run: ./gradlew -Prelease.releaseVersion=${{ github.event.release.name }} -Pversion=${{github.event.release.name}} artifactoryPublish \ No newline at end of file diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f8842123330..0a50ec57c2d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -12,15 +12,12 @@ name: "CodeQL" on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - paths-ignore: - - '**/*.json' - - '**/*.md' - - '**/*.properties' - - '**/*.txt' + workflow_dispatch: + schedule: + # * is a special character in YAML so you have to quote this string + # expression evaluates to midnight every night + - cron: '0 0 * * *' + jobs: analyze: name: Analyze @@ -31,15 +28,15 @@ jobs: security-events: write steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Set up Java - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: 'temurin' java-version: 17 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@2f93e4319b2f04a2efc38fa7f78bd681bc3f7b2f with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -48,9 +45,11 @@ jobs: queries: security-and-quality,security-extended - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: compileJava noscan run: | JAVA_OPTS="-Xmx2048M" ./gradlew --no-scan compileJava - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@2f93e4319b2f04a2efc38fa7f78bd681bc3f7b2f diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c487ed84b28..5cb78f215d4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,31 +4,25 @@ on: types: - prereleased env: - registry: ghcr.io + registry: docker.io jobs: hadolint: runs-on: ubuntu-22.04 steps: - name: Checkout Repo - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Set up Java - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 17 - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 - - name: hadoLint_openj9-jdk_17 - run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile - - name: hadoLint_openjdk_17 - run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile - - name: hadoLint_openjdk_17_debug - run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile - - name: hadoLint_openjdk_latest - run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile - - name: hadoLint_graalvm - run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true + - name: hadoLint + run: docker run --rm -i hadolint/hadolint < docker/Dockerfile buildDocker: needs: hadolint permissions: @@ -55,37 +49,40 @@ jobs: echo "ARCH=arm64" >> $GITHUB_OUTPUT fi - name: Checkout Repo - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: short sha id: shortSha run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: Set up Java - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 17 - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: install goss run: | mkdir -p docker/reports curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} + - name: login to ${{ env.registry }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d + with: + registry: ${{ env.registry }} + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: build and test docker - uses: gradle/gradle-build-action@v2.12.0 + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 env: architecture: ${{ steps.prep.outputs.ARCH }} with: - arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }} - - name: login to ghcr - uses: docker/login-action@v3.0.0 - with: - registry: ${{ env.registry }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + cache-disabled: true + arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{github.event.release.name}} -Prelease.releaseVersion=${{ github.event.release.name }} - name: publish env: architecture: ${{ steps.prep.outputs.ARCH }} - run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }} + run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{github.event.release.name}} -Prelease.releaseVersion=${{ github.event.release.name }} multiArch: needs: buildDocker runs-on: ubuntu-22.04 @@ -94,20 +91,33 @@ jobs: packages: write steps: - name: Checkout Repo - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Set up Java - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 17 - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 - - name: login to ghcr - uses: docker/login-action@v3.0.0 + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true + - name: login to ${{ env.registry }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d with: registry: ${{ env.registry }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: multi-arch docker - run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Prelease.releaseVersion=${{ github.ref_name }} - + run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{github.event.release.name}} -Prelease.releaseVersion=${{ github.event.release.name }} + amendNotes: + needs: multiArch + runs-on: ubuntu-22.04 + permissions: + contents: write + steps: + - name: add pull command to release notes + uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 + with: + append_body: true + body: | + `docker pull ${{env.registry}}/${{secrets.DOCKER_ORG}}/besu:${{github.event.release.name}}` diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 68115ea40bf..8a60d09017d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,73 +1,40 @@ name: integration-tests on: + workflow_dispatch: pull_request: - pull_request_review: - types: - - submitted + branches: + - main + - release-* + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true env: - GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.caching=true" jobs: - shouldRun: - name: checks to ensure we should run - runs-on: ubuntu-22.04 - outputs: - shouldRun: ${{steps.shouldRun.outputs.result}} - steps: - - name: required check - id: shouldRun - uses: actions/github-script@v7.0.1 - env: - # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main - RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - with: - script: | - const { RELEVANT_SHA } = process.env; - const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: RELEVANT_SHA, - }); - - const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tests'); - const alreadyRun = intTested && intTested.find(({ state }) => state === 'success') > 0; - const { data: reviews } = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - }); - const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); - - console.log("tests should be run = %j", shouldRun); - console.log("alreadyRun = %j", alreadyRun); - console.log("approvingReviews = %j", approvingReviews.length); - - return shouldRun; integration-tests: + name: "integration-passed" runs-on: ubuntu-22.04 - needs: shouldRun - if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} permissions: statuses: write checks: write steps: - name: Checkout Repo - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Java - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 17 - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 - - name: run integration tests - run: ./gradlew integrationTest compileJmh -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: Publish Test Report - uses: mikepenz/action-junit-report@v4 - if: (success() || failure()) + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 with: - report_paths: '**/build/test-results/integrationTest/TEST-*.xml' + cache-disabled: true + - name: run integration tests + run: ./gradlew integrationTest compileJmh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index 9d2778fba47..00000000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: nightly - -on: - workflow_dispatch: - schedule: - # * is a special character in YAML so you have to quote this string - # expression evaluates to midnight every night - - cron: '0 0 * * *' - -env: - nightly-tag: develop - registry: ghcr.io - -jobs: - hadolint: - runs-on: ubuntu-22.04 - steps: - - name: Checkout Repo - uses: actions/checkout@v4.1.1 - - name: Set up Java - uses: actions/setup-java@v4.0.0 - with: - distribution: temurin - java-version: 17 - - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 - - name: hadoLint_openj9-jdk_17 - run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile - - name: hadoLint_openjdk_17 - run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile - - name: hadoLint_openjdk_17_debug - run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile - - name: hadoLint_openjdk_latest - run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile - - name: hadoLint_graalvm - run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile - buildDocker: - needs: hadolint - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - platform: - - ubuntu-22.04 - - [self-hosted, ARM64] - runs-on: ${{ matrix.platform }} - steps: - - name: Prepare - id: prep - run: | - platform=${{ matrix.platform }} - if [ "$platform" = 'ubuntu-22.04' ]; then - echo "PLATFORM_PAIR=linux-amd64" >> $GITHUB_OUTPUT - echo "ARCH=amd64" >> $GITHUB_OUTPUT - else - echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT - echo "ARCH=arm64" >> $GITHUB_OUTPUT - fi - - name: Checkout Repo - uses: actions/checkout@v4.1.1 - - name: short sha - id: shortSha - run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Set up Java - uses: actions/setup-java@v4.0.0 - with: - distribution: temurin - java-version: 17 - - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 - - name: build image - uses: gradle/gradle-build-action@v2.12.0 - with: - arguments: distDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main - - name: install goss - run: | - mkdir -p docker/reports - curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} - - name: test docker - uses: gradle/gradle-build-action@v2.12.0 - env: - architecture: ${{ steps.prep.outputs.ARCH }} - with: - arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main - - name: login to ghcr - uses: docker/login-action@v3.0.0 - with: - registry: ${{ env.registry }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: publish - env: - architecture: ${{ steps.prep.outputs.ARCH }} - run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main - multiArch: - permissions: - contents: read - packages: write - needs: buildDocker - runs-on: ubuntu-22.04 - steps: - - name: Checkout Repo - uses: actions/checkout@v4.1.1 - - name: Set up Java - uses: actions/setup-java@v4.0.0 - with: - distribution: temurin - java-version: 17 - - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 - - name: Login to DockerHub - uses: docker/login-action@v3.0.0 - with: - registry: ${{ env.registry }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: multi-arch docker - run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ github.repository_owner }} -Pbranch=main - diff --git a/.github/workflows/parallel-unit-tests.yml b/.github/workflows/parallel-unit-tests.yml deleted file mode 100644 index b12fa43655b..00000000000 --- a/.github/workflows/parallel-unit-tests.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: parallel-unit-tests -#experimental work in progress - trying to figure out how to split tests across multi-modules by runtime -on: - workflow_dispatch: - -env: - GRADLE_OPTS: "-Dorg.gradle.daemon=false" - total-runners: 4 -jobs: - junit: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - runner_index: - - 0 - - 1 - - 2 - - 3 - steps: - - name: Checkout Repo - uses: actions/checkout@v4.1.1 - - name: Split tests - id: split-tests - uses: chaosaffe/split-tests@v1-alpha.1 - with: - glob: '**/src/test/java/**/*.java' - split-total: ${{ env.total-runners }} - split-index: ${{ matrix.runner_index }} - line-count: true - - name: Set up Java - uses: actions/setup-java@v4.0.0 - with: - distribution: adopt - java-version: 17 - cache: gradle - - name: write out test list - run: echo "${{ steps.split-tests.outputs.test-suite }}" >> testList.txt - - name: debug testfile paths - run: cat testList.txt - - name: format gradle args - # regex means: truncate file paths to align with package name, replacing with tests switch, then drop file extension, - # then swap path delimiter with package delimiter - run: cat testList.txt | sed -e 's/[^ ]*src\/test\/java\//--tests\ /g' -e 's/\.java//g' -e 's/\//\./g' >> gradleArgs.txt - - name: debug test class list - run: cat gradleArgs.txt - - name: run unit tests - run: ./gradlew test `cat gradleArgs.txt` - diff --git a/.github/workflows/pr-checklist-on-open.yml b/.github/workflows/pr-checklist-on-open.yml deleted file mode 100644 index f849ba760bb..00000000000 --- a/.github/workflows/pr-checklist-on-open.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "comment on pr with checklist" -on: - pull_request_target: - types: [ opened ] - branches: [ main ] -jobs: - checklist: - name: "add checklist as a comment on newly opened PRs" - runs-on: ubuntu-22.04 - permissions: - pull-requests: write - steps: - - uses: actions/github-script@v7.0.1 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: '- [ ] I thought about documentation and added the `doc-change-required` label to this PR if [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).\n- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).\n- [ ] If my PR includes database changes (e.g. KeyValueSegmentIdentifier) I have thought about compatibility and performed forwards and backwards compatibility tests\n- [ ] I thought about running CI.\n- [ ] If I did not run CI, I ran as much locally as possible before pushing.\n-' - }) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 5abbcb6d0a3..1fa7d3d31d8 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -2,11 +2,16 @@ name: pre-review on: pull_request: - workflow_dispatch: + branches: + - main + - release-* -permissions: - statuses: write - checks: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true" jobs: repolint: @@ -15,46 +20,57 @@ jobs: container: ghcr.io/todogroup/repolinter:v0.11.2 steps: - name: Checkout Code - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Lint Repo run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown gradle-wrapper: name: "Gradle Wrapper Validation" runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4.1.1 - - uses: gradle/wrapper-validation-action@v1.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: gradle/wrapper-validation-action@56b90f209b02bf6d1deae490e9ef18b21a389cd4 spotless: runs-on: ubuntu-22.04 - if: ${{ github.actor != 'dependabot[bot]' }} steps: - name: Checkout Repo - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Java - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 17 - name: Setup Gradle - uses: gradle/gradle-build-action@v2.12.0 + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: run spotless - run: ./gradlew spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew spotlessCheck compile: runs-on: ubuntu-22.04 timeout-minutes: 30 needs: [spotless, gradle-wrapper, repolint] steps: - name: Checkout Repo - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Java - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 17 - name: Setup Gradle - uses: gradle/gradle-build-action@v2.12.0 + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: Gradle Compile - run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew build -x test -x spotlessCheck unitTests: env: GRADLEW_UNIT_TEST_ARGS: ${{matrix.gradle_args}} @@ -72,32 +88,62 @@ jobs: - "ethereum:api:testBonsai" - "ethereum:api:testForest" - "ethereum:api:testRemainder" + - "ethereum:eth:test" - "ethereum:core:test" + #includes will need exact strings from gradle args above + include: + - gradle_args: "test -x besu:test -x consensus:test -x crypto:test -x ethereum:eth:test -x ethereum:api:test -x ethereum:core:test" + filename: "everythingElse" + - gradle_args: "besu:test consensus:test crypto:test" + filename: "consensusCrypto" + - gradle_args: "ethereum:api:testBonsai" + filename: "apiBonsai" + - gradle_args: "ethereum:api:testRemainder" + filename: "apiForest" + - gradle_args: "ethereum:api:testRemainder" + filename: "apiRemainder" + - gradle_args: "ethereum:eth:test" + filename: "eth" + - gradle_args: "ethereum:core:test" + filename: "core" steps: - name: Checkout Repo - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Java - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: temurin java-version: 17 - name: Setup Gradle - uses: gradle/gradle-build-action@v2.12.0 + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: run unit tests id: unitTest - run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: Publish Test Report - uses: mikepenz/action-junit-report@v4 - if: success() || failure() # always run even if the build step fails + run: ./gradlew $GRADLEW_UNIT_TEST_ARGS + - name: Upload Unit Test Results + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 with: - report_paths: '**/test-results/**/TEST-*.xml' - annotate_only: true - pre-review: + name: unit-${{matrix.filename}}-test-results + path: '**/test-results/**/TEST-*.xml' + unittests-passed: + name: "unittests-passed" runs-on: ubuntu-22.04 needs: [unitTests] permissions: checks: write statuses: write + if: always() steps: - - name: consolidation - run: echo "consolidating statuses" + # Fail if any `needs` job was not a success. + # Along with `if: always()`, this allows this job to act as a single required status check for the entire workflow. + - name: Fail on workflow error + run: exit 1 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + }} diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 1f2cac7b159..68310944af4 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -1,147 +1,83 @@ name: reference-tests on: + workflow_dispatch: pull_request: - pull_request_review: - types: - - submitted + branches: + - main + - release-* env: - GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" - total-runners: 6 + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.caching=true" + total-runners: 10 -jobs: - shouldRun: - name: checks to ensure we should run - # necessary because there is no single PR approved event, need to check all comments/approvals/denials - # might also be a job running, and additional approvals - runs-on: ubuntu-22.04 - outputs: - shouldRun: ${{steps.shouldRun.outputs.result}} - steps: - - name: required check - id: shouldRun - uses: actions/github-script@v7.0.1 - env: - # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main - RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - with: - script: | - const { RELEVANT_SHA } = process.env; - const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: RELEVANT_SHA, - }); - - - const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tests'); - const alreadyRun = refTested && refTested.find(({ state }) => state === 'success') > 0; - const { data: reviews } = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - }); - const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); - - console.log("tests should be run = %j", shouldRun); - console.log("alreadyRun = %j", alreadyRun); - console.log("approvingReviews = %j", approvingReviews.length); - - return shouldRun; - - prepareReferenceTestEthereum: - runs-on: ubuntu-22.04 - needs: shouldRun - if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4.1.1 - with: - submodules: recursive - set-safe-directory: true - - name: Set up Java - uses: actions/setup-java@v4.0.0 - with: - distribution: temurin - java-version: 17 - - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 - - name: execute generate reference tests - run: ./gradlew ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: store generated tests - uses: actions/upload-artifact@v3 - with: - name: 'reference-tests' - path: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: referenceTestEthereum: runs-on: ubuntu-22.04 permissions: statuses: write checks: write - needs: - - prepareReferenceTestEthereum - if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} + packages: read strategy: fail-fast: true matrix: - runner_index: [0,1,2,3,4,5] + runner_index: [1,2,3,4,5,6,7,8,9,10] steps: - name: Checkout Repo - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} submodules: recursive - name: Set up Java - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: adopt-openj9 java-version: 17 - - name: retrieve generated tests - uses: actions/download-artifact@v3.0.2 - with: - name: 'reference-tests' - path: 'ethereum/referencetests/build/generated/sources/reference-test/' - - name: get reference test report - uses: dawidd6/action-download-artifact@v2 - with: - branch: main - name_is_regexp: true - name: 'reference-test-node-\d*\d-results' - path: tmp/ref-xml-reports-downloaded - if_no_artifact_found: true - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 - - name: Split tests - id: split-tests - uses: r7kamura/split-tests-by-timings@v0 + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 with: - reports: tmp/ref-xml-reports-downloaded - glob: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java' - total: ${{env.total-runners}} - index: ${{ matrix.runner_index }} + cache-disabled: true + - name: execute generate reference tests + run: ./gradlew ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: list test files generated + run: find ethereum/referencetests/build/generated/sources/reference-test -name "*.java" | sort >> filenames.txt + - name: Split tests + run: ./.github/workflows/splitList.sh filenames.txt ${{env.total-runners}} + - name: echo test file count + run: cat group_${{matrix.runner_index}}.txt | wc + - name: convert to test suite classnames + run: cat group_${{matrix.runner_index}}.txt | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /' > testClasses.txt - name: compose gradle args - run: echo ${{ steps.split-tests.outputs.paths }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /' > refTestArgs.txt + run: tr '\n' ' ' < testClasses.txt > refTestArgs.txt + - name: refTestArgs.txt + run: cat refTestArgs.txt - name: run reference tests - run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` - name: Upload Test Report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 if: always() # always run even if the previous step fails with: name: reference-test-node-${{matrix.runner_index}}-results path: '**/build/test-results/referenceTests/TEST-*.xml' - - name: Publish Test Report - uses: mikepenz/action-junit-report@v4 - if: success() || failure() # always run even if the build step fails - with: - report_paths: '**/build/test-results/referenceTest/TEST-*.xml' - reference-tests: + reftests-passed: + name: "reftests-passed" runs-on: ubuntu-22.04 needs: [ referenceTestEthereum ] permissions: checks: write statuses: write + if: always() steps: - - name: consolidation - run: echo "consolidating statuses" - + # Fail if any `needs` job was not a success. + # Along with `if: always()`, this allows this job to act as a single required status check for the entire workflow. + - name: Fail on workflow error + run: exit 1 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e0c4f0bb298..8d8b4c8e79c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,23 +1,30 @@ name: release besu on: - workflow_dispatch: release: types: [released] +env: + registry: docker.io jobs: dockerPromoteX64: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4.1.1 - - uses: actions/setup-java@v4.0.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: 'temurin' # See 'Supported distributions' for available options java-version: '17' cache: gradle - - name: Login to DockerHub - run: echo '${{ secrets.DOCKER_PASSWORD_RW }}' | docker login -u '${{ secrets.DOCKER_USER_RW }}' --password-stdin + - name: login to ${{ env.registry }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d + with: + registry: ${{ env.registry }} + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: Setup Gradle - uses: gradle/gradle-build-action@v2.12.0 + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: Docker upload - run: ./gradlew "-Prelease.releaseVersion=${{ github.ref_name }}" "-PdockerOrgName=${{ secrets.DOCKER_ORG }}" dockerUploadRelease + run: ./gradlew "-Prelease.releaseVersion=${{ github.event.release.name }}" "-PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }}" dockerUploadRelease - name: Docker manifest - run: ./gradlew "-Prelease.releaseVersion=${{ github.ref_name }}" "-PdockerOrgName=${{ secrets.DOCKER_ORG }}" manifestDockerRelease + run: ./gradlew "-Prelease.releaseVersion=${{ github.event.release.name }}" "-PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }}" manifestDockerRelease diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 7c4acee7e40..9f70e699178 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -16,24 +16,26 @@ jobs: runs-on: ubuntu-latest steps: - name: checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Set up JDK 17 - uses: actions/setup-java@v4.0.0 + uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 with: distribution: 'temurin' java-version: '17' - name: Cache SonarCloud packages - uses: actions/cache@v3 + uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - name: setup gradle - uses: gradle/gradle-build-action@v2.12.0 + uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1 + with: + cache-disabled: true - name: Build and analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - SONAR_ORGANIZATION: ${{ env.SONAR_ORGANIZATION }} - SONAR_PROJECT_KEY: $ {{ env.SONAR_PROJECT_KEY }} + SONAR_ORGANIZATION: ${{ vars.SONAR_ORGANIZATION }} + SONAR_PROJECT_KEY: ${{ vars.SONAR_PROJECT_KEY }} run: ./gradlew build sonarqube --continue --info -Dorg.gradle.parallel=true -Dorg.gradle.caching=true diff --git a/CHANGELOG.md b/CHANGELOG.md index 861e1977720..ed7d0aa12f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,77 @@ # Changelog -## 24.1.2-SNAPSHOT +## Next Release ### Breaking Changes -- Following the OpenMetrics convention, the updated Prometheus client adds the `_total` suffix to every metrics of type counter, with the effect that some existing metrics have been renamed to have this suffix. If you are using the official Besu Grafana dashboard [(available here)](https://grafana.com/grafana/dashboards/16455-besu-full/), just update it to the latest revision, that accepts the old and the new name of the affected metrics. If you have a custom dashboards or use the metrics in other ways, then you need to manually update it to support the new naming. +- RocksDB database metadata format has changed to be more expressive, the migration of an existing metadata file to the new format is automatic at startup. Before performing a downgrade to a previous version it is mandatory to revert to the original format using the subcommand `besu --data-path=/path/to/besu/datadir storage revert-metadata v2-to-v1`. +- BFT networks won't start with SNAP or CHECKPOINT sync (previously Besu would start with this config but quietly fail to sync, so it's now more obvious that it won't work) [#6625](https://github.com/hyperledger/besu/pull/6625), [#6667](https://github.com/hyperledger/besu/pull/6667) +- Forest pruning has been removed, it was deprecated since 24.1.0. In case you are still using it you must now remove any of the following options: `pruning-enabled`, `pruning-blocks-retained` and `pruning-block-confirmations`, from your configuration, and you may want to consider switching to Bonsai. + +### Upcoming Breaking Changes +- Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version. + +### Deprecations + +### Additions and Improvements +- Update "host allow list" logic to transition from deprecated `host()` method to suggested `authority()` method.[#6878](https://github.com/hyperledger/besu/issues/6878) +- `txpool_besuPendingTransactions`change parameter `numResults` to optional parameter [#6708](https://github.com/hyperledger/besu/pull/6708) +- Extend `Blockchain` service [#6592](https://github.com/hyperledger/besu/pull/6592) +- Add bft-style `blockperiodseconds` transitions to Clique [#6596](https://github.com/hyperledger/besu/pull/6596) +- Add `createemptyblocks` transitions to Clique [#6608](https://github.com/hyperledger/besu/pull/6608) +- RocksDB database metadata refactoring [#6555](https://github.com/hyperledger/besu/pull/6555) +- Make layered txpool aware of `minGasPrice` and `minPriorityFeePerGas` dynamic options [#6611](https://github.com/hyperledger/besu/pull/6611) +- Update commons-compress to 1.26.0 [#6648](https://github.com/hyperledger/besu/pull/6648) +- Update Vert.x to 4.5.4 [#6666](https://github.com/hyperledger/besu/pull/6666) +- Refactor and extend `TransactionPoolValidatorService` [#6636](https://github.com/hyperledger/besu/pull/6636) +- Introduce `TransactionSimulationService` [#6686](https://github.com/hyperledger/besu/pull/6686) +- Transaction call object to accept both `input` and `data` field simultaneously if they are set to equal values [#6702](https://github.com/hyperledger/besu/pull/6702) +- `eth_call` for blob tx allows for empty `maxFeePerBlobGas` [#6731](https://github.com/hyperledger/besu/pull/6731) +- Extend error handling of plugin RPC methods [#6759](https://github.com/hyperledger/besu/pull/6759) +- Added engine_newPayloadV4 and engine_getPayloadV4 methods [#6783](https://github.com/hyperledger/besu/pull/6783) +- Reduce storage size of receipts [#6602](https://github.com/hyperledger/besu/pull/6602) +- Dedicated log marker for invalid txs removed from the txpool [#6826](https://github.com/hyperledger/besu/pull/6826) +- Prevent startup with BONSAI and privacy enabled [#6809](https://github.com/hyperledger/besu/pull/6809) +- Remove deprecated Forest pruning [#6810](https://github.com/hyperledger/besu/pull/6810) +- Experimental Snap Sync Server [#6640](https://github.com/hyperledger/besu/pull/6640) +- Upgrade Reference Tests to 13.2 [#6854](https://github.com/hyperledger/besu/pull/6854) +- Update Web3j dependencies [#6811](https://github.com/hyperledger/besu/pull/6811) +- Add `tx-pool-blob-price-bump` option to configure the price bump percentage required to replace blob transactions (by default 100%) [#6874](https://github.com/hyperledger/besu/pull/6874) + +### Bug fixes +- Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665) +- Make block transaction selection max time aware of PoA transitions [#6676](https://github.com/hyperledger/besu/pull/6676) +- Don't enable the BFT mining coordinator when running sub commands such as `blocks export` [#6675](https://github.com/hyperledger/besu/pull/6675) +- In JSON-RPC return optional `v` fields for type 1 and type 2 transactions [#6762](https://github.com/hyperledger/besu/pull/6762) +- Fix Shanghai/QBFT block import bug when syncing new nodes [#6765](https://github.com/hyperledger/besu/pull/6765) +- Fix to avoid broadcasting full blob txs, instead of only the tx announcement, to a subset of nodes [#6835](https://github.com/hyperledger/besu/pull/6835) +- Snap client fixes discovered during snap server testing [#6847](https://github.com/hyperledger/besu/pull/6847) +- Correctly initialize the txpool as disabled on creation [#6890](https://github.com/hyperledger/besu/pull/6890) + +### Download Links + +## 24.3.0 + +### Breaking Changes +- SNAP - Snap sync is now the default for named networks [#6530](https://github.com/hyperledger/besu/pull/6530) + - if you want to use the previous default behavior, you'll need to specify `--sync-mode=FAST` +- BONSAI - Default data storage format is now Bonsai [#6536](https://github.com/hyperledger/besu/pull/6536) + - if you had previously used the default (FOREST), at startup you will get an error indicating the mismatch + `Mismatch: DB at '/your-path' is FOREST (Version 1) but config expects BONSAI (Version 2). Please check your config.` + - to fix this mismatch, specify the format explicitly using `--data-storage-format=FOREST` +- Following the OpenMetrics convention, the updated Prometheus client adds the `_total` suffix to every metrics of type counter, with the effect that some existing metrics have been renamed to have this suffix. If you are using the official Besu Grafana dashboard [(available here)](https://grafana.com/grafana/dashboards/16455-besu-full/), just update it to the latest revision, that accepts the old and the new name of the affected metrics. If you have a custom dashboard or use the metrics in other ways, then you need to manually update it to support the new naming. - The `trace-filter` method in JSON-RPC API now has a default block range limit of 1000, adjustable with `--rpc-max-trace-filter-range` (thanks @alyokaz) [#6446](https://github.com/hyperledger/besu/pull/6446) - Requesting the Ethereum Node Record (ENR) to acquire the fork id from bonded peers is now enabled by default, so the following change has been made [#5628](https://github.com/hyperledger/besu/pull/5628): - `--Xfilter-on-enr-fork-id` has been removed. To disable the feature use `--filter-on-enr-fork-id=false`. - `--engine-jwt-enabled` has been removed. Use `--engine-jwt-disabled` instead. [#6491](https://github.com/hyperledger/besu/pull/6491) - +- Release docker images now provided at ghcr.io instead of dockerhub ### Deprecations +- X_SNAP and X_CHECKPOINT are marked for deprecation and will be removed in 24.6.0 in favor of SNAP and CHECKPOINT [#6405](https://github.com/hyperledger/besu/pull/6405) +- `--Xp2p-peer-lower-bound` is deprecated. [#6501](https://github.com/hyperledger/besu/pull/6501) + +### Upcoming Breaking Changes +- `--Xbonsai-limit-trie-logs-enabled` will be removed. You will need to use `--bonsai-limit-trie-logs-enabled` instead. Additionally, this limit will change to be enabled by default. + - If you do not want the limit enabled (eg you have `--bonsai-historical-block-limit` set < 512), you need to explicitly disable it using `--bonsai-limit-trie-logs-enabled=false` or increase the limit. [#6561](https://github.com/hyperledger/besu/pull/6561) ### Additions and Improvements - Upgrade Prometheus and Opentelemetry dependencies [#6422](https://github.com/hyperledger/besu/pull/6422) @@ -19,16 +80,50 @@ - Log blob count when importing a block via Engine API [#6466](https://github.com/hyperledger/besu/pull/6466) - Introduce `--Xbonsai-limit-trie-logs-enabled` experimental feature which by default will only retain the latest 512 trie logs, saving about 3GB per week in database growth [#5390](https://github.com/hyperledger/besu/issues/5390) - Introduce `besu storage x-trie-log prune` experimental offline subcommand which will prune all redundant trie logs except the latest 512 [#6303](https://github.com/hyperledger/besu/pull/6303) +- Improve flat trace generation performance [#6472](https://github.com/hyperledger/besu/pull/6472) +- SNAP and CHECKPOINT sync - early access flag removed so now simply SNAP and CHECKPOINT [#6405](https://github.com/hyperledger/besu/pull/6405) +- X_SNAP and X_CHECKPOINT are marked for deprecation and will be removed in 24.4.0 - Github Actions based build. - Introduce caching mechanism to optimize Keccak hash calculations for account storage slots during block processing [#6452](https://github.com/hyperledger/besu/pull/6452) - Added configuration options for `pragueTime` to genesis file for Prague fork development [#6473](https://github.com/hyperledger/besu/pull/6473) - Moving trielog storage to RocksDB's blobdb to improve write amplications [#6289](https://github.com/hyperledger/besu/pull/6289) +- Support for `shanghaiTime` fork and Shanghai EVM smart contracts in QBFT/IBFT chains [#6353](https://github.com/hyperledger/besu/pull/6353) +- Change ExecutionHaltReason for contract creation collision case to return ILLEGAL_STATE_CHANGE [#6518](https://github.com/hyperledger/besu/pull/6518) +- Experimental feature `--Xbonsai-code-using-code-hash-enabled` for storing Bonsai code storage by code hash [#6505](https://github.com/hyperledger/besu/pull/6505) +- More accurate column size `storage rocksdb usage` subcommand [#6540](https://github.com/hyperledger/besu/pull/6540) +- Adds `storage rocksdb x-stats` subcommand [#6540](https://github.com/hyperledger/besu/pull/6540) +- New `eth_blobBaseFee`JSON-RPC method [#6581](https://github.com/hyperledger/besu/pull/6581) +- Add blob transaction support to `eth_call` [#6661](https://github.com/hyperledger/besu/pull/6661) +- Add blobs to `eth_feeHistory` [#6679](https://github.com/hyperledger/besu/pull/6679) +- Upgrade reference tests to version 13.1 [#6574](https://github.com/hyperledger/besu/pull/6574) +- Extend `BesuConfiguration` service [#6584](https://github.com/hyperledger/besu/pull/6584) +- Add `ethereum_min_gas_price` and `ethereum_min_priority_fee` metrics to track runtime values of `min-gas-price` and `min-priority-fee` [#6587](https://github.com/hyperledger/besu/pull/6587) +- Option to perform version incompatibility checks when starting Besu. In this first release of the feature, if `--version-compatibility-protection` is set to true it checks that the version of Besu being started is the same or higher than the previous version. [6307](https://github.com/hyperledger/besu/pull/6307) +- Moved account frame warming from GasCalculator into the Call operations [#6557](https://github.com/hyperledger/besu/pull/6557) ### Bug fixes - Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) - Fix `poa-block-txs-selection-max-time` option that was inadvertently reset to its default after being configured [#6444](https://github.com/hyperledger/besu/pull/6444) +- Fix for tx incorrectly discarded when there is a timeout during block creation [#6563](https://github.com/hyperledger/besu/pull/6563) +- Fix traces so that call gas costing in traces matches other clients traces [#6525](https://github.com/hyperledger/besu/pull/6525) + +### Download Links +https://github.com/hyperledger/besu/releases/tag/24.3.0 +https://github.com/hyperledger/besu/releases/download/24.3.0/besu-24.3.0.tar.gz / sha256 8037ce51bb5bb396d29717a812ea7ff577b0d6aa341d67d1e5b77cbc55b15f84 +https://github.com/hyperledger/besu/releases/download/24.3.0/besu-24.3.0.zip / sha256 41ea2ca734a3b377f43ee178166b5b809827084789378dbbe4e5b52bbd8e0674 + +## 24.1.2 + +### Bug fixes +- Fix ETC Spiral upgrade breach of consensus [#6524](https://github.com/hyperledger/besu/pull/6524) + +### Additions and Improvements +- Adds timestamp to enable Cancun upgrade on mainnet [#6545](https://github.com/hyperledger/besu/pull/6545) +- Github Actions based build.[#6427](https://github.com/hyperledger/besu/pull/6427) ### Download Links +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.2/besu-24.1.2.zip / sha256 9033f300edd81c770d3aff27a29f59dd4b6142a113936886a8f170718e412971 +https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.2/besu-24.1.2.tar.gz / sha256 082db8cf4fb67527aa0dd757e5d254b3b497f5027c23287f9c0a74a6a743bf08 ## 24.1.1 @@ -36,7 +131,7 @@ - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 (thanks @h4l) [#6360](https://github.com/hyperledger/besu/pull/6360) - The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) -- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis. This is to prevent possible DoS attacks in case a single transaction is taking too long to execute, and to have a stable block production rate. This could be a breaking change if an existing network needs to accept transactions that take more time to executed than the newly introduced limit. If it is mandatory for these networks to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. [#6423](https://github.com/hyperledger/besu/pull/6423) +- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis. This is to prevent possible DoS attacks in case a single transaction is taking too long to execute, and to have a stable block production rate. This could be a breaking change if an existing network needs to accept transactions that take more time to execute than the newly introduced limit. If it is mandatory for these networks to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. [#6423](https://github.com/hyperledger/besu/pull/6423) ### Deprecations @@ -85,6 +180,7 @@ https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.t - Add custom genesis file name to config overview if specified [#6297](https://github.com/hyperledger/besu/pull/6297) - Update Gradle plugins and replace unmaintained License Gradle Plugin with the actively maintained Gradle License Report [#6275](https://github.com/hyperledger/besu/pull/6275) - Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328) +- Add a cache on senders by transaction hash [#6375](https://github.com/hyperledger/besu/pull/6375) ### Bug fixes - Hotfix for selfdestruct preimages on bonsai [#6359]((https://github.com/hyperledger/besu/pull/6359) @@ -363,7 +459,7 @@ https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.4.4/besu-23.4.4.z - Early access - layered transaction pool implementation [#5290](https://github.com/hyperledger/besu/pull/5290) - New RPC method `debug_getRawReceipts` [#5476](https://github.com/hyperledger/besu/pull/5476) - Add TrieLogFactory plugin support [#5440](https://github.com/hyperledger/besu/pull/5440) -- Ignore `min-block-occupancy-ratio` option when on PoS networks, since in some cases, it prevents to have full blocks even if enough transactions are present [#5491](https://github.com/hyperledger/besu/pull/5491) +- Ignore `min-block-occupancy-ratio` option when on PoS networks, since in some cases, it prevents to have full blocks even if enough transactions are present [#5491](https://github.com/hyperledger/besu/pull/5491) ### Bug Fixes - Fix eth_feeHistory response for the case in which blockCount is higher than highestBlock requested. [#5397](https://github.com/hyperledger/besu/pull/5397) @@ -2249,7 +2345,7 @@ Workaround - Limit the number of blocks queried by each `eth_getLogs` call. - Implemented private contract log filters including JSON-RPC methods to interact with private filters. [\#735](https://github.com/hyperledger/besu/pull/735) - Implemented EIP-2315: Simple Subroutines for the EVM [\#717](https://github.com/hyperledger/besu/pull/717) - Implemented Splunk logging. [\#725](https://github.com/hyperledger/besu/pull/725) -- Implemented optional native library encryption. [\#675](https://github.com/hyperledger/besu/pull/675). To enable add `--Xsecp256k1-native-enabled` (for transaciton signatures) and/or `--Xaltbn128-native-enabled` (for altbn128 precomiled contracts) as command line options. +- Implemented optional native library encryption. [\#675](https://github.com/hyperledger/besu/pull/675). To enable add `--Xsecp256k1-native-enabled` (for transaction signatures) and/or `--Xaltbn128-native-enabled` (for altbn128 precomiled contracts) as command line options. ### Bug Fixes @@ -2431,7 +2527,7 @@ Early access features are available features that are not recommended for produc have unstable interfaces. * [Onchain privacy groups](https://besu.hyperledger.org/en/latest/Concepts/Privacy/Onchain-PrivacyGroups/) with add and remove members. - Not being able to to re-add a member to an onchain privacy group is a [known issue](https://github.com/hyperledger/besu/issues/455) + Not being able to re-add a member to an onchain privacy group is a [known issue](https://github.com/hyperledger/besu/issues/455) with the add and remove functionality. ### Known Issues diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java index cac4deb9d9b..0e1f6d65d5f 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java @@ -56,15 +56,12 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import org.junit.After; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Superclass for acceptance tests. For now (transition to junit5 is ongoing) this class supports - * junit4 format. - */ +/** Superclass for acceptance tests. */ @ExtendWith(AcceptanceTestBaseTestWatcher.class) public class AcceptanceTestBase { @@ -131,7 +128,7 @@ protected AcceptanceTestBase() { exitedSuccessfully = new ExitedWithCode(0); } - @After + @AfterEach public void tearDownAcceptanceTestBase() { reportMemory(); cluster.close(); diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/BlockUtils.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/BlockUtils.java index 3a4ca71002b..29f60dd74a6 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/BlockUtils.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/BlockUtils.java @@ -58,6 +58,7 @@ public static BlockHeader createBlockHeader( null, null, null, + null, blockHeaderFunctions); } } diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java index ba00ac8f22f..b2f9bc0abf7 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java @@ -33,6 +33,7 @@ import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration; import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.pki.config.PkiKeyStoreConfiguration; import org.hyperledger.besu.tests.acceptance.dsl.condition.Condition; @@ -61,6 +62,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Properties; @@ -109,6 +111,7 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable private final WebSocketConfiguration webSocketConfiguration; private final JsonRpcIpcConfiguration jsonRpcIpcConfiguration; private final MetricsConfiguration metricsConfiguration; + private final DataStorageConfiguration dataStorageConfiguration; private Optional permissioningConfiguration; private final ApiConfiguration apiConfiguration; private final GenesisConfigurationProvider genesisConfigProvider; @@ -145,6 +148,7 @@ public BesuNode( final MetricsConfiguration metricsConfiguration, final Optional permissioningConfiguration, final ApiConfiguration apiConfiguration, + final DataStorageConfiguration dataStorageConfiguration, final Optional keyfilePath, final boolean devMode, final NetworkName network, @@ -195,6 +199,7 @@ public BesuNode( this.metricsConfiguration = metricsConfiguration; this.permissioningConfiguration = permissioningConfiguration; this.apiConfiguration = apiConfiguration; + this.dataStorageConfiguration = dataStorageConfiguration; this.genesisConfigProvider = genesisConfigProvider; this.devMode = devMode; this.network = network; @@ -427,11 +432,14 @@ public NodeRequests nodeRequests() { getGenesisConfig() .map( gc -> - gc.toLowerCase().contains("ibft") ? ConsensusType.IBFT2 : ConsensusType.QBFT) + gc.toLowerCase(Locale.ROOT).contains("ibft") + ? ConsensusType.IBFT2 + : ConsensusType.QBFT) .orElse(ConsensusType.IBFT2); nodeRequests = new NodeRequests( + web3jService, new JsonRpc2_0Web3j(web3jService, 2000, Async.defaultExecutorService()), new CliqueRequestFactory(web3jService), new BftRequestFactory(web3jService, bftType), @@ -690,6 +698,10 @@ public void setPrivacyParameters(final PrivacyParameters privacyParameters) { this.privacyParameters = privacyParameters; } + public DataStorageConfiguration getDataStorageConfiguration() { + return dataStorageConfiguration; + } + public boolean isDevMode() { return devMode; } @@ -777,6 +789,21 @@ public void stop() { nodeRequests.shutdown(); nodeRequests = null; } + + deleteRuntimeFiles(); + } + + private void deleteRuntimeFiles() { + try { + Files.deleteIfExists(homeDirectory.resolve("besu.networks")); + } catch (IOException e) { + LOG.error("Failed to clean up besu.networks file in {}", homeDirectory, e); + } + try { + Files.deleteIfExists(homeDirectory.resolve("besu.ports")); + } catch (IOException e) { + LOG.error("Failed to clean up besu.ports file in {}", homeDirectory, e); + } } @Override diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java index 538cb89e126..adf872e19cc 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java @@ -18,11 +18,14 @@ import static java.nio.charset.StandardCharsets.UTF_8; import org.hyperledger.besu.cli.options.TransactionPoolOptions; +import org.hyperledger.besu.cli.options.stable.DataStorageOptions; import org.hyperledger.besu.cli.options.unstable.NetworkingOptions; import org.hyperledger.besu.ethereum.api.jsonrpc.ipc.JsonRpcIpcConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration; import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import org.hyperledger.besu.tests.acceptance.dsl.StaticNodesUtils; @@ -49,6 +52,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.apache.commons.lang3.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -74,8 +78,15 @@ public void startNode(final BesuNode node) { final Path dataDir = node.homeDirectory(); + final var workingDir = + new File(System.getProperty("user.dir")).getParentFile().getParentFile().toPath(); + final List params = new ArrayList<>(); - params.add("build/install/besu/bin/besu"); + if (SystemUtils.IS_OS_WINDOWS) { + params.add(workingDir.resolve("build\\install\\besu\\bin\\besu.bat").toString()); + } else { + params.add("build/install/besu/bin/besu"); + } params.add("--data-path"); params.add(dataDir.toAbsolutePath().toString()); @@ -109,6 +120,13 @@ public void startNode(final BesuNode node) { .build()) .getCLIOptions()); + params.addAll( + DataStorageOptions.fromConfig( + ImmutableDataStorageConfiguration.builder() + .from(DataStorageConfiguration.DEFAULT_FOREST_CONFIG) + .build()) + .getCLIOptions()); + if (node.getMiningParameters().isMiningEnabled()) { params.add("--miner-enabled"); params.add("--miner-coinbase"); @@ -412,15 +430,13 @@ public void startNode(final BesuNode node) { LOG.info("Creating besu process with params {}", params); final ProcessBuilder processBuilder = new ProcessBuilder(params) - .directory(new File(System.getProperty("user.dir")).getParentFile().getParentFile()) + .directory(workingDir.toFile()) .redirectErrorStream(true) .redirectInput(Redirect.INHERIT); if (!node.getPlugins().isEmpty()) { processBuilder .environment() - .put( - "BESU_OPTS", - "-Dbesu.plugins.dir=" + dataDir.resolve("plugins").toAbsolutePath().toString()); + .put("BESU_OPTS", "-Dbesu.plugins.dir=" + dataDir.resolve("plugins").toAbsolutePath()); } // Use non-blocking randomness for acceptance tests processBuilder @@ -562,7 +578,7 @@ private void killBesuProcess(final String name) { LOG.info("Killing {} process, pid {}", name, process.pid()); - process.destroy(); + process.descendants().forEach(ProcessHandle::destroy); try { process.waitFor(30, TimeUnit.SECONDS); } catch (final InterruptedException e) { diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java index 429c91d2b99..0f943ea0797 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java @@ -28,7 +28,9 @@ import org.hyperledger.besu.cryptoservices.KeyPairSecurityModule; import org.hyperledger.besu.cryptoservices.NodeKey; import org.hyperledger.besu.ethereum.GasLimitCalculator; +import org.hyperledger.besu.ethereum.api.ApiConfiguration; import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration; +import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; @@ -36,31 +38,38 @@ import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; +import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.MetricsSystemFactory; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.data.EnodeURL; import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.BesuEvents; +import org.hyperledger.besu.plugin.services.BlockchainService; +import org.hyperledger.besu.plugin.services.PermissioningService; import org.hyperledger.besu.plugin.services.PicoCLIOptions; -import org.hyperledger.besu.plugin.services.PluginTransactionValidatorService; +import org.hyperledger.besu.plugin.services.PrivacyPluginService; import org.hyperledger.besu.plugin.services.RpcEndpointService; import org.hyperledger.besu.plugin.services.SecurityModuleService; import org.hyperledger.besu.plugin.services.StorageService; +import org.hyperledger.besu.plugin.services.TransactionPoolValidatorService; import org.hyperledger.besu.plugin.services.TransactionSelectionService; +import org.hyperledger.besu.plugin.services.TransactionSimulationService; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin; -import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; -import org.hyperledger.besu.plugin.services.txvalidator.PluginTransactionValidatorFactory; import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.services.BesuEventsImpl; import org.hyperledger.besu.services.BesuPluginContextImpl; +import org.hyperledger.besu.services.BlockchainServiceImpl; import org.hyperledger.besu.services.PermissioningServiceImpl; import org.hyperledger.besu.services.PicoCLIOptionsImpl; -import org.hyperledger.besu.services.PluginTransactionValidatorServiceImpl; +import org.hyperledger.besu.services.PrivacyPluginServiceImpl; import org.hyperledger.besu.services.RpcEndpointServiceImpl; import org.hyperledger.besu.services.SecurityModuleServiceImpl; import org.hyperledger.besu.services.StorageServiceImpl; +import org.hyperledger.besu.services.TransactionPoolValidatorServiceImpl; import org.hyperledger.besu.services.TransactionSelectionServiceImpl; +import org.hyperledger.besu.services.TransactionSimulationServiceImpl; import java.io.File; import java.nio.file.Path; @@ -70,7 +79,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -93,17 +101,28 @@ private BesuPluginContextImpl buildPluginContext( final BesuNode node, final StorageServiceImpl storageService, final SecurityModuleServiceImpl securityModuleService, - final BesuConfiguration commonPluginConfiguration) { + final TransactionSimulationServiceImpl transactionSimulationServiceImpl, + final TransactionSelectionServiceImpl transactionSelectionServiceImpl, + final TransactionPoolValidatorServiceImpl transactionPoolValidatorServiceImpl, + final BlockchainServiceImpl blockchainServiceImpl, + final RpcEndpointServiceImpl rpcEndpointServiceImpl, + final BesuConfiguration commonPluginConfiguration, + final PermissioningServiceImpl permissioningService) { final CommandLine commandLine = new CommandLine(CommandSpec.create()); final BesuPluginContextImpl besuPluginContext = new BesuPluginContextImpl(); besuPluginContext.addService(StorageService.class, storageService); besuPluginContext.addService(SecurityModuleService.class, securityModuleService); besuPluginContext.addService(PicoCLIOptions.class, new PicoCLIOptionsImpl(commandLine)); - besuPluginContext.addService(RpcEndpointService.class, new RpcEndpointServiceImpl()); + besuPluginContext.addService(RpcEndpointService.class, rpcEndpointServiceImpl); besuPluginContext.addService( - TransactionSelectionService.class, new TransactionSelectionServiceImpl()); + TransactionSelectionService.class, transactionSelectionServiceImpl); besuPluginContext.addService( - PluginTransactionValidatorService.class, new PluginTransactionValidatorServiceImpl()); + TransactionPoolValidatorService.class, transactionPoolValidatorServiceImpl); + besuPluginContext.addService( + TransactionSimulationService.class, transactionSimulationServiceImpl); + besuPluginContext.addService(BlockchainService.class, blockchainServiceImpl); + besuPluginContext.addService(BesuConfiguration.class, commonPluginConfiguration); + final Path pluginsPath; final String pluginDir = System.getProperty("besu.plugins.dir"); if (pluginDir == null || pluginDir.isEmpty()) { @@ -117,15 +136,16 @@ private BesuPluginContextImpl buildPluginContext( } else { pluginsPath = Path.of(pluginDir); } - besuPluginContext.registerPlugins(pluginsPath); - - commandLine.parseArgs(node.getConfiguration().getExtraCLIOptions().toArray(new String[0])); besuPluginContext.addService(BesuConfiguration.class, commonPluginConfiguration); + besuPluginContext.addService(PermissioningService.class, permissioningService); + besuPluginContext.addService(PrivacyPluginService.class, new PrivacyPluginServiceImpl()); + + besuPluginContext.registerPlugins(pluginsPath); + commandLine.parseArgs(node.getConfiguration().getExtraCLIOptions().toArray(new String[0])); // register built-in plugins new RocksDBPlugin().register(besuPluginContext); - return besuPluginContext; } @@ -143,15 +163,43 @@ public void startNode(final BesuNode node) { final StorageServiceImpl storageService = new StorageServiceImpl(); final SecurityModuleServiceImpl securityModuleService = new SecurityModuleServiceImpl(); + final TransactionSimulationServiceImpl transactionSimulationServiceImpl = + new TransactionSimulationServiceImpl(); + final TransactionSelectionServiceImpl transactionSelectionServiceImpl = + new TransactionSelectionServiceImpl(); + final TransactionPoolValidatorServiceImpl transactionPoolValidatorServiceImpl = + new TransactionPoolValidatorServiceImpl(); + final BlockchainServiceImpl blockchainServiceImpl = new BlockchainServiceImpl(); + final RpcEndpointServiceImpl rpcEndpointServiceImpl = new RpcEndpointServiceImpl(); final Path dataDir = node.homeDirectory(); - final BesuConfiguration commonPluginConfiguration = - new BesuConfigurationImpl(dataDir, dataDir.resolve(DATABASE_PATH)); + final BesuConfigurationImpl commonPluginConfiguration = new BesuConfigurationImpl(); + final PermissioningServiceImpl permissioningService = new PermissioningServiceImpl(); + + final var miningParameters = + ImmutableMiningParameters.builder() + .from(node.getMiningParameters()) + .transactionSelectionService(transactionSelectionServiceImpl) + .build(); + commonPluginConfiguration.init( + dataDir, + dataDir.resolve(DATABASE_PATH), + node.getDataStorageConfiguration(), + miningParameters); final BesuPluginContextImpl besuPluginContext = besuPluginContextMap.computeIfAbsent( node, n -> buildPluginContext( - node, storageService, securityModuleService, commonPluginConfiguration)); + node, + storageService, + securityModuleService, + transactionSimulationServiceImpl, + transactionSelectionServiceImpl, + transactionPoolValidatorServiceImpl, + blockchainServiceImpl, + rpcEndpointServiceImpl, + commonPluginConfiguration, + permissioningService)); GlobalOpenTelemetry.resetForTest(); final ObservableMetricsSystem metricsSystem = @@ -184,23 +232,20 @@ public void startNode(final BesuNode node) { ImmutableTransactionPoolConfiguration.builder() .from(node.getTransactionPoolConfiguration()) .strictTransactionReplayProtectionEnabled(node.isStrictTxReplayProtectionEnabled()) + .transactionPoolValidatorService(transactionPoolValidatorServiceImpl) .build(); final int maxPeers = 25; - final Optional transactionSelectorFactory = - getTransactionSelectorFactory(besuPluginContext); - - final PluginTransactionValidatorFactory pluginTransactionValidatorFactory = - getPluginTransactionValidatorFactory(besuPluginContext); builder .synchronizerConfiguration(new SynchronizerConfiguration.Builder().build()) .dataDirectory(node.homeDirectory()) - .miningParameters(node.getMiningParameters()) + .miningParameters(miningParameters) .privacyParameters(node.getPrivacyParameters()) .nodeKey(new NodeKey(new KeyPairSecurityModule(KeyPairUtil.loadKeyPair(dataDir)))) .metricsSystem(metricsSystem) .transactionPoolConfiguration(txPoolConfig) + .dataStorageConfiguration(DataStorageConfiguration.DEFAULT_FOREST_CONFIG) .ethProtocolConfiguration(EthProtocolConfiguration.defaultConfig()) .clock(Clock.systemUTC()) .isRevertReasonEnabled(node.isRevertReasonEnabled()) @@ -211,12 +256,9 @@ public void startNode(final BesuNode node) { .map(pkiConfig -> new PkiBlockCreationConfigurationProvider().load(pkiConfig))) .evmConfiguration(EvmConfiguration.DEFAULT) .maxPeers(maxPeers) - .lowerBoundPeers(maxPeers) .maxRemotelyInitiatedPeers(15) .networkConfiguration(node.getNetworkingConfiguration()) - .randomPeerPriority(false) - .transactionSelectorFactory(transactionSelectorFactory) - .pluginTransactionValidatorFactory(pluginTransactionValidatorFactory); + .randomPeerPriority(false); node.getGenesisConfig() .map(GenesisConfigFile::fromConfig) @@ -224,6 +266,10 @@ public void startNode(final BesuNode node) { final BesuController besuController = builder.build(); + initTransactionSimulationService( + transactionSimulationServiceImpl, besuController, node.getApiConfiguration()); + initBlockchainService(blockchainServiceImpl, besuController); + final RunnerBuilder runnerBuilder = new RunnerBuilder(); runnerBuilder.permissioningConfiguration(node.getPermissioningConfiguration()); runnerBuilder.apiConfiguration(node.getApiConfiguration()); @@ -241,7 +287,7 @@ public void startNode(final BesuNode node) { .jsonRpcIpcConfiguration(node.jsonRpcIpcConfiguration()) .dataDir(node.homeDirectory()) .metricsSystem(metricsSystem) - .permissioningService(new PermissioningServiceImpl()) + .permissioningService(permissioningService) .metricsConfiguration(node.getMetricsConfiguration()) .p2pEnabled(node.isP2pEnabled()) .p2pTLSConfiguration(node.getTLSConfiguration()) @@ -250,15 +296,14 @@ public void startNode(final BesuNode node) { node.getStaticNodes().stream() .map(EnodeURLImpl::fromString) .collect(Collectors.toList())) - .besuPluginContext(new BesuPluginContextImpl()) + .besuPluginContext(besuPluginContext) .autoLogBloomCaching(false) .storageProvider(storageProvider) - .rpcEndpointService(new RpcEndpointServiceImpl()); + .rpcEndpointService(rpcEndpointServiceImpl); node.engineRpcConfiguration().ifPresent(runnerBuilder::engineJsonRpcConfiguration); - final Runner runner = runnerBuilder.build(); - besuPluginContext.beforeExternalServices(); + final Runner runner = runnerBuilder.build(); runner.startExternalServices(); @@ -277,6 +322,25 @@ public void startNode(final BesuNode node) { MDC.remove("node"); } + private void initBlockchainService( + final BlockchainServiceImpl blockchainServiceImpl, final BesuController besuController) { + blockchainServiceImpl.init( + besuController.getProtocolContext(), besuController.getProtocolSchedule()); + } + + private void initTransactionSimulationService( + final TransactionSimulationServiceImpl transactionSimulationService, + final BesuController besuController, + final ApiConfiguration apiConfiguration) { + transactionSimulationService.init( + besuController.getProtocolContext().getBlockchain(), + new TransactionSimulator( + besuController.getProtocolContext().getBlockchain(), + besuController.getProtocolContext().getWorldStateArchive(), + besuController.getProtocolSchedule(), + apiConfiguration.getGasCap())); + } + @Override public void stopNode(final BesuNode node) { final BesuPluginContextImpl pluginContext = besuPluginContextMap.remove(node); @@ -326,18 +390,4 @@ public void startConsoleCapture() { public String getConsoleContents() { throw new RuntimeException("Console contents can only be captured in process execution"); } - - private Optional getTransactionSelectorFactory( - final BesuPluginContextImpl besuPluginContext) { - final Optional txSelectionService = - besuPluginContext.getService(TransactionSelectionService.class); - return txSelectionService.isPresent() ? txSelectionService.get().get() : Optional.empty(); - } - - private PluginTransactionValidatorFactory getPluginTransactionValidatorFactory( - final BesuPluginContextImpl besuPluginContext) { - final Optional txValidatorService = - besuPluginContext.getService(PluginTransactionValidatorService.class); - return txValidatorService.map(PluginTransactionValidatorService::get).orElse(null); - } } diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java index 0c0d66be6a8..d1e398f7215 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration; import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.pki.config.PkiKeyStoreConfiguration; import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.GenesisConfigurationProvider; @@ -48,6 +49,7 @@ public class BesuNodeConfiguration { private final MetricsConfiguration metricsConfiguration; private final Optional permissioningConfiguration; private final ApiConfiguration apiConfiguration; + private final DataStorageConfiguration dataStorageConfiguration; private final Optional keyFilePath; private final boolean devMode; private final GenesisConfigurationProvider genesisConfigProvider; @@ -84,6 +86,7 @@ public class BesuNodeConfiguration { final MetricsConfiguration metricsConfiguration, final Optional permissioningConfiguration, final ApiConfiguration apiConfiguration, + final DataStorageConfiguration dataStorageConfiguration, final Optional keyFilePath, final boolean devMode, final NetworkName network, @@ -117,6 +120,7 @@ public class BesuNodeConfiguration { this.metricsConfiguration = metricsConfiguration; this.permissioningConfiguration = permissioningConfiguration; this.apiConfiguration = apiConfiguration; + this.dataStorageConfiguration = dataStorageConfiguration; this.keyFilePath = keyFilePath; this.dataPath = dataPath; this.devMode = devMode; @@ -183,6 +187,10 @@ public ApiConfiguration getApiConfiguration() { return apiConfiguration; } + public DataStorageConfiguration getDataStorageConfiguration() { + return dataStorageConfiguration; + } + public Optional getKeyFilePath() { return keyFilePath; } diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java index c20e7ec6d11..1a9a16f36f5 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java @@ -39,6 +39,7 @@ import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration; import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; import org.hyperledger.besu.pki.config.PkiKeyStoreConfiguration; import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.GenesisConfigurationProvider; @@ -73,6 +74,8 @@ public class BesuNodeConfigurationBuilder { private MetricsConfiguration metricsConfiguration = MetricsConfiguration.builder().build(); private Optional permissioningConfiguration = Optional.empty(); private ApiConfiguration apiConfiguration = ImmutableApiConfiguration.builder().build(); + private DataStorageConfiguration dataStorageConfiguration = + DataStorageConfiguration.DEFAULT_FOREST_CONFIG; private String keyFilePath = null; private boolean devMode = true; private GenesisConfigurationProvider genesisConfigProvider = ignore -> Optional.empty(); @@ -506,6 +509,12 @@ public BesuNodeConfigurationBuilder apiConfiguration(final ApiConfiguration apiC return this; } + public BesuNodeConfigurationBuilder dataStorageConfiguration( + final DataStorageConfiguration dataStorageConfiguration) { + this.dataStorageConfiguration = dataStorageConfiguration; + return this; + } + public BesuNodeConfiguration build() { return new BesuNodeConfiguration( name, @@ -519,6 +528,7 @@ public BesuNodeConfiguration build() { metricsConfiguration, permissioningConfiguration, apiConfiguration, + dataStorageConfiguration, Optional.ofNullable(keyFilePath), devMode, network, diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java index ed26587812c..ff6dc2ac9b1 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java @@ -49,6 +49,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.UnaryOperator; import io.vertx.core.Vertx; @@ -72,6 +73,7 @@ public BesuNode create(final BesuNodeConfiguration config) throws IOException { config.getMetricsConfiguration(), config.getPermissioningConfiguration(), config.getApiConfiguration(), + config.getDataStorageConfiguration(), config.getKeyFilePath(), config.isDevMode(), config.getNetwork(), @@ -375,17 +377,27 @@ public BesuNode createCliqueNode(final String name) throws IOException { public BesuNode createCliqueNode(final String name, final CliqueOptions cliqueOptions) throws IOException { - return createCliqueNodeWithExtraCliOptions(name, cliqueOptions, List.of()); + return createCliqueNodeWithExtraCliOptionsAndRpcApis(name, cliqueOptions, List.of()); } - public BesuNode createCliqueNodeWithExtraCliOptions( + public BesuNode createCliqueNodeWithExtraCliOptionsAndRpcApis( final String name, final CliqueOptions cliqueOptions, final List extraCliOptions) throws IOException { + return createCliqueNodeWithExtraCliOptionsAndRpcApis( + name, cliqueOptions, extraCliOptions, Set.of()); + } + + public BesuNode createCliqueNodeWithExtraCliOptionsAndRpcApis( + final String name, + final CliqueOptions cliqueOptions, + final List extraCliOptions, + final Set extraRpcApis) + throws IOException { return create( new BesuNodeConfigurationBuilder() .name(name) .miningEnabled() - .jsonRpcConfiguration(node.createJsonRpcWithCliqueEnabledConfig()) + .jsonRpcConfiguration(node.createJsonRpcWithCliqueEnabledConfig(extraRpcApis)) .webSocketConfiguration(node.createWebSocketEnabledConfig()) .devMode(false) .jsonRpcTxPool() @@ -583,7 +595,7 @@ public BesuNode createCliqueNodeWithValidators(final String name, final String.. new BesuNodeConfigurationBuilder() .name(name) .miningEnabled() - .jsonRpcConfiguration(node.createJsonRpcWithCliqueEnabledConfig()) + .jsonRpcConfiguration(node.createJsonRpcWithCliqueEnabledConfig(Set.of())) .webSocketConfiguration(node.createWebSocketEnabledConfig()) .jsonRpcTxPool() .devMode(false) diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/NodeConfigurationFactory.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/NodeConfigurationFactory.java index f2682993f8a..219d15d1adf 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/NodeConfigurationFactory.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/NodeConfigurationFactory.java @@ -30,8 +30,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; public class NodeConfigurationFactory { @@ -44,8 +46,10 @@ public Optional createGenesisConfigForValidators( return genesisConfigProvider.create(nodes); } - public JsonRpcConfiguration createJsonRpcWithCliqueEnabledConfig() { - return createJsonRpcWithRpcApiEnabledConfig(CLIQUE.name()); + public JsonRpcConfiguration createJsonRpcWithCliqueEnabledConfig(final Set extraRpcApis) { + final var enabledApis = new HashSet<>(extraRpcApis); + enabledApis.add(CLIQUE.name()); + return createJsonRpcWithRpcApiEnabledConfig(enabledApis.toArray(String[]::new)); } public JsonRpcConfiguration createJsonRpcWithIbft2EnabledConfig(final boolean minerEnabled) { diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java index 4ec4396d6b5..1c0df5ace6d 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java @@ -76,6 +76,7 @@ public class PrivacyNode implements AutoCloseable { private final boolean isFlexiblePrivacyEnabled; private final boolean isMultitenancyEnabled; private final boolean isPrivacyPluginEnabled; + private final BesuNodeConfiguration besuConfig; public PrivacyNode( final PrivacyNodeConfiguration privacyConfiguration, @@ -89,7 +90,7 @@ public PrivacyNode( selectEnclave(enclaveType, enclaveDir, config, privacyConfiguration, containerNetwork); this.vertx = vertx; - final BesuNodeConfiguration besuConfig = config; + this.besuConfig = config; isFlexiblePrivacyEnabled = privacyConfiguration.isFlexiblePrivacyGroupEnabled(); isMultitenancyEnabled = privacyConfiguration.isMultitenancyEnabled(); @@ -108,6 +109,7 @@ public PrivacyNode( besuConfig.getMetricsConfiguration(), besuConfig.getPermissioningConfiguration(), besuConfig.getApiConfiguration(), + besuConfig.getDataStorageConfiguration(), besuConfig.getKeyFilePath(), besuConfig.isDevMode(), besuConfig.getNetwork(), @@ -272,6 +274,8 @@ public NodeConfiguration getConfiguration() { private PrivacyStorageProvider createKeyValueStorageProvider( final Path dataLocation, final Path dbLocation) { + final var besuConfiguration = new BesuConfigurationImpl(); + besuConfiguration.init(dataLocation, dbLocation, null, besuConfig.getMiningParameters()); return new PrivacyKeyValueStorageProviderBuilder() .withStorageFactory( new RocksDBKeyValuePrivacyStorageFactory( @@ -284,7 +288,7 @@ private PrivacyStorageProvider createKeyValueStorageProvider( DEFAULT_IS_HIGH_SPEC), Arrays.asList(KeyValueSegmentIdentifier.values()), RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS))) - .withCommonConfiguration(new BesuConfigurationImpl(dataLocation, dbLocation)) + .withCommonConfiguration(besuConfiguration) .withMetricsSystem(new NoOpMetricsSystem()) .build(); } diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/pubsub/WebSocket.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/pubsub/WebSocket.java index 381800f5282..3976912e097 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/pubsub/WebSocket.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/pubsub/WebSocket.java @@ -53,6 +53,6 @@ public void unsubscribe(final Subscription subscription) { public void verifyTotalEventsReceived(final int expectedTotalEventCount) { WaitUtils.waitFor( - () -> assertThat(connection.getSubscriptionEvents()).hasSize(expectedTotalEventCount)); + 60, () -> assertThat(connection.getSubscriptionEvents()).hasSize(expectedTotalEventCount)); } } diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java index 14bc9a550cb..97e5c9d3878 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java @@ -20,6 +20,7 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.stream.Collectors; import org.web3j.crypto.Credentials; @@ -83,7 +84,7 @@ && parameterTypesAreEqual(i.getParameterTypes(), parameterObjects)) @SuppressWarnings("rawtypes") private boolean parameterTypesAreEqual( - final Class[] expectedTypes, final ArrayList actualObjects) { + final Class[] expectedTypes, final List actualObjects) { if (expectedTypes.length != actualObjects.size()) { return false; } diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/NodeRequests.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/NodeRequests.java index 4741e397ddc..1151100065b 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/NodeRequests.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/NodeRequests.java @@ -27,10 +27,11 @@ import java.util.Optional; import org.web3j.protocol.Web3j; +import org.web3j.protocol.Web3jService; import org.web3j.protocol.websocket.WebSocketService; public class NodeRequests { - + private final Web3jService web3jService; private final Web3j netEth; private final CliqueRequestFactory clique; private final BftRequestFactory bft; @@ -44,6 +45,7 @@ public class NodeRequests { private final TxPoolRequestFactory txPool; public NodeRequests( + final Web3jService web3jService, final Web3j netEth, final CliqueRequestFactory clique, final BftRequestFactory bft, @@ -55,6 +57,7 @@ public NodeRequests( final TxPoolRequestFactory txPool, final Optional websocketService, final LoginRequestFactory login) { + this.web3jService = web3jService; this.netEth = netEth; this.clique = clique; this.bft = bft; @@ -116,4 +119,8 @@ public void shutdown() { netEth.shutdown(); websocketService.ifPresent(WebSocketService::close); } + + public Web3jService getWeb3jService() { + return web3jService; + } } diff --git a/acceptance-tests/tests/build.gradle b/acceptance-tests/tests/build.gradle index b20b96b2735..10b3138384a 100644 --- a/acceptance-tests/tests/build.gradle +++ b/acceptance-tests/tests/build.gradle @@ -12,24 +12,19 @@ */ plugins { - id 'org.web3j' version '4.9.2' - id 'org.web3j.solidity' version '0.3.5' -} - -configurations.all { - resolutionStrategy.eachDependency { DependencyResolveDetails details -> - if (details.requested.group == 'org.web3j' && details.requested.version == '4.9.2') { - details.useVersion '4.9.4' - details.because 'Plugin version is 4.9.2 (latest), but we want it to use web3j libs version 4.9.4' - } - } + id 'org.web3j' version '4.11.1' + id 'org.web3j.solidity' version '0.4.0' } web3j { generatedPackageName = 'org.hyperledger.besu.tests.web3j.generated' } sourceSets.main.solidity.srcDirs = ["$projectDir/contracts"] -solidity { resolvePackages = false } +solidity { + resolvePackages = false + // TODO: remove the forced version, when DEV network is upgraded to support latest forks + version '0.8.19' +} dependencies { api 'org.slf4j:slf4j-api' @@ -155,6 +150,33 @@ task acceptanceTestMainnet(type: Test) { doFirst { mkdir "${buildDir}/jvmErrorLogs" } } +task acceptanceTestNotPrivacy(type: Test) { + inputs.property "integration.date", LocalTime.now() // so it runs at every invocation + exclude '**/privacy/**' + + useJUnitPlatform {} + + dependsOn(rootProject.installDist) + setSystemProperties(test.getSystemProperties()) + systemProperty 'acctests.runBesuAsProcess', 'true' + systemProperty 'java.security.properties', "${buildDir}/resources/test/acceptanceTesting.security" + mustRunAfter rootProject.subprojects*.test + description = 'Runs MAINNET Besu acceptance tests (excluding privacy since they run nightly, and are being refactored).' + group = 'verification' + + jvmArgs "-XX:ErrorFile=${buildDir}/jvmErrorLogs/java_err_pid%p.log" + + testLogging { + exceptionFormat = 'full' + showStackTraces = true + showStandardStreams = Boolean.getBoolean('acctests.showStandardStreams') + showExceptions = true + showCauses = true + } + + doFirst { mkdir "${buildDir}/jvmErrorLogs" } +} + task acceptanceTestCliqueBft(type: Test) { inputs.property "integration.date", LocalTime.now() // so it runs at every invocation include '**/bft/**' diff --git a/acceptance-tests/tests/contracts/SimpleStorage.sol b/acceptance-tests/tests/contracts/SimpleStorage.sol index 35dc7c3932f..0b4838f1c26 100644 --- a/acceptance-tests/tests/contracts/SimpleStorage.sol +++ b/acceptance-tests/tests/contracts/SimpleStorage.sol @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -pragma solidity >=0.7.0 <0.9.0; +pragma solidity >=0.7.0 <0.8.20; // compile with: // solc SimpleStorage.sol --bin --abi --optimize --overwrite -o . diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftMiningAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftMiningAcceptanceTest.java index 71eb3908a7c..1e741a5cc68 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftMiningAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftMiningAcceptanceTest.java @@ -150,6 +150,36 @@ public void shouldMineOnSingleNodeWithFreeGas_London( cluster.verify(receiver.balanceEquals(3)); } + @ParameterizedTest(name = "{index}: {0}") + @MethodSource("factoryFunctions") + public void shouldMineOnSingleNodeWithFreeGas_Shanghai( + final String testName, final BftAcceptanceTestParameterization nodeFactory) throws Exception { + setUp(testName, nodeFactory); + final BesuNode minerNode = nodeFactory.createNode(besu, "miner1"); + updateGenesisConfigToShanghai(minerNode, true); + + cluster.start(minerNode); + + cluster.verify(blockchain.reachesHeight(minerNode, 1)); + + final Account sender = accounts.createAccount("account1"); + final Account receiver = accounts.createAccount("account2"); + + minerNode.execute(accountTransactions.createTransfer(sender, 50, Amount.ZERO)); + cluster.verify(sender.balanceEquals(50)); + + minerNode.execute(accountTransactions.create1559Transfer(sender, 50, 4, Amount.ZERO)); + cluster.verify(sender.balanceEquals(100)); + + minerNode.execute( + accountTransactions.createIncrementalTransfers(sender, receiver, 1, Amount.ZERO)); + cluster.verify(receiver.balanceEquals(1)); + + minerNode.execute( + accountTransactions.create1559IncrementalTransfers(sender, receiver, 2, 4, Amount.ZERO)); + cluster.verify(receiver.balanceEquals(3)); + } + @ParameterizedTest(name = "{index}: {0}") @MethodSource("factoryFunctions") public void shouldMineOnMultipleNodes( @@ -245,4 +275,16 @@ private static void updateGenesisConfigToLondon( config.put("zeroBaseFee", zeroBaseFeeEnabled); minerNode.setGenesisConfig(genesisConfigNode.toString()); } + + private static void updateGenesisConfigToShanghai( + final BesuNode minerNode, final boolean zeroBaseFeeEnabled) { + final Optional genesisConfig = + minerNode.getGenesisConfigProvider().create(List.of(minerNode)); + final ObjectNode genesisConfigNode = JsonUtil.objectNodeFromString(genesisConfig.orElseThrow()); + final ObjectNode config = (ObjectNode) genesisConfigNode.get("config"); + config.remove("berlinBlock"); + config.put("shanghaiTime", 100); + config.put("zeroBaseFee", zeroBaseFeeEnabled); + minerNode.setGenesisConfig(genesisConfigNode.toString()); + } } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/clique/CliqueMiningAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/clique/CliqueMiningAcceptanceTest.java index c2048c61d9c..50c58f8b34b 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/clique/CliqueMiningAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/clique/CliqueMiningAcceptanceTest.java @@ -14,14 +14,23 @@ */ package org.hyperledger.besu.tests.acceptance.clique; +import static java.util.stream.Collectors.joining; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.data.Percentage.withPercentage; + import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBaseJunit5; import org.hyperledger.besu.tests.acceptance.dsl.account.Account; import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode; import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.GenesisConfigurationFactory.CliqueOptions; import java.io.IOException; +import java.math.BigInteger; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.junit.jupiter.api.Test; +import org.web3j.protocol.core.DefaultBlockParameter; public class CliqueMiningAcceptanceTest extends AcceptanceTestBaseJunit5 { @@ -55,7 +64,7 @@ public void shouldNotMineBlocksIfNoTransactionsWhenCreateEmptyBlockIsFalse() thr } @Test - public void shouldMineBlocksOnlyWhenTransactionsArePresentWhenCreateEmptyBlockIsFalse() + public void shouldMineBlocksOnlyWhenTransactionsArePresentWhenCreateEmptyBlocksIsFalse() throws IOException { final var cliqueOptionsNoEmptyBlocks = new CliqueOptions( @@ -123,4 +132,173 @@ public void shouldStillMineWhenANodeFailsAndHasSufficientValidators() throws IOE cluster.verifyOnActiveNodes(clique.blockIsCreatedByProposer(minerNode1)); cluster.verifyOnActiveNodes(clique.blockIsCreatedByProposer(minerNode2)); } + + @Test + public void shouldMineBlocksAccordingToBlockPeriodTransitions() throws IOException { + + final var cliqueOptions = new CliqueOptions(3, CliqueOptions.DEFAULT.epochLength(), true); + final BesuNode minerNode = besu.createCliqueNode("miner1", cliqueOptions); + + // setup transitions + final Map decreasePeriodTo2_Transition = + Map.of("block", 3, "blockperiodseconds", 2); + final Map decreasePeriodTo1_Transition = + Map.of("block", 4, "blockperiodseconds", 1); + // ensure previous blockperiodseconds transition is carried over + final Map dummy_Transition = Map.of("block", 5, "createemptyblocks", true); + final Map increasePeriodTo2_Transition = + Map.of("block", 6, "blockperiodseconds", 2); + + final Optional initialGenesis = + minerNode.getGenesisConfigProvider().create(List.of(minerNode)); + final String genesisWithTransitions = + prependTransitionsToCliqueOptions( + initialGenesis.orElseThrow(), + List.of( + decreasePeriodTo2_Transition, + decreasePeriodTo1_Transition, + dummy_Transition, + increasePeriodTo2_Transition)); + minerNode.setGenesisConfig(genesisWithTransitions); + + // Mine 6 blocks + cluster.start(minerNode); + minerNode.verify(blockchain.reachesHeight(minerNode, 5)); + + // Assert the block period decreased/increased after each transition + final long block1Timestamp = getTimestampForBlock(minerNode, 1); + final long block2Timestamp = getTimestampForBlock(minerNode, 2); + final long block3Timestamp = getTimestampForBlock(minerNode, 3); + final long block4Timestamp = getTimestampForBlock(minerNode, 4); + final long block5Timestamp = getTimestampForBlock(minerNode, 5); + final long block6Timestamp = getTimestampForBlock(minerNode, 6); + assertThat(block2Timestamp - block1Timestamp).isCloseTo(3, withPercentage(20)); + assertThat(block3Timestamp - block2Timestamp).isCloseTo(2, withPercentage(20)); + assertThat(block4Timestamp - block3Timestamp).isCloseTo(1, withPercentage(20)); + assertThat(block5Timestamp - block4Timestamp).isCloseTo(1, withPercentage(20)); + assertThat(block6Timestamp - block5Timestamp).isCloseTo(2, withPercentage(20)); + } + + @Test + public void shouldMineBlocksAccordingToCreateEmptyBlocksTransitions() throws IOException { + + final var cliqueOptionsEmptyBlocks = + new CliqueOptions(2, CliqueOptions.DEFAULT.epochLength(), true); + final BesuNode minerNode = besu.createCliqueNode("miner1", cliqueOptionsEmptyBlocks); + + // setup transitions + final Map noEmptyBlocks_Transition = + Map.of("block", 3, "createemptyblocks", false); + final Map emptyBlocks_Transition = + Map.of("block", 4, "createemptyblocks", true); + final Map secondNoEmptyBlocks_Transition = + Map.of("block", 6, "createemptyblocks", false); + // ensure previous createemptyblocks transition is carried over + final Map dummy_Transition = Map.of("block", 7, "blockperiodseconds", 1); + + final Optional initialGenesis = + minerNode.getGenesisConfigProvider().create(List.of(minerNode)); + final String genesisWithTransitions = + prependTransitionsToCliqueOptions( + initialGenesis.orElseThrow(), + List.of( + noEmptyBlocks_Transition, + emptyBlocks_Transition, + secondNoEmptyBlocks_Transition, + dummy_Transition)); + minerNode.setGenesisConfig(genesisWithTransitions); + + final Account sender = accounts.createAccount("account1"); + + // Mine 2 blocks + cluster.start(minerNode); + minerNode.verify(blockchain.reachesHeight(minerNode, 1)); + + // tx required to mine block + cluster.verify(clique.noNewBlockCreated(minerNode)); + minerNode.execute(accountTransactions.createTransfer(sender, 50)); + minerNode.verify(clique.blockIsCreatedByProposer(minerNode)); + + // Mine 2 more blocks so chain head is 5 + minerNode.verify(blockchain.reachesHeight(minerNode, 2)); + + // tx required to mine block 6 + cluster.verify(clique.noNewBlockCreated(minerNode)); + minerNode.execute(accountTransactions.createTransfer(sender, 50)); + minerNode.verify(clique.blockIsCreatedByProposer(minerNode)); + + // check createemptyblocks transition carried over when other transition activated... + // tx required to mine block 7 + cluster.verify(clique.noNewBlockCreated(minerNode)); + } + + private long getTimestampForBlock(final BesuNode minerNode, final int blockNumber) { + return minerNode + .execute( + ethTransactions.block(DefaultBlockParameter.valueOf(BigInteger.valueOf(blockNumber)))) + .getTimestamp() + .longValue(); + } + + private String prependTransitionsToCliqueOptions( + final String originalOptions, final List> transitions) { + final StringBuilder stringBuilder = + new StringBuilder() + .append(formatCliqueTransitionsOptions(transitions)) + .append(",\n") + .append(quote("clique")) + .append(": {"); + + return originalOptions.replace(quote("clique") + ": {", stringBuilder.toString()); + } + + private String formatCliqueTransitionsOptions(final List> transitions) { + final StringBuilder stringBuilder = new StringBuilder(); + + stringBuilder.append(quote("transitions")); + stringBuilder.append(": {\n"); + stringBuilder.append(quote("clique")); + stringBuilder.append(": ["); + final String formattedTransitions = + transitions.stream().map(this::formatTransition).collect(joining(",\n")); + stringBuilder.append(formattedTransitions); + stringBuilder.append("\n]"); + stringBuilder.append("}\n"); + + return stringBuilder.toString(); + } + + private String quote(final Object value) { + return '"' + value.toString() + '"'; + } + + private String formatTransition(final Map transition) { + final StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("{"); + String formattedTransition = + transition.keySet().stream() + .map(key -> formatKeyValues(key, transition.get(key))) + .collect(joining(",")); + stringBuilder.append(formattedTransition); + stringBuilder.append("}"); + return stringBuilder.toString(); + } + + private String formatKeyValues(final Object... keyOrValue) { + if (keyOrValue.length % 2 == 1) { + // An odd number of strings cannot form a set of key-value pairs + throw new IllegalArgumentException("Must supply key-value pairs"); + } + final StringBuilder stringBuilder = new StringBuilder(); + for (int i = 0; i < keyOrValue.length; i += 2) { + if (i > 0) { + stringBuilder.append(", "); + } + final String key = keyOrValue[i].toString(); + final Object value = keyOrValue[i + 1]; + final String valueStr = value instanceof String ? quote(value) : value.toString(); + stringBuilder.append(String.format("\n%s: %s", quote(key), valueStr)); + } + return stringBuilder.toString(); + } } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java index 4e4330da03b..30c32546c5a 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java @@ -44,6 +44,17 @@ public void setUp() throws Exception { strictNode = besu.createArchiveNode("strictNode", configureNode((true))); miningNode = besu.createMinerNode("strictMiningNode", configureNode((true))); cluster.start(lenientNode, strictNode, miningNode); + + // verify nodes are fully connected otherwise tx could not be propagated + lenientNode.verify(net.awaitPeerCount(2)); + strictNode.verify(net.awaitPeerCount(2)); + miningNode.verify(net.awaitPeerCount(2)); + + // verify that the miner started producing blocks and all other nodes are syncing from it + waitForBlockHeight(miningNode, 1); + final var minerChainHead = miningNode.execute(ethTransactions.block()); + lenientNode.verify(blockchain.minimumHeight(minerChainHead.getNumber().longValue())); + strictNode.verify(blockchain.minimumHeight(minerChainHead.getNumber().longValue())); } @Test @@ -53,6 +64,7 @@ public void shouldSendSuccessfullyToLenientNodeWithoutChainId() { final String txHash = tx.transactionHash(); lenientNode.verify(eth.expectSuccessfulEthRawTransaction(rawTx)); + // Tx should be included on-chain miningNode.verify(eth.expectSuccessfulTransactionReceipt(txHash)); } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineEip6110AcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineEip6110AcceptanceTest.java deleted file mode 100644 index e77f011d9b4..00000000000 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineEip6110AcceptanceTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.tests.acceptance.jsonrpc; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.stream.Stream; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.params.provider.Arguments; - -public class ExecutionEngineEip6110AcceptanceTest extends AbstractJsonRpcTest { - private static final String GENESIS_FILE = "/jsonrpc/engine/eip6110/genesis.json"; - private static final String TEST_CASE_PATH = "/jsonrpc/engine/eip6110/test-cases/"; - - private static JsonRpcTestsContext testsContext; - - public ExecutionEngineEip6110AcceptanceTest() { - super(testsContext); - } - - @BeforeAll - public static void init() throws IOException { - testsContext = new JsonRpcTestsContext(GENESIS_FILE); - } - - public static Stream testCases() throws URISyntaxException { - return testCasesFromPath(TEST_CASE_PATH); - } - - @AfterAll - public static void tearDown() { - testsContext.cluster.close(); - } -} diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java index b04bad5644d..e638614ad58 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java @@ -47,10 +47,7 @@ public void setUp() { permissionedNode.execute(allowNode(permissionedNode)); permissionedNode.verify(connectionIsAllowed(permissionedNode)); - allowedNode.verify(eth.syncingStatus(false)); - bootnode.verify(eth.syncingStatus(false)); - permissionedNode.verify(eth.syncingStatus(false)); - forbiddenNode.verify(eth.syncingStatus(false)); + verifyAllNodesHaveFinishedSyncing(); } @Test @@ -92,6 +89,8 @@ public void permissioningUpdatesPropagateThroughNetwork() { permissionedNode.verify(admin.addPeer(allowedNode)); permissionedNode.verify(net.awaitPeerCount(2)); + verifyAllNodesHaveFinishedSyncing(); + // permissioning changes in peer should propagate to permissioned node allowedNode.execute(allowNode(forbiddenNode)); allowedNode.verify(connectionIsAllowed(forbiddenNode)); @@ -101,6 +100,13 @@ public void permissioningUpdatesPropagateThroughNetwork() { permissionedNode.verify(net.awaitPeerCount(3)); } + private void verifyAllNodesHaveFinishedSyncing() { + allowedNode.verify(eth.syncingStatus(false)); + bootnode.verify(eth.syncingStatus(false)); + permissionedNode.verify(eth.syncingStatus(false)); + forbiddenNode.verify(eth.syncingStatus(false)); + } + @Test public void onchainPermissioningAllowlistShouldPersistAcrossRestarts() { permissionedCluster.stop(); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PermissioningPluginTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PermissioningPluginTest.java index 146110237da..60047c16579 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PermissioningPluginTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/plugins/PermissioningPluginTest.java @@ -37,22 +37,13 @@ public class PermissioningPluginTest extends AcceptanceTestBaseJunit5 { @BeforeEach public void setUp() throws Exception { - final BesuNodeConfigurationBuilder builder = - new BesuNodeConfigurationBuilder() - .miningEnabled(false) - .plugins(List.of("testPlugins")) - .extraCLIOptions(List.of("--plugin-permissioning-test-enabled=true")) - .jsonRpcEnabled() - .jsonRpcTxPool() - .jsonRpcAdmin(); + minerNode = besu.create(createNodeBuilder().name("miner").build()); - minerNode = besu.create(builder.name("miner").build()); + aliceNode = besu.create(createNodeBuilder().name("alice").keyFilePath("key").build()); - aliceNode = besu.create(builder.name("alice").keyFilePath("key").build()); + bobNode = besu.create(createNodeBuilder().name("bob").keyFilePath("key1").build()); - bobNode = besu.create(builder.name("bob").keyFilePath("key1").build()); - - charlieNode = besu.create(builder.name("charlie").keyFilePath("key2").build()); + charlieNode = besu.create(createNodeBuilder().name("charlie").keyFilePath("key2").build()); cluster.start(minerNode, charlieNode); @@ -63,6 +54,16 @@ public void setUp() throws Exception { bobNode.awaitPeerDiscovery(net.awaitPeerCount(2)); } + private BesuNodeConfigurationBuilder createNodeBuilder() { + return new BesuNodeConfigurationBuilder() + .miningEnabled(false) + .plugins(List.of("testPlugins")) + .extraCLIOptions(List.of("--plugin-permissioning-test-enabled=true")) + .jsonRpcEnabled() + .jsonRpcTxPool() + .jsonRpcAdmin(); + } + @Test public void blockedConnectionNodeCanOnlyConnectToTransactionNode() { minerNode.verify(admin.hasPeer(aliceNode)); diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/PrivacyClusterAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/PrivacyClusterAcceptanceTest.java index 419286f1689..e1bb0a980ea 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/PrivacyClusterAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/privacy/PrivacyClusterAcceptanceTest.java @@ -187,6 +187,10 @@ public void aliceCanUsePrivDistributeTransaction() { EnclaveEncryptorType.EC.equals(enclaveEncryptorType) ? "0x3e5d325a03ad3ce5640502219833d30b89ce3ce1" : "0xebf56429e6500e84442467292183d4d621359838"; + final String receiptPrivacyGroupId = + EnclaveEncryptorType.EC.equals(enclaveEncryptorType) + ? "MjuFB4b9Hz+f8zvkWWasxZWRjHWXU4t7B2nOHo4mekA=" + : "DyAOiF/ynpc+JXa2YAGB0bCitSlOMNm+ShmB/7M6C4w="; final RawPrivateTransaction rawPrivateTransaction = RawPrivateTransaction.createContractTransaction( @@ -196,6 +200,7 @@ public void aliceCanUsePrivDistributeTransaction() { Numeric.prependHexPrefix(EventEmitter.BINARY), Base64String.wrap(alice.getEnclaveKey()), Collections.singletonList(Base64String.wrap(bob.getEnclaveKey())), + Base64String.wrap(receiptPrivacyGroupId), RESTRICTED); final String signedPrivateTransaction = @@ -243,10 +248,6 @@ public void aliceCanUsePrivDistributeTransaction() { "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEXIgZqRA25V+3nN+Do6b5r0jiUunub6ubjPhqwHpPxP44uUYh9RKCQNRnsqCJ9PjeTnC8R3ieJk7HWAlycU1bug==")) : new ArrayList<>( Collections.singletonList("Ko2bVqD+nNlNYL5EE7y3IdOnviftjiizpjRt+HTuFBs=")); - final String receiptPrivacyGroupId = - EnclaveEncryptorType.EC.equals(enclaveEncryptorType) - ? "MjuFB4b9Hz+f8zvkWWasxZWRjHWXU4t7B2nOHo4mekA=" - : "DyAOiF/ynpc+JXa2YAGB0bCitSlOMNm+ShmB/7M6C4w="; final PrivateTransactionReceipt expectedReceipt = new PrivateTransactionReceipt( diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java index 4ae94924b25..f230054dcfd 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java @@ -41,9 +41,19 @@ public void setUp() throws Exception { minerNode = besu.createMinerNode("miner-node1"); archiveNode = besu.createArchiveNode("full-node1"); cluster.start(minerNode, archiveNode); + + // verify nodes are fully connected otherwise tx could not be propagated + minerNode.verify(net.awaitPeerCount(1)); + archiveNode.verify(net.awaitPeerCount(1)); + accountOne = accounts.createAccount("account-one"); minerWebSocket = new WebSocket(vertx, minerNode.getConfiguration()); archiveWebSocket = new WebSocket(vertx, archiveNode.getConfiguration()); + + // verify that the miner started producing blocks and all other nodes are syncing from it + waitForBlockHeight(minerNode, 1); + final var minerChainHead = minerNode.execute(ethTransactions.block()); + archiveNode.verify(blockchain.minimumHeight(minerChainHead.getNumber().longValue())); } @AfterEach diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/genesis.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/genesis.json deleted file mode 100644 index 232c577f6c8..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/genesis.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "config": { - "chainId":6110, - "homesteadBlock":0, - "eip150Block":0, - "eip155Block":0, - "eip158Block":0, - "byzantiumBlock":0, - "constantinopleBlock":0, - "petersburgBlock":0, - "istanbulBlock":0, - "muirGlacierBlock":0, - "berlinBlock":0, - "londonBlock":0, - "terminalTotalDifficulty":0, - "cancunTime":0, - "experimentalEipsTime":20, - "clique": { - "period": 5, - "epoch": 30000 - }, - "depositContractAddress": "0x4242424242424242424242424242424242424242" - }, - "nonce":"0x42", - "timestamp":"0x0", - "extraData":"0x0000000000000000000000000000000000000000000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit":"0x1C9C380", - "difficulty":"0x400000000", - "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000", - "coinbase":"0x0000000000000000000000000000000000000000", - "alloc":{ - "0xa4664C40AACeBD82A2Db79f0ea36C06Bc6A19Adb": { - "balance": "1000000000000000000000000000" - }, - "0x4242424242424242424242424242424242424242": { - "balance": "0", - "code": "0x60806040526004361061003f5760003560e01c806301ffc9a71461004457806322895118146100a4578063621fd130146101ba578063c5f2892f14610244575b600080fd5b34801561005057600080fd5b506100906004803603602081101561006757600080fd5b50357fffffffff000000000000000000000000000000000000000000000000000000001661026b565b604080519115158252519081900360200190f35b6101b8600480360360808110156100ba57600080fd5b8101906020810181356401000000008111156100d557600080fd5b8201836020820111156100e757600080fd5b8035906020019184600183028401116401000000008311171561010957600080fd5b91939092909160208101903564010000000081111561012757600080fd5b82018360208201111561013957600080fd5b8035906020019184600183028401116401000000008311171561015b57600080fd5b91939092909160208101903564010000000081111561017957600080fd5b82018360208201111561018b57600080fd5b803590602001918460018302840111640100000000831117156101ad57600080fd5b919350915035610304565b005b3480156101c657600080fd5b506101cf6110b5565b6040805160208082528351818301528351919283929083019185019080838360005b838110156102095781810151838201526020016101f1565b50505050905090810190601f1680156102365780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561025057600080fd5b506102596110c7565b60408051918252519081900360200190f35b60007fffffffff0000000000000000000000000000000000000000000000000000000082167f01ffc9a70000000000000000000000000000000000000000000000000000000014806102fe57507fffffffff0000000000000000000000000000000000000000000000000000000082167f8564090700000000000000000000000000000000000000000000000000000000145b92915050565b6030861461035d576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118056026913960400191505060405180910390fd5b602084146103b6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252603681526020018061179c6036913960400191505060405180910390fd5b6060821461040f576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260298152602001806118786029913960400191505060405180910390fd5b670de0b6b3a7640000341015610470576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118526026913960400191505060405180910390fd5b633b9aca003406156104cd576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260338152602001806117d26033913960400191505060405180910390fd5b633b9aca00340467ffffffffffffffff811115610535576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602781526020018061182b6027913960400191505060405180910390fd5b6060610540826114ba565b90507f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c589898989858a8a6105756020546114ba565b6040805160a0808252810189905290819060208201908201606083016080840160c085018e8e80828437600083820152601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690910187810386528c815260200190508c8c808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690920188810386528c5181528c51602091820193918e019250908190849084905b83811015610648578181015183820152602001610630565b50505050905090810190601f1680156106755780820380516001836020036101000a031916815260200191505b5086810383528881526020018989808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169092018881038452895181528951602091820193918b019250908190849084905b838110156106ef5781810151838201526020016106d7565b50505050905090810190601f16801561071c5780820380516001836020036101000a031916815260200191505b509d505050505050505050505050505060405180910390a1600060028a8a600060801b604051602001808484808284377fffffffffffffffffffffffffffffffff0000000000000000000000000000000090941691909301908152604080517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0818403018152601090920190819052815191955093508392506020850191508083835b602083106107fc57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016107bf565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610859573d6000803e3d6000fd5b5050506040513d602081101561086e57600080fd5b5051905060006002806108846040848a8c6116fe565b6040516020018083838082843780830192505050925050506040516020818303038152906040526040518082805190602001908083835b602083106108f857805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016108bb565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610955573d6000803e3d6000fd5b5050506040513d602081101561096a57600080fd5b5051600261097b896040818d6116fe565b60405160009060200180848480828437919091019283525050604080518083038152602092830191829052805190945090925082918401908083835b602083106109f457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016109b7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610a51573d6000803e3d6000fd5b5050506040513d6020811015610a6657600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610ada57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610a9d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610b37573d6000803e3d6000fd5b5050506040513d6020811015610b4c57600080fd5b50516040805160208101858152929350600092600292839287928f928f92018383808284378083019250505093505050506040516020818303038152906040526040518082805190602001908083835b60208310610bd957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610b9c565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610c36573d6000803e3d6000fd5b5050506040513d6020811015610c4b57600080fd5b50516040518651600291889160009188916020918201918291908601908083835b60208310610ca957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610c6c565b6001836020036101000a0380198251168184511680821785525050505050509050018367ffffffffffffffff191667ffffffffffffffff1916815260180182815260200193505050506040516020818303038152906040526040518082805190602001908083835b60208310610d4e57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610d11565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610dab573d6000803e3d6000fd5b5050506040513d6020811015610dc057600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610e3457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610df7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610e91573d6000803e3d6000fd5b5050506040513d6020811015610ea657600080fd5b50519050858114610f02576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260548152602001806117486054913960600191505060405180910390fd5b60205463ffffffff11610f60576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260218152602001806117276021913960400191505060405180910390fd5b602080546001019081905560005b60208110156110a9578160011660011415610fa0578260008260208110610f9157fe5b0155506110ac95505050505050565b600260008260208110610faf57fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061102557805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610fe8565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015611082573d6000803e3d6000fd5b5050506040513d602081101561109757600080fd5b50519250600282049150600101610f6e565b50fe5b50505050505050565b60606110c26020546114ba565b905090565b6020546000908190815b60208110156112f05781600116600114156111e6576002600082602081106110f557fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061116b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161112e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156111c8573d6000803e3d6000fd5b5050506040513d60208110156111dd57600080fd5b505192506112e2565b600283602183602081106111f657fe5b015460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061126b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161122e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156112c8573d6000803e3d6000fd5b5050506040513d60208110156112dd57600080fd5b505192505b6002820491506001016110d1565b506002826112ff6020546114ba565b600060401b6040516020018084815260200183805190602001908083835b6020831061135a57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161131d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790527fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000095909516920191825250604080518083037ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8018152601890920190819052815191955093508392850191508083835b6020831061143f57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101611402565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa15801561149c573d6000803e3d6000fd5b5050506040513d60208110156114b157600080fd5b50519250505090565b60408051600880825281830190925260609160208201818036833701905050905060c082901b8060071a60f81b826000815181106114f457fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060061a60f81b8260018151811061153757fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060051a60f81b8260028151811061157a57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060041a60f81b826003815181106115bd57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060031a60f81b8260048151811061160057fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060021a60f81b8260058151811061164357fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060011a60f81b8260068151811061168657fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060001a60f81b826007815181106116c957fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a90535050919050565b6000808585111561170d578182fd5b83861115611719578182fd5b505082019391909203915056fe4465706f736974436f6e74726163743a206d65726b6c6520747265652066756c6c4465706f736974436f6e74726163743a207265636f6e7374727563746564204465706f7369744461746120646f6573206e6f74206d6174636820737570706c696564206465706f7369745f646174615f726f6f744465706f736974436f6e74726163743a20696e76616c6964207769746864726177616c5f63726564656e7469616c73206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c7565206e6f74206d756c7469706c65206f6620677765694465706f736974436f6e74726163743a20696e76616c6964207075626b6579206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f20686967684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f206c6f774465706f736974436f6e74726163743a20696e76616c6964207369676e6174757265206c656e677468a26469706673582212201dd26f37a621703009abf16e77e69c93dc50c79db7f6cc37543e3e0e3decdc9764736f6c634300060b0033", - "storage": { - "0x0000000000000000000000000000000000000000000000000000000000000022": "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", - "0x0000000000000000000000000000000000000000000000000000000000000023": "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", - "0x0000000000000000000000000000000000000000000000000000000000000024": "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", - "0x0000000000000000000000000000000000000000000000000000000000000025": "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", - "0x0000000000000000000000000000000000000000000000000000000000000026": "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", - "0x0000000000000000000000000000000000000000000000000000000000000027": "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", - "0x0000000000000000000000000000000000000000000000000000000000000028": "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", - "0x0000000000000000000000000000000000000000000000000000000000000029": "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", - "0x000000000000000000000000000000000000000000000000000000000000002a": "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", - "0x000000000000000000000000000000000000000000000000000000000000002b": "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", - "0x000000000000000000000000000000000000000000000000000000000000002c": "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", - "0x000000000000000000000000000000000000000000000000000000000000002d": "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", - "0x000000000000000000000000000000000000000000000000000000000000002e": "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", - "0x000000000000000000000000000000000000000000000000000000000000002f": "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", - "0x0000000000000000000000000000000000000000000000000000000000000030": "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", - "0x0000000000000000000000000000000000000000000000000000000000000031": "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", - "0x0000000000000000000000000000000000000000000000000000000000000032": "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", - "0x0000000000000000000000000000000000000000000000000000000000000033": "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", - "0x0000000000000000000000000000000000000000000000000000000000000034": "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", - "0x0000000000000000000000000000000000000000000000000000000000000035": "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", - "0x0000000000000000000000000000000000000000000000000000000000000036": "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", - "0x0000000000000000000000000000000000000000000000000000000000000037": "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", - "0x0000000000000000000000000000000000000000000000000000000000000038": "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", - "0x0000000000000000000000000000000000000000000000000000000000000039": "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", - "0x000000000000000000000000000000000000000000000000000000000000003a": "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", - "0x000000000000000000000000000000000000000000000000000000000000003b": "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", - "0x000000000000000000000000000000000000000000000000000000000000003c": "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", - "0x000000000000000000000000000000000000000000000000000000000000003d": "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", - "0x000000000000000000000000000000000000000000000000000000000000003e": "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", - "0x000000000000000000000000000000000000000000000000000000000000003f": "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", - "0x0000000000000000000000000000000000000000000000000000000000000040": "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7" - } - } - }, - "number":"0x0", - "gasUsed":"0x0", - "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", - "baseFeePerGas":"0x7" -} \ No newline at end of file diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/01_cancun_prepare_payload.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/01_cancun_prepare_payload.json deleted file mode 100644 index 47063a25e4b..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/01_cancun_prepare_payload.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "request": { - "jsonrpc": "2.0", - "method": "engine_forkchoiceUpdatedV3", - "params": [ - { - "headBlockHash": "0x78a301e0d846bd169889c9755c9aa4ce2972dfc4bd63de61f3303887d3e81f98", - "safeBlockHash": "0x78a301e0d846bd169889c9755c9aa4ce2972dfc4bd63de61f3303887d3e81f98", - "finalizedBlockHash": "0x0000000000000000000000000000000000000000000000000000000000000000" - }, - { - "timestamp": "0x10", - "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", - "suggestedFeeRecipient": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "withdrawals": [], - "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000" - } - ], - "id": 67 - }, - "response": { - "jsonrpc": "2.0", - "id": 67, - "result": { - "payloadStatus": { - "status": "VALID", - "latestValidHash": "0x78a301e0d846bd169889c9755c9aa4ce2972dfc4bd63de61f3303887d3e81f98", - "validationError": null - }, - "payloadId": "0x282643d459a6f711" - } - }, - "statusCode": 200 -} \ No newline at end of file diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/02_cancun_getPayloadV3.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/02_cancun_getPayloadV3.json deleted file mode 100644 index c6180edaad3..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/02_cancun_getPayloadV3.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "request": { - "jsonrpc": "2.0", - "method": "engine_getPayloadV3", - "params": [ - "0x282643d459a6f711" - ], - "id": 67 - }, - "response": { - "jsonrpc": "2.0", - "id": 67, - "result": { - "executionPayload": { - "parentHash": "0x78a301e0d846bd169889c9755c9aa4ce2972dfc4bd63de61f3303887d3e81f98", - "feeRecipient": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "stateRoot": "0x9b8c4a9a86cb49252075c0db2f0e72fb1e49350a0f70ea36f26f700201961e62", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x1c9c380", - "gasUsed": "0x0", - "timestamp": "0x10", - "extraData": "0x", - "baseFeePerGas": "0x7", - "excessBlobGas": "0x0", - "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactions": [], - "withdrawals": [], - "blockNumber": "0x1", - "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "blobGasUsed": "0x0", - "blockHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315" - }, - "blockValue": "0x0", - "blobsBundle": { - "commitments": [], - "proofs": [], - "blobs": [] - }, - "shouldOverrideBuilder": false - } - }, - "statusCode": 200 -} \ No newline at end of file diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/03_cancun_newPayloadV3.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/03_cancun_newPayloadV3.json deleted file mode 100644 index b8ae8dbd89a..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/03_cancun_newPayloadV3.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "request": { - "jsonrpc": "2.0", - "method": "engine_newPayloadV3", - "params": [ - { - "parentHash": "0x78a301e0d846bd169889c9755c9aa4ce2972dfc4bd63de61f3303887d3e81f98", - "feeRecipient": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "stateRoot": "0x9b8c4a9a86cb49252075c0db2f0e72fb1e49350a0f70ea36f26f700201961e62", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x1c9c380", - "gasUsed": "0x0", - "timestamp": "0x10", - "extraData": "0x", - "baseFeePerGas": "0x7", - "transactions": [], - "withdrawals": [], - "blockNumber": "0x1", - "blockHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315", - "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "excessBlobGas": "0x0", - "blobGasUsed": "0x0" - }, - [], - "0x0000000000000000000000000000000000000000000000000000000000000000" - ], - "id": 67 - }, - "response": { - "jsonrpc": "2.0", - "id": 67, - "result": { - "status": "VALID", - "latestValidHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315", - "validationError": null - } - }, - "statusCode": 200 -} \ No newline at end of file diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/04_cancun_forkchoiceUpdatedV3.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/04_cancun_forkchoiceUpdatedV3.json deleted file mode 100644 index 3e7d9adcb42..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/04_cancun_forkchoiceUpdatedV3.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "request": { - "jsonrpc": "2.0", - "method": "engine_forkchoiceUpdatedV3", - "params": [ - { - "headBlockHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315", - "safeBlockHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315", - "finalizedBlockHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315" - }, - null - ], - "id": 67 - }, - "response": { - "jsonrpc": "2.0", - "id": 67, - "result": { - "payloadStatus": { - "status": "VALID", - "latestValidHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315", - "validationError": null - }, - "payloadId": null - } - }, - "statusCode": 200 -} \ No newline at end of file diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/05_eip6110_forkchoiceUpdatedV3.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/05_eip6110_forkchoiceUpdatedV3.json deleted file mode 100644 index 30317597867..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/05_eip6110_forkchoiceUpdatedV3.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "request": { - "jsonrpc": "2.0", - "method": "engine_forkchoiceUpdatedV3", - "params": [ - { - "headBlockHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315", - "safeBlockHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315", - "finalizedBlockHash": "0x0000000000000000000000000000000000000000000000000000000000000000" - }, - { - "timestamp": "0x20", - "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", - "suggestedFeeRecipient": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "withdrawals": [], - "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000" - } - ], - "id": 67 - }, - "response": { - "jsonrpc": "2.0", - "id": 67, - "result": { - "payloadStatus": { - "status": "VALID", - "latestValidHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315", - "validationError": null - }, - "payloadId": "0x282643b909febddf" - } - }, - "statusCode": 200 -} \ No newline at end of file diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/06_eip6110_getPayloadV6110.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/06_eip6110_getPayloadV6110.json deleted file mode 100644 index e31b2f8f8a9..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/06_eip6110_getPayloadV6110.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "request": { - "jsonrpc": "2.0", - "method": "engine_getPayloadV6110", - "params": [ - "0x282643b909febddf" - ], - "id": 67 - }, - "response": { - "jsonrpc": "2.0", - "id": 67, - "result": { - "executionPayload": { - "parentHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315", - "feeRecipient": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "stateRoot": "0x9b8c4a9a86cb49252075c0db2f0e72fb1e49350a0f70ea36f26f700201961e62", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x1c9c380", - "gasUsed": "0x0", - "timestamp": "0x20", - "extraData": "0x", - "baseFeePerGas": "0x7", - "excessBlobGas": "0x0", - "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactions": [], - "withdrawals": [], - "depositReceipts": [], - "blockNumber": "0x2", - "blockHash": "0xc8255831601171a628ef17f6601d3d1d30ff9b382e77592ed1af32354f6dafbb", - "blobGasUsed": "0x0", - "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" - }, - "blockValue": "0x0", - "blobsBundle": { - "commitments": [], - "proofs": [], - "blobs": [] - }, - "shouldOverrideBuilder": false - } - }, - "statusCode": 200 -} \ No newline at end of file diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/07_eip6110_send_raw_transaction.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/07_eip6110_send_raw_transaction.json deleted file mode 100644 index 90072c28e0e..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/07_eip6110_send_raw_transaction.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "request": { - "jsonrpc": "2.0", - "method": "eth_sendRawTransaction", - "params": ["0x02f9021c8217de808459682f008459682f0e830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120749715de5d1226545c6b3790f515d551a5cc5bf1d49c87a696860554d2fc4f14000000000000000000000000000000000000000000000000000000000000003096a96086cff07df17668f35f7418ef8798079167e3f4f9b72ecde17b28226137cf454ab1dd20ef5d924786ab3483c2f9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef20000000000000000000000000000000000000000000000000000000000000060b1acdb2c4d3df3f1b8d3bfd33421660df358d84d78d16c4603551935f4b67643373e7eb63dcb16ec359be0ec41fee33b03a16e80745f2374ff1d3c352508ac5d857c6476d3c3bcf7e6ca37427c9209f17be3af5264c0e2132b3dd1156c28b4e9c080a09f597089338d7f44f5c59f8230bb38f243849228a8d4e9d2e2956e6050f5b2c7a076486996c7e62802b8f95eee114783e4b403fd11093ba96286ff42c595f24452"], - "id": 67 - }, - "response": { - "jsonrpc": "2.0", - "id": 67, - "result": "0x8ff1a50169f52f14cc1cf0300ec037c054a9b99df462e6372c7ca655bf1f00cd" - }, - "statusCode": 200 -} \ No newline at end of file diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/08_eip6110_invalid_null_deposits_execute_payload.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/08_eip6110_invalid_null_deposits_execute_payload.json deleted file mode 100644 index ebf5de77500..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/08_eip6110_invalid_null_deposits_execute_payload.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "request": { - "jsonrpc": "2.0", - "method": "engine_newPayloadV6110", - "params": [ - { - "parentHash": "0x45811fa27a100ce9035e5e086b9669275041a4ec0ebbd920be028fd7b0aa2356", - "feeRecipient": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "stateRoot": "0x9b8c4a9a86cb49252075c0db2f0e72fb1e49350a0f70ea36f26f700201961e62", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x1c9c380", - "gasUsed": "0x0", - "timestamp": "0x20", - "extraData": "0x", - "baseFeePerGas": "0x7", - "excessBlobGas": "0x0", - "transactions": [], - "withdrawals": [], - "depositReceipts" : null, - "blockNumber": "0x2", - "blockHash": "0xf6c3f1180ba58d6ea4c69c9328c7afb1fda41df06c368741c1f8310567879de7", - "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "blobGasUsed": "0x0" - }, - [], - "0x0000000000000000000000000000000000000000000000000000000000000000" - ], - "id": 67 - }, - "response": { - "jsonrpc": "2.0", - "id": 67, - "error": { - "code": -32602, - "message": "Invalid params", - "data" : "Missing deposit field" - } - }, - "statusCode": 200 -} \ No newline at end of file diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/09_eip6110_newPayloadV6110.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/09_eip6110_newPayloadV6110.json deleted file mode 100644 index 760b9481295..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/09_eip6110_newPayloadV6110.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "request": { - "jsonrpc": "2.0", - "method": "engine_newPayloadV6110", - "params": [ - { - "parentHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315", - "feeRecipient": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "stateRoot": "0x14208ac0e218167936e220b72d5d5887a963cb858ea2f2d268518f014a3da3fa", - "logsBloom": "0x10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000", - "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x1c9c380", - "gasUsed": "0x14B6E", - "timestamp": "0x20", - "extraData": "0x", - "baseFeePerGas": "0x7", - "excessBlobGas": "0x0", - "transactions": [ - "0x02f9021c8217de808459682f008459682f0e830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120749715de5d1226545c6b3790f515d551a5cc5bf1d49c87a696860554d2fc4f14000000000000000000000000000000000000000000000000000000000000003096a96086cff07df17668f35f7418ef8798079167e3f4f9b72ecde17b28226137cf454ab1dd20ef5d924786ab3483c2f9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef20000000000000000000000000000000000000000000000000000000000000060b1acdb2c4d3df3f1b8d3bfd33421660df358d84d78d16c4603551935f4b67643373e7eb63dcb16ec359be0ec41fee33b03a16e80745f2374ff1d3c352508ac5d857c6476d3c3bcf7e6ca37427c9209f17be3af5264c0e2132b3dd1156c28b4e9c080a09f597089338d7f44f5c59f8230bb38f243849228a8d4e9d2e2956e6050f5b2c7a076486996c7e62802b8f95eee114783e4b403fd11093ba96286ff42c595f24452" - ], - "withdrawals": [], - "depositReceipts" : [ - {"amount":"0x773594000","index":"0x0","pubkey":"0x96a96086cff07df17668f35f7418ef8798079167e3f4f9b72ecde17b28226137cf454ab1dd20ef5d924786ab3483c2f9","signature":"0xb1acdb2c4d3df3f1b8d3bfd33421660df358d84d78d16c4603551935f4b67643373e7eb63dcb16ec359be0ec41fee33b03a16e80745f2374ff1d3c352508ac5d857c6476d3c3bcf7e6ca37427c9209f17be3af5264c0e2132b3dd1156c28b4e9","withdrawalCredentials":"0x003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef2"} - ], - "blockNumber": "0x2", - "blockHash": "0xddb65a684b9b8980b6231ee0e388566c10a9c4583bbddf16f8d68bbc0b8ed965", - "receiptsRoot": "0x79ee3424eb720a3ad4b1c5a372bb8160580cbe4d893778660f34213c685627a9", - "blobGasUsed": "0x0" - }, - [], - "0x0000000000000000000000000000000000000000000000000000000000000000" - ], - "id": 67 - }, - "response": { - "jsonrpc": "2.0", - "id": 67, - "result": { - "status": "VALID", - "latestValidHash": "0xddb65a684b9b8980b6231ee0e388566c10a9c4583bbddf16f8d68bbc0b8ed965", - "validationError": null - } - }, - "statusCode": 200 -} \ No newline at end of file diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/10_eip6110_forkchoiceUpdatedV3.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/10_eip6110_forkchoiceUpdatedV3.json deleted file mode 100644 index ef779c959f5..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/10_eip6110_forkchoiceUpdatedV3.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "request": { - "jsonrpc": "2.0", - "method": "engine_forkchoiceUpdatedV3", - "params": [ - { - "headBlockHash": "0xddb65a684b9b8980b6231ee0e388566c10a9c4583bbddf16f8d68bbc0b8ed965", - "safeBlockHash": "0xddb65a684b9b8980b6231ee0e388566c10a9c4583bbddf16f8d68bbc0b8ed965", - "finalizedBlockHash": "0x0000000000000000000000000000000000000000000000000000000000000000" - }, - { - "timestamp": "0x30", - "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", - "suggestedFeeRecipient": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "withdrawals": [], - "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000" - } - ], - "id": 67 - }, - "response": { - "jsonrpc": "2.0", - "id": 67, - "result": { - "payloadStatus": { - "status": "VALID", - "latestValidHash": "0xddb65a684b9b8980b6231ee0e388566c10a9c4583bbddf16f8d68bbc0b8ed965", - "validationError": null - }, - "payloadId": "0x282643db882670cf" - } - }, - "statusCode" : 200 -} \ No newline at end of file diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/11_eip6110_getPayloadV6110.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/11_eip6110_getPayloadV6110.json deleted file mode 100644 index 5214ee7cd5a..00000000000 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/11_eip6110_getPayloadV6110.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "request": { - "jsonrpc": "2.0", - "method": "engine_getPayloadV6110", - "params": [ - "0x282643db882670cf" - ], - "id": 67 - }, - "response": { - "jsonrpc": "2.0", - "id": 67, - "result": { - "executionPayload": { - "parentHash": "0xddb65a684b9b8980b6231ee0e388566c10a9c4583bbddf16f8d68bbc0b8ed965", - "feeRecipient": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "stateRoot": "0x14208ac0e218167936e220b72d5d5887a963cb858ea2f2d268518f014a3da3fa", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "prevRandao": "0x0000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x1c9c380", - "gasUsed": "0x0", - "timestamp": "0x30", - "extraData": "0x", - "baseFeePerGas": "0x7", - "excessBlobGas": "0x0", - "parentBeaconBlockRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactions": [], - "withdrawals": [], - "depositReceipts": [], - "blockNumber": "0x3", - "blockHash": "0xf1e7093b5d229885caab11a3acb95412af80f9077b742020a8014cf81c8c75f2", - "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "blobGasUsed": "0x0" - }, - "blockValue": "0x0", - "blobsBundle": { - "commitments": [], - "proofs": [], - "blobs": [] - }, - "shouldOverrideBuilder": false - } - }, - "statusCode": 200 -} \ No newline at end of file diff --git a/besu/build.gradle b/besu/build.gradle index 0b31c4a43c2..85c17e7ff35 100644 --- a/besu/build.gradle +++ b/besu/build.gradle @@ -28,6 +28,8 @@ jar { } dependencies { + api project(':datatypes') + api 'org.slf4j:slf4j-api' implementation project(':config') diff --git a/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java b/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java index 8d0a693e27f..6379e2c1a64 100644 --- a/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java @@ -133,6 +133,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -800,7 +801,7 @@ public Runner build() { metricsSystem, supportedCapabilities, jsonRpcConfiguration.getRpcApis().stream() - .filter(apiGroup -> !apiGroup.toLowerCase().startsWith("engine")) + .filter(apiGroup -> !apiGroup.toLowerCase(Locale.ROOT).startsWith("engine")) .collect(Collectors.toList()), filterManager, accountLocalConfigPermissioningController, @@ -938,7 +939,7 @@ public Runner build() { metricsSystem, supportedCapabilities, webSocketConfiguration.getRpcApis().stream() - .filter(apiGroup -> !apiGroup.toLowerCase().startsWith("engine")) + .filter(apiGroup -> !apiGroup.toLowerCase(Locale.ROOT).startsWith("engine")) .collect(Collectors.toList()), filterManager, accountLocalConfigPermissioningController, @@ -1021,7 +1022,7 @@ public Runner build() { metricsSystem, supportedCapabilities, jsonRpcIpcConfiguration.getEnabledApis().stream() - .filter(apiGroup -> !apiGroup.toLowerCase().startsWith("engine")) + .filter(apiGroup -> !apiGroup.toLowerCase(Locale.ROOT).startsWith("engine")) .collect(Collectors.toList()), filterManager, accountLocalConfigPermissioningController, diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index b24a8e1c2a1..624699f70c5 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -21,6 +21,7 @@ import static java.util.Collections.singletonList; import static org.hyperledger.besu.cli.DefaultCommandValues.getDefaultBesuDataPath; import static org.hyperledger.besu.cli.config.NetworkName.MAINNET; +import static org.hyperledger.besu.cli.options.unstable.NetworkingOptions.PEER_LOWER_BOUND_FLAG; import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; import static org.hyperledger.besu.cli.util.CommandLineUtils.isOptionSet; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; @@ -48,6 +49,7 @@ import org.hyperledger.besu.cli.error.BesuParameterExceptionHandler; import org.hyperledger.besu.cli.options.MiningOptions; import org.hyperledger.besu.cli.options.TransactionPoolOptions; +import org.hyperledger.besu.cli.options.stable.ApiConfigurationOptions; import org.hyperledger.besu.cli.options.stable.DataStorageOptions; import org.hyperledger.besu.cli.options.stable.EthstatsOptions; import org.hyperledger.besu.cli.options.stable.GraphQlOptions; @@ -107,7 +109,6 @@ import org.hyperledger.besu.enclave.EnclaveFactory; import org.hyperledger.besu.ethereum.GasLimitCalculator; import org.hyperledger.besu.ethereum.api.ApiConfiguration; -import org.hyperledger.besu.ethereum.api.ImmutableApiConfiguration; import org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration; import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration; import org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis; @@ -117,7 +118,9 @@ import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.MiningParameters; +import org.hyperledger.besu.ethereum.core.MiningParametersMetrics; import org.hyperledger.besu.ethereum.core.PrivacyParameters; +import org.hyperledger.besu.ethereum.core.VersionMetadata; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; @@ -136,8 +139,8 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; -import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; -import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract; import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract; import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract; @@ -157,33 +160,35 @@ import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.PermissioningService; import org.hyperledger.besu.plugin.services.PicoCLIOptions; -import org.hyperledger.besu.plugin.services.PluginTransactionValidatorService; import org.hyperledger.besu.plugin.services.PrivacyPluginService; import org.hyperledger.besu.plugin.services.RpcEndpointService; import org.hyperledger.besu.plugin.services.SecurityModuleService; import org.hyperledger.besu.plugin.services.StorageService; import org.hyperledger.besu.plugin.services.TraceService; +import org.hyperledger.besu.plugin.services.TransactionPoolValidatorService; import org.hyperledger.besu.plugin.services.TransactionSelectionService; +import org.hyperledger.besu.plugin.services.TransactionSimulationService; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import org.hyperledger.besu.plugin.services.metrics.MetricCategoryRegistry; import org.hyperledger.besu.plugin.services.securitymodule.SecurityModule; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.PrivacyKeyValueStorageFactory; import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin; -import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelectorFactory; -import org.hyperledger.besu.plugin.services.txvalidator.PluginTransactionValidatorFactory; +import org.hyperledger.besu.services.BesuConfigurationImpl; import org.hyperledger.besu.services.BesuEventsImpl; import org.hyperledger.besu.services.BesuPluginContextImpl; import org.hyperledger.besu.services.BlockchainServiceImpl; import org.hyperledger.besu.services.PermissioningServiceImpl; import org.hyperledger.besu.services.PicoCLIOptionsImpl; -import org.hyperledger.besu.services.PluginTransactionValidatorServiceImpl; import org.hyperledger.besu.services.PrivacyPluginServiceImpl; import org.hyperledger.besu.services.RpcEndpointServiceImpl; import org.hyperledger.besu.services.SecurityModuleServiceImpl; import org.hyperledger.besu.services.StorageServiceImpl; import org.hyperledger.besu.services.TraceServiceImpl; +import org.hyperledger.besu.services.TransactionPoolValidatorServiceImpl; import org.hyperledger.besu.services.TransactionSelectionServiceImpl; +import org.hyperledger.besu.services.TransactionSimulationServiceImpl; import org.hyperledger.besu.services.kvstore.InMemoryStoragePlugin; import org.hyperledger.besu.util.InvalidConfigurationException; import org.hyperledger.besu.util.LogConfigurator; @@ -231,7 +236,6 @@ import io.vertx.core.metrics.MetricsOptions; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import picocli.AutoComplete; import picocli.CommandLine; @@ -254,8 +258,12 @@ synopsisHeading = "%n", descriptionHeading = "%n@|bold,fg(cyan) Description:|@%n%n", optionListHeading = "%n@|bold,fg(cyan) Options:|@%n", - footerHeading = "%n", - footer = "Besu is licensed under the Apache License 2.0") + footerHeading = "%nBesu is licensed under the Apache License 2.0%n", + footer = { + "%n%n@|fg(cyan) To get started quickly, just choose a network to sync and a profile to run with suggested defaults:|@", + "%n@|fg(cyan) for Mainnet|@ --network=mainnet --profile=[minimalist_staker|staker]", + "%nMore info and other profiles at https://besu.hyperledger.org%n" + }) public class BesuCommand implements DefaultCommandValues, Runnable { @SuppressWarnings("PrivateStaticFinalLoggers") @@ -320,7 +328,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable { private int maxPeers; private int maxRemoteInitiatedPeers; - private int peersLowerBound; // CLI options defined by user at runtime. // Options parsing is done with CLI library Picocli https://picocli.info/ @@ -340,16 +347,16 @@ public class BesuCommand implements DefaultCommandValues, Runnable { description = "The path to Besu data directory (default: ${DEFAULT-VALUE})") final Path dataPath = getDefaultBesuDataPath(this); - // Genesis file path with null default option if the option - // is not defined on command line as this default is handled by Runner + // Genesis file path with null default option. + // This default is handled by Runner // to use mainnet json file from resources as indicated in the // default network option - // Then we have no control over genesis default value here. + // Then we ignore genesis default value here. @CommandLine.Option( names = {"--genesis-file"}, paramLabel = MANDATORY_FILE_FORMAT_HELP, description = - "Genesis file. Setting this option makes --network option ignored and requires --network-id to be set.") + "Genesis file for your custom network. Setting this option requires --network-id to be set. (Cannot be used with --network)") private final File genesisFile = null; @Option( @@ -364,7 +371,9 @@ public class BesuCommand implements DefaultCommandValues, Runnable { P2PDiscoveryOptionGroup p2PDiscoveryOptionGroup = new P2PDiscoveryOptionGroup(); private final TransactionSelectionServiceImpl transactionSelectionServiceImpl; - private final PluginTransactionValidatorServiceImpl transactionValidatorServiceImpl; + private final TransactionPoolValidatorServiceImpl transactionValidatorServiceImpl; + private final TransactionSimulationServiceImpl transactionSimulationServiceImpl; + private final BlockchainServiceImpl blockchainServiceImpl; static class P2PDiscoveryOptionGroup { @@ -505,7 +514,7 @@ private InetAddress autoDiscoverDefaultIP() { names = {"--sync-mode"}, paramLabel = MANDATORY_MODE_FORMAT_HELP, description = - "Synchronization mode, possible values are ${COMPLETION-CANDIDATES} (default: FAST if a --network is supplied and privacy isn't enabled. FULL otherwise.)") + "Synchronization mode, possible values are ${COMPLETION-CANDIDATES} (default: SNAP if a --network is supplied and privacy isn't enabled. FULL otherwise.)") private SyncMode syncMode = null; @Option( @@ -555,6 +564,12 @@ private InetAddress autoDiscoverDefaultIP() { arity = "1") private final Path kzgTrustedSetupFile = null; + @Option( + names = {"--version-compatibility-protection"}, + description = + "Perform compatibility checks between the version of Besu being started and the version of Besu that last started with this data directory. (default: ${DEFAULT-VALUE})") + private Boolean versionCompatibilityProtection = null; + @CommandLine.ArgGroup(validate = false, heading = "@|bold GraphQL Options|@%n") GraphQlOptions graphQlOptions = new GraphQlOptions(); @@ -781,12 +796,6 @@ static class MetricsOptionGroup { "How deep a chain reorganization must be in order for it to be logged (default: ${DEFAULT-VALUE})") private final Long reorgLoggingThreshold = 6L; - @Option( - names = {"--pruning-enabled"}, - description = - "Enable disk-space saving optimization that removes old state that is unlikely to be required (default: ${DEFAULT-VALUE})") - private final Boolean pruningEnabled = false; - // Permission Option Group @CommandLine.ArgGroup(validate = false, heading = "@|bold Permissions Options|@%n") PermissionsOptions permissionsOptions = new PermissionsOptions(); @@ -836,66 +845,15 @@ static class MetricsOptionGroup { private final Map genesisConfigOverrides = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - @Option( - names = {"--pruning-blocks-retained"}, - paramLabel = "", - description = - "Minimum number of recent blocks for which to keep entire world state (default: ${DEFAULT-VALUE})", - arity = "1") - private final Integer pruningBlocksRetained = PrunerConfiguration.DEFAULT_PRUNING_BLOCKS_RETAINED; - - @Option( - names = {"--pruning-block-confirmations"}, - paramLabel = "", - description = - "Minimum number of confirmations on a block before marking begins (default: ${DEFAULT-VALUE})", - arity = "1") - private final Integer pruningBlockConfirmations = - PrunerConfiguration.DEFAULT_PRUNING_BLOCK_CONFIRMATIONS; - @CommandLine.Option( names = {"--pid-path"}, paramLabel = MANDATORY_PATH_FORMAT_HELP, description = "Path to PID file (optional)") private final Path pidPath = null; - @CommandLine.Option( - names = {"--api-gas-price-blocks"}, - description = "Number of blocks to consider for eth_gasPrice (default: ${DEFAULT-VALUE})") - private final Long apiGasPriceBlocks = 100L; - - @CommandLine.Option( - names = {"--api-gas-price-percentile"}, - description = "Percentile value to measure for eth_gasPrice (default: ${DEFAULT-VALUE})") - private final Double apiGasPricePercentile = 50.0; - - @CommandLine.Option( - names = {"--api-gas-price-max"}, - description = "Maximum gas price for eth_gasPrice (default: ${DEFAULT-VALUE})") - private final Long apiGasPriceMax = 500_000_000_000L; - - @CommandLine.Option( - names = {"--api-gas-and-priority-fee-limiting-enabled"}, - hidden = true, - description = - "Set to enable gas price and minimum priority fee limit in eth_getGasPrice and eth_feeHistory (default: ${DEFAULT-VALUE})") - private final Boolean apiGasAndPriorityFeeLimitingEnabled = false; - - @CommandLine.Option( - names = {"--api-gas-and-priority-fee-lower-bound-coefficient"}, - hidden = true, - description = - "Coefficient for setting the lower limit of gas price and minimum priority fee in eth_getGasPrice and eth_feeHistory (default: ${DEFAULT-VALUE})") - private final Long apiGasAndPriorityFeeLowerBoundCoefficient = - ApiConfiguration.DEFAULT_LOWER_BOUND_GAS_AND_PRIORITY_FEE_COEFFICIENT; - - @CommandLine.Option( - names = {"--api-gas-and-priority-fee-upper-bound-coefficient"}, - hidden = true, - description = - "Coefficient for setting the upper limit of gas price and minimum priority fee in eth_getGasPrice and eth_feeHistory (default: ${DEFAULT-VALUE})") - private final Long apiGasAndPriorityFeeUpperBoundCoefficient = - ApiConfiguration.DEFAULT_UPPER_BOUND_GAS_AND_PRIORITY_FEE_COEFFICIENT; + // API Configuration Option Group + @CommandLine.ArgGroup(validate = false, heading = "@|bold API Configuration Options|@%n") + ApiConfigurationOptions apiConfigurationOptions = new ApiConfigurationOptions(); @CommandLine.Option( names = {"--static-nodes-file"}, @@ -904,29 +862,11 @@ static class MetricsOptionGroup { "Specifies the static node file containing the static nodes for this node to connect to") private final Path staticNodesFile = null; - @CommandLine.Option( - names = {"--rpc-max-logs-range"}, - description = - "Specifies the maximum number of blocks to retrieve logs from via RPC. Must be >=0. 0 specifies no limit (default: ${DEFAULT-VALUE})") - private final Long rpcMaxLogsRange = 5000L; - - @CommandLine.Option( - names = {"--rpc-gas-cap"}, - description = - "Specifies the gasLimit cap for transaction simulation RPC methods. Must be >=0. 0 specifies no limit (default: ${DEFAULT-VALUE})") - private final Long rpcGasCap = 0L; - @CommandLine.Option( names = {"--cache-last-blocks"}, description = "Specifies the number of last blocks to cache (default: ${DEFAULT-VALUE})") private final Integer numberOfblocksToCache = 0; - @Option( - names = {"--rpc-max-trace-filter-range"}, - description = - "Specifies the maximum number of blocks for the trace_filter method. Must be >=0. 0 specifies no limit (default: $DEFAULT-VALUE)") - private final Long maxTraceFilterRange = 1000L; - @Mixin private P2PTLSConfigOptions p2pTLSConfigOptions; @Mixin private PkiBlockCreationOptions pkiBlockCreationOptions; @@ -941,9 +881,10 @@ static class MetricsOptionGroup { private MetricsConfiguration metricsConfiguration; private Optional permissioningConfiguration; private Optional p2pTLSConfiguration; + private DataStorageConfiguration dataStorageConfiguration; private Collection staticNodes; private BesuController besuController; - private BesuConfiguration pluginCommonConfiguration; + private BesuConfigurationImpl pluginCommonConfiguration; private MiningParameters miningParameters; private BesuComponent besuComponent; @@ -994,7 +935,9 @@ public BesuCommand( new PkiBlockCreationConfigurationProvider(), new RpcEndpointServiceImpl(), new TransactionSelectionServiceImpl(), - new PluginTransactionValidatorServiceImpl()); + new TransactionPoolValidatorServiceImpl(), + new TransactionSimulationServiceImpl(), + new BlockchainServiceImpl()); } /** @@ -1016,6 +959,8 @@ public BesuCommand( * @param rpcEndpointServiceImpl instance of RpcEndpointServiceImpl * @param transactionSelectionServiceImpl instance of TransactionSelectionServiceImpl * @param transactionValidatorServiceImpl instance of TransactionValidatorServiceImpl + * @param transactionSimulationServiceImpl instance of TransactionSimulationServiceImpl + * @param blockchainServiceImpl instance of BlockchainServiceImpl */ @VisibleForTesting protected BesuCommand( @@ -1034,7 +979,9 @@ protected BesuCommand( final PkiBlockCreationConfigurationProvider pkiBlockCreationConfigProvider, final RpcEndpointServiceImpl rpcEndpointServiceImpl, final TransactionSelectionServiceImpl transactionSelectionServiceImpl, - final PluginTransactionValidatorServiceImpl transactionValidatorServiceImpl) { + final TransactionPoolValidatorServiceImpl transactionValidatorServiceImpl, + final TransactionSimulationServiceImpl transactionSimulationServiceImpl, + final BlockchainServiceImpl blockchainServiceImpl) { this.besuComponent = besuComponent; this.logger = besuComponent.getBesuCommandLogger(); this.rlpBlockImporter = rlpBlockImporter; @@ -1048,12 +995,14 @@ protected BesuCommand( this.securityModuleService = securityModuleService; this.permissioningService = permissioningService; this.privacyPluginService = privacyPluginService; - pluginCommonConfiguration = new BesuCommandConfigurationService(); + this.pluginCommonConfiguration = new BesuConfigurationImpl(); besuPluginContext.addService(BesuConfiguration.class, pluginCommonConfiguration); this.pkiBlockCreationConfigProvider = pkiBlockCreationConfigProvider; this.rpcEndpointServiceImpl = rpcEndpointServiceImpl; this.transactionSelectionServiceImpl = transactionSelectionServiceImpl; this.transactionValidatorServiceImpl = transactionValidatorServiceImpl; + this.transactionSimulationServiceImpl = transactionSimulationServiceImpl; + this.blockchainServiceImpl = blockchainServiceImpl; } /** @@ -1075,6 +1024,9 @@ public int parse( toCommandLine(); + // use terminal width for usage message + commandLine.getCommandSpec().usageMessage().autoWidth(true); + handleStableOptions(); addSubCommands(in); registerConverters(); @@ -1119,9 +1071,15 @@ public void run() { vertx = createVertx(createVertxOptions(metricsSystem.get())); validateOptions(); + configure(); + + // If we're not running against a named network, or if version compat protection has been + // explicitly enabled, perform compatibility check + VersionMetadata.versionCompatibilityChecks(versionCompatibilityProtection, dataDir()); + configureNativeLibs(); - besuController = initController(); + besuController = buildController(); besuPluginContext.beforeExternalServices(); @@ -1143,7 +1101,7 @@ public void run() { } @VisibleForTesting - void setBesuConfiguration(final BesuConfiguration pluginCommonConfiguration) { + void setBesuConfiguration(final BesuConfigurationImpl pluginCommonConfiguration) { this.pluginCommonConfiguration = pluginCommonConfiguration; } @@ -1235,7 +1193,10 @@ private void preparePlugins() { besuPluginContext.addService( TransactionSelectionService.class, transactionSelectionServiceImpl); besuPluginContext.addService( - PluginTransactionValidatorService.class, transactionValidatorServiceImpl); + TransactionPoolValidatorService.class, transactionValidatorServiceImpl); + besuPluginContext.addService( + TransactionSimulationService.class, transactionSimulationServiceImpl); + besuPluginContext.addService(BlockchainService.class, blockchainServiceImpl); // register built-in plugins rocksDBPlugin = new RocksDBPlugin(); @@ -1316,6 +1277,16 @@ private Runner buildRunner() { } private void startPlugins() { + blockchainServiceImpl.init( + besuController.getProtocolContext(), besuController.getProtocolSchedule()); + transactionSimulationServiceImpl.init( + besuController.getProtocolContext().getBlockchain(), + new TransactionSimulator( + besuController.getProtocolContext().getBlockchain(), + besuController.getProtocolContext().getWorldStateArchive(), + besuController.getProtocolSchedule(), + apiConfiguration.getGasCap())); + besuPluginContext.addService( BesuEvents.class, new BesuEventsImpl( @@ -1325,10 +1296,6 @@ private void startPlugins() { besuController.getSyncState())); besuPluginContext.addService(MetricsSystem.class, getMetricsSystem()); - besuPluginContext.addService( - BlockchainService.class, - new BlockchainServiceImpl(besuController.getProtocolContext().getBlockchain())); - besuPluginContext.addService( TraceService.class, new TraceServiceImpl( @@ -1481,10 +1448,15 @@ private void validateOptions() { validateTransactionPoolOptions(); validateDataStorageOptions(); validateGraphQlOptions(); + validateApiOptions(); p2pTLSConfigOptions.checkP2PTLSOptionsDependencies(logger, commandLine); pkiBlockCreationOptions.checkPkiBlockCreationOptionsDependencies(logger, commandLine); } + private void validateApiOptions() { + apiConfigurationOptions.validate(commandLine, logger); + } + private void validateTransactionPoolOptions() { transactionPoolOptions.validate(commandLine, getActualGenesisConfigOptions()); } @@ -1568,31 +1540,13 @@ private void validateDnsOptionsParams() { } } - private void checkApiOptionsDependencies() { - CommandLineUtils.checkOptionDependencies( - logger, - commandLine, - "--api-gas-and-priority-fee-limiting-enabled", - !apiGasAndPriorityFeeLimitingEnabled, - asList( - "--api-gas-and-priority-fee-upper-bound-coefficient", - "--api-gas-and-priority-fee-lower-bound-coefficient")); - } - private void ensureValidPeerBoundParams() { maxPeers = p2PDiscoveryOptionGroup.maxPeers; - peersLowerBound = unstableNetworkingOptions.toDomainObject().getPeerLowerBound(); - if (peersLowerBound > maxPeers) { - logger.warn( - "`--Xp2p-peer-lower-bound` " - + peersLowerBound - + " must not exceed --max-peers " - + maxPeers); - logger.warn("setting --Xp2p-peer-lower-bound=" + maxPeers); - peersLowerBound = maxPeers; - } final Boolean isLimitRemoteWireConnectionsEnabled = p2PDiscoveryOptionGroup.isLimitRemoteWireConnectionsEnabled; + if (isOptionSet(commandLine, PEER_LOWER_BOUND_FLAG)) { + logger.warn(PEER_LOWER_BOUND_FLAG + " is deprecated and will be removed soon."); + } if (isLimitRemoteWireConnectionsEnabled) { final float fraction = Fraction.fromPercentage(p2PDiscoveryOptionGroup.maxRemoteConnectionsPercentage) @@ -1600,7 +1554,7 @@ private void ensureValidPeerBoundParams() { checkState( fraction >= 0.0 && fraction <= 1.0, "Fraction of remote connections allowed must be between 0.0 and 1.0 (inclusive)."); - maxRemoteInitiatedPeers = (int) Math.floor(fraction * maxPeers); + maxRemoteInitiatedPeers = Math.round(fraction * maxPeers); } else { maxRemoteInitiatedPeers = maxPeers; } @@ -1644,6 +1598,19 @@ private GenesisConfigOptions readGenesisConfigOptions() { throw new ParameterException( this.commandLine, "Unable to load genesis file. " + e.getCause()); } + // snap and checkpoint can't be used with BFT but can for clique + if (genesisConfigOptions.isIbftLegacy() + || genesisConfigOptions.isIbft2() + || genesisConfigOptions.isQbft()) { + final String errorSuffix = "can't be used with BFT networks"; + if (SyncMode.CHECKPOINT.equals(syncMode) || SyncMode.X_CHECKPOINT.equals(syncMode)) { + throw new ParameterException( + commandLine, String.format("%s %s", "Checkpoint sync", errorSuffix)); + } + if (syncMode == SyncMode.SNAP || syncMode == SyncMode.X_SNAP) { + throw new ParameterException(commandLine, String.format("%s %s", "Snap sync", errorSuffix)); + } + } return genesisConfigOptions; } @@ -1673,8 +1640,8 @@ && isOptionSet(commandLine, "--sync-min-peers")) { CommandLineUtils.failIfOptionDoesntMeetRequirement( commandLine, - "--Xcheckpoint-post-merge-enabled can only be used with X_CHECKPOINT sync-mode", - SyncMode.X_CHECKPOINT.equals(getDefaultSyncModeIfNotSet()), + "--Xcheckpoint-post-merge-enabled can only be used with CHECKPOINT sync-mode", + SyncMode.isCheckpointSync(getDefaultSyncModeIfNotSet()), singletonList("--Xcheckpoint-post-merge-enabled")); CommandLineUtils.failIfOptionDoesntMeetRequirement( @@ -1692,24 +1659,13 @@ && isOptionSet(commandLine, "--sync-min-peers")) { "--node-private-key-file", "--security-module=" + DEFAULT_SECURITY_MODULE); } - - if (isPruningEnabled()) { - if (dataStorageOptions - .toDomainObject() - .getDataStorageFormat() - .equals(DataStorageFormat.BONSAI)) { - logger.warn("Forest pruning is ignored with Bonsai data storage format."); - } else { - logger.warn( - "Forest pruning is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format."); - } - } } private void configure() throws Exception { checkPortClash(); checkIfRequiredPortsAreAvailable(); syncMode = getDefaultSyncModeIfNotSet(); + versionCompatibilityProtection = getDefaultVersionCompatibilityProtectionIfNotSet(); ethNetworkConfig = updateNetworkConfig(network); @@ -1739,7 +1695,8 @@ private void configure() throws Exception { unstableIpcOptions.isEnabled(), unstableIpcOptions.getIpcPath(), unstableIpcOptions.getRpcIpcApis()); - apiConfiguration = apiConfiguration(); + apiConfiguration = apiConfigurationOptions.apiConfiguration(getMiningParameters()); + dataStorageConfiguration = getDataStorageConfiguration(); // hostsWhitelist is a hidden option. If it is specified, add the list to hostAllowlist if (!hostsWhitelist.isEmpty()) { // if allowlist == default values, remove the default values @@ -1802,10 +1759,6 @@ private void ensureAllNodesAreInAllowlist( } } - private BesuController initController() { - return buildController(); - } - /** * Builds BesuController * @@ -1827,6 +1780,11 @@ public BesuController buildController() { * @return instance of BesuControllerBuilder */ public BesuControllerBuilder getControllerBuilder() { + pluginCommonConfiguration.init( + dataDir(), + dataDir().resolve(DATABASE_PATH), + getDataStorageConfiguration(), + getMiningParameters()); final KeyValueStorageProvider storageProvider = keyValueStorageProvider(keyValueStorageName); return controllerBuilderFactory .fromEthNetworkConfig( @@ -1834,9 +1792,8 @@ public BesuControllerBuilder getControllerBuilder() { .synchronizerConfiguration(buildSyncConfig()) .ethProtocolConfiguration(unstableEthProtocolOptions.toDomainObject()) .networkConfiguration(unstableNetworkingOptions.toDomainObject()) - .transactionSelectorFactory(getTransactionSelectorFactory()) - .pluginTransactionValidatorFactory(getPluginTransactionValidatorFactory()) .dataDirectory(dataDir()) + .dataStorageConfiguration(getDataStorageConfiguration()) .miningParameters(getMiningParameters()) .transactionPoolConfiguration(buildTransactionPoolConfiguration()) .nodeKey(new NodeKey(securityModule())) @@ -1847,9 +1804,6 @@ public BesuControllerBuilder getControllerBuilder() { .clock(Clock.systemUTC()) .isRevertReasonEnabled(isRevertReasonEnabled) .storageProvider(storageProvider) - .isPruningEnabled(isPruningEnabled()) - .pruningConfiguration( - new PrunerConfiguration(pruningBlockConfirmations, pruningBlocksRetained)) .genesisConfigOverrides(genesisConfigOverrides) .gasLimitCalculator( getMiningParameters().getTargetGasLimit().isPresent() @@ -1858,28 +1812,13 @@ public BesuControllerBuilder getControllerBuilder() { .requiredBlocks(requiredBlocks) .reorgLoggingThreshold(reorgLoggingThreshold) .evmConfiguration(unstableEvmOptions.toDomainObject()) - .dataStorageConfiguration(dataStorageOptions.toDomainObject()) .maxPeers(p2PDiscoveryOptionGroup.maxPeers) - .lowerBoundPeers(peersLowerBound) .maxRemotelyInitiatedPeers(maxRemoteInitiatedPeers) .randomPeerPriority(p2PDiscoveryOptionGroup.randomPeerPriority) .chainPruningConfiguration(unstableChainPruningOptions.toDomainObject()) .cacheLastBlocks(numberOfblocksToCache); } - @NotNull - private Optional getTransactionSelectorFactory() { - final Optional txSelectionService = - besuPluginContext.getService(TransactionSelectionService.class); - return txSelectionService.isPresent() ? txSelectionService.get().get() : Optional.empty(); - } - - private PluginTransactionValidatorFactory getPluginTransactionValidatorFactory() { - final Optional txSValidatorService = - besuPluginContext.getService(PluginTransactionValidatorService.class); - return txSValidatorService.map(PluginTransactionValidatorService::get).orElse(null); - } - private JsonRpcConfiguration createEngineJsonRpcConfiguration( final Integer engineListenPort, final List allowCallsFrom) { jsonRpcHttpOptions.checkDependencies(logger, commandLine); @@ -1917,32 +1856,6 @@ private void checkPrivacyTlsOptionsDependencies() { "--privacy-tls-known-enclave-file")); } - private ApiConfiguration apiConfiguration() { - checkApiOptionsDependencies(); - var builder = - ImmutableApiConfiguration.builder() - .gasPriceBlocks(apiGasPriceBlocks) - .gasPricePercentile(apiGasPricePercentile) - .gasPriceMinSupplier( - getMiningParameters().getMinTransactionGasPrice().getAsBigInteger()::longValueExact) - .gasPriceMax(apiGasPriceMax) - .maxLogsRange(rpcMaxLogsRange) - .gasCap(rpcGasCap) - .isGasAndPriorityFeeLimitingEnabled(apiGasAndPriorityFeeLimitingEnabled) - .maxTraceFilterRange(maxTraceFilterRange); - if (apiGasAndPriorityFeeLimitingEnabled) { - if (apiGasAndPriorityFeeLowerBoundCoefficient > apiGasAndPriorityFeeUpperBoundCoefficient) { - throw new ParameterException( - this.commandLine, - "--api-gas-and-priority-fee-lower-bound-coefficient cannot be greater than the value of --api-gas-and-priority-fee-upper-bound-coefficient"); - } - builder - .lowerBoundGasAndPriorityFeeCoefficient(apiGasAndPriorityFeeLowerBoundCoefficient) - .upperBoundGasAndPriorityFeeCoefficient(apiGasAndPriorityFeeUpperBoundCoefficient); - } - return builder.build(); - } - /** * Metrics Configuration for Besu * @@ -2020,8 +1933,15 @@ private PrivacyParameters privacyParameters() { if (syncMode == SyncMode.FAST) { throw new ParameterException(commandLine, String.format("%s %s", "Fast sync", errorSuffix)); } - if (isPruningEnabled()) { - throw new ParameterException(commandLine, String.format("%s %s", "Pruning", errorSuffix)); + if (syncMode == SyncMode.SNAP || syncMode == SyncMode.X_SNAP) { + throw new ParameterException(commandLine, String.format("%s %s", "Snap sync", errorSuffix)); + } + if (syncMode == SyncMode.CHECKPOINT || syncMode == SyncMode.X_CHECKPOINT) { + throw new ParameterException( + commandLine, String.format("%s %s", "Checkpoint sync", errorSuffix)); + } + if (getDataStorageConfiguration().getDataStorageFormat().equals(DataStorageFormat.BONSAI)) { + throw new ParameterException(commandLine, String.format("%s %s", "Bonsai", errorSuffix)); } if (Boolean.TRUE.equals(privacyOptionGroup.isPrivacyMultiTenancyEnabled) @@ -2165,6 +2085,7 @@ private SynchronizerConfiguration buildSyncConfig() { } private TransactionPoolConfiguration buildTransactionPoolConfiguration() { + transactionPoolOptions.setPluginTransactionValidatorService(transactionValidatorServiceImpl); final var txPoolConf = transactionPoolOptions.toDomainObject(); final var txPoolConfBuilder = ImmutableTransactionPoolConfiguration.builder() @@ -2206,13 +2127,26 @@ private TransactionPoolConfiguration buildTransactionPoolConfiguration() { private MiningParameters getMiningParameters() { if (miningParameters == null) { - miningOptions.setGenesisBlockPeriodSeconds( - getGenesisBlockPeriodSeconds(getActualGenesisConfigOptions())); + miningOptions.setTransactionSelectionService(transactionSelectionServiceImpl); miningParameters = miningOptions.toDomainObject(); + getGenesisBlockPeriodSeconds(getActualGenesisConfigOptions()) + .ifPresent(miningParameters::setBlockPeriodSeconds); + initMiningParametersMetrics(miningParameters); } return miningParameters; } + private DataStorageConfiguration getDataStorageConfiguration() { + if (dataStorageConfiguration == null) { + dataStorageConfiguration = dataStorageOptions.toDomainObject(); + } + return dataStorageConfiguration; + } + + private void initMiningParametersMetrics(final MiningParameters miningParameters) { + new MiningParametersMetrics(getMetricsSystem(), miningParameters); + } + private OptionalInt getGenesisBlockPeriodSeconds( final GenesisConfigOptions genesisConfigOptions) { if (genesisConfigOptions.isClique()) { @@ -2230,10 +2164,6 @@ private OptionalInt getGenesisBlockPeriodSeconds( return OptionalInt.empty(); } - private boolean isPruningEnabled() { - return pruningEnabled; - } - // Blockchain synchronization from peers. private Runner synchronize( final BesuController controller, @@ -2563,11 +2493,12 @@ protected void checkIfRequiredPortsAreAvailable() { .forEach( port -> { if (port.equals(p2PDiscoveryOptionGroup.p2pPort) - && !NetworkUtility.isPortAvailable(port)) { + && (NetworkUtility.isPortUnavailableForTcp(port) + || NetworkUtility.isPortUnavailableForUdp(port))) { unavailablePorts.add(port); } if (!port.equals(p2PDiscoveryOptionGroup.p2pPort) - && !NetworkUtility.isPortAvailableForTcp(port)) { + && NetworkUtility.isPortUnavailableForTcp(port)) { unavailablePorts.add(port); } }); @@ -2623,22 +2554,14 @@ String getLogLevel() { return loggingLevelOption.getLogLevel(); } - private class BesuCommandConfigurationService implements BesuConfiguration { - - @Override - public Path getStoragePath() { - return dataDir().resolve(DATABASE_PATH); - } - - @Override - public Path getDataPath() { - return dataDir(); - } - - @Override - public int getDatabaseVersion() { - return dataStorageOptions.toDomainObject().getDataStorageFormat().getDatabaseVersion(); - } + /** + * Returns the flag indicating that version compatibility checks will be made. + * + * @return true if compatibility checks should be made, otherwise false + */ + @VisibleForTesting + public Boolean getVersionCompatibilityProtection() { + return versionCompatibilityProtection; } private void instantiateSignatureAlgorithmFactory() { @@ -2657,10 +2580,7 @@ private void instantiateSignatureAlgorithmFactory() { SignatureAlgorithmFactory.setInstance(SignatureAlgorithmType.create(ecCurve.get())); } catch (final IllegalArgumentException e) { throw new CommandLine.InitializationException( - new StringBuilder() - .append("Invalid genesis file configuration for ecCurve. ") - .append(e.getMessage()) - .toString()); + "Invalid genesis file configuration for ecCurve. " + e.getMessage()); } } @@ -2744,11 +2664,17 @@ private SyncMode getDefaultSyncModeIfNotSet() { .orElse( genesisFile == null && !privacyOptionGroup.isPrivacyEnabled - && Optional.ofNullable(network).map(NetworkName::canFastSync).orElse(false) - ? SyncMode.FAST + && Optional.ofNullable(network).map(NetworkName::canSnapSync).orElse(false) + ? SyncMode.SNAP : SyncMode.FULL); } + private Boolean getDefaultVersionCompatibilityProtectionIfNotSet() { + // Version compatibility protection is enabled by default for non-named networks + return Optional.ofNullable(versionCompatibilityProtection) + .orElse(commandLine.getParseResult().hasMatchedOption("network") ? false : true); + } + private String generateConfigurationOverview() { final ConfigurationOverviewBuilder builder = new ConfigurationOverviewBuilder(logger); @@ -2799,14 +2725,15 @@ private String generateConfigurationOverview() { builder.setHighSpecEnabled(); } - if (dataStorageOptions.toDomainObject().getUnstable().getBonsaiLimitTrieLogsEnabled()) { + if (getDataStorageConfiguration().getUnstable().getBonsaiLimitTrieLogsEnabled()) { builder.setLimitTrieLogsEnabled(); - builder.setTrieLogRetentionLimit( - dataStorageOptions.toDomainObject().getBonsaiMaxLayersToLoad()); + builder.setTrieLogRetentionLimit(getDataStorageConfiguration().getBonsaiMaxLayersToLoad()); builder.setTrieLogsPruningWindowSize( - dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningWindowSize()); + getDataStorageConfiguration().getUnstable().getBonsaiTrieLogPruningWindowSize()); } + builder.setSnapServerEnabled(this.unstableSynchronizerOptions.isSnapsyncServerEnabled()); + builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation()); builder.setWorldStateUpdateMode(unstableEvmOptions.toDomainObject().worldUpdaterMode()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java index bf03c675d8d..2b3dfd7dd51 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java @@ -55,6 +55,7 @@ public class ConfigurationOverviewBuilder { private boolean isBonsaiLimitTrieLogsEnabled = false; private long trieLogRetentionLimit = 0; private Integer trieLogsPruningWindowSize = null; + private boolean isSnapServerEnabled = false; private TransactionPoolConfiguration.Implementation txPoolImplementation; private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode; private Map environment; @@ -219,6 +220,17 @@ public ConfigurationOverviewBuilder setTrieLogRetentionLimit(final long limit) { return this; } + /** + * Sets snap server enabled/disabled + * + * @param snapServerEnabled bool to indicate if snap server is enabled + * @return the builder + */ + public ConfigurationOverviewBuilder setSnapServerEnabled(final boolean snapServerEnabled) { + isSnapServerEnabled = snapServerEnabled; + return this; + } + /** * Sets trie logs pruning window size * @@ -339,6 +351,10 @@ public String build() { lines.add("Using " + worldStateUpdateMode + " worldstate update mode"); + if (isSnapServerEnabled) { + lines.add("Experimental Snap Sync server enabled"); + } + if (isBonsaiLimitTrieLogsEnabled) { final StringBuilder trieLogPruningString = new StringBuilder(); trieLogPruningString diff --git a/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java b/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java index 3a0db478160..e62d76baf38 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java @@ -74,8 +74,6 @@ public interface DefaultCommandValues { int SYNC_MIN_PEER_COUNT = 5; /** The constant DEFAULT_MAX_PEERS. */ int DEFAULT_MAX_PEERS = 25; - /** The constant DEFAULT_P2P_PEER_LOWER_BOUND. */ - int DEFAULT_P2P_PEER_LOWER_BOUND = 25; /** The constant DEFAULT_HTTP_MAX_CONNECTIONS. */ int DEFAULT_HTTP_MAX_CONNECTIONS = 80; /** The constant DEFAULT_HTTP_MAX_BATCH_SIZE. */ diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java b/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java index 6638ddfa44e..2e7050c3946 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.cli.config; import java.math.BigInteger; +import java.util.Locale; import java.util.Optional; import org.apache.commons.lang3.StringUtils; @@ -29,6 +30,7 @@ public enum NetworkName { GOERLI("/goerli.json", BigInteger.valueOf(5)), /** Holešky network name. */ HOLESKY("/holesky.json", BigInteger.valueOf(17000)), + KAUSTINEN("/kaustinen.json", BigInteger.valueOf(69420)), /** Dev network name. */ DEV("/dev.json", BigInteger.valueOf(2018), false), @@ -43,17 +45,17 @@ public enum NetworkName { private final String genesisFile; private final BigInteger networkId; - private final boolean canFastSync; + private final boolean canSnapSync; private final String deprecationDate; NetworkName(final String genesisFile, final BigInteger networkId) { this(genesisFile, networkId, true); } - NetworkName(final String genesisFile, final BigInteger networkId, final boolean canFastSync) { + NetworkName(final String genesisFile, final BigInteger networkId, final boolean canSnapSync) { this.genesisFile = genesisFile; this.networkId = networkId; - this.canFastSync = canFastSync; + this.canSnapSync = canSnapSync; // no deprecations planned this.deprecationDate = null; } @@ -77,12 +79,12 @@ public BigInteger getNetworkId() { } /** - * Can fast sync boolean. + * Can SNAP sync boolean. * * @return the boolean */ - public boolean canFastSync() { - return canFastSync; + public boolean canSnapSync() { + return canSnapSync; } /** @@ -91,7 +93,7 @@ public boolean canFastSync() { * @return the string */ public String normalize() { - return StringUtils.capitalize(name().toLowerCase()); + return StringUtils.capitalize(name().toLowerCase(Locale.ROOT)); } /** diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java index 823dd9f142c..111261d5c80 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java @@ -14,9 +14,21 @@ */ package org.hyperledger.besu.cli.config; +import java.util.Locale; + +import org.apache.commons.lang3.StringUtils; + /** Enum for profile names. Each profile corresponds to a configuration file. */ public enum ProfileName { - /** The 'DEV' profile. Corresponds to the 'profiles/dev.toml' configuration file. */ + /** The 'STAKER' profile */ + STAKER("profiles/staker.toml"), + /** The 'MINIMALIST_STAKER' profile */ + MINIMALIST_STAKER("profiles/minimalist-staker.toml"), + /** The 'ENTERPRISE' profile */ + ENTERPRISE("profiles/enterprise-private.toml"), + /** The 'PRIVATE' profile */ + PRIVATE("profiles/enterprise-private.toml"), + /** The 'DEV' profile. */ DEV("profiles/dev.toml"); private final String configFile; @@ -38,4 +50,9 @@ public enum ProfileName { public String getConfigFile() { return configFile; } + + @Override + public String toString() { + return StringUtils.capitalize(name().replaceAll("_", " ").toLowerCase(Locale.ROOT)); + } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java b/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java index 684c3ce30c0..91ca381b50d 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java @@ -18,6 +18,7 @@ import java.util.EnumSet; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import com.google.common.annotations.VisibleForTesting; @@ -54,7 +55,7 @@ public & MetricCategory> void addCategories(final Class ca * @param metricCategory the metric category */ public void addRegistryCategory(final MetricCategory metricCategory) { - metricCategories.put(metricCategory.getName().toUpperCase(), metricCategory); + metricCategories.put(metricCategory.getName().toUpperCase(Locale.ROOT), metricCategory); } /** diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java index 2e5a239f16d..bcde10077bb 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.cli.options; +import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_NON_POA_BLOCK_TXS_SELECTION_MAX_TIME; @@ -37,10 +38,10 @@ import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters; import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues; import org.hyperledger.besu.ethereum.core.MiningParameters; +import org.hyperledger.besu.plugin.services.TransactionSelectionService; import org.hyperledger.besu.util.number.PositiveNumber; import java.util.List; -import java.util.OptionalInt; import org.apache.tuweni.bytes.Bytes; import org.slf4j.Logger; @@ -189,7 +190,7 @@ static class Unstable { DEFAULT_POS_BLOCK_CREATION_REPETITION_MIN_DURATION; } - private OptionalInt maybeGenesisBlockPeriodSeconds; + private TransactionSelectionService transactionSelectionService; private MiningOptions() {} @@ -203,13 +204,13 @@ public static MiningOptions create() { } /** - * Set the optional genesis block period per seconds + * Set the transaction selection service * - * @param genesisBlockPeriodSeconds if the network is PoA then the block period in seconds - * specified in the genesis file, otherwise empty. + * @param transactionSelectionService the transaction selection service */ - public void setGenesisBlockPeriodSeconds(final OptionalInt genesisBlockPeriodSeconds) { - maybeGenesisBlockPeriodSeconds = genesisBlockPeriodSeconds; + public void setTransactionSelectionService( + final TransactionSelectionService transactionSelectionService) { + this.transactionSelectionService = transactionSelectionService; } /** @@ -298,7 +299,7 @@ public void validate( static MiningOptions fromConfig(final MiningParameters miningParameters) { final MiningOptions miningOptions = MiningOptions.create(); - miningOptions.setGenesisBlockPeriodSeconds(miningParameters.getGenesisBlockPeriodSeconds()); + miningOptions.setTransactionSelectionService(miningParameters.getTransactionSelectionService()); miningOptions.isMiningEnabled = miningParameters.isMiningEnabled(); miningOptions.iStratumMiningEnabled = miningParameters.isStratumMiningEnabled(); miningOptions.stratumNetworkInterface = miningParameters.getStratumNetworkInterface(); @@ -333,10 +334,9 @@ static MiningOptions fromConfig(final MiningParameters miningParameters) { @Override public MiningParameters toDomainObject() { - if (maybeGenesisBlockPeriodSeconds == null) { - throw new IllegalStateException( - "genesisBlockPeriodSeconds must be set before using this object"); - } + checkNotNull( + transactionSelectionService, + "transactionSelectionService must be set before using this object"); final var updatableInitValuesBuilder = MutableInitValues.builder() @@ -354,7 +354,7 @@ public MiningParameters toDomainObject() { } return ImmutableMiningParameters.builder() - .genesisBlockPeriodSeconds(maybeGenesisBlockPeriodSeconds) + .transactionSelectionService(transactionSelectionService) .mutableInitValues(updatableInitValuesBuilder.build()) .isStratumMiningEnabled(iStratumMiningEnabled) .stratumNetworkInterface(stratumNetworkInterface) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/TransactionPoolOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/TransactionPoolOptions.java index 0b6718387b1..0ededa3c5a5 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/TransactionPoolOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/TransactionPoolOptions.java @@ -30,6 +30,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.plugin.services.TransactionPoolValidatorService; import org.hyperledger.besu.util.number.Fraction; import org.hyperledger.besu.util.number.Percentage; @@ -51,12 +52,15 @@ public class TransactionPoolOptions implements CLIOptions description = "Format to store trie data in. Either FOREST or BONSAI (default: ${DEFAULT-VALUE}).", arity = "1") - private DataStorageFormat dataStorageFormat = DataStorageFormat.FOREST; + private DataStorageFormat dataStorageFormat = DataStorageFormat.BONSAI; @Option( names = {BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"}, @@ -60,6 +63,12 @@ public class DataStorageOptions implements CLIOptions arity = "1") private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; + @Option( + names = "--receipt-compaction-enabled", + description = "Enables compact storing of receipts (default: ${DEFAULT-VALUE}).", + arity = "1") + private Boolean receiptCompactionEnabled = DEFAULT_RECEIPT_COMPACTION_ENABLED; + @CommandLine.ArgGroup(validate = false) private final DataStorageOptions.Unstable unstableOptions = new Unstable(); @@ -74,7 +83,7 @@ public static class Unstable { @CommandLine.Option( hidden = true, - names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED}, + names = {BONSAI_LIMIT_TRIE_LOGS_ENABLED, "--Xbonsai-trie-log-pruning-enabled"}, description = "Limit the number of trie logs that are retained. (default: ${DEFAULT-VALUE})") private boolean bonsaiLimitTrieLogsEnabled = DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; @@ -85,6 +94,14 @@ public static class Unstable { description = "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})") private int bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; + + @CommandLine.Option( + hidden = true, + names = {"--Xbonsai-code-using-code-hash-enabled"}, + arity = "1", + description = + "Enables code storage using code hash instead of by account hash. (default: ${DEFAULT-VALUE})") + private boolean bonsaiCodeUsingCodeHashEnabled = DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED; } /** * Create data storage options. @@ -130,14 +147,23 @@ public void validate(final CommandLine commandLine) { } } - static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject) { + /** + * Converts to options from the configuration + * + * @param domainObject to be reversed + * @return the options that correspond to the configuration + */ + public static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject) { final DataStorageOptions dataStorageOptions = DataStorageOptions.create(); dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat(); dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad(); + dataStorageOptions.receiptCompactionEnabled = domainObject.getReceiptCompactionEnabled(); dataStorageOptions.unstableOptions.bonsaiLimitTrieLogsEnabled = domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled(); dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize = domainObject.getUnstable().getBonsaiTrieLogPruningWindowSize(); + dataStorageOptions.unstableOptions.bonsaiCodeUsingCodeHashEnabled = + domainObject.getUnstable().getBonsaiCodeStoredByCodeHashEnabled(); return dataStorageOptions; } @@ -147,10 +173,12 @@ public DataStorageConfiguration toDomainObject() { return ImmutableDataStorageConfiguration.builder() .dataStorageFormat(dataStorageFormat) .bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad) + .receiptCompactionEnabled(receiptCompactionEnabled) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() .bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled) .bonsaiTrieLogPruningWindowSize(unstableOptions.bonsaiTrieLogPruningWindowSize) + .bonsaiCodeStoredByCodeHashEnabled(unstableOptions.bonsaiCodeUsingCodeHashEnabled) .build()) .build(); } @@ -166,6 +194,6 @@ public List getCLIOptions() { * @return the normalized string */ public String normalizeDataStorageFormat() { - return StringUtils.capitalize(dataStorageFormat.toString().toLowerCase()); + return StringUtils.capitalize(dataStorageFormat.toString().toLowerCase(Locale.ROOT)); } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java index ad119ea61a0..ce5198084cf 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.cli.options.stable; +import java.util.Locale; import java.util.Set; import picocli.CommandLine; @@ -52,8 +53,8 @@ public void setLogLevel(final String logLevel) { if ("FATAL".equalsIgnoreCase(logLevel)) { System.out.println("FATAL level is deprecated"); this.logLevel = "ERROR"; - } else if (ACCEPTED_VALUES.contains(logLevel.toUpperCase())) { - this.logLevel = logLevel.toUpperCase(); + } else if (ACCEPTED_VALUES.contains(logLevel.toUpperCase(Locale.ROOT))) { + this.logLevel = logLevel.toUpperCase(Locale.ROOT); } else { throw new CommandLine.ParameterException( spec.commandLine(), "Unknown logging value: " + logLevel); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java index 82dffbbe136..35e37d06ddc 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/NetworkingOptions.java @@ -80,8 +80,8 @@ public class NetworkingOptions implements CLIOptions { hidden = true, names = PEER_LOWER_BOUND_FLAG, description = - "Lower bound on the target number of P2P connections (default: ${DEFAULT-VALUE})") - private Integer peerLowerBoundConfig = DefaultCommandValues.DEFAULT_P2P_PEER_LOWER_BOUND; + "(Deprecated) Lower bound on the target number of P2P connections (default: ${DEFAULT-VALUE})") + private final Integer peerLowerBoundConfig = DefaultCommandValues.DEFAULT_MAX_PEERS; private NetworkingOptions() {} @@ -107,7 +107,6 @@ public static NetworkingOptions fromConfig(final NetworkingConfiguration network cliOptions.initiateConnectionsFrequencySec = networkingConfig.getInitiateConnectionsFrequencySec(); cliOptions.dnsDiscoveryServerOverride = networkingConfig.getDnsDiscoveryServerOverride(); - cliOptions.peerLowerBoundConfig = networkingConfig.getPeerLowerBound(); return cliOptions; } @@ -120,7 +119,6 @@ public NetworkingConfiguration toDomainObject() { config.setDnsDiscoveryServerOverride(dnsDiscoveryServerOverride); config.getDiscovery().setDiscoveryV5Enabled(isPeerDiscoveryV5Enabled); config.getDiscovery().setFilterOnEnrForkId(filterOnEnrForkId); - config.setPeerLowerBound(peerLowerBoundConfig); return config; } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java index 8760258fba2..da5199f85dc 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java @@ -82,6 +82,8 @@ public class SynchronizerOptions implements CLIOptions