diff --git a/CHANGELOG.md b/CHANGELOG.md index 1602655f1dd..8eee086d0e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,11 +30,11 @@ - Moving trielog storage to RocksDB's blobdb to improve write amplications [#6289](https://github.com/hyperledger/besu/pull/6289) - Support for `shanghaiTime` fork and Shanghai EVM smart contracts in QBFT/IBFT chains [#6353](https://github.com/hyperledger/besu/pull/6353) - Change ExecutionHaltReason for contract creation collision case to return ILLEGAL_STATE_CHANGE [#6518](https://github.com/hyperledger/besu/pull/6518) +- Experimental feature `--Xbonsai-code-using-code-hash-enabled` for storing Bonsai code storage by code hash [#6505](https://github.com/hyperledger/besu/pull/6505) ### Bug fixes - Fix the way an advertised host configured with `--p2p-host` is treated when communicating with the originator of a PING packet [#6225](https://github.com/hyperledger/besu/pull/6225) - Fix `poa-block-txs-selection-max-time` option that was inadvertently reset to its default after being configured [#6444](https://github.com/hyperledger/besu/pull/6444) -- Fix ETC Spiral upgrade breach of consensus [#6524](https://github.com/hyperledger/besu/pull/6524) ### Download Links diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index 72331394889..e2f6d5fcfab 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.cli.options.stable; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT; @@ -85,6 +86,14 @@ public static class Unstable { description = "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})") private int bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; + + @CommandLine.Option( + hidden = true, + names = {"--Xbonsai-code-using-code-hash-enabled"}, + arity = "1", + description = + "Enables code storage using code hash instead of by account hash. (default: ${DEFAULT-VALUE})") + private boolean bonsaiCodeUsingCodeHashEnabled = DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED; } /** * Create data storage options. @@ -138,6 +147,8 @@ static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled(); dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize = domainObject.getUnstable().getBonsaiTrieLogPruningWindowSize(); + dataStorageOptions.unstableOptions.bonsaiCodeUsingCodeHashEnabled = + domainObject.getUnstable().getBonsaiCodeStoredByCodeHashEnabled(); return dataStorageOptions; } @@ -151,6 +162,7 @@ public DataStorageConfiguration toDomainObject() { ImmutableDataStorageConfiguration.Unstable.builder() .bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled) .bonsaiTrieLogPruningWindowSize(unstableOptions.bonsaiTrieLogPruningWindowSize) + .bonsaiCodeStoredByCodeHashEnabled(unstableOptions.bonsaiCodeUsingCodeHashEnabled) .build()) .build(); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index 8aed9ee2d70..8dc2ef87803 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -88,6 +88,28 @@ public void bonsaiTrieLogRetentionLimitShouldBeAboveMinimum() { "511"); } + @Test + public void bonsaiCodeUsingCodeHashEnabledCanBeEnabled() { + internalTestSuccess( + dataStorageConfiguration -> + assertThat( + dataStorageConfiguration.getUnstable().getBonsaiCodeStoredByCodeHashEnabled()) + .isEqualTo(true), + "--Xbonsai-code-using-code-hash-enabled", + "true"); + } + + @Test + public void bonsaiCodeUsingCodeHashEnabledCanBeDisabled() { + internalTestSuccess( + dataStorageConfiguration -> + assertThat( + dataStorageConfiguration.getUnstable().getBonsaiCodeStoredByCodeHashEnabled()) + .isEqualTo(false), + "--Xbonsai-code-using-code-hash-enabled", + "false"); + } + @Override protected DataStorageConfiguration createDefaultDomainObject() { return DataStorageConfiguration.DEFAULT_CONFIG; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index b9643cdc49b..c357f1d56c2 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -79,7 +79,7 @@ public Optional getAccount(final Hash accountHash) { } @Override - public Optional getCode(final Bytes32 codeHash, final Hash accountHash) { + public Optional getCode(final Hash codeHash, final Hash accountHash) { return isClosedGet() ? Optional.empty() : super.getCode(codeHash, accountHash); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index efe97f32c60..f950e7c9163 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -110,7 +110,7 @@ public FlatDbMode getFlatDbMode() { } @Override - public Optional getCode(final Bytes32 codeHash, final Hash accountHash) { + public Optional getCode(final Hash codeHash, final Hash accountHash) { if (codeHash.equals(Hash.EMPTY)) { return Optional.of(Bytes.EMPTY); } else { @@ -323,7 +323,7 @@ public FlatDbStrategy getFlatDbStrategy() { } public interface BonsaiUpdater extends WorldStateStorage.Updater { - BonsaiUpdater removeCode(final Hash accountHash); + BonsaiUpdater removeCode(final Hash accountHash, final Hash codeHash); BonsaiUpdater removeAccountInfoState(final Hash accountHash); @@ -356,14 +356,14 @@ public Updater( } @Override - public BonsaiUpdater removeCode(final Hash accountHash) { - flatDbStrategy.removeFlatCode(composedWorldStateTransaction, accountHash); + public BonsaiUpdater removeCode(final Hash accountHash, final Hash codeHash) { + flatDbStrategy.removeFlatCode(composedWorldStateTransaction, accountHash, codeHash); return this; } @Override - public BonsaiUpdater putCode(final Hash accountHash, final Bytes32 codeHash, final Bytes code) { - if (code.size() == 0) { + public BonsaiUpdater putCode(final Hash accountHash, final Hash codeHash, final Bytes code) { + if (code.isEmpty()) { // Don't save empty values return this; } @@ -379,7 +379,7 @@ public BonsaiUpdater removeAccountInfoState(final Hash accountHash) { @Override public BonsaiUpdater putAccountInfoState(final Hash accountHash, final Bytes accountValue) { - if (accountValue.size() == 0) { + if (accountValue.isEmpty()) { // Don't save empty values return this; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java new file mode 100644 index 00000000000..7d77ff1a6fc --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java @@ -0,0 +1,54 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; + +public class AccountHashCodeStorageStrategy implements CodeStorageStrategy { + @Override + public Optional getFlatCode( + final Hash codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage) { + return storage + .get(CODE_STORAGE, accountHash.toArrayUnsafe()) + .map(Bytes::wrap) + .filter(b -> Hash.hash(b).equals(codeHash)); + } + + @Override + public void putFlatCode( + final SegmentedKeyValueStorageTransaction transaction, + final Hash accountHash, + final Hash codeHash, + final Bytes code) { + transaction.put(CODE_STORAGE, accountHash.toArrayUnsafe(), code.toArrayUnsafe()); + } + + @Override + public void removeFlatCode( + final SegmentedKeyValueStorageTransaction transaction, + final Hash accountHash, + final Hash codeHash) { + transaction.remove(CODE_STORAGE, accountHash.toArrayUnsafe()); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java new file mode 100644 index 00000000000..6f4d8bb8147 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java @@ -0,0 +1,55 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; + +public class CodeHashCodeStorageStrategy implements CodeStorageStrategy { + + @Override + public Optional getFlatCode( + final Hash codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage) { + return storage.get(CODE_STORAGE, codeHash.toArrayUnsafe()).map(Bytes::wrap); + } + + @Override + public void putFlatCode( + final SegmentedKeyValueStorageTransaction transaction, + final Hash accountHash, + final Hash codeHash, + final Bytes code) { + transaction.put(CODE_STORAGE, codeHash.toArrayUnsafe(), code.toArrayUnsafe()); + } + + @Override + public void removeFlatCode( + final SegmentedKeyValueStorageTransaction transaction, + final Hash accountHash, + final Hash codeHash) {} + + public static boolean isCodeHashValue(final byte[] key, final byte[] value) { + final Hash valueHash = Hash.hash(Bytes.wrap(value)); + return Bytes.wrap(key).equals(valueHash); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java new file mode 100644 index 00000000000..8767be24788 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java @@ -0,0 +1,41 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; + +public interface CodeStorageStrategy { + + Optional getFlatCode( + final Hash codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage); + + void putFlatCode( + final SegmentedKeyValueStorageTransaction transaction, + final Hash accountHash, + final Hash codeHash, + final Bytes code); + + void removeFlatCode( + final SegmentedKeyValueStorageTransaction transaction, + final Hash accountHash, + final Hash codeHash); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java index 6c1c918172b..d85f4932516 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java @@ -53,9 +53,12 @@ public abstract class FlatDbStrategy { protected final Counter getStorageValueCounter; protected final Counter getStorageValueFlatDatabaseCounter; + protected final CodeStorageStrategy codeStorageStrategy; - public FlatDbStrategy(final MetricsSystem metricsSystem) { + public FlatDbStrategy( + final MetricsSystem metricsSystem, final CodeStorageStrategy codeStorageStrategy) { this.metricsSystem = metricsSystem; + this.codeStorageStrategy = codeStorageStrategy; getAccountCounter = metricsSystem.createCounter( @@ -107,14 +110,11 @@ public abstract Optional getFlatStorageValueByStorageSlotKey( * Retrieves the code data for the given code hash and account hash. */ public Optional getFlatCode( - final Bytes32 codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage) { + final Hash codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage) { if (codeHash.equals(Hash.EMPTY)) { return Optional.of(Bytes.EMPTY); } else { - return storage - .get(CODE_STORAGE, accountHash.toArrayUnsafe()) - .map(Bytes::wrap) - .filter(b -> Hash.hash(b).equals(codeHash)); + return codeStorageStrategy.getFlatCode(codeHash, accountHash, storage); } } @@ -162,8 +162,10 @@ public void removeFlatAccountStorageValueByStorageSlotHash( * Removes code for the given account hash. */ public void removeFlatCode( - final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash) { - transaction.remove(CODE_STORAGE, accountHash.toArrayUnsafe()); + final SegmentedKeyValueStorageTransaction transaction, + final Hash accountHash, + final Hash codeHash) { + codeStorageStrategy.removeFlatCode(transaction, accountHash, codeHash); } /* @@ -172,9 +174,9 @@ public void removeFlatCode( public void putFlatCode( final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash, - final Bytes32 codeHash, + final Hash codeHash, final Bytes code) { - transaction.put(CODE_STORAGE, accountHash.toArrayUnsafe(), code.toArrayUnsafe()); + codeStorageStrategy.putFlatCode(transaction, accountHash, codeHash, code); } public void clearAll(final SegmentedKeyValueStorage storage) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java index 55f89fc663f..e251f28c175 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; @@ -24,7 +25,9 @@ import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; import java.nio.charset.StandardCharsets; +import java.util.Optional; +import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,12 +38,14 @@ public class FlatDbStrategyProvider { // 0x666C61744462537461747573 public static final byte[] FLAT_DB_MODE = "flatDbStatus".getBytes(StandardCharsets.UTF_8); private final MetricsSystem metricsSystem; + private final DataStorageConfiguration dataStorageConfiguration; protected FlatDbMode flatDbMode; protected FlatDbStrategy flatDbStrategy; public FlatDbStrategyProvider( final MetricsSystem metricsSystem, final DataStorageConfiguration dataStorageConfiguration) { this.metricsSystem = metricsSystem; + this.dataStorageConfiguration = dataStorageConfiguration; } public void loadFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStateStorage) { @@ -50,16 +55,20 @@ public void loadFlatDbStrategy(final SegmentedKeyValueStorage composedWorldState // if flatDbMode is not loaded or has changed, reload flatDbStrategy if (this.flatDbMode == null || !this.flatDbMode.equals(newFlatDbMode)) { this.flatDbMode = newFlatDbMode; + final CodeStorageStrategy codeStorageStrategy = + deriveUseCodeStorageByHash(composedWorldStateStorage) + ? new CodeHashCodeStorageStrategy() + : new AccountHashCodeStorageStrategy(); if (flatDbMode == FlatDbMode.FULL) { - this.flatDbStrategy = new FullFlatDbStrategy(metricsSystem); + this.flatDbStrategy = new FullFlatDbStrategy(metricsSystem, codeStorageStrategy); } else { - this.flatDbStrategy = new PartialFlatDbStrategy(metricsSystem); + this.flatDbStrategy = new PartialFlatDbStrategy(metricsSystem, codeStorageStrategy); } } } - private FlatDbMode deriveFlatDbStrategy( - final SegmentedKeyValueStorage composedWorldStateStorage) { + @VisibleForTesting + FlatDbMode deriveFlatDbStrategy(final SegmentedKeyValueStorage composedWorldStateStorage) { var flatDbMode = FlatDbMode.fromVersion( composedWorldStateStorage @@ -71,6 +80,37 @@ private FlatDbMode deriveFlatDbStrategy( return flatDbMode; } + private boolean deriveUseCodeStorageByHash( + final SegmentedKeyValueStorage composedWorldStateStorage) { + final boolean configCodeUsingHash = + dataStorageConfiguration.getUnstable().getBonsaiCodeStoredByCodeHashEnabled(); + boolean codeUsingCodeByHash = + detectCodeStorageByHash(composedWorldStateStorage) + .map( + dbCodeUsingHash -> { + if (dbCodeUsingHash != configCodeUsingHash) { + LOG.warn( + "Bonsai db is using code storage mode {} but config specifies mode {}. Using mode from database", + dbCodeUsingHash, + configCodeUsingHash); + } + return dbCodeUsingHash; + }) + .orElse(configCodeUsingHash); + LOG.info("Bonsai db mode with code stored using code hash enabled = {}", codeUsingCodeByHash); + return codeUsingCodeByHash; + } + + private Optional detectCodeStorageByHash( + final SegmentedKeyValueStorage composedWorldStateStorage) { + return composedWorldStateStorage.stream(CODE_STORAGE) + .limit(1) + .findFirst() + .map( + keypair -> + CodeHashCodeStorageStrategy.isCodeHashValue(keypair.getKey(), keypair.getValue())); + } + public FlatDbStrategy getFlatDbStrategy( final SegmentedKeyValueStorage composedWorldStateStorage) { if (flatDbStrategy == null) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java index 1885069e3d4..83f7f030c73 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java @@ -37,8 +37,9 @@ public class FullFlatDbStrategy extends FlatDbStrategy { protected final Counter getStorageValueNotFoundInFlatDatabaseCounter; - public FullFlatDbStrategy(final MetricsSystem metricsSystem) { - super(metricsSystem); + public FullFlatDbStrategy( + final MetricsSystem metricsSystem, final CodeStorageStrategy codeStorageStrategy) { + super(metricsSystem, codeStorageStrategy); getAccountNotFoundInFlatDatabaseCounter = metricsSystem.createCounter( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java index 632f433f9b0..cfe18dbe2ad 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java @@ -52,8 +52,9 @@ public class PartialFlatDbStrategy extends FlatDbStrategy { protected final Counter getStorageValueMerkleTrieCounter; protected final Counter getStorageValueMissingMerkleTrieCounter; - public PartialFlatDbStrategy(final MetricsSystem metricsSystem) { - super(metricsSystem); + public PartialFlatDbStrategy( + final MetricsSystem metricsSystem, final CodeStorageStrategy codeStorageStrategy) { + super(metricsSystem, codeStorageStrategy); getAccountMerkleTrieCounter = metricsSystem.createCounter( BesuMetricCategory.BLOCKCHAIN, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java index 8abcd442cfd..93c0ab30902 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java @@ -51,12 +51,14 @@ import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Stream; import javax.annotation.Nonnull; +import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.rlp.RLP; @@ -253,7 +255,8 @@ private void updateTheAccounts( } } - private void updateCode( + @VisibleForTesting + protected void updateCode( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { maybeStateUpdater.ifPresent( @@ -262,15 +265,29 @@ private void updateCode( worldStateUpdater.getCodeToUpdate().entrySet()) { final Bytes updatedCode = codeUpdate.getValue().getUpdated(); final Hash accountHash = codeUpdate.getKey().addressHash(); - if (updatedCode == null || updatedCode.isEmpty()) { - bonsaiUpdater.removeCode(accountHash); + final Bytes priorCode = codeUpdate.getValue().getPrior(); + + // code hasn't changed then do nothing + if (Objects.equals(priorCode, updatedCode) + || (codeIsEmpty(priorCode) && codeIsEmpty(updatedCode))) { + continue; + } + + if (codeIsEmpty(updatedCode)) { + final Hash priorCodeHash = Hash.hash(priorCode); + bonsaiUpdater.removeCode(accountHash, priorCodeHash); } else { - bonsaiUpdater.putCode(accountHash, null, updatedCode); + final Hash codeHash = Hash.hash(codeUpdate.getValue().getUpdated()); + bonsaiUpdater.putCode(accountHash, codeHash, updatedCode); } } }); } + private boolean codeIsEmpty(final Bytes value) { + return value == null || value.isEmpty(); + } + private void updateAccountStorageState( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestWorldStateKeyValueStorage.java index f53d3e53ce2..294164e3d9b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestWorldStateKeyValueStorage.java @@ -51,7 +51,7 @@ public DataStorageFormat getDataStorageFormat() { } @Override - public Optional getCode(final Bytes32 codeHash, final Hash accountHash) { + public Optional getCode(final Hash codeHash, final Hash accountHash) { if (codeHash.equals(Hash.EMPTY)) { return Optional.of(Bytes.EMPTY); } else { @@ -172,7 +172,7 @@ public Updater( @Override public WorldStateStorage.Updater putCode( - final Hash accountHash, final Bytes32 codeHash, final Bytes code) { + final Hash accountHash, final Hash codeHash, final Bytes code) { if (code.size() == 0) { // Don't save empty values return this; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java index fba4420226d..548ef60b6de 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java @@ -52,6 +52,7 @@ interface Unstable { boolean DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED = false; long MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; int DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE = 30_000; + boolean DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED = false; DataStorageConfiguration.Unstable DEFAULT = ImmutableDataStorageConfiguration.Unstable.builder().build(); @@ -65,5 +66,10 @@ default boolean getBonsaiLimitTrieLogsEnabled() { default int getBonsaiTrieLogPruningWindowSize() { return DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; } + + @Value.Default + default boolean getBonsaiCodeStoredByCodeHashEnabled() { + return DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED; + } } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java index 3bb6a0dfe78..c37b6334b7e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java @@ -27,7 +27,7 @@ public interface WorldStateStorage { - Optional getCode(Bytes32 codeHash, Hash accountHash); + Optional getCode(Hash codeHash, Hash accountHash); Optional getAccountStateTrieNode(Bytes location, Bytes32 nodeHash); @@ -98,7 +98,7 @@ default Map streamFlatStorages( interface Updater { - Updater putCode(Hash accountHash, Bytes32 nodeHash, Bytes code); + Updater putCode(Hash accountHash, Hash nodeHash, Bytes code); default Updater putCode(final Hash accountHash, final Bytes code) { // Skip the hash calculation for empty code diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java index b3dfbd32b05..fbdfa7724f7 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java @@ -17,6 +17,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -37,7 +38,9 @@ import org.hyperledger.besu.ethereum.trie.StorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; @@ -59,27 +62,45 @@ public class BonsaiWorldStateKeyValueStorageTest { - public static Collection data() { + public static Collection flatDbMode() { return Arrays.asList(new Object[][] {{FlatDbMode.FULL}, {FlatDbMode.PARTIAL}}); } - final BonsaiWorldStateKeyValueStorage storage = emptyStorage(); + public static Collection flatDbModeAndCodeStorageMode() { + return Arrays.asList( + new Object[][] { + {FlatDbMode.FULL, false}, + {FlatDbMode.PARTIAL, false}, + {FlatDbMode.FULL, true}, + {FlatDbMode.PARTIAL, true} + }); + } + + BonsaiWorldStateKeyValueStorage storage; public void setUp(final FlatDbMode flatDbMode) { + storage = emptyStorage(); + if (flatDbMode.equals(FlatDbMode.FULL)) { + storage.upgradeToFullFlatDbMode(); + } + } + + public void setUp(final FlatDbMode flatDbMode, final boolean useCodeHashStorage) { + storage = emptyStorage(useCodeHashStorage); if (flatDbMode.equals(FlatDbMode.FULL)) { storage.upgradeToFullFlatDbMode(); } } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getCode_returnsEmpty(final FlatDbMode flatDbMode) { setUp(flatDbMode); assertThat(storage.getCode(Hash.EMPTY, Hash.EMPTY)).contains(Bytes.EMPTY); } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getAccountStateTrieNode_returnsEmptyNode(final FlatDbMode flatDbMode) { setUp(flatDbMode); assertThat(storage.getAccountStateTrieNode(Bytes.EMPTY, MerkleTrie.EMPTY_TRIE_NODE_HASH)) @@ -87,7 +108,7 @@ void getAccountStateTrieNode_returnsEmptyNode(final FlatDbMode flatDbMode) { } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getAccountStorageTrieNode_returnsEmptyNode(final FlatDbMode flatDbMode) { setUp(flatDbMode); assertThat( @@ -97,23 +118,25 @@ void getAccountStorageTrieNode_returnsEmptyNode(final FlatDbMode flatDbMode) { } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getNodeData_returnsEmptyValue(final FlatDbMode flatDbMode) { setUp(flatDbMode); assertThat(storage.getNodeData(null, null)).isEmpty(); } @ParameterizedTest - @MethodSource("data") - void getNodeData_returnsEmptyNode(final FlatDbMode flatDbMode) { - setUp(flatDbMode); + @MethodSource("flatDbModeAndCodeStorageMode") + void getNodeData_returnsEmptyNode( + final FlatDbMode flatDbMode, final boolean accountHashCodeStorage) { + setUp(flatDbMode, accountHashCodeStorage); assertThat(storage.getNodeData(Bytes.EMPTY, MerkleTrie.EMPTY_TRIE_NODE_HASH)).isEmpty(); } @ParameterizedTest - @MethodSource("data") - void getCode_saveAndGetSpecialValues(final FlatDbMode flatDbMode) { - setUp(flatDbMode); + @MethodSource("flatDbModeAndCodeStorageMode") + void getCode_saveAndGetSpecialValues( + final FlatDbMode flatDbMode, final boolean accountHashCodeStorage) { + setUp(flatDbMode, accountHashCodeStorage); storage .updater() .putCode(Hash.EMPTY, MerkleTrie.EMPTY_TRIE_NODE) @@ -125,9 +148,10 @@ void getCode_saveAndGetSpecialValues(final FlatDbMode flatDbMode) { } @ParameterizedTest - @MethodSource("data") - void getCode_saveAndGetRegularValue(final FlatDbMode flatDbMode) { - setUp(flatDbMode); + @MethodSource("flatDbModeAndCodeStorageMode") + void getCode_saveAndGetRegularValue( + final FlatDbMode flatDbMode, final boolean accountHashCodeStorage) { + setUp(flatDbMode, accountHashCodeStorage); final Bytes bytes = Bytes.fromHexString("0x123456"); storage.updater().putCode(Hash.EMPTY, bytes).commit(); @@ -135,7 +159,7 @@ void getCode_saveAndGetRegularValue(final FlatDbMode flatDbMode) { } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getAccountStateTrieNode_saveAndGetSpecialValues(final FlatDbMode flatDbMode) { setUp(flatDbMode); storage @@ -151,7 +175,7 @@ void getAccountStateTrieNode_saveAndGetSpecialValues(final FlatDbMode flatDbMode } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getAccountStateTrieNode_saveAndGetRegularValue(final FlatDbMode flatDbMode) { setUp(flatDbMode); final Bytes location = Bytes.fromHexString("0x01"); @@ -163,7 +187,7 @@ void getAccountStateTrieNode_saveAndGetRegularValue(final FlatDbMode flatDbMode) } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getAccountStorageTrieNode_saveAndGetSpecialValues(final FlatDbMode flatDbMode) { setUp(flatDbMode); @@ -186,7 +210,7 @@ void getAccountStorageTrieNode_saveAndGetSpecialValues(final FlatDbMode flatDbMo } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getAccountStorageTrieNode_saveAndGetRegularValue(final FlatDbMode flatDbMode) { setUp(flatDbMode); final Hash accountHash = Address.fromHexString("0x1").addressHash(); @@ -203,7 +227,7 @@ void getAccountStorageTrieNode_saveAndGetRegularValue(final FlatDbMode flatDbMod } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getAccount_notLoadFromTrieWhenEmptyAndFlatDbFullMode(final FlatDbMode flatDbMode) { setUp(flatDbMode); Assumptions.assumeTrue(flatDbMode == FlatDbMode.FULL); @@ -235,7 +259,7 @@ void getAccount_notLoadFromTrieWhenEmptyAndFlatDbFullMode(final FlatDbMode flatD } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getAccount_loadFromTrieWhenEmptyAndFlatDbPartialMode(final FlatDbMode flatDbMode) { setUp(flatDbMode); Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL); @@ -264,7 +288,7 @@ void getAccount_loadFromTrieWhenEmptyAndFlatDbPartialMode(final FlatDbMode flatD } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void shouldUsePartialDBStrategyAfterDowngradingMode(final FlatDbMode flatDbMode) { setUp(flatDbMode); Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL); @@ -296,7 +320,7 @@ void shouldUsePartialDBStrategyAfterDowngradingMode(final FlatDbMode flatDbMode) } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getStorage_loadFromTrieWhenEmptyWithPartialMode(final FlatDbMode flatDbMode) { setUp(flatDbMode); Assumptions.assumeTrue(flatDbMode == FlatDbMode.PARTIAL); @@ -345,7 +369,7 @@ void getStorage_loadFromTrieWhenEmptyWithPartialMode(final FlatDbMode flatDbMode } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void getStorage_loadFromTrieWhenEmptyWithFullMode(final FlatDbMode flatDbMode) { setUp(flatDbMode); Assumptions.assumeTrue(flatDbMode == FlatDbMode.FULL); @@ -365,7 +389,7 @@ void getStorage_loadFromTrieWhenEmptyWithFullMode(final FlatDbMode flatDbMode) { } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void clear_reloadFlatDbStrategy(final FlatDbMode flatDbMode) { setUp(flatDbMode); final BonsaiWorldStateKeyValueStorage storage = spy(emptyStorage()); @@ -385,7 +409,7 @@ void clear_reloadFlatDbStrategy(final FlatDbMode flatDbMode) { } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void reconcilesNonConflictingUpdaters(final FlatDbMode flatDbMode) { setUp(flatDbMode); final Hash accountHashA = Address.fromHexString("0x1").addressHash(); @@ -411,14 +435,14 @@ void reconcilesNonConflictingUpdaters(final FlatDbMode flatDbMode) { } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void isWorldStateAvailable_defaultIsFalse(final FlatDbMode flatDbMode) { setUp(flatDbMode); assertThat(emptyStorage().isWorldStateAvailable(UInt256.valueOf(1), Hash.EMPTY)).isFalse(); } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void isWorldStateAvailable_StateAvailableByRootHash(final FlatDbMode flatDbMode) { setUp(flatDbMode); @@ -433,7 +457,7 @@ void isWorldStateAvailable_StateAvailableByRootHash(final FlatDbMode flatDbMode) } @ParameterizedTest - @MethodSource("data") + @MethodSource("flatDbMode") void isWorldStateAvailable_afterCallingSaveWorldstate(final FlatDbMode flatDbMode) { setUp(flatDbMode); @@ -458,6 +482,20 @@ private BonsaiWorldStateKeyValueStorage emptyStorage() { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG); } + private BonsaiWorldStateKeyValueStorage emptyStorage(final boolean useCodeHashStorage) { + return new BonsaiWorldStateKeyValueStorage( + new InMemoryKeyValueStorageProvider(), + new NoOpMetricsSystem(), + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(DataStorageFormat.BONSAI) + .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiCodeStoredByCodeHashEnabled(useCodeHashStorage) + .build()) + .build()); + } + @Test void successfulPruneReturnsTrue() { final KeyValueStorage mockTrieLogStorage = mock(KeyValueStorage.class); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java index 8d2984879b0..1a115defd75 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java @@ -16,10 +16,14 @@ package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; +import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; @@ -27,10 +31,13 @@ import java.util.List; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -38,7 +45,10 @@ class FlatDbStrategyProviderTest { private final FlatDbStrategyProvider flatDbStrategyProvider = new FlatDbStrategyProvider(new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_CONFIG); private final SegmentedKeyValueStorage composedWorldStateStorage = - new SegmentedInMemoryKeyValueStorage(List.of(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE)); + new SegmentedInMemoryKeyValueStorage( + List.of( + KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE, + KeyValueSegmentIdentifier.CODE_STORAGE)); @ParameterizedTest @EnumSource(FlatDbMode.class) @@ -64,6 +74,92 @@ void upgradesFlatDbStrategyToFullFlatDbMode() { assertThat(flatDbStrategyProvider.flatDbStrategy).isNotNull(); assertThat(flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)) .isInstanceOf(FullFlatDbStrategy.class); + assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy) + .isInstanceOf(AccountHashCodeStorageStrategy.class); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void emptyDbCreatesFlatDbStrategyUsingCodeByHashConfig(final boolean codeByHashEnabled) { + final DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(DataStorageFormat.BONSAI) + .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiCodeStoredByCodeHashEnabled(codeByHashEnabled) + .build()) + .build(); + final FlatDbStrategyProvider flatDbStrategyProvider = + new FlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration); + + flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); + final Class expectedCodeStorageClass = + codeByHashEnabled + ? CodeHashCodeStorageStrategy.class + : AccountHashCodeStorageStrategy.class; + assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL); + assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy) + .isInstanceOf(expectedCodeStorageClass); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void existingAccountHashDbUsesAccountHash(final boolean codeByHashEnabled) { + final DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(DataStorageFormat.BONSAI) + .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiCodeStoredByCodeHashEnabled(codeByHashEnabled) + .build()) + .build(); + final FlatDbStrategyProvider flatDbStrategyProvider = + new FlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration); + + final SegmentedKeyValueStorageTransaction transaction = + composedWorldStateStorage.startTransaction(); + final AccountHashCodeStorageStrategy accountHashCodeStorageStrategy = + new AccountHashCodeStorageStrategy(); + // key representing account hash just needs to not be the code hash + final Hash accountHash = Hash.wrap(Bytes32.fromHexString("0001")); + accountHashCodeStorageStrategy.putFlatCode(transaction, accountHash, null, Bytes.of(2)); + transaction.commit(); + + flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); + assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL); + assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy) + .isInstanceOf(AccountHashCodeStorageStrategy.class); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void existingCodeHashDbUsesCodeHash(final boolean codeByHashEnabled) { + final DataStorageConfiguration dataStorageConfiguration = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(DataStorageFormat.BONSAI) + .bonsaiMaxLayersToLoad(DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) + .unstable( + ImmutableDataStorageConfiguration.Unstable.builder() + .bonsaiCodeStoredByCodeHashEnabled(codeByHashEnabled) + .build()) + .build(); + final FlatDbStrategyProvider flatDbStrategyProvider = + new FlatDbStrategyProvider(new NoOpMetricsSystem(), dataStorageConfiguration); + + final SegmentedKeyValueStorageTransaction transaction = + composedWorldStateStorage.startTransaction(); + + final CodeHashCodeStorageStrategy codeHashCodeStorageStrategy = + new CodeHashCodeStorageStrategy(); + codeHashCodeStorageStrategy.putFlatCode(transaction, null, Hash.hash(Bytes.of(1)), Bytes.of(1)); + transaction.commit(); + + flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); + assertThat(flatDbStrategyProvider.flatDbMode).isEqualTo(FlatDbMode.PARTIAL); + assertThat(flatDbStrategyProvider.flatDbStrategy.codeStorageStrategy) + .isInstanceOf(CodeHashCodeStorageStrategy.class); } @Test diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java new file mode 100644 index 00000000000..c3ec2e18b37 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java @@ -0,0 +1,143 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.bonsai.worldview; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.evm.internal.EvmConfiguration; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class BonsaiWorldStateTest { + @Mock BonsaiWorldStateUpdateAccumulator bonsaiWorldStateUpdateAccumulator; + @Mock BonsaiWorldStateKeyValueStorage.BonsaiUpdater bonsaiUpdater; + @Mock Blockchain blockchain; + @Mock BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage; + + private static final Bytes CODE = Bytes.of(10); + private static final Hash CODE_HASH = Hash.hash(CODE); + private static final Hash ACCOUNT_HASH = Hash.hash(Address.ZERO); + private static final Address ACCOUNT = Address.ZERO; + + private BonsaiWorldState worldState; + + @BeforeEach + void setup() { + worldState = + new BonsaiWorldState( + InMemoryKeyValueStorageProvider.createBonsaiInMemoryWorldStateArchive(blockchain), + bonsaiWorldStateKeyValueStorage, + EvmConfiguration.DEFAULT); + } + + @ParameterizedTest + @MethodSource("priorAndUpdatedEmptyAndNullBytes") + void codeUpdateDoesNothingWhenMarkedAsDeletedButAlreadyDeleted( + final Bytes prior, final Bytes updated) { + final Map> codeToUpdate = + Map.of(Address.ZERO, new BonsaiValue<>(prior, updated)); + when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); + worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); + + verifyNoInteractions(bonsaiUpdater); + } + + @Test + void codeUpdateDoesNothingWhenAddingSameAsExistingValue() { + final Map> codeToUpdate = + Map.of(Address.ZERO, new BonsaiValue<>(CODE, CODE)); + when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); + worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); + + verifyNoInteractions(bonsaiUpdater); + } + + @ParameterizedTest + @MethodSource("emptyAndNullBytes") + void removesCodeWhenMarkedAsDeleted(final Bytes updated) { + final Map> codeToUpdate = + Map.of(Address.ZERO, new BonsaiValue<>(CODE, updated)); + when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); + worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); + + verify(bonsaiUpdater).removeCode(ACCOUNT_HASH, CODE_HASH); + } + + @ParameterizedTest + @MethodSource("codeValueAndEmptyAndNullBytes") + void addsCodeForNewCodeValue(final Bytes prior) { + final Map> codeToUpdate = + Map.of(ACCOUNT, new BonsaiValue<>(prior, CODE)); + + when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); + worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); + + verify(bonsaiUpdater).putCode(ACCOUNT_HASH, CODE_HASH, CODE); + } + + @Test + void updateCodeForMultipleValues() { + final Map> codeToUpdate = new HashMap<>(); + codeToUpdate.put(Address.fromHexString("0x1"), new BonsaiValue<>(null, CODE)); + codeToUpdate.put(Address.fromHexString("0x2"), new BonsaiValue<>(CODE, null)); + codeToUpdate.put(Address.fromHexString("0x3"), new BonsaiValue<>(Bytes.of(9), CODE)); + + when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); + worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); + + verify(bonsaiUpdater).putCode(Address.fromHexString("0x1").addressHash(), CODE_HASH, CODE); + verify(bonsaiUpdater).removeCode(Address.fromHexString("0x2").addressHash(), CODE_HASH); + verify(bonsaiUpdater).putCode(Address.fromHexString("0x3").addressHash(), CODE_HASH, CODE); + } + + private static Stream emptyAndNullBytes() { + return Stream.of(Bytes.EMPTY, null); + } + + private static Stream codeValueAndEmptyAndNullBytes() { + return Stream.of(Bytes.EMPTY, null); + } + + private static Stream priorAndUpdatedEmptyAndNullBytes() { + return Stream.of( + Arguments.of(null, Bytes.EMPTY), + Arguments.of(Bytes.EMPTY, null), + Arguments.of(null, null), + Arguments.of(Bytes.EMPTY, Bytes.EMPTY)); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java index fa57209a00f..bc505cd0e5d 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java @@ -66,8 +66,7 @@ public void getCode_saveAndGetSpecialValues() { final ForestWorldStateKeyValueStorage storage = emptyStorage(); storage.updater().putCode(null, MerkleTrie.EMPTY_TRIE_NODE).putCode(null, Bytes.EMPTY).commit(); - assertThat(storage.getCode(MerkleTrie.EMPTY_TRIE_NODE_HASH, null)) - .contains(MerkleTrie.EMPTY_TRIE_NODE); + assertThat(storage.getCode(Hash.EMPTY_TRIE_HASH, null)).contains(MerkleTrie.EMPTY_TRIE_NODE); assertThat(storage.getCode(Hash.EMPTY, null)).contains(Bytes.EMPTY); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java index 3b8bc7f2193..d5df4344769 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java @@ -97,7 +97,8 @@ private void assertDataPersisted(final List> tasks) { } else if (task.getData() instanceof BytecodeRequest) { final BytecodeRequest data = (BytecodeRequest) task.getData(); assertThat( - worldStateStorage.getCode(data.getCodeHash(), Hash.wrap(data.getAccountHash()))) + worldStateStorage.getCode( + Hash.wrap(data.getCodeHash()), Hash.wrap(data.getAccountHash()))) .isPresent(); } else { fail("not expected message");