diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index 4dd89cd7ca1..578f92e0a42 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -40,7 +40,14 @@ public BonsaiWorldStateUpdateAccumulator( final Consumer> accountPreloader, final Consumer storagePreloader, final EvmConfiguration evmConfiguration) { - super(world, accountPreloader, storagePreloader, evmConfiguration); + super( + world, + accountPreloader, + storagePreloader, + (__, ___) -> { + /*nothing to preload for the code*/ + }, + evmConfiguration); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java index 6f3c9d4a3af..826afd4d169 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.AccountConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.CodeConsumingMap; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.evm.account.Account; @@ -62,9 +63,10 @@ public abstract class DiffBasedWorldStateUpdateAccumulator> accountPreloader; protected final Consumer storagePreloader; + private final Consumer codePreloader; private final AccountConsumingMap> accountsToUpdate; - private final Map> codeToUpdate = new ConcurrentHashMap<>(); + private final CodeConsumingMap codeToUpdate; private final Set
storageToClear = Collections.synchronizedSet(new HashSet<>()); protected final EvmConfiguration evmConfiguration; @@ -81,11 +83,14 @@ public DiffBasedWorldStateUpdateAccumulator( final DiffBasedWorldView world, final Consumer> accountPreloader, final Consumer storagePreloader, + final Consumer codePreloader, final EvmConfiguration evmConfiguration) { super(world, evmConfiguration); this.accountsToUpdate = new AccountConsumingMap<>(new ConcurrentHashMap<>(), accountPreloader); + this.codeToUpdate = new CodeConsumingMap(new ConcurrentHashMap<>(), codePreloader); this.accountPreloader = accountPreloader; this.storagePreloader = storagePreloader; + this.codePreloader = codePreloader; this.isAccumulatorStateChanged = false; this.evmConfiguration = evmConfiguration; } @@ -108,6 +113,10 @@ protected Consumer getStoragePreloader() { return storagePreloader; } + public Consumer getCodePreloader() { + return codePreloader; + } + protected EvmConfiguration getEvmConfiguration() { return evmConfiguration; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/CodeConsumingMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/CodeConsumingMap.java new file mode 100644 index 00000000000..68b191b3ec8 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/CodeConsumingMap.java @@ -0,0 +1,53 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import javax.annotation.Nonnull; + +import com.google.common.collect.ForwardingMap; +import org.apache.tuweni.bytes.Bytes; + +public class CodeConsumingMap extends ForwardingMap> { + + private final ConcurrentMap> codes; + private final Consumer consumer; + + public CodeConsumingMap( + final ConcurrentMap> codes, final Consumer consumer) { + this.codes = codes; + this.consumer = consumer; + } + + @Override + public DiffBasedValue put( + @Nonnull final Address address, @Nonnull final DiffBasedValue value) { + consumer.process(address, value.getUpdated() != null ? value.getUpdated() : value.getPrior()); + return codes.put(address, value); + } + + public Consumer getConsumer() { + return consumer; + } + + @Override + protected Map> delegate() { + return codes; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/StemPreloader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/StemPreloader.java new file mode 100644 index 00000000000..f9aff616071 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/StemPreloader.java @@ -0,0 +1,380 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher; +import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.stream.IntStream; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +/** + * Preloads stems for accounts, storage slots, and code. This class is designed to optimize block + * processing by caching trie stems associated with specific accounts, storage slots, and code data. + * By preloading these stems, the class aims to reduce the computational overhead and access times + * typically required during the state root computation. + * + *

The preloading process involves generating and caching stems for: + * + *

    + *
  • Account-related keys (e.g., version, balance, nonce, and code hash keys) + *
  • Storage slot keys for a given account + *
  • Code chunk keys for smart contract code associated with an account + *
+ * + *

This class utilizes a {@link Cache} to store preloaded stems, with the cache size configurable + * through the {@code STEM_CACHE_SIZE} constant. The {@link Hasher} used for stem generation is + * configurable, allowing for different hashing strategies (e.g., Pedersen hashing) to be employed. + * + * @see org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter + * @see org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher + * @see org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher + */ +public class StemPreloader { + private static final int STEM_CACHE_SIZE = 10_000; + private final Cache> stemByAddressCache = + CacheBuilder.newBuilder().maximumSize(STEM_CACHE_SIZE).build(); + + private final TrieKeyAdapter trieKeyAdapter; + + private final Hasher hasher; + + private long missedStemCounter = 0; + + private long cachedStemCounter = 0; + + public StemPreloader() { + this.hasher = new PedersenHasher(); + this.trieKeyAdapter = new TrieKeyAdapter(hasher); + this.trieKeyAdapter.versionKey( + Address.ZERO); // TODO REMOVE is just to preload the native library for performance check + } + + /** + * Creates a preloaded hasher context for a given updated address, storage, and code. This method + * preloads stems for account keys, storage slot keys, and code chunk keys based on the provided + * updates. It then returns a {@link HasherContext} containing the preloaded stems and flags + * indicating whether storage or code trie keys are present. + * + *

This method is particularly useful for optimizing the hashing process during state root + * computation by ensuring that stems are preloaded and readily available, thereby reducing the + * need for on-the-fly stem generation. + * + * @param address the address for which to preload trie stems + * @param storageUpdate a map of storage slot keys to their updated values + * @param codeUpdate the updated code value, encapsulated in a {@link DiffBasedValue} + * @return a {@link HasherContext} containing the preloaded stems and flags for storage and code + * trie key presence + */ + public HasherContext createPreloadedHasher( + final Address address, + final Map> storageUpdate, + final DiffBasedValue codeUpdate) { + + // generate account triekeys + final List accountKeyIds = new ArrayList<>(generateAccountKeyIds()); + + // generate storage triekeys + final List storageKeyIds = new ArrayList<>(); + boolean isStorageUpdateNeeded; + if (storageUpdate != null) { + final Set storageSlotKeys = storageUpdate.keySet(); + isStorageUpdateNeeded = !storageSlotKeys.isEmpty(); + if (isStorageUpdateNeeded) { + storageKeyIds.addAll(generateStorageKeyIds(storageSlotKeys)); + } + } + + // generate code triekeys + final List codeChunkIds = new ArrayList<>(); + boolean isCodeUpdateNeeded; + if (codeUpdate != null) { + final Bytes previousCode = codeUpdate.getPrior(); + final Bytes updatedCode = codeUpdate.getUpdated(); + isCodeUpdateNeeded = + !codeUpdate.isUnchanged() && !(codeIsEmpty(previousCode) && codeIsEmpty(updatedCode)); + if (isCodeUpdateNeeded) { + accountKeyIds.add(Parameters.CODE_SIZE_LEAF_KEY); + codeChunkIds.addAll( + generateCodeChunkKeyIds(updatedCode == null ? previousCode : updatedCode)); + } + } + + return new HasherContext( + new CachedPedersenHasher( + STEM_CACHE_SIZE, + generateManyStems(address, accountKeyIds, storageKeyIds, codeChunkIds, true)), + !storageKeyIds.isEmpty(), + !codeChunkIds.isEmpty()); + } + + /** + * Asynchronously preloads stems for a given account address. This method is designed to optimize + * the access to account-related stems by caching them ahead of time, thus reducing the + * computational overhead during state root computation. + * + * @param account the address of the account for which stems are to be preloaded + */ + public void preLoadAccount(final Address account) { + CompletableFuture.runAsync(() -> cacheAccountStems(account)); + } + + /** + * Asynchronously preloads stems for a specific storage slot associated with an account. This + * method enhances the efficiency of accessing storage-related stems by ensuring they are cached + * in advance, thereby facilitating faster state root computation. + * + * @param account the address of the account associated with the storage slot + * @param slotKey the key of the storage slot for which stems are to be preloaded + */ + public void preLoadStorageSlot(final Address account, final StorageSlotKey slotKey) { + CompletableFuture.runAsync(() -> cacheSlotStems(account, slotKey)); + } + + /** + * Asynchronously preloads stems for the code associated with an account. + * + * @param account the address of the account associated with the code + * @param code the smart contract code for which stems are to be preloaded + */ + public void preLoadCode(final Address account, final Bytes code) { + CompletableFuture.runAsync(() -> cacheCodeStems(account, code)); + } + + /** + * Generates a comprehensive set of stems for an account, including stems for account keys, + * storage slot keys, and code chunk keys. + * + * @param address the address for which stems are to be generated + * @param accountKeyIds a list of trie keys associated with the account + * @param storageKeyIds a list of trie keys associated with the account's storage slots + * @param codeChunkIds a list of trie keys associated with the account's code chunks + * @param checkCacheBeforeGeneration flag indicating whether to check the cache before generating + * new stems + * @return a map of trie key index to their corresponding stems + */ + public Map generateManyStems( + final Address address, + final List accountKeyIds, + final List storageKeyIds, + final List codeChunkIds, + final boolean checkCacheBeforeGeneration) { + + final Set trieIndexes = new HashSet<>(); + + if (!accountKeyIds.isEmpty()) { + trieIndexes.add(UInt256.ZERO); + } + for (Bytes32 storageKey : storageKeyIds) { + trieIndexes.add(trieKeyAdapter.getStorageKeyTrieIndex(storageKey)); + } + for (Bytes32 codeChunkId : codeChunkIds) { + trieIndexes.add(trieKeyAdapter.getCodeChunkKeyTrieIndex(codeChunkId)); + } + + if (checkCacheBeforeGeneration) { + + // not regenerate stem already preloaded + final Map stemCached = getCachedStemForAccount(address); + stemCached.forEach( + (key, __) -> { + trieIndexes.remove(key); + cachedStemCounter++; + }); + missedStemCounter += trieIndexes.size(); + if (trieIndexes.isEmpty()) { + return stemCached; + } + + final Map stems = hasher.manyStems(address, new ArrayList<>(trieIndexes)); + stems.putAll(stemCached); + + return stems; + + } else { + return hasher.manyStems(address, new ArrayList<>(trieIndexes)); + } + } + + /** + * Retrieves the cache that maps account addresses to their corresponding cached stems. + * + * @return the cache mapping account addresses to trie stems + */ + @VisibleForTesting + public Cache> getStemByAddressCache() { + return stemByAddressCache; + } + + /** + * Retrieves the current number of stems that were not found in the cache and had to be generated. + * This counter can be used to monitor the effectiveness of the stem preloading mechanism. + * + * @return the number of stems that were missed and subsequently generated + */ + public long getNbMissedStems() { + return missedStemCounter; + } + + /** + * Retrieves the current number of stems that were found in the cache and had not to be generated. + * This counter can be used to monitor the effectiveness of the stem preloading mechanism. + * + * @return the number of stems that were present in the cache + */ + public long getNbCachedStems() { + return cachedStemCounter; + } + + /** + * Resets the counter of missed trie stems. This method can be used to clear the count of trie + * stems that were not found in the cache and had to be generated, typically used in conjunction + * with performance monitoring or testing. + */ + public void reset() { + cachedStemCounter = 0; + missedStemCounter = 0; + } + + /** + * Caches stems for all account-related keys for a given account address. + * + * @param account the address of the account for which stems are to be cached + */ + @VisibleForTesting + public void cacheAccountStems(final Address account) { + try { + final Map cache = stemByAddressCache.get(account, ConcurrentHashMap::new); + cache.putAll( + generateManyStems( + account, + generateAccountKeyIds(), + Collections.emptyList(), + Collections.emptyList(), + false)); + } catch (ExecutionException e) { + // no op + } + } + + /** + * Caches stems for a specific storage slot key associated with an address. + * + * @param account the address of the account associated with the storage slot + * @param slotKey the storage slot key for which trie stems are to be cached + */ + @VisibleForTesting + public void cacheSlotStems(final Address account, final StorageSlotKey slotKey) { + try { + final Map cache = stemByAddressCache.get(account, ConcurrentHashMap::new); + if (slotKey.getSlotKey().isPresent()) { + cache.putAll( + generateManyStems( + account, + Collections.emptyList(), + List.of(slotKey.getSlotKey().get()), + Collections.emptyList(), + false)); + } + } catch (ExecutionException e) { + // no op + } + } + + /** + * Caches stems for the code associated with an account address. + * + * @param account the address of the account associated with the code + * @param code the smart contract code for which trie stems are to be cached + */ + @VisibleForTesting + public void cacheCodeStems(final Address account, final Bytes code) { + try { + final Map cache = stemByAddressCache.get(account, ConcurrentHashMap::new); + cache.putAll( + generateManyStems( + account, + Collections.emptyList(), + Collections.emptyList(), + generateCodeChunkKeyIds(code), + false)); + } catch (ExecutionException e) { + // no op + } + } + + private List generateAccountKeyIds() { + final List keys = new ArrayList<>(); + keys.add(Parameters.VERSION_LEAF_KEY); + keys.add(Parameters.BALANCE_LEAF_KEY); + keys.add(Parameters.NONCE_LEAF_KEY); + keys.add(Parameters.CODE_KECCAK_LEAF_KEY); + return keys; + } + + private List generateCodeChunkKeyIds(final Bytes code) { + return new ArrayList<>( + IntStream.range(0, trieKeyAdapter.getNbChunk(code)).mapToObj(UInt256::valueOf).toList()); + } + + private List generateStorageKeyIds(final Set storageSlotKeys) { + return storageSlotKeys.stream() + .map(storageSlotKey -> storageSlotKey.getSlotKey().orElseThrow()) + .map(Bytes32::wrap) + .toList(); + } + + private Map getCachedStemForAccount(final Address address) { + return Optional.ofNullable(stemByAddressCache.getIfPresent(address)) + .orElseGet(ConcurrentHashMap::new); + } + + private boolean codeIsEmpty(final Bytes value) { + return value == null || value.isEmpty(); + } + + /** + * Represents the context for a hasher, including the hasher instance and flags indicating the + * presence of storage trie keys and code trie keys. This record is used to encapsulate the state + * and configuration of the hasher in relation to preloaded trie stems. + * + * @param hasher the hasher instance used for generating stems + * @param hasStorageTrieKeys flag indicating whether storage trie keys are present in the cache + * @param hasCodeTrieKeys flag indicating whether code trie keys are present in the cache + */ + public record HasherContext(Hasher hasher, boolean hasStorageTrieKeys, boolean hasCodeTrieKeys) {} +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java index 3e3bf89bd1c..0dc64c5ac00 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java @@ -31,11 +31,10 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.StemPreloader; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.StemPreloader.HasherContext; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleLayeredWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; -import org.hyperledger.besu.ethereum.verkletrie.TrieKeyPreloader; -import org.hyperledger.besu.ethereum.verkletrie.TrieKeyPreloader.HasherContext; import org.hyperledger.besu.ethereum.verkletrie.VerkleEntryFactory; import org.hyperledger.besu.ethereum.verkletrie.VerkleTrie; import org.hyperledger.besu.evm.account.Account; @@ -43,11 +42,8 @@ import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nonnull; @@ -63,7 +59,7 @@ public class VerkleWorldState extends DiffBasedWorldState { private static final Logger LOG = LoggerFactory.getLogger(VerkleWorldState.class); - private final TrieKeyPreloader trieKeyPreloader; + private final StemPreloader stemPreloader; public VerkleWorldState( final VerkleWorldStateProvider archive, @@ -89,10 +85,14 @@ public VerkleWorldState( cachedWorldStorageManager, trieLogManager, diffBasedWorldStateConfig); - this.trieKeyPreloader = new TrieKeyPreloader(); + this.stemPreloader = new StemPreloader(); this.setAccumulator( new VerkleWorldStateUpdateAccumulator( - this, (addr, value) -> {}, (addr, value) -> {}, evmConfiguration)); + this, + (addr, value) -> stemPreloader.preLoadAccount(addr), + stemPreloader::preLoadStorageSlot, + stemPreloader::preLoadCode, + evmConfiguration)); } @Override @@ -124,46 +124,15 @@ protected Hash internalCalculateRootHash( .forEach( accountUpdate -> { final Address accountKey = accountUpdate.getKey(); - // generate account triekeys - final List accountKeyIds = - new ArrayList<>(trieKeyPreloader.generateAccountKeyIds()); - - // generate storage triekeys - final List storageKeyIds = new ArrayList<>(); final StorageConsumingMap> storageAccountUpdate = worldStateUpdater.getStorageToUpdate().get(accountKey); - boolean isStorageUpdateNeeded; - if (storageAccountUpdate != null) { - final Set storageSlotKeys = storageAccountUpdate.keySet(); - isStorageUpdateNeeded = !storageSlotKeys.isEmpty(); - if (isStorageUpdateNeeded) { - storageKeyIds.addAll(trieKeyPreloader.generateStorageKeyIds(storageSlotKeys)); - } - } - - // generate code triekeys - final List codeKeyIds = new ArrayList<>(); final DiffBasedValue codeUpdate = worldStateUpdater.getCodeToUpdate().get(accountKey); - boolean isCodeUpdateNeeded; - if (codeUpdate != null) { - final Bytes previousCode = codeUpdate.getPrior(); - final Bytes updatedCode = codeUpdate.getUpdated(); - isCodeUpdateNeeded = - !codeUpdate.isUnchanged() - && !(codeIsEmpty(previousCode) && codeIsEmpty(updatedCode)); - if (isCodeUpdateNeeded) { - accountKeyIds.add(Parameters.CODE_SIZE_LEAF_KEY); - codeKeyIds.addAll( - trieKeyPreloader.generateCodeChunkKeyIds( - updatedCode == null ? previousCode : updatedCode)); - } - } preloadedHashers.put( accountKey, - trieKeyPreloader.createPreloadedHasher( - accountKey, accountKeyIds, storageKeyIds, codeKeyIds)); + stemPreloader.createPreloadedHasher( + accountKey, storageAccountUpdate, codeUpdate)); }); for (final Map.Entry> accountUpdate : @@ -198,11 +167,16 @@ protected Hash internalCalculateRootHash( value); })); - LOG.info("end commit "); // LOG.info(stateTrie.toDotTree()); final Bytes32 rootHash = stateTrie.getRootHash(); - LOG.info("end commit " + rootHash); - + LOG.info( + "end commit " + + rootHash + + " " + + stemPreloader.getNbMissedStems() + + "/" + + stemPreloader.getNbCachedStems()); + stemPreloader.reset(); return Hash.wrap(rootHash); } @@ -287,36 +261,19 @@ private void updateCode( maybeStateUpdater.ifPresent( bonsaiUpdater -> bonsaiUpdater.removeCode(accountHash, priorCodeHash)); } else { - if (updatedCode.isEmpty()) { - final Hash codeHash = Hash.hash(updatedCode); - verkleEntryFactory - .generateKeyValuesForCode(accountKey, updatedCode) - .forEach( - (bytes, bytes2) -> { - // System.out.println("add code " + bytes + " " + bytes2); - stateTrie.put(bytes, bytes2); - }); - maybeStateUpdater.ifPresent( - bonsaiUpdater -> bonsaiUpdater.removeCode(accountHash, codeHash)); - } else { - final Hash codeHash = Hash.hash(updatedCode); - verkleEntryFactory - .generateKeyValuesForCode(accountKey, updatedCode) - .forEach( - (bytes, bytes2) -> { - System.out.println("add code " + bytes + " " + bytes2); - stateTrie.put(bytes, bytes2); - }); - maybeStateUpdater.ifPresent( - bonsaiUpdater -> bonsaiUpdater.putCode(accountHash, codeHash, updatedCode)); - } + final Hash codeHash = Hash.hash(updatedCode); + verkleEntryFactory + .generateKeyValuesForCode(accountKey, updatedCode) + .forEach( + (bytes, bytes2) -> { + System.out.println("add code " + bytes + " " + bytes2); + stateTrie.put(bytes, bytes2); + }); + maybeStateUpdater.ifPresent( + bonsaiUpdater -> bonsaiUpdater.putCode(accountHash, codeHash, updatedCode)); } } - private boolean codeIsEmpty(final Bytes value) { - return value == null || value.isEmpty(); - } - private void updateAccountStorageState( final Address accountKey, final VerkleTrie stateTrie, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java index 1bb15200415..1a685336f6a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldStateUpdateAccumulator.java @@ -30,6 +30,7 @@ import java.util.Optional; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; public class VerkleWorldStateUpdateAccumulator @@ -39,8 +40,9 @@ public VerkleWorldStateUpdateAccumulator( final DiffBasedWorldView world, final Consumer> accountPreloader, final Consumer storagePreloader, + final Consumer codePreloader, final EvmConfiguration evmConfiguration) { - super(world, accountPreloader, storagePreloader, evmConfiguration); + super(world, accountPreloader, storagePreloader, codePreloader, evmConfiguration); } @Override @@ -50,6 +52,7 @@ public DiffBasedWorldStateUpdateAccumulator copy() { wrappedWorldView(), getAccountPreloader(), getStoragePreloader(), + getCodePreloader(), getEvmConfiguration()); copy.cloneFromUpdater(this); return copy; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java index a474fe412ff..ed788fc562b 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java @@ -474,7 +474,6 @@ void rollBackFourTimes() { worldState.persist(headerThree); final TrieLogLayer layerThree = getTrieLogLayer(trieLogStorage, headerThree.getHash()); - System.out.println(layerThree.dump()); final WorldUpdater updater4 = worldState.updater(); final MutableAccount mutableAccount4 = updater4.getAccount(addressOne); mutableAccount4.setStorageValue(UInt256.ONE, UInt256.valueOf(1)); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/StemPreloaderTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/StemPreloaderTests.java new file mode 100644 index 00000000000..1fcdb5bb612 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/StemPreloaderTests.java @@ -0,0 +1,131 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.StemPreloader; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class StemPreloaderTests { + + private StemPreloader stemPreLoader; + private final List

accounts = + List.of(Address.fromHexString("0xdeadbeef"), Address.fromHexString("0xdeadbeee")); + + @BeforeEach + public void setup() { + stemPreLoader = new StemPreloader(); + } + + // Test to verify that a single account's trie key is correctly added to the cache during preload + @Test + void shouldAddAccountsStemsInCacheDuringPreload() { + stemPreLoader.cacheAccountStems(accounts.get(0)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is exactly 1, indicating a single trie key has + // been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + } + + // Test to verify that stems for multiple accounts are correctly added to the cache during + // preload + @Test + void shouldAddMultipleAccountsStemsInCacheDuringPreload() { + stemPreLoader.cacheAccountStems(accounts.get(0)); + stemPreLoader.cacheAccountStems(accounts.get(1)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(2); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that each account's cache size is 1, indicating that stems for both accounts have + // been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + final Map account2Cache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(1)); + Assertions.assertThat(Objects.requireNonNull(account2Cache).size()).isEqualTo(1); + } + + // Test to verify that header slots stems are correctly added to the cache for a single + // account during preload + @Test + void shouldAddHeaderSlotsStemsInCacheDuringPreload() { + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.ZERO)); + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.ONE)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 1, indicating that two slots in the header + // storage have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + } + + // Test to verify that multiple slots stems are correctly added to the cache for a single + // account during preload + @Test + void shouldAddMultipleSlotsStemsInCacheDuringPreload() { + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.ONE)); + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.valueOf(64))); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 2, indicating that slots in both the header and + // main storage have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); + } + + // Test to verify that code stems are correctly added to the cache for a single account during + // preload + @Test + void shouldAddCodeStemsInCacheDuringPreload() { + stemPreLoader.cacheCodeStems(accounts.get(0), Bytes.repeat((byte) 0x01, 4096)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 2, indicating that the code is too big and + // multiple offsets have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); + } + + // Test to verify that missing stems are correctly generated and accounted for + @Test + void shouldGenerateMissingStems() { + stemPreLoader.cacheAccountStems(accounts.get(0)); + Map stems = + stemPreLoader.generateManyStems( + accounts.get(0), + Collections.emptyList(), + List.of(UInt256.valueOf(64)), + List.of(UInt256.valueOf(128)), + true); + // Assert that two stems (slot and code) are missing from the cache + Assertions.assertThat(stemPreLoader.getNbMissedStems()).isEqualTo(2); + // Assert that three stems have been generated: one for the header, one for the slot in main + // storage, and one for the code chunk + Assertions.assertThat(stems.size()).isEqualTo(3); + } +} diff --git a/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/TrieKeyPreloader.java b/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/TrieKeyPreloader.java deleted file mode 100644 index 2bdf4961ed9..00000000000 --- a/ethereum/verkletrie/src/main/java/org/hyperledger/besu/ethereum/verkletrie/TrieKeyPreloader.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.verkletrie; - -import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyBatchAdapter; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher; -import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.IntStream; - -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; - -public class TrieKeyPreloader { - - private final TrieKeyBatchAdapter trieKeyAdapter; - - private final Hasher hasher; - - public TrieKeyPreloader() { - this.hasher = new PedersenHasher(); - trieKeyAdapter = new TrieKeyBatchAdapter(hasher); - trieKeyAdapter.versionKey( - Address.ZERO); // TODO REMOVE is just to preload the native library for performance check - } - - public List generateAccountKeyIds() { - final List keys = new ArrayList<>(); - keys.add(Parameters.VERSION_LEAF_KEY); - keys.add(Parameters.BALANCE_LEAF_KEY); - keys.add(Parameters.NONCE_LEAF_KEY); - keys.add(Parameters.CODE_KECCAK_LEAF_KEY); - return keys; - } - - public List generateCodeChunkKeyIds(final Bytes code) { - return new ArrayList<>( - IntStream.range(0, trieKeyAdapter.getNbChunk(code)).mapToObj(UInt256::valueOf).toList()); - } - - public List generateStorageKeyIds(final Set storageSlotKeys) { - return storageSlotKeys.stream() - .map(storageSlotKey -> storageSlotKey.getSlotKey().orElseThrow()) - .map(Bytes32::wrap) - .toList(); - } - - public HasherContext createPreloadedHasher( - final Address address, - final List accountKeyIds, - final List storageKeyIds, - final List codeChunkIds) { - return new HasherContext( - new CachedPedersenHasher( - trieKeyAdapter.manyTrieKeyHashes(address, accountKeyIds, storageKeyIds, codeChunkIds)), - !storageKeyIds.isEmpty(), - !codeChunkIds.isEmpty()); - } - - public record HasherContext(Hasher hasher, boolean hasStorageTrieKeys, boolean hasCodeTrieKeys) {} -} diff --git a/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java b/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java index 0001654862e..a73058f8b0c 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java @@ -422,7 +422,7 @@ public ChunkAccessKey( @Override public List getStorageSlotTreeIndexes(final UInt256 storageKey) { return List.of( - TRIE_KEY_ADAPTER.locateStorageKeyOffset(storageKey), - TRIE_KEY_ADAPTER.locateStorageKeySuffix(storageKey)); + TRIE_KEY_ADAPTER.getStorageKeyTrieIndex(storageKey), + TRIE_KEY_ADAPTER.getStorageKeySuffix(storageKey)); } } diff --git a/gradle/versions.gradle b/gradle/versions.gradle index e6f6d5f5bc8..c23ca37312c 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -164,7 +164,7 @@ dependencyManagement { entry 'blake2bf' } - dependencySet(group: 'org.hyperledger.besu', version: '0.8.4-SNAPSHOT') { + dependencySet(group: 'org.hyperledger.besu', version: '0.8.5') { entry 'ipa-multipoint' }