diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/StemPreloader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/StemPreloader.java new file mode 100644 index 00000000000..f9aff616071 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/StemPreloader.java @@ -0,0 +1,380 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher; +import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher; +import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.stream.IntStream; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +/** + * Preloads stems for accounts, storage slots, and code. This class is designed to optimize block + * processing by caching trie stems associated with specific accounts, storage slots, and code data. + * By preloading these stems, the class aims to reduce the computational overhead and access times + * typically required during the state root computation. + * + *

The preloading process involves generating and caching stems for: + * + *

+ * + *

This class utilizes a {@link Cache} to store preloaded stems, with the cache size configurable + * through the {@code STEM_CACHE_SIZE} constant. The {@link Hasher} used for stem generation is + * configurable, allowing for different hashing strategies (e.g., Pedersen hashing) to be employed. + * + * @see org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter + * @see org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher + * @see org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher + */ +public class StemPreloader { + private static final int STEM_CACHE_SIZE = 10_000; + private final Cache> stemByAddressCache = + CacheBuilder.newBuilder().maximumSize(STEM_CACHE_SIZE).build(); + + private final TrieKeyAdapter trieKeyAdapter; + + private final Hasher hasher; + + private long missedStemCounter = 0; + + private long cachedStemCounter = 0; + + public StemPreloader() { + this.hasher = new PedersenHasher(); + this.trieKeyAdapter = new TrieKeyAdapter(hasher); + this.trieKeyAdapter.versionKey( + Address.ZERO); // TODO REMOVE is just to preload the native library for performance check + } + + /** + * Creates a preloaded hasher context for a given updated address, storage, and code. This method + * preloads stems for account keys, storage slot keys, and code chunk keys based on the provided + * updates. It then returns a {@link HasherContext} containing the preloaded stems and flags + * indicating whether storage or code trie keys are present. + * + *

This method is particularly useful for optimizing the hashing process during state root + * computation by ensuring that stems are preloaded and readily available, thereby reducing the + * need for on-the-fly stem generation. + * + * @param address the address for which to preload trie stems + * @param storageUpdate a map of storage slot keys to their updated values + * @param codeUpdate the updated code value, encapsulated in a {@link DiffBasedValue} + * @return a {@link HasherContext} containing the preloaded stems and flags for storage and code + * trie key presence + */ + public HasherContext createPreloadedHasher( + final Address address, + final Map> storageUpdate, + final DiffBasedValue codeUpdate) { + + // generate account triekeys + final List accountKeyIds = new ArrayList<>(generateAccountKeyIds()); + + // generate storage triekeys + final List storageKeyIds = new ArrayList<>(); + boolean isStorageUpdateNeeded; + if (storageUpdate != null) { + final Set storageSlotKeys = storageUpdate.keySet(); + isStorageUpdateNeeded = !storageSlotKeys.isEmpty(); + if (isStorageUpdateNeeded) { + storageKeyIds.addAll(generateStorageKeyIds(storageSlotKeys)); + } + } + + // generate code triekeys + final List codeChunkIds = new ArrayList<>(); + boolean isCodeUpdateNeeded; + if (codeUpdate != null) { + final Bytes previousCode = codeUpdate.getPrior(); + final Bytes updatedCode = codeUpdate.getUpdated(); + isCodeUpdateNeeded = + !codeUpdate.isUnchanged() && !(codeIsEmpty(previousCode) && codeIsEmpty(updatedCode)); + if (isCodeUpdateNeeded) { + accountKeyIds.add(Parameters.CODE_SIZE_LEAF_KEY); + codeChunkIds.addAll( + generateCodeChunkKeyIds(updatedCode == null ? previousCode : updatedCode)); + } + } + + return new HasherContext( + new CachedPedersenHasher( + STEM_CACHE_SIZE, + generateManyStems(address, accountKeyIds, storageKeyIds, codeChunkIds, true)), + !storageKeyIds.isEmpty(), + !codeChunkIds.isEmpty()); + } + + /** + * Asynchronously preloads stems for a given account address. This method is designed to optimize + * the access to account-related stems by caching them ahead of time, thus reducing the + * computational overhead during state root computation. + * + * @param account the address of the account for which stems are to be preloaded + */ + public void preLoadAccount(final Address account) { + CompletableFuture.runAsync(() -> cacheAccountStems(account)); + } + + /** + * Asynchronously preloads stems for a specific storage slot associated with an account. This + * method enhances the efficiency of accessing storage-related stems by ensuring they are cached + * in advance, thereby facilitating faster state root computation. + * + * @param account the address of the account associated with the storage slot + * @param slotKey the key of the storage slot for which stems are to be preloaded + */ + public void preLoadStorageSlot(final Address account, final StorageSlotKey slotKey) { + CompletableFuture.runAsync(() -> cacheSlotStems(account, slotKey)); + } + + /** + * Asynchronously preloads stems for the code associated with an account. + * + * @param account the address of the account associated with the code + * @param code the smart contract code for which stems are to be preloaded + */ + public void preLoadCode(final Address account, final Bytes code) { + CompletableFuture.runAsync(() -> cacheCodeStems(account, code)); + } + + /** + * Generates a comprehensive set of stems for an account, including stems for account keys, + * storage slot keys, and code chunk keys. + * + * @param address the address for which stems are to be generated + * @param accountKeyIds a list of trie keys associated with the account + * @param storageKeyIds a list of trie keys associated with the account's storage slots + * @param codeChunkIds a list of trie keys associated with the account's code chunks + * @param checkCacheBeforeGeneration flag indicating whether to check the cache before generating + * new stems + * @return a map of trie key index to their corresponding stems + */ + public Map generateManyStems( + final Address address, + final List accountKeyIds, + final List storageKeyIds, + final List codeChunkIds, + final boolean checkCacheBeforeGeneration) { + + final Set trieIndexes = new HashSet<>(); + + if (!accountKeyIds.isEmpty()) { + trieIndexes.add(UInt256.ZERO); + } + for (Bytes32 storageKey : storageKeyIds) { + trieIndexes.add(trieKeyAdapter.getStorageKeyTrieIndex(storageKey)); + } + for (Bytes32 codeChunkId : codeChunkIds) { + trieIndexes.add(trieKeyAdapter.getCodeChunkKeyTrieIndex(codeChunkId)); + } + + if (checkCacheBeforeGeneration) { + + // not regenerate stem already preloaded + final Map stemCached = getCachedStemForAccount(address); + stemCached.forEach( + (key, __) -> { + trieIndexes.remove(key); + cachedStemCounter++; + }); + missedStemCounter += trieIndexes.size(); + if (trieIndexes.isEmpty()) { + return stemCached; + } + + final Map stems = hasher.manyStems(address, new ArrayList<>(trieIndexes)); + stems.putAll(stemCached); + + return stems; + + } else { + return hasher.manyStems(address, new ArrayList<>(trieIndexes)); + } + } + + /** + * Retrieves the cache that maps account addresses to their corresponding cached stems. + * + * @return the cache mapping account addresses to trie stems + */ + @VisibleForTesting + public Cache> getStemByAddressCache() { + return stemByAddressCache; + } + + /** + * Retrieves the current number of stems that were not found in the cache and had to be generated. + * This counter can be used to monitor the effectiveness of the stem preloading mechanism. + * + * @return the number of stems that were missed and subsequently generated + */ + public long getNbMissedStems() { + return missedStemCounter; + } + + /** + * Retrieves the current number of stems that were found in the cache and had not to be generated. + * This counter can be used to monitor the effectiveness of the stem preloading mechanism. + * + * @return the number of stems that were present in the cache + */ + public long getNbCachedStems() { + return cachedStemCounter; + } + + /** + * Resets the counter of missed trie stems. This method can be used to clear the count of trie + * stems that were not found in the cache and had to be generated, typically used in conjunction + * with performance monitoring or testing. + */ + public void reset() { + cachedStemCounter = 0; + missedStemCounter = 0; + } + + /** + * Caches stems for all account-related keys for a given account address. + * + * @param account the address of the account for which stems are to be cached + */ + @VisibleForTesting + public void cacheAccountStems(final Address account) { + try { + final Map cache = stemByAddressCache.get(account, ConcurrentHashMap::new); + cache.putAll( + generateManyStems( + account, + generateAccountKeyIds(), + Collections.emptyList(), + Collections.emptyList(), + false)); + } catch (ExecutionException e) { + // no op + } + } + + /** + * Caches stems for a specific storage slot key associated with an address. + * + * @param account the address of the account associated with the storage slot + * @param slotKey the storage slot key for which trie stems are to be cached + */ + @VisibleForTesting + public void cacheSlotStems(final Address account, final StorageSlotKey slotKey) { + try { + final Map cache = stemByAddressCache.get(account, ConcurrentHashMap::new); + if (slotKey.getSlotKey().isPresent()) { + cache.putAll( + generateManyStems( + account, + Collections.emptyList(), + List.of(slotKey.getSlotKey().get()), + Collections.emptyList(), + false)); + } + } catch (ExecutionException e) { + // no op + } + } + + /** + * Caches stems for the code associated with an account address. + * + * @param account the address of the account associated with the code + * @param code the smart contract code for which trie stems are to be cached + */ + @VisibleForTesting + public void cacheCodeStems(final Address account, final Bytes code) { + try { + final Map cache = stemByAddressCache.get(account, ConcurrentHashMap::new); + cache.putAll( + generateManyStems( + account, + Collections.emptyList(), + Collections.emptyList(), + generateCodeChunkKeyIds(code), + false)); + } catch (ExecutionException e) { + // no op + } + } + + private List generateAccountKeyIds() { + final List keys = new ArrayList<>(); + keys.add(Parameters.VERSION_LEAF_KEY); + keys.add(Parameters.BALANCE_LEAF_KEY); + keys.add(Parameters.NONCE_LEAF_KEY); + keys.add(Parameters.CODE_KECCAK_LEAF_KEY); + return keys; + } + + private List generateCodeChunkKeyIds(final Bytes code) { + return new ArrayList<>( + IntStream.range(0, trieKeyAdapter.getNbChunk(code)).mapToObj(UInt256::valueOf).toList()); + } + + private List generateStorageKeyIds(final Set storageSlotKeys) { + return storageSlotKeys.stream() + .map(storageSlotKey -> storageSlotKey.getSlotKey().orElseThrow()) + .map(Bytes32::wrap) + .toList(); + } + + private Map getCachedStemForAccount(final Address address) { + return Optional.ofNullable(stemByAddressCache.getIfPresent(address)) + .orElseGet(ConcurrentHashMap::new); + } + + private boolean codeIsEmpty(final Bytes value) { + return value == null || value.isEmpty(); + } + + /** + * Represents the context for a hasher, including the hasher instance and flags indicating the + * presence of storage trie keys and code trie keys. This record is used to encapsulate the state + * and configuration of the hasher in relation to preloaded trie stems. + * + * @param hasher the hasher instance used for generating stems + * @param hasStorageTrieKeys flag indicating whether storage trie keys are present in the cache + * @param hasCodeTrieKeys flag indicating whether code trie keys are present in the cache + */ + public record HasherContext(Hasher hasher, boolean hasStorageTrieKeys, boolean hasCodeTrieKeys) {} +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java deleted file mode 100644 index 75e946dc27c..00000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/cache/TrieKeyPreloader.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache; - -import static org.hyperledger.besu.ethereum.trie.verkle.util.Parameters.VERKLE_NODE_WIDTH; - -import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; -import org.hyperledger.besu.ethereum.trie.verkle.adapter.TrieKeyAdapter; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.CachedPedersenHasher; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.Hasher; -import org.hyperledger.besu.ethereum.trie.verkle.hasher.PedersenHasher; -import org.hyperledger.besu.ethereum.trie.verkle.util.Parameters; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.stream.IntStream; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; - -public class TrieKeyPreloader { - private static final int TRIE_KEY_CACHE_SIZE = 10_000; - private final Cache> trieKeysCache = - CacheBuilder.newBuilder().maximumSize(TRIE_KEY_CACHE_SIZE).build(); - - private final TrieKeyAdapter trieKeyAdapter; - - private final Hasher hasher; - - private long missedTrieKeys = 0; - - public TrieKeyPreloader() { - this.hasher = new PedersenHasher(); - this.trieKeyAdapter = new TrieKeyAdapter(hasher); - this.trieKeyAdapter.versionKey( - Address.ZERO); // TODO REMOVE is just to preload the native library for performance check - } - - public HasherContext createPreloadedHasher( - final Address address, - final Map> storageUpdate, - final DiffBasedValue codeUpdate) { - - // generate account triekeys - final List accountKeyIds = new ArrayList<>(generateAccountKeyIds()); - - // generate storage triekeys - final List storageKeyIds = new ArrayList<>(); - boolean isStorageUpdateNeeded; - if (storageUpdate != null) { - final Set storageSlotKeys = storageUpdate.keySet(); - isStorageUpdateNeeded = !storageSlotKeys.isEmpty(); - if (isStorageUpdateNeeded) { - storageKeyIds.addAll(generateStorageKeyIds(storageSlotKeys)); - } - } - - // generate code triekeys - final List codeChunkIds = new ArrayList<>(); - boolean isCodeUpdateNeeded; - if (codeUpdate != null) { - final Bytes previousCode = codeUpdate.getPrior(); - final Bytes updatedCode = codeUpdate.getUpdated(); - isCodeUpdateNeeded = - !codeUpdate.isUnchanged() && !(codeIsEmpty(previousCode) && codeIsEmpty(updatedCode)); - if (isCodeUpdateNeeded) { - accountKeyIds.add(Parameters.CODE_SIZE_LEAF_KEY); - codeChunkIds.addAll( - generateCodeChunkKeyIds(updatedCode == null ? previousCode : updatedCode)); - } - } - - return new HasherContext( - new CachedPedersenHasher( - generateManyTrieKeyHashes(address, accountKeyIds, storageKeyIds, codeChunkIds, true)), - !storageKeyIds.isEmpty(), - !codeChunkIds.isEmpty()); - } - - public void preLoadAccount(final Address account) { - CompletableFuture.runAsync(() -> cacheAccountTrieKeys(account)); - } - - public void preLoadStorageSlot(final Address account, final StorageSlotKey slotKey) { - CompletableFuture.runAsync(() -> cacheSlotTrieKeys(account, slotKey)); - } - - public void preLoadCode(final Address account, final Bytes code) { - CompletableFuture.runAsync(() -> cacheCodeTrieKeys(account, code)); - } - - public Map generateManyTrieKeyHashes( - final Address address, - final List accountKeyIds, - final List storageKeyIds, - final List codeChunkIds, - final boolean checkCacheBeforeGeneration) { - - final Set offsetsToGenerate = new HashSet<>(); - - if (!accountKeyIds.isEmpty()) { - offsetsToGenerate.add(UInt256.ZERO); - } - for (Bytes32 storageKey : storageKeyIds) { - offsetsToGenerate.add(trieKeyAdapter.locateStorageKeyOffset(storageKey)); - } - for (Bytes32 codeChunkId : codeChunkIds) { - final UInt256 codeChunkOffset = trieKeyAdapter.locateCodeChunkKeyOffset(codeChunkId); - offsetsToGenerate.add(codeChunkOffset.divide(VERKLE_NODE_WIDTH)); - } - - if (checkCacheBeforeGeneration) { - - // not regenerate data already preloaded - final Map cachedTrieKeys = getCachedTrieKeysForAccount(address); - cachedTrieKeys.forEach((key, __) -> offsetsToGenerate.remove(key)); - missedTrieKeys += offsetsToGenerate.size(); - - if (offsetsToGenerate.isEmpty()) { - return cachedTrieKeys; - } - - final Map trieKeyHashes = - hasher.manyTrieKeyHashes(address, new ArrayList<>(offsetsToGenerate)); - trieKeyHashes.putAll(cachedTrieKeys); - - return trieKeyHashes; - - } else { - return hasher.manyTrieKeyHashes(address, new ArrayList<>(offsetsToGenerate)); - } - } - - @VisibleForTesting - public Cache> getTrieKeysCache() { - return trieKeysCache; - } - - public long getMissedTrieKeys() { - return missedTrieKeys; - } - - public void reset() { - missedTrieKeys = 0; - } - - @VisibleForTesting - public void cacheAccountTrieKeys(final Address account) { - try { - final Map cache = trieKeysCache.get(account, this::createAccountCache); - cache.putAll( - generateManyTrieKeyHashes( - account, - generateAccountKeyIds(), - Collections.emptyList(), - Collections.emptyList(), - false)); - } catch (ExecutionException e) { - // no op - } - } - - @VisibleForTesting - public void cacheSlotTrieKeys(final Address account, final StorageSlotKey slotKey) { - try { - final Map cache = trieKeysCache.get(account, this::createAccountCache); - if (slotKey.getSlotKey().isPresent()) { - cache.putAll( - generateManyTrieKeyHashes( - account, - Collections.emptyList(), - List.of(slotKey.getSlotKey().get()), - Collections.emptyList(), - false)); - } - } catch (ExecutionException e) { - // no op - } - } - - @VisibleForTesting - public void cacheCodeTrieKeys(final Address account, final Bytes code) { - try { - final Map cache = trieKeysCache.get(account, this::createAccountCache); - cache.putAll( - generateManyTrieKeyHashes( - account, - Collections.emptyList(), - Collections.emptyList(), - generateCodeChunkKeyIds(code), - false)); - } catch (ExecutionException e) { - // no op - } - } - - private List generateAccountKeyIds() { - final List keys = new ArrayList<>(); - keys.add(Parameters.VERSION_LEAF_KEY); - keys.add(Parameters.BALANCE_LEAF_KEY); - keys.add(Parameters.NONCE_LEAF_KEY); - keys.add(Parameters.CODE_KECCAK_LEAF_KEY); - return keys; - } - - private List generateCodeChunkKeyIds(final Bytes code) { - return new ArrayList<>( - IntStream.range(0, trieKeyAdapter.getNbChunk(code)).mapToObj(UInt256::valueOf).toList()); - } - - private List generateStorageKeyIds(final Set storageSlotKeys) { - return storageSlotKeys.stream() - .map(storageSlotKey -> storageSlotKey.getSlotKey().orElseThrow()) - .map(Bytes32::wrap) - .toList(); - } - - private Map createAccountCache() { - return new ConcurrentHashMap<>(); - } - - private Map getCachedTrieKeysForAccount(final Address address) { - return Optional.ofNullable(trieKeysCache.getIfPresent(address)) - .orElseGet(ConcurrentHashMap::new); - } - - private boolean codeIsEmpty(final Bytes value) { - return value == null || value.isEmpty(); - } - - public record HasherContext(Hasher hasher, boolean hasStorageTrieKeys, boolean hasCodeTrieKeys) {} -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java index f2142b43f46..a2c766cbc8d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/worldview/VerkleWorldState.java @@ -30,8 +30,8 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleAccount; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.VerkleWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.TrieKeyPreloader; -import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.TrieKeyPreloader.HasherContext; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.StemPreloader; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.StemPreloader.HasherContext; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleLayeredWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.diffbased.verkle.storage.VerkleWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.verkletrie.VerkleEntryFactory; @@ -58,7 +58,7 @@ public class VerkleWorldState extends DiffBasedWorldState { private static final Logger LOG = LoggerFactory.getLogger(VerkleWorldState.class); - private final TrieKeyPreloader trieKeyPreloader; + private final StemPreloader stemPreloader; public VerkleWorldState( final VerkleWorldStateProvider archive, @@ -77,13 +77,13 @@ public VerkleWorldState( final TrieLogManager trieLogManager, final EvmConfiguration evmConfiguration) { super(worldStateKeyValueStorage, cachedWorldStorageManager, trieLogManager); - this.trieKeyPreloader = new TrieKeyPreloader(); + this.stemPreloader = new StemPreloader(); this.setAccumulator( new VerkleWorldStateUpdateAccumulator( this, - (addr, value) -> trieKeyPreloader.preLoadAccount(addr), - trieKeyPreloader::preLoadStorageSlot, - trieKeyPreloader::preLoadCode, + (addr, value) -> stemPreloader.preLoadAccount(addr), + stemPreloader::preLoadStorageSlot, + stemPreloader::preLoadCode, evmConfiguration)); } @@ -123,7 +123,7 @@ protected Hash internalCalculateRootHash( preloadedHashers.put( accountKey, - trieKeyPreloader.createPreloadedHasher( + stemPreloader.createPreloadedHasher( accountKey, storageAccountUpdate, codeUpdate)); }); @@ -161,8 +161,14 @@ protected Hash internalCalculateRootHash( // LOG.info(stateTrie.toDotTree()); final Bytes32 rootHash = stateTrie.getRootHash(); - LOG.info("end commit " + rootHash); - trieKeyPreloader.reset(); + LOG.info( + "end commit " + + rootHash + + " " + + stemPreloader.getNbMissedStems() + + "/" + + stemPreloader.getNbCachedStems()); + stemPreloader.reset(); return Hash.wrap(rootHash); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java index 9db0811f70c..c0bcf968131 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/LogRollingTests.java @@ -464,7 +464,6 @@ void rollBackFourTimes() { worldState.persist(headerThree); final TrieLogLayer layerThree = getTrieLogLayer(trieLogStorage, headerThree.getHash()); - System.out.println(layerThree.dump()); final WorldUpdater updater4 = worldState.updater(); final MutableAccount mutableAccount4 = updater4.getAccount(addressOne); mutableAccount4.setStorageValue(UInt256.ONE, UInt256.valueOf(1)); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/StemPreloaderTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/StemPreloaderTests.java new file mode 100644 index 00000000000..1fcdb5bb612 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/StemPreloaderTests.java @@ -0,0 +1,131 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.verkle; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.StemPreloader; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class StemPreloaderTests { + + private StemPreloader stemPreLoader; + private final List

accounts = + List.of(Address.fromHexString("0xdeadbeef"), Address.fromHexString("0xdeadbeee")); + + @BeforeEach + public void setup() { + stemPreLoader = new StemPreloader(); + } + + // Test to verify that a single account's trie key is correctly added to the cache during preload + @Test + void shouldAddAccountsStemsInCacheDuringPreload() { + stemPreLoader.cacheAccountStems(accounts.get(0)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is exactly 1, indicating a single trie key has + // been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + } + + // Test to verify that stems for multiple accounts are correctly added to the cache during + // preload + @Test + void shouldAddMultipleAccountsStemsInCacheDuringPreload() { + stemPreLoader.cacheAccountStems(accounts.get(0)); + stemPreLoader.cacheAccountStems(accounts.get(1)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(2); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that each account's cache size is 1, indicating that stems for both accounts have + // been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + final Map account2Cache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(1)); + Assertions.assertThat(Objects.requireNonNull(account2Cache).size()).isEqualTo(1); + } + + // Test to verify that header slots stems are correctly added to the cache for a single + // account during preload + @Test + void shouldAddHeaderSlotsStemsInCacheDuringPreload() { + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.ZERO)); + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.ONE)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 1, indicating that two slots in the header + // storage have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); + } + + // Test to verify that multiple slots stems are correctly added to the cache for a single + // account during preload + @Test + void shouldAddMultipleSlotsStemsInCacheDuringPreload() { + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.ONE)); + stemPreLoader.cacheSlotStems(accounts.get(0), new StorageSlotKey(UInt256.valueOf(64))); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 2, indicating that slots in both the header and + // main storage have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); + } + + // Test to verify that code stems are correctly added to the cache for a single account during + // preload + @Test + void shouldAddCodeStemsInCacheDuringPreload() { + stemPreLoader.cacheCodeStems(accounts.get(0), Bytes.repeat((byte) 0x01, 4096)); + Assertions.assertThat(stemPreLoader.getStemByAddressCache().size()).isEqualTo(1); + final Map accountCache = + stemPreLoader.getStemByAddressCache().getIfPresent(accounts.get(0)); + // Assert that the cache size for the account is 2, indicating that the code is too big and + // multiple offsets have been cached + Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); + } + + // Test to verify that missing stems are correctly generated and accounted for + @Test + void shouldGenerateMissingStems() { + stemPreLoader.cacheAccountStems(accounts.get(0)); + Map stems = + stemPreLoader.generateManyStems( + accounts.get(0), + Collections.emptyList(), + List.of(UInt256.valueOf(64)), + List.of(UInt256.valueOf(128)), + true); + // Assert that two stems (slot and code) are missing from the cache + Assertions.assertThat(stemPreLoader.getNbMissedStems()).isEqualTo(2); + // Assert that three stems have been generated: one for the header, one for the slot in main + // storage, and one for the code chunk + Assertions.assertThat(stems.size()).isEqualTo(3); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/TrieKeyPreloaderTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/TrieKeyPreloaderTests.java deleted file mode 100644 index 4b9d5026f9f..00000000000 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/verkle/TrieKeyPreloaderTests.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright contributors to Hyperledger Besu. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie.diffbased.verkle; - -import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.diffbased.verkle.cache.TrieKeyPreloader; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class TrieKeyPreloaderTests { - - private TrieKeyPreloader trieKeyPreLoader; - private final List
accounts = - List.of(Address.fromHexString("0xdeadbeef"), Address.fromHexString("0xdeadbeee")); - - @BeforeEach - public void setup() { - trieKeyPreLoader = new TrieKeyPreloader(); - } - - // Test to verify that a single account's trie key is correctly added to the cache during preload - @Test - void shouldAddAccountsTrieKeyInCacheDuringPreload() { - trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(0)); - Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); - final Map accountCache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); - // Assert that the cache size for the account is exactly 1, indicating a single trie key has - // been cached - Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); - } - - // Test to verify that trie keys for multiple accounts are correctly added to the cache during - // preload - @Test - void shouldAddMultipleAccountsTrieKeyInCacheDuringPreload() { - trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(0)); - trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(1)); - Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(2); - final Map accountCache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); - // Assert that each account's cache size is 1, indicating that trie keys for both accounts have - // been cached - Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); - final Map account2Cache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(1)); - Assertions.assertThat(Objects.requireNonNull(account2Cache).size()).isEqualTo(1); - } - - // Test to verify that header slots trie keys are correctly added to the cache for a single - // account during preload - @Test - void shouldAddHeaderSlotsTrieKeysInCacheDuringPreload() { - trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.ZERO)); - trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.ONE)); - Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); - final Map accountCache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); - // Assert that the cache size for the account is 1, indicating that two slots in the header - // storage have been cached - Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(1); - } - - // Test to verify that multiple slots trie keys are correctly added to the cache for a single - // account during preload - @Test - void shouldAddMultipleSlotsTrieKeysInCacheDuringPreload() { - trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.ONE)); - trieKeyPreLoader.cacheSlotTrieKeys(accounts.get(0), new StorageSlotKey(UInt256.valueOf(64))); - Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); - final Map accountCache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); - // Assert that the cache size for the account is 2, indicating that slots in both the header and - // main storage have been cached - Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); - } - - // Test to verify that code trie keys are correctly added to the cache for a single account during - // preload - @Test - void shouldAddCodeTrieKeysInCacheDuringPreload() { - trieKeyPreLoader.cacheCodeTrieKeys(accounts.get(0), Bytes.repeat((byte) 0x01, 4096)); - Assertions.assertThat(trieKeyPreLoader.getTrieKeysCache().size()).isEqualTo(1); - final Map accountCache = - trieKeyPreLoader.getTrieKeysCache().getIfPresent(accounts.get(0)); - // Assert that the cache size for the account is 2, indicating that the code is too big and - // multiple offsets have been cached - Assertions.assertThat(Objects.requireNonNull(accountCache).size()).isEqualTo(2); - } - - // Test to verify that missing trie keys are correctly generated and accounted for - @Test - void shouldGenerateMissingTrieKeys() { - trieKeyPreLoader.cacheAccountTrieKeys(accounts.get(0)); - Map trieKeyHashes = - trieKeyPreLoader.generateManyTrieKeyHashes( - accounts.get(0), - Collections.emptyList(), - List.of(UInt256.valueOf(64)), - List.of(UInt256.valueOf(128)), - true); - // Assert that two trie keys (slot and code) are missing from the cache - Assertions.assertThat(trieKeyPreLoader.getMissedTrieKeys()).isEqualTo(2); - // Assert that three trie keys have been generated: one for the header, one for the slot in main - // storage, and one for the code chunk - Assertions.assertThat(trieKeyHashes.size()).isEqualTo(3); - } -} diff --git a/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java b/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java index 0001654862e..a73058f8b0c 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/gascalculator/stateless/Eip4762AccessWitness.java @@ -422,7 +422,7 @@ public ChunkAccessKey( @Override public List getStorageSlotTreeIndexes(final UInt256 storageKey) { return List.of( - TRIE_KEY_ADAPTER.locateStorageKeyOffset(storageKey), - TRIE_KEY_ADAPTER.locateStorageKeySuffix(storageKey)); + TRIE_KEY_ADAPTER.getStorageKeyTrieIndex(storageKey), + TRIE_KEY_ADAPTER.getStorageKeySuffix(storageKey)); } }