From 75f565e0c7d8bd2b1716c9d408dbe03dbca183a7 Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Thu, 27 Jun 2024 23:20:51 -0600 Subject: [PATCH 01/11] T8n support for isStateTest and empty accounts (#7275) Update t8n executor to support new isStateTest env flag that will disable extra-transactional processing such as block rewards and beacon root. Also, make sure such extra-transactional commits don't create empty accounts. Signed-off-by: Danno Ferrin Co-authored-by: Usman Saleem --- .../hyperledger/besu/evmtool/T8nExecutor.java | 53 ++++++++++++------- .../BlockchainReferenceTestCaseSpec.java | 3 +- .../referencetests/ReferenceTestEnv.java | 23 ++++++-- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java index ccabce833a6..caa5cb36724 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java @@ -51,6 +51,7 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedAccount; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.gascalculator.GasCalculator; import org.hyperledger.besu.evm.log.Log; @@ -265,9 +266,11 @@ static T8nResult runTest( .blobGasPricePerGas(calculateExcessBlobGasForParent(protocolSpec, blockHeader)); long blobGasLimit = protocolSpec.getGasLimitCalculator().currentBlobGasLimit(); - protocolSpec - .getBlockHashProcessor() - .processBlockHashes(blockchain, worldState, referenceTestEnv); + if (!referenceTestEnv.isStateTest()) { + protocolSpec + .getBlockHashProcessor() + .processBlockHashes(blockchain, worldState, referenceTestEnv); + } final WorldUpdater rootWorldStateUpdater = worldState.updater(); List receipts = new ArrayList<>(); @@ -318,13 +321,12 @@ static T8nResult runTest( timer.stop(); if (shouldClearEmptyAccounts(fork)) { - final Account coinbase = worldStateUpdater.getOrCreate(blockHeader.getCoinbase()); - if (coinbase != null && coinbase.isEmpty()) { - worldStateUpdater.deleteAccount(coinbase.getAddress()); - } - final Account txSender = worldStateUpdater.getAccount(transaction.getSender()); - if (txSender != null && txSender.isEmpty()) { - worldStateUpdater.deleteAccount(txSender.getAddress()); + var entries = new ArrayList<>(worldState.getAccumulator().getAccountsToUpdate().entrySet()); + for (var entry : entries) { + DiffBasedAccount updated = entry.getValue().getUpdated(); + if (updated != null && updated.isEmpty()) { + worldState.getAccumulator().deleteAccount(entry.getKey()); + } } } if (result.isInvalid()) { @@ -397,7 +399,9 @@ static T8nResult runTest( // block reward // The max production reward was 5 Eth, longs can hold over 18 Eth. - if (!validTransactions.isEmpty() && (rewardString == null || Long.decode(rewardString) > 0)) { + if (!referenceTestEnv.isStateTest() + && !validTransactions.isEmpty() + && (rewardString == null || Long.decode(rewardString) > 0)) { Wei reward = (rewardString == null) ? protocolSpec.getBlockReward() @@ -408,15 +412,24 @@ static T8nResult runTest( } rootWorldStateUpdater.commit(); - // Invoke the withdrawal processor to handle CL withdrawals. - if (!referenceTestEnv.getWithdrawals().isEmpty()) { - try { - protocolSpec - .getWithdrawalsProcessor() - .ifPresent( - p -> p.processWithdrawals(referenceTestEnv.getWithdrawals(), worldState.updater())); - } catch (RuntimeException re) { - resultObject.put("exception", re.getMessage()); + + if (referenceTestEnv.isStateTest()) { + if (!referenceTestEnv.getWithdrawals().isEmpty()) { + resultObject.put("exception", "withdrawals are not supported in state tests"); + } + } else { + // Invoke the withdrawal processor to handle CL withdrawals. + if (!referenceTestEnv.getWithdrawals().isEmpty()) { + try { + protocolSpec + .getWithdrawalsProcessor() + .ifPresent( + p -> + p.processWithdrawals( + referenceTestEnv.getWithdrawals(), worldState.updater())); + } catch (RuntimeException re) { + resultObject.put("exception", re.getMessage()); + } } } diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java index 378637f3634..1048f6238a2 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java @@ -249,7 +249,8 @@ public CandidateBlock( @JsonProperty("uncleHeaders") final Object uncleHeaders, @JsonProperty("withdrawals") final Object withdrawals, @JsonProperty("depositRequests") final Object depositRequests, - @JsonProperty("withdrawalRequests") final Object withdrawalRequests) { + @JsonProperty("withdrawalRequests") final Object withdrawalRequests, + @JsonProperty("consolidationRequests") final Object consolidationRequests) { boolean blockVaid = true; // The BLOCK__WrongCharAtRLP_0 test has an invalid character in its rlp string. Bytes rlpAttempt = null; diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java index 3cd248e3aef..636e23f8786 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java @@ -83,6 +83,8 @@ Withdrawal asWithdrawal() { private final Bytes32 beaconRoot; + private final boolean isStateTest; + /** * Public constructor. * @@ -120,7 +122,8 @@ public ReferenceTestEnv( @JsonProperty("parentGasLimit") final String parentGasLimit, @JsonProperty("parentGasUsed") final String parentGasUsed, @JsonProperty("parentTimestamp") final String parentTimestamp, - @JsonProperty("parentUncleHash") final String _parentUncleHash) { + @JsonProperty("parentUncleHash") final String _parentUncleHash, + @JsonProperty("isStateTest") final String isStateTest) { super( generateTestBlockHash(previousHash, number), Hash.EMPTY_LIST_HASH, // ommersHash @@ -164,10 +167,16 @@ public ReferenceTestEnv( Map.entry( Long.decode(entry.getKey()), Hash.fromHexString(entry.getValue()))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - this.beaconRoot = - beaconRoot == null - ? (currentBeaconRoot == null ? null : Hash.fromHexString(currentBeaconRoot)) - : Hash.fromHexString(beaconRoot); + if (beaconRoot == null) { + if (currentBeaconRoot == null) { + this.beaconRoot = null; + } else { + this.beaconRoot = Hash.fromHexString(currentBeaconRoot); + } + } else { + this.beaconRoot = Hash.fromHexString(beaconRoot); + } + this.isStateTest = Boolean.parseBoolean(isStateTest); } @Override @@ -239,6 +248,10 @@ public Map getBlockHashes() { return blockHashes; } + public boolean isStateTest() { + return isStateTest; + } + @Override public boolean equals(final Object o) { if (this == o) return true; From 6d55f91a9c8269cd797fec0954c727c087007b1e Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Fri, 28 Jun 2024 16:09:52 +1000 Subject: [PATCH 02/11] Promote storage x-trie-log subcommand to trie-log (#7278) Signed-off-by: Simon Dudley --- CHANGELOG.md | 8 ++++++++ .../besu/cli/subcommands/storage/TrieLogSubCommand.java | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4eb0b5166d7..5ba6b4dc5dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## Next Release +### Upcoming Breaking Changes +- Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version. +- PKI-backed QBFT will be removed in a future version of Besu. Other forms of QBFT will remain unchanged. +- --Xbonsai-limit-trie-logs-enabled is deprecated, use --bonsai-limit-trie-logs-enabled instead +- --Xbonsai-trie-logs-pruning-window-size is deprecated, use --bonsai-trie-logs-pruning-window-size instead +- `besu storage x-trie-log` subcommand is deprecated, use `besu storage trie-log` instead + ### Breaking Changes - `Xp2p-peer-lower-bound` has been removed. [#7247](https://github.com/hyperledger/besu/pull/7247) @@ -16,6 +23,7 @@ - Add LUKSO as predefined network name [#7223](https://github.com/hyperledger/besu/pull/7223) - Refactored how code, initcode, and max stack size are configured in forks. [#7245](https://github.com/hyperledger/besu/pull/7245) - Nodes in a permissioned chain maintain (and retry) connections to bootnodes [#7257](https://github.com/hyperledger/besu/pull/7257) +- Promote experimental `besu storage x-trie-log` subcommand to production-ready [#7278](https://github.com/hyperledger/besu/pull/7278) ### Bug fixes - Validation errors ignored in accounts-allowlist and empty list [#7138](https://github.com/hyperledger/besu/issues/7138) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index 6b0507895c3..de42616822f 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -49,7 +49,8 @@ /** The Trie Log subcommand. */ @Command( - name = "x-trie-log", + name = "trie-log", + aliases = "x-trie-log", description = "Manipulate trie logs", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, From c298a9e77e00aa3745a47a155bf5b6448e53bac4 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Fri, 28 Jun 2024 08:37:16 -0400 Subject: [PATCH 03/11] Evm tool readme update (#7274) * update paths to binary. update docker build to use java 21 * updated suggested jdk --------- Signed-off-by: Justin Florentine --- ethereum/evmtool/README.md | 71 ++++++--------------- ethereum/evmtool/build.gradle | 41 ------------ ethereum/evmtool/src/main/docker/Dockerfile | 2 +- 3 files changed, 20 insertions(+), 94 deletions(-) diff --git a/ethereum/evmtool/README.md b/ethereum/evmtool/README.md index a1a703f9132..50db1066f38 100644 --- a/ethereum/evmtool/README.md +++ b/ethereum/evmtool/README.md @@ -8,13 +8,9 @@ and enclosing data structures. Using EVM Tool in execution-specification-tests ----------------------------------------------- -To use EVM Tool in Execution Spec tests it is recommended that you use -the GraalVM build, as the framework incurs significant startup penalties -for each invocation when run via the Java runtime. - ### Building Execution Tests on macOS -Current as of 24 Jun 2023. +Current as of 26 Jun 2024. MacOS users will typically encounter two problems,one relating to the version of Python used and one relating to zsh. @@ -35,7 +31,7 @@ install python packages needs to escape the brackets pip install -e .\[docs,lint,test\] ``` -An all-in-one script, including homebrew, would look like +An all-in-one script, using homebrew, would look like ```zsh brew install ethereum solidity @@ -43,85 +39,56 @@ git clone https://github.com/ethereum/execution-spec-tests cd execution-spec-tests python3 -m venv ./venv/ source ./venv/bin/activate -pip install -e .[docs,lint,test] +pip install -e .\[docs,lint,test\] ``` -### Building EvmTool with GraalVM on macOS +### Building EvmTool on macOS -First you need a GraalVM JDK installed, if not already installed. +First you need a Java 21+ JDK installed, if not already installed. It is recommended you install [SDKMAN](https://sdkman.io/install) to -manage the graalVM install, homebrew has issues with native attributes -and code signing. +manage the jvm install. ```zsh -sdk install java 22.3.r17-grl -sdk use java 22.3.r17-grl +sdk install java 21.0.3-tem +sdk use java 21.0.3-tem ``` -You can also manually install GraalVM from -the [GraalVM website](https://www.graalvm.org/downloads).. - -Once GraalVM is installed you use the `nativeCompile` target. +Once a JVM is installed you use the gradle target: ```zsh -./gradlew nativeCompile +./gradlew installDist -x test ``` The resulting binary -is `./ethereum/evmtool/build/native/nativeCompile/evmtool` +is `build/install/besu/bin/evmtool` If the testing repository and besu are installed in the same parent directory, the command to run the execution tests is ```zsh -fill -v tests --evm-bin ../besu/ethereum/evmtool/build/install/evmtool/bin/evm +fill -v tests --evm-bin ../besu/build/install/besu/bin/evmtool ``` Assuming homebrew and SDKMan are both installed, the complete script is ```zsh -sdk install java 22.3.r17-grl -sdk use java 22.3.r17-grl +sdk install java 21.0.3-tem +sdk use java 21.0.3-tem git clone https://github.com/hyperledger/besu cd besu -./gradlew nativeCompile +./gradlew installDist -x test cd .. brew install ethereum solidity +solc-select install latest +solc-select use latest git clone https://github.com/ethereum/execution-spec-tests cd execution-spec-tests python3 -m venv ./venv/ source ./venv/bin/activate -pip install -e .[docs,lint,test] +pip install -e .\[docs,lint,test\] -fill -v tests --evm-bin ../besu/ethereum/evmtool/build/install/evmtool/bin/evm +fill -v tests --evm-bin ../besu/build/install/besu/bin/evmtool ``` -If you don't want to use the GraalVM tool the binary that is compatible -is generated by the `ethereum:evmtool:installdist` target and is located -at `./ethereum/evmtool/build/install/evmtool/bin/evm` - -Why not GraalVM for everything? -------------------------------- - -Using GraalVM in execution-spec-tests results in over 20x performance -increase in execution. It will be faster to build GraalVM from scratch -and run the execution-spec-tests than to run just the Java version. - -It is demonstrably faster to run the Java version for a node. -All the test execution gains are the result of reduced startup -penalties. Larger benchmarks will show that performance intensive EVM -code will be slower in GraalVM than the Java version due to the adaptive -compiler. - -For contracts that execute 30 million gas in small operations it is -often faster to run the Java EVMTool than the GraalVM EVMTool, including -startup penalty. The execution tests focus on smaller VM tests that -demonstrate specification conformance. - -We would also need to reconsider some library choices. GraalVM does not -work with Log4J, and we would have to ban that library across Besu and -all dependents. Libraries such as Netty also have some problematic entry -points that interact poorly with how SLF4J needs to initialize. - diff --git a/ethereum/evmtool/build.gradle b/ethereum/evmtool/build.gradle index 0d1700082fa..aaddc354d5f 100644 --- a/ethereum/evmtool/build.gradle +++ b/ethereum/evmtool/build.gradle @@ -72,16 +72,6 @@ dependencies { mainClassName = 'org.hyperledger.besu.evmtool.EvmTool' -startScripts { - applicationName = 'evm' - defaultJvmOpts = [ - "-Dsecp256k1.randomize=false" - ] - doLast { - unixScript.text = unixScript.text.replace('BESU_HOME', '\$APP_HOME') - windowsScript.text = windowsScript.text.replace('BESU_HOME', '%~dp0..') - } -} // rename the top level dir from besu- to besu and this makes it really // simple for use in docker @@ -148,34 +138,3 @@ tasks.register('dockerUpload', Exec) { executable "sh" args "-c", cmd } - -graalvmNative { - binaries { - main { - sharedLibrary = false - buildArgs.addAll( - "-H:ReflectionConfigurationFiles=${projectDir}/src/main/graal/reflection-config.json", - "-H:AdditionalSecurityProviders=org.bouncycastle.jce.provider.BouncyCastleProvider", - '-H:+TraceSecurityServices' - ) - - - // Netty drags in older versions of bouncy castle, exclude it so there are no conflicts - excludeConfig.put("io.netty:netty-buffer", [".*"]) - } - } -} - - -configurations.nativeImageClasspath { - // netty statically allocates some problematic classes - exclude group: 'io.netty', module: 'netty-buffer' - exclude group: 'io.netty', module: 'netty-common' - exclude group: 'io.netty', module: 'netty-transport' - - // keep log4j from sneaking in. GraalVM has an aleric reaction if it sees even one class - exclude group: 'org.slf4j', module: 'log4j-over-slf4j:1.7.36' - exclude group: "log4j", module: "log4j" - exclude group: "org.apache.logging.log4j" - exclude group: 'org.bouncycastle', module: 'bcprov-jdk18on' -} diff --git a/ethereum/evmtool/src/main/docker/Dockerfile b/ethereum/evmtool/src/main/docker/Dockerfile index 9d9bf4aafa1..9d7b48d3cb1 100644 --- a/ethereum/evmtool/src/main/docker/Dockerfile +++ b/ethereum/evmtool/src/main/docker/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update $NO_PROXY_CACHE && \ apt-get -o Acquire::BrokenProxy=true -o Acquire::http::No-Cache=true -o Acquire::http::Pipeline-Depth=0 \ --no-install-recommends -q --assume-yes install ca-certificates-java=20190909* && \ apt-get -o Acquire::BrokenProxy=true -o Acquire::http::No-Cache=true -o Acquire::http::Pipeline-Depth=0 \ - --no-install-recommends -q --assume-yes install openjdk-17-jre-headless=17* && \ + --no-install-recommends -q --assume-yes install openjdk-21-jre-headless=21* && \ # Clean apt cache \ apt-get clean && \ rm -rf /var/cache/apt/archives/* /var/cache/apt/archives/partial/* && \ From fa63fc45b469b309fb6a4c66e438e22f33a198a0 Mon Sep 17 00:00:00 2001 From: Usman Saleem Date: Sat, 29 Jun 2024 02:55:47 +1000 Subject: [PATCH 04/11] javadoc - Add missing javadoc for evmtool module (#7277) Signed-off-by: Usman Saleem --- build.gradle | 2 - .../besu/evmtool/B11rSubCommand.java | 11 ++++ .../besu/evmtool/BenchmarkSubCommand.java | 15 +++++ .../besu/evmtool/BlockchainModule.java | 8 +++ .../besu/evmtool/CodeValidateSubCommand.java | 22 +++++++ .../besu/evmtool/DataStoreModule.java | 11 ++++ .../besu/evmtool/EOFTestSubCommand.java | 15 +++++ .../org/hyperledger/besu/evmtool/EvmTool.java | 9 +++ .../besu/evmtool/EvmToolCommand.java | 43 ++++++++++++ .../evmtool/EvmToolCommandOptionsModule.java | 13 ++++ .../besu/evmtool/EvmToolComponent.java | 47 +++++++++++++ .../besu/evmtool/GenesisFileModule.java | 15 +++++ .../hyperledger/besu/evmtool/JsonUtils.java | 6 ++ .../besu/evmtool/PrettyPrintSubCommand.java | 28 ++++++++ .../besu/evmtool/ProtocolModule.java | 15 +++++ .../besu/evmtool/StateTestSubCommand.java | 21 ++++++ .../hyperledger/besu/evmtool/T8nExecutor.java | 66 ++++++++++++++++++- .../besu/evmtool/T8nServerSubCommand.java | 17 +++++ .../besu/evmtool/T8nSubCommand.java | 23 +++++++ .../besu/evmtool/VersionProvider.java | 18 +++++ 20 files changed, 402 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 1adc3b992ce..7d2293b61e3 100644 --- a/build.gradle +++ b/build.gradle @@ -401,8 +401,6 @@ allprojects { '-org.hyperledger.besu.ethereum.eth.*,' + '-org.hyperledger.besu.ethereum.eth,' + '-org.hyperledger.besu.consensus.merge,' + - // evmtool module - '-org.hyperledger.besu.evmtool,' + // p2p module '-org.hyperledger.besu.ethereum.p2p,' + '-org.hyperledger.besu.ethereum.p2p.*,' + diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java index 9a27a8a40d9..0bb7949754f 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java @@ -51,6 +51,11 @@ import picocli.CommandLine.Parameters; import picocli.CommandLine.ParentCommand; +/** + * This class implements the Runnable interface and represents the B11rSubCommand. It is responsible + * for handling the block builder subcommand in the EVM tool. It provides methods to read headers, + * move fields, and run the command. + */ @Command( name = COMMAND_NAME, aliases = {COMMAND_ALIAS}, @@ -138,12 +143,18 @@ public void consumeParameters( } } + /** Default constructor for the B11rSubCommand class. This is required by PicoCLI. */ @SuppressWarnings("unused") public B11rSubCommand() { // PicoCLI requires this parentCommand = null; } + /** + * Constructs a new B11rSubCommand with the given parent command. + * + * @param parentCommand the parent command of this subcommand + */ @SuppressWarnings("unused") public B11rSubCommand(final EvmToolCommand parentCommand) { // PicoCLI requires this too diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BenchmarkSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BenchmarkSubCommand.java index 05d90a7b358..829bf2d58e7 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BenchmarkSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BenchmarkSubCommand.java @@ -33,13 +33,22 @@ import picocli.CommandLine.Parameters; import picocli.CommandLine.ParentCommand; +/** + * This class represents the BenchmarkSubCommand. It is responsible for executing an Ethereum State + * Test. + */ @CommandLine.Command( name = COMMAND_NAME, description = "Execute an Ethereum State Test.", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class) public class BenchmarkSubCommand implements Runnable { + /** + * The command name for the BenchmarkSubCommand. This constant is used as the name attribute in + * the {@code CommandLine.Command} annotation. + */ public static final String COMMAND_NAME = "benchmark"; + private final PrintStream output; enum Benchmark { @@ -68,11 +77,17 @@ enum Benchmark { @ParentCommand EvmToolCommand parentCommand; + /** Default constructor for the BenchmarkSubCommand class. This is required by PicoCLI. */ public BenchmarkSubCommand() { // PicoCLI requires this this(System.out); } + /** + * Constructs a new BenchmarkSubCommand with the given output stream. + * + * @param output the output stream to be used + */ public BenchmarkSubCommand(final PrintStream output) { this.output = output; } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BlockchainModule.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BlockchainModule.java index b3a1531133a..7b463fe08e3 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BlockchainModule.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BlockchainModule.java @@ -39,10 +39,18 @@ import dagger.Provides; import org.apache.tuweni.bytes.Bytes32; +/** + * This class is a Dagger module that provides dependencies related to the blockchain. It includes + * the GenesisFileModule and DataStoreModule for providing the genesis block and data store + * respectively. The class is annotated with {@code @Module} to indicate that it is a Dagger module. + */ @SuppressWarnings("WeakerAccess") @Module(includes = {GenesisFileModule.class, DataStoreModule.class}) public class BlockchainModule { + /** Default constructor for the BlockchainModule class. */ + public BlockchainModule() {} + @Singleton @Provides Blockchain provideBlockchain( diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/CodeValidateSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/CodeValidateSubCommand.java index 10e167f4069..e16d4014f0b 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/CodeValidateSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/CodeValidateSubCommand.java @@ -40,6 +40,11 @@ import picocli.CommandLine; import picocli.CommandLine.ParentCommand; +/** + * This class represents the CodeValidateSubCommand. It is responsible for validating EVM code for + * fuzzing. It implements the Runnable interface and is annotated with the {@code + * CommandLine.Command} annotation. + */ @SuppressWarnings({"ConstantValue", "DataFlowIssue"}) @CommandLine.Command( name = COMMAND_NAME, @@ -47,6 +52,10 @@ mixinStandardHelpOptions = true, versionProvider = VersionProvider.class) public class CodeValidateSubCommand implements Runnable { + /** + * The command name for the CodeValidateSubCommand. This constant is used as the name attribute in + * the CommandLine.Command annotation. + */ public static final String COMMAND_NAME = "code-validate"; @ParentCommand EvmToolCommand parentCommand; @@ -62,6 +71,7 @@ public class CodeValidateSubCommand implements Runnable { @CommandLine.Parameters private final List cliCode = new ArrayList<>(); + /** Default constructor for the CodeValidateSubCommand class. This is required by PicoCLI. */ @SuppressWarnings("unused") public CodeValidateSubCommand() { // PicoCLI requires this @@ -111,6 +121,18 @@ private void checkCodeFromBufferedReader(final BufferedReader in) { } } + /** + * This method is responsible for validating the EVM code. It takes a hexadecimal string + * representation of the EVM code as input. The method first converts the hexadecimal string to + * Bytes. It then checks if the code follows the EOF layout. If the layout is valid, it retrieves + * the code from the EVM. If the code is invalid, it returns an error message with the reason for + * the invalidity. If the code is valid, it returns a string with "OK" followed by the hexadecimal + * string representation of each code section. + * + * @param hexCode the hexadecimal string representation of the EVM code + * @return a string indicating whether the code is valid or not, and in case of validity, the + * hexadecimal string representation of each code section + */ public String considerCode(final String hexCode) { Bytes codeBytes; try { diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java index 28b212024c5..7629b318476 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java @@ -38,6 +38,14 @@ import dagger.Module; import dagger.Provides; +/** + * This class is a Dagger module that provides dependencies related to the data store. It includes + * the GenesisFileModule for providing the genesis block. The class is annotated with + * {@code @Module} to indicate that it is a Dagger module. It provides various key-value storages + * such as variables, blockchain, world state, world state preimage, and pruning. The type of + * key-value storage (e.g., rocksdb, memory) can be specified. The class also provides a + * BlockchainStorage which is a prefixed key blockchain storage. + */ @SuppressWarnings({"CloseableProvides"}) @Module(includes = GenesisFileModule.class) public class DataStoreModule { @@ -50,6 +58,9 @@ public class DataStoreModule { List.of(KeyValueSegmentIdentifier.values()), RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS)); + /** Default constructor for the DataStoreModule class. */ + public DataStoreModule() {} + @Provides @Singleton @Named("variables") diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EOFTestSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EOFTestSubCommand.java index ab53b41e1d4..bfa873e61cf 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EOFTestSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EOFTestSubCommand.java @@ -45,13 +45,16 @@ import org.apache.tuweni.bytes.Bytes; import picocli.CommandLine; +/** A PicoCli annotated command for running EOF validation reference tests. */ @CommandLine.Command( name = COMMAND_NAME, description = "Runs EOF validation reference tests", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class) public class EOFTestSubCommand implements Runnable { + /** The name of the EOF validation reference test command. */ public static final String COMMAND_NAME = "eof-test"; + @CommandLine.ParentCommand private final EvmToolCommand parentCommand; // picocli does it magically @@ -65,10 +68,16 @@ public class EOFTestSubCommand implements Runnable { EVM evm; String fork = null; + /** Default constructor for the EOFTestSubCommand class. Sets the parent command to null. */ public EOFTestSubCommand() { this(null); } + /** + * Constructor for the EOFTestSubCommand class with a parent command. + * + * @param parentCommand The parent command for this sub command. + */ public EOFTestSubCommand(final EvmToolCommand parentCommand) { this.parentCommand = parentCommand; } @@ -199,6 +208,12 @@ private void executeEOFTest(final String fileName, final MapEvmToolCommand implements the Runnable interface, making it the entrypoint for PicoCLI to + * execute this command. + * + *

The class provides various options for setting up and executing EVM transactions. These + * options include, but are not limited to, setting the gas price, sender address, receiver address, + * and the data to be sent with the transaction. + * + *

Key methods in this class include 'run()' for executing the command, 'execute()' for setting + * up and running the EVM transaction, and 'dumpWorldState()' for outputting the current state of + * the Ethereum world state. + */ @Command( description = "This command evaluates EVM transactions.", abbreviateSynopsis = true, @@ -247,12 +262,22 @@ void setBytes(final String optionValue) { PrintWriter out; InputStream in; + /** + * Default constructor for the EvmToolCommand class. It initializes the input stream with an empty + * byte array and the output stream with the standard output. + */ public EvmToolCommand() { this( new ByteArrayInputStream(new byte[0]), new PrintWriter(new BufferedWriter(new OutputStreamWriter(System.out, UTF_8)), true)); } + /** + * Constructor for the EvmToolCommand class with custom input and output streams. + * + * @param in The input stream to be used. + * @param out The output stream to be used. + */ public EvmToolCommand(final InputStream in, final PrintWriter out) { this.in = in; this.out = out; @@ -322,10 +347,21 @@ private static void addForkHelp(final CommandLine subCommandLine) { subCommandLine.setHelpSectionKeys(keys); } + /** + * Returns the fork name provided by the Dagger options. If no fork is provided, it returns the + * name of the default EVM specification version. + * + * @return The fork name. + */ public String getFork() { return daggerOptions.provideFork().orElse(EvmSpecVersion.defaultVersion().getName()); } + /** + * Checks if a fork is provided in the Dagger options. + * + * @return True if a fork is provided, false otherwise. + */ public boolean hasFork() { return daggerOptions.provideFork().isPresent(); } @@ -506,6 +542,13 @@ public void run() { } } + /** + * Dumps the current state of the Ethereum world state to the provided PrintWriter. The state + * includes account balances, nonces, codes, and storage. The output is in JSON format. + * + * @param worldState The Ethereum world state to be dumped. + * @param out The PrintWriter to which the state is dumped. + */ public static void dumpWorldState(final MutableWorldState worldState, final PrintWriter out) { out.println("{"); worldState diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommandOptionsModule.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommandOptionsModule.java index fe229dc3cf3..820b11442e9 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommandOptionsModule.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommandOptionsModule.java @@ -32,6 +32,16 @@ import picocli.CommandLine; import picocli.CommandLine.Option; +/** + * This class, EvmToolCommandOptionsModule, is a Dagger module that provides dependencies for the + * EvmToolCommand. It contains options for setting up the EVM tool, such as whether revert reasons + * should be persisted, the fork to evaluate, the key-value storage to be used, the data path, the + * block number to evaluate against, and the world state update mode. + * + *

The class uses PicoCLI annotations to define these options, which can be provided via the + * command line when running the EVM tool. Each option has a corresponding provider method that + * Dagger uses to inject the option's value where needed. + */ @SuppressWarnings("WeakerAccess") @Module public class EvmToolCommandOptionsModule { @@ -133,4 +143,7 @@ BlockParameter provideBlockParameter() { EvmConfiguration provideEvmConfiguration() { return new EvmConfiguration(jumpDestCacheWeightKilobytes, worldstateUpdateMode); } + + /** Default constructor for the EvmToolCommandOptionsModule class. */ + public EvmToolCommandOptionsModule() {} } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolComponent.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolComponent.java index 176700ee8ee..cc84b59480c 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolComponent.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolComponent.java @@ -26,6 +26,22 @@ import dagger.Component; +/** + * This is a Dagger component interface for the EVM (Ethereum Virtual Machine) tool. It is annotated + * with @Singleton to ensure that only a single instance of this component exists within the Dagger + * component graph. + * + *

The component is composed of several modules that provide the necessary dependencies for the + * EVM tool: - ProtocolModule: Provides the protocol specification. - GenesisFileModule: Provides + * the genesis file for the blockchain. - DataStoreModule: Provides the data store for blockchain + * data. - BlockchainModule: Provides the blockchain instance. - EvmToolCommandOptionsModule: + * Provides the command options for the EVM tool. - MetricsConfigurationModule and + * MetricsSystemModule: Provide the metrics system and its configuration. + * + *

The interface defines methods to get instances of key classes like ProtocolSpec, EVM, + * WorldUpdater, MutableWorldState, and Blockchain. These methods are used by Dagger to inject the + * returned instances where needed. + */ @Singleton @Component( modules = { @@ -39,13 +55,44 @@ }) public interface EvmToolComponent { + /** + * Retrieves the ProtocolSpec instance. ProtocolSpec defines the Ethereum protocol specifications, + * which includes the precompiled contracts, the gas calculator, the EVM, and the private nonce + * calculator. + * + * @return The ProtocolSpec instance. + */ ProtocolSpec getProtocolSpec(); + /** + * Retrieves the EVM instance. EVM (Ethereum Virtual Machine) is responsible for executing the + * bytecode of smart contracts in Ethereum. + * + * @return The EVM instance. + */ EVM getEVM(); + /** + * Retrieves the WorldUpdater instance. WorldUpdater is used to modify the world state, which + * includes the accounts and their associated storage and code. + * + * @return The WorldUpdater instance. + */ WorldUpdater getWorldUpdater(); + /** + * Retrieves the MutableWorldState instance. MutableWorldState represents the world state of + * Ethereum, which includes all accounts, their balances, nonces, codes, and storage. + * + * @return The MutableWorldState instance. + */ MutableWorldState getWorldState(); + /** + * Retrieves the Blockchain instance. Blockchain represents the Ethereum blockchain, which + * includes blocks, transactions, and the world state. + * + * @return The Blockchain instance. + */ Blockchain getBlockchain(); } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/GenesisFileModule.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/GenesisFileModule.java index 506c3911d8f..1283ec039fc 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/GenesisFileModule.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/GenesisFileModule.java @@ -36,11 +36,26 @@ import dagger.Provides; import io.vertx.core.json.JsonObject; +/** + * This class, GenesisFileModule, is a Dagger module that provides dependencies for the GenesisFile. + * It contains options for setting up the GenesisFile, such as the genesis configuration, genesis + * state, block header functions, and the genesis block. + * + *

The class uses Dagger annotations to define these options, which can be provided via the + * command line when running the EVM tool. Each option has a corresponding provider method that + * Dagger uses to inject the option's value where needed. + */ @Module public class GenesisFileModule { private final String genesisConfig; + /** + * Constructs a new GenesisFileModule with the specified genesis configuration. + * + * @param genesisConfig The configuration for the genesis file. This is typically a JSON string + * that specifies various parameters for the genesis block of the blockchain. + */ protected GenesisFileModule(final String genesisConfig) { this.genesisConfig = genesisConfig; } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/JsonUtils.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/JsonUtils.java index 42da8fd950f..7af31054a65 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/JsonUtils.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/JsonUtils.java @@ -24,6 +24,12 @@ import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; import org.apache.tuweni.bytes.Bytes; +/** + * Utility class for JSON related operations. This class provides a method to create an ObjectMapper + * with standard configurations needed for evmtool. The ObjectMapper is configured to match the + * standard JSON output of Go, and it does not auto close the source. It also registers serializers + * for Address and Bytes classes. This class is not meant to be instantiated. + */ public class JsonUtils { private JsonUtils() {} diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/PrettyPrintSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/PrettyPrintSubCommand.java index 10d31960a82..465f2c48951 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/PrettyPrintSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/PrettyPrintSubCommand.java @@ -30,13 +30,31 @@ import org.apache.tuweni.bytes.Bytes; import picocli.CommandLine; +/** + * This class, PrettyPrintSubCommand, is a command-line interface (CLI) command that pretty prints + * EOF (Ethereum Object Format) code. It implements the Runnable interface, meaning it can be used + * in a thread of execution. + * + *

The class is annotated with {@code @CommandLine.Command}, which is a PicoCLI annotation that + * designates this class as a command-line command. The annotation parameters define the command's + * name, description, whether it includes standard help options, and the version provider. + * + *

The command's functionality is defined in the run() method, which is overridden from the + * Runnable interface. + */ @CommandLine.Command( name = COMMAND_NAME, description = "Pretty Prints EOF Code", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class) public class PrettyPrintSubCommand implements Runnable { + /** + * The name of the command for the PrettyPrintSubCommand. This constant is used as the name + * parameter in the @CommandLine.Command annotation. It defines the command name that users should + * enter on the command line to invoke this command. + */ public static final String COMMAND_NAME = "pretty-print"; + @CommandLine.ParentCommand private final EvmToolCommand parentCommand; @CommandLine.Option( @@ -48,10 +66,20 @@ public class PrettyPrintSubCommand implements Runnable { // picocli does it magically @CommandLine.Parameters private final List codeList = new ArrayList<>(); + /** + * Default constructor for the PrettyPrintSubCommand class. This constructor initializes the + * parentCommand to null. + */ public PrettyPrintSubCommand() { this(null); } + /** + * Constructs a new PrettyPrintSubCommand with the specified parent command. + * + * @param parentCommand The parent command for this subcommand. This is typically an instance of + * EvmToolCommand. + */ public PrettyPrintSubCommand(final EvmToolCommand parentCommand) { this.parentCommand = parentCommand; } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/ProtocolModule.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/ProtocolModule.java index 1b8f557b3b8..a196a35f5ba 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/ProtocolModule.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/ProtocolModule.java @@ -24,10 +24,25 @@ import dagger.Module; import dagger.Provides; +/** + * This class, ProtocolModule, is a Dagger module that provides dependencies for the ProtocolSpec + * and EVM. It includes the GenesisFileModule, which provides the genesis configuration for the + * blockchain. + * + *

The class uses Dagger annotations to define these dependencies, which can be provided via the + * command line when running the EVM tool. Each dependency has a corresponding provider method that + * Dagger uses to inject the dependency's value where needed. + */ @SuppressWarnings("WeakerAccess") @Module(includes = GenesisFileModule.class) public class ProtocolModule { + /** + * Default constructor for the ProtocolModule class. This constructor doesn't take any arguments + * and doesn't perform any initialization. + */ + public ProtocolModule() {} + @Provides @Singleton ProtocolSpec getProtocolSpec(final ProtocolSchedule protocolSchedule) { diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java index 60a09b26e8e..064a093e840 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java @@ -65,12 +65,29 @@ import picocli.CommandLine.Parameters; import picocli.CommandLine.ParentCommand; +/** + * This class, StateTestSubCommand, is a command-line interface (CLI) command that executes an + * Ethereum State Test. It implements the Runnable interface, meaning it can be used in a thread of + * execution. + * + *

The class is annotated with @CommandLine.Command, which is a PicoCLI annotation that + * designates this class as a command-line command. The annotation parameters define the command's + * name, description, whether it includes standard help options, and the version provider. + * + *

The command's functionality is defined in the run() method, which is overridden from the + * Runnable interface. + */ @Command( name = COMMAND_NAME, description = "Execute an Ethereum State Test.", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class) public class StateTestSubCommand implements Runnable { + /** + * The name of the command for the StateTestSubCommand. This constant is used as the name + * parameter in the @CommandLine.Command annotation. It defines the command name that users should + * enter on the command line to invoke this command. + */ public static final String COMMAND_NAME = "state-test"; static final Supplier referenceTestProtocolSchedules = @@ -112,6 +129,10 @@ public class StateTestSubCommand implements Runnable { // picocli does it magically @Parameters private final List stateTestFiles = new ArrayList<>(); + /** + * Default constructor for the StateTestSubCommand class. This constructor doesn't take any + * arguments and initializes the parentCommand to null. PicoCLI requires this constructor. + */ @SuppressWarnings("unused") public StateTestSubCommand() { // PicoCLI requires this diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java index caa5cb36724..40c73e383fc 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java @@ -86,12 +86,43 @@ import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; +/** + * The T8nExecutor class is responsible for executing transactions in the context of the Ethereum + * Virtual Machine (EVM). It extracts transactions from a given input, runs tests on them, and + * generates results including stateRoot, txRoot, receiptsRoot, and logsHash. It also handles block + * rewards and withdrawal processing. This class is part of the EVM tooling within the Hyperledger + * Besu project. + */ public class T8nExecutor { private static final Set

EMPTY_ADDRESS_SET = Set.of(); + /** + * A record that represents a transaction that has been rejected. It contains the index of the + * transaction and the error message explaining why it was rejected. + * + * @param index The index of the rejected transaction. + * @param error The error message explaining why the transaction was rejected. + */ public record RejectedTransaction(int index, String error) {} + /** + * Default constructor for the T8nExecutor class. This constructor does not perform any + * operations. + */ + public T8nExecutor() {} + + /** + * Extracts transactions from a given JSON iterator and adds them to the provided transactions + * list. If a transaction cannot be parsed or is invalid, it is added to the rejections list with + * its index and error message. + * + * @param out PrintWriter used for outputting information or errors. + * @param it Iterator over JSON nodes, each representing a transaction. + * @param transactions List of transactions to which parsed transactions are added. + * @param rejections List of RejectedTransaction records to which rejected transactions are added. + * @return The updated list of transactions after parsing and validation. + */ protected static List extractTransactions( final PrintWriter out, final Iterator it, @@ -547,12 +578,45 @@ static T8nResult runTest( return new T8nResult(allocObject, bodyBytes, resultObject); } - interface TracerManager { + /** + * The TracerManager interface provides methods for managing OperationTracer instances. It is used + * in the context of Ethereum Virtual Machine (EVM) execution to trace operations. + * + *

The interface defines two methods: - getManagedTracer: This method is used to get a managed + * OperationTracer instance for a specific transaction. - disposeTracer: This method is used to + * dispose of an OperationTracer instance when it is no longer needed. + */ + public interface TracerManager { + + /** + * Retrieves a managed OperationTracer instance for a specific transaction. + * + * @param txIndex The index of the transaction for which the tracer is to be retrieved. + * @param txHash The hash of the transaction for which the tracer is to be retrieved. + * @return The managed OperationTracer instance. + * @throws Exception If an error occurs while retrieving the tracer. + */ OperationTracer getManagedTracer(int txIndex, Hash txHash) throws Exception; + /** + * Disposes of an OperationTracer instance when it is no longer needed. + * + * @param tracer The OperationTracer instance to be disposed. + * @throws IOException If an error occurs while disposing the tracer. + */ void disposeTracer(OperationTracer tracer) throws IOException; } + /** + * A record that represents the result of a transaction test run in the Ethereum Virtual Machine + * (EVM). It contains the final state of the accounts (allocObject), the raw bytes of the + * transactions (bodyBytes), and the result of the test run (resultObject). + * + * @param allocObject The final state of the accounts after the test run. + * @param bodyBytes The raw bytes of the transactions that were run. + * @param resultObject The result of the test run, including stateRoot, txRoot, receiptsRoot, + * logsHash, and other details. + */ @SuppressWarnings("unused") record T8nResult(ObjectNode allocObject, TextNode bodyBytes, ObjectNode resultObject) {} } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nServerSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nServerSubCommand.java index 1c0a49438e0..ffa6e67b72f 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nServerSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nServerSubCommand.java @@ -55,6 +55,19 @@ import picocli.CommandLine; import picocli.CommandLine.ParentCommand; +/** + * The T8nServerSubCommand class is responsible for running an Ethereum State Test server. It reads + * the initial state, transactions, and environment from input files or stdin, executes the + * transactions in the Ethereum Virtual Machine (EVM), and writes the final state, transaction + * results, and traces to output files or stdout. + * + *

The class uses the Vert.x library for handling HTTP requests and the picocli library for + * command line argument parsing. It includes options for specifying the host and port to bind to, + * and the base directory for output. + * + *

The class also includes a TracerManager for managing OperationTracer instances, which are used + * to trace EVM operations when the --json flag is specified. + */ @SuppressWarnings("java:S106") // using standard output is the point of this class @CommandLine.Command( name = "t8n-server", @@ -80,6 +93,10 @@ public class T8nServerSubCommand implements Runnable { @ParentCommand private final EvmToolCommand parentCommand; + /** + * Default constructor for the T8nServerSubCommand class. This constructor is required by PicoCLI + * and assigns null to parentCommand. + */ @SuppressWarnings("unused") public T8nServerSubCommand() { // PicoCLI requires this diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java index 95dabb3183f..d6db1a1e8d2 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java @@ -59,6 +59,19 @@ import picocli.CommandLine.Parameters; import picocli.CommandLine.ParentCommand; +/** + * The T8nSubCommand class is responsible for executing an Ethereum State Test. It reads the initial + * state, transactions, and environment from input files or stdin, executes the transactions in the + * Ethereum Virtual Machine (EVM), and writes the final state, transaction results, and traces to + * output files or stdout. + * + *

The class uses the picocli library for command line argument parsing and includes options for + * specifying the input and output files, the fork to run the transition against, the chain ID, and + * the block reward. + * + *

The class also includes a TracerManager for managing OperationTracer instances, which are used + * to trace EVM operations when the --json flag is specified. + */ @Command( name = COMMAND_NAME, aliases = COMMAND_ALIAS, @@ -154,12 +167,22 @@ public void consumeParameters( } } + /** + * Default constructor for the T8nSubCommand class. This constructor is required by PicoCLI and + * assigns parent command to 'null'. + */ @SuppressWarnings("unused") public T8nSubCommand() { // PicoCLI requires this parentCommand = null; } + /** + * Constructor for the T8nSubCommand class with a parent command. This constructor is required by + * PicoCLI. + * + * @param parentCommand The parent command for this sub command. + */ @SuppressWarnings("unused") public T8nSubCommand(final EvmToolCommand parentCommand) { // PicoCLI requires this too diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/VersionProvider.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/VersionProvider.java index d97fded8825..6f3ff330717 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/VersionProvider.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/VersionProvider.java @@ -18,8 +18,26 @@ import picocli.CommandLine; +/** + * The VersionProvider class is responsible for providing the version of the Hyperledger Besu EVM + * tool. It implements the IVersionProvider interface from the picocli library. + * + *

The getVersion method returns a string array containing the version of the Hyperledger Besu + * EVM tool. + */ public class VersionProvider implements CommandLine.IVersionProvider { + /** + * Default constructor for the VersionProvider class. This constructor does not perform any + * operations. + */ + public VersionProvider() {} + + /** + * This method returns the version of the Hyperledger Besu EVM tool. + * + * @return A string array containing the version of the Hyperledger Besu EVM tool. + */ @Override public String[] getVersion() { return new String[] {"Hyperledger Besu evm " + BesuInfo.shortVersion()}; From 20b82a40c9be873cd8ace5f1ae76ae564b00d789 Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Fri, 28 Jun 2024 17:07:26 -0600 Subject: [PATCH 05/11] Rename ValidatorPublicKey to ValidatorPubKey (#7280) Adapt to EIP-7002 name change for validator public key in all places. Signed-off-by: Danno Ferrin --- .../WithdrawalRequestParameter.java | 2 +- .../besu/ethereum/core/WithdrawalRequest.java | 18 ++++---- .../encoding/WithdrawalRequestDecoder.java | 4 +- .../encoding/WithdrawalRequestEncoder.java | 2 +- .../WithdrawalRequestContractHelper.java | 4 +- .../WithdrawalRequestContractHelperTest.java | 22 +++++----- .../hyperledger/besu/evmtool/T8nExecutor.java | 6 ++- .../besu/evmtool/t8n/prague-deposit.json | 8 +--- .../besu/evmtool/t8n/prague-withdrawal.json | 42 +++++++++++-------- plugin-api/build.gradle | 2 +- .../besu/plugin/data/WithdrawalRequest.java | 2 +- 11 files changed, 56 insertions(+), 56 deletions(-) diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/WithdrawalRequestParameter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/WithdrawalRequestParameter.java index 67fa13f3f89..3561da37a80 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/WithdrawalRequestParameter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/WithdrawalRequestParameter.java @@ -45,7 +45,7 @@ public static WithdrawalRequestParameter fromWithdrawalRequest( final WithdrawalRequest withdrawalRequest) { return new WithdrawalRequestParameter( withdrawalRequest.getSourceAddress().toHexString(), - withdrawalRequest.getValidatorPublicKey().toHexString(), + withdrawalRequest.getValidatorPubkey().toHexString(), withdrawalRequest.getAmount().toShortHexString()); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/WithdrawalRequest.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/WithdrawalRequest.java index decdf22c736..0e7afc1595b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/WithdrawalRequest.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/WithdrawalRequest.java @@ -26,13 +26,13 @@ public class WithdrawalRequest extends Request implements org.hyperledger.besu.plugin.data.WithdrawalRequest { private final Address sourceAddress; - private final BLSPublicKey validatorPublicKey; + private final BLSPublicKey validatorPubkey; private final GWei amount; public WithdrawalRequest( - final Address sourceAddress, final BLSPublicKey validatorPublicKey, final GWei amount) { + final Address sourceAddress, final BLSPublicKey validatorPubkey, final GWei amount) { this.sourceAddress = sourceAddress; - this.validatorPublicKey = validatorPublicKey; + this.validatorPubkey = validatorPubkey; this.amount = amount; } @@ -47,8 +47,8 @@ public Address getSourceAddress() { } @Override - public PublicKey getValidatorPublicKey() { - return validatorPublicKey; + public PublicKey getValidatorPubkey() { + return validatorPubkey; } @Override @@ -61,8 +61,8 @@ public String toString() { return "WithdrawalRequest{" + "sourceAddress=" + sourceAddress - + " validatorPublicKey=" - + validatorPublicKey + + " validatorPubkey=" + + validatorPubkey + " amount=" + amount + '}'; @@ -78,12 +78,12 @@ public boolean equals(final Object o) { } final WithdrawalRequest that = (WithdrawalRequest) o; return Objects.equals(sourceAddress, that.sourceAddress) - && Objects.equals(validatorPublicKey, that.validatorPublicKey) + && Objects.equals(validatorPubkey, that.validatorPubkey) && Objects.equals(amount, that.amount); } @Override public int hashCode() { - return Objects.hash(sourceAddress, validatorPublicKey, amount); + return Objects.hash(sourceAddress, validatorPubkey, amount); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/WithdrawalRequestDecoder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/WithdrawalRequestDecoder.java index 7ade9947b44..031209284e8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/WithdrawalRequestDecoder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/WithdrawalRequestDecoder.java @@ -28,11 +28,11 @@ public class WithdrawalRequestDecoder { public static WithdrawalRequest decode(final RLPInput rlpInput) { rlpInput.enterList(); final Address sourceAddress = Address.readFrom(rlpInput); - final BLSPublicKey validatorPublicKey = BLSPublicKey.readFrom(rlpInput); + final BLSPublicKey validatorPubkey = BLSPublicKey.readFrom(rlpInput); final GWei amount = GWei.of(rlpInput.readUInt64Scalar()); rlpInput.leaveList(); - return new WithdrawalRequest(sourceAddress, validatorPublicKey, amount); + return new WithdrawalRequest(sourceAddress, validatorPubkey, amount); } public static WithdrawalRequest decodeOpaqueBytes(final Bytes input) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/WithdrawalRequestEncoder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/WithdrawalRequestEncoder.java index 26be22d4882..c9fdb37fcea 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/WithdrawalRequestEncoder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/WithdrawalRequestEncoder.java @@ -48,7 +48,7 @@ private static void encodeWithdrawalRequest( final WithdrawalRequest withdrawalRequest, final RLPOutput rlpOutput) { rlpOutput.startList(); rlpOutput.writeBytes(withdrawalRequest.getSourceAddress()); - rlpOutput.writeBytes(withdrawalRequest.getValidatorPublicKey()); + rlpOutput.writeBytes(withdrawalRequest.getValidatorPubkey()); rlpOutput.writeUInt64Scalar(withdrawalRequest.getAmount()); rlpOutput.endList(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalRequestContractHelper.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalRequestContractHelper.java index 6bb0e81a75e..c0b7302e867 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalRequestContractHelper.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalRequestContractHelper.java @@ -153,7 +153,7 @@ private static List peekExpectedWithdrawalRequests( queueHeadIndex.plus(i).multiply(WITHDRAWAL_REQUEST_STORAGE_SLOT_SIZE)); final Address sourceAddress = Address.wrap(account.getStorageValue(queueStorageSlot).toBytes().slice(12, 20)); - final BLSPublicKey validatorPublicKey = + final BLSPublicKey validatorPubkey = BLSPublicKey.wrap( Bytes.concatenate( account @@ -165,7 +165,7 @@ private static List peekExpectedWithdrawalRequests( UInt64.fromBytes(account.getStorageValue(queueStorageSlot.plus(2)).slice(16, 8)); withdrawalRequests.add( - new WithdrawalRequest(sourceAddress, validatorPublicKey, GWei.of(amount))); + new WithdrawalRequest(sourceAddress, validatorPubkey, GWei.of(amount))); } return withdrawalRequests; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalRequestContractHelperTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalRequestContractHelperTest.java index 6d21744e7bd..7e1e8571350 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalRequestContractHelperTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/WithdrawalRequestContractHelperTest.java @@ -31,7 +31,6 @@ import org.hyperledger.besu.evm.worldstate.WorldUpdater; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.tuweni.bytes.Bytes; @@ -47,12 +46,12 @@ class WithdrawalRequestContractHelperTest { private MutableAccount contract; @BeforeEach - public void setUp() { + void setUp() { worldState = createInMemoryWorldStateArchive().getMutable(); } @Test - public void popWithdrawalRequestsFromQueue_ReadWithdrawalRequestsCorrectly() { + void popWithdrawalRequestsFromQueue_ReadWithdrawalRequestsCorrectly() { final List validatorWithdrawalRequests = List.of(createExit(), createExit(), createExit()); loadContractStorage(worldState, validatorWithdrawalRequests); @@ -64,7 +63,7 @@ public void popWithdrawalRequestsFromQueue_ReadWithdrawalRequestsCorrectly() { } @Test - public void + void popWithdrawalRequestsFromQueue_whenContractCodeIsEmpty_ReturnsEmptyListOfWithdrawalRequests() { // Create account with empty code final WorldUpdater updater = worldState.updater(); @@ -76,10 +75,10 @@ public void popWithdrawalRequestsFromQueue_ReadWithdrawalRequestsCorrectly() { } @Test - public void popWithdrawalRequestsFromQueue_WhenMoreWithdrawalRequests_UpdatesQueuePointers() { + void popWithdrawalRequestsFromQueue_WhenMoreWithdrawalRequests_UpdatesQueuePointers() { // Loading contract with more than 16 WithdrawalRequests final List validatorWithdrawalRequests = - IntStream.range(0, 30).mapToObj(__ -> createExit()).collect(Collectors.toList()); + IntStream.range(0, 30).mapToObj(__ -> createExit()).toList(); loadContractStorage(worldState, validatorWithdrawalRequests); // After loading the contract, the WithdrawalRequests count since last block should match the // size of the list @@ -102,7 +101,7 @@ public void popWithdrawalRequestsFromQueue_WhenMoreWithdrawalRequests_UpdatesQue } @Test - public void popWithdrawalRequestsFromQueue_WhenNoMoreWithdrawalRequests_ZeroQueuePointers() { + void popWithdrawalRequestsFromQueue_WhenNoMoreWithdrawalRequests_ZeroQueuePointers() { final List withdrawalRequests = List.of(createExit(), createExit(), createExit()); loadContractStorage(worldState, withdrawalRequests); @@ -126,7 +125,7 @@ public void popWithdrawalRequestsFromQueue_WhenNoMoreWithdrawalRequests_ZeroQueu } @Test - public void popWithdrawalRequestsFromQueue_WhenNoWithdrawalRequests_DoesNothing() { + void popWithdrawalRequestsFromQueue_WhenNoWithdrawalRequests_DoesNothing() { // Loading contract with 0 WithdrawalRequests loadContractStorage(worldState, List.of()); // After loading storage, we have the WithdrawalRequests count as zero because no @@ -135,7 +134,7 @@ public void popWithdrawalRequestsFromQueue_WhenNoWithdrawalRequests_DoesNothing( final List poppedWithdrawalRequests = WithdrawalRequestContractHelper.popWithdrawalRequestsFromQueue(worldState); - assertThat(poppedWithdrawalRequests).hasSize(0); + assertThat(poppedWithdrawalRequests).isEmpty(); // Check that queue pointers are correct (head and tail are zero) assertContractStorageValue(WITHDRAWAL_REQUEST_QUEUE_HEAD_STORAGE_SLOT, 0); @@ -186,14 +185,13 @@ private void loadContractStorage( Bytes.fromHexString("0x000000000000000000000000"), request.getSourceAddress()))); // validator_pubkey contract.setStorageValue( - UInt256.valueOf(offset++), - UInt256.fromBytes(request.getValidatorPublicKey().slice(0, 32))); + UInt256.valueOf(offset++), UInt256.fromBytes(request.getValidatorPubkey().slice(0, 32))); contract.setStorageValue( // set public key to slot, with 16 bytes padding on the right UInt256.valueOf(offset++), UInt256.fromBytes( Bytes.concatenate( - request.getValidatorPublicKey().slice(32, 16), + request.getValidatorPubkey().slice(32, 16), request.getAmount().toBytes(), // 8 bytes for amount Bytes.fromHexString("0x0000000000000000")))); } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java index 40c73e383fc..d5239848821 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java @@ -110,7 +110,9 @@ public record RejectedTransaction(int index, String error) {} * Default constructor for the T8nExecutor class. This constructor does not perform any * operations. */ - public T8nExecutor() {} + public T8nExecutor() { + // Default constructor required for Javadoc linting + } /** * Extracts transactions from a given JSON iterator and adds them to the provided transactions @@ -531,7 +533,7 @@ static T8nResult runTest( wr -> { var obj = withdrawlRequests.addObject(); obj.put("sourceAddress", wr.getSourceAddress().toHexString()); - obj.put("validatorPublicKey", wr.getValidatorPublicKey().toHexString()); + obj.put("validatorPubkey", wr.getValidatorPubkey().toHexString()); obj.put("amount", wr.getAmount().toHexString()); }); } diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-deposit.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-deposit.json index 80e415b3fc2..034325fbe86 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-deposit.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-deposit.json @@ -180,12 +180,6 @@ "balance": "0x0", "nonce": "0x1" }, - "0x25a219378dad9b3503c8268c9ca836a52427a4fb": { - "storage": { - "0x0000000000000000000000000000000000000000000000000000000000000000": "0xe4fb5d47f70d54b4f36777ea4c882cf767f93d8f8170285d97a1b8275dfe4dbb" - }, - "balance": "0x0" - }, "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { "balance": "0xaa00be18c288efd690", "nonce": "0x2" @@ -193,7 +187,7 @@ }, "body": "0xf90404f901ff8007830f42409400000000219ab540356cbb839cbe05303d7705fa8901bc16d674ec800000b9019422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000d0000000000000000000000000000000000000000000000000000000000000011085acb6376c2707b118225da41825974c12b5924a05c6a53b988c9cbc33c55b05000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000325a0ffeb1d6e7ef8e9ee9b64dcdc3b056f9a1d2b94c1572f1949954e712364604575a03d0f42bad795205de84db8d4ab10b9abd0d081ffe560cbf45f6c281768112a69f901ff0107830f42409400000000219ab540356cbb839cbe05303d7705fa8901bc16d674ec800000b9019422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000d0000000000000000000000000000000000000000000000000000000000000011085acb6376c2707b118225da41825974c12b5924a05c6a53b988c9cbc33c55b05000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000326a05bb08e348c9c4b0a2e15d04f4a89d1a210d013205de8d3d93e38e5c1b0c8d8aba04300c0f575d9908d2cbc3413ab82895678bb8f3ef356224dd1e7cb972f2c4855", "result": { - "stateRoot": "0x0dd3e32e0081ff7ca5a0236b257676f277999ec2b8da5b31e19400715de907f1", + "stateRoot": "0x83bc620ec3d5ff77109b241f561e233a5c4f35bfc3337594c12958f51ff2c1c8", "txRoot": "0x2b790bf82ef7259a0e4513d1b89a77d81e99672ba68758ef2ba3fde32851d023", "receiptsRoot": "0x9c8d7a917ecb3ff2566f264abbf39131e51b08b07eb2b69cb46989d79d985593", "logsHash": "0x43e31613bfefc1f55d8b3ca2b61f933f3838d523dc11cb5d7ffdd2ecf0ab5d49", diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-withdrawal.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-withdrawal.json index 85ad0d7e9ab..cf593cbb763 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-withdrawal.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-withdrawal.json @@ -64,6 +64,12 @@ "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500", "storage": {} }, + "0x25a219378dad9b3503c8268c9ca836a52427a4fb": { + "nonce": "0x00", + "balance": "0x1", + "code": "0x", + "storage": {} + }, "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { "nonce": "0x00", "balance": "0xad78ebc5ac62000000", @@ -213,7 +219,7 @@ "storage": { "0x0000000000000000000000000000000000000000000000000000000000000000": "0x10715cfbefdb8a0cb2f7d7ca5ee6d1ea65515ecb41cff0a22d1e11716a9d27fb" }, - "balance": "0x0" + "balance": "0x1" }, "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { "balance": "0xad78ebc5ac619bbcea", @@ -222,7 +228,7 @@ }, "body": "0xf903e5f903e28007830f424094000000000000000000000000000000000000020080b90380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001fffffffffffffffe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fffffffffffffffe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005fffffffffffffffe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007fffffffffffffffe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009fffffffffffffffe00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000bfffffffffffffffe00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000dfffffffffffffffe00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ffffffffffffffffe000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000026a0963cb6620fe5828cbc93bb7139d3f4501067d76275dff648bf48c3c100ca8dd4a04ac396104a5be4643406718f59a6e74d62a32777f5f6135e55e805e87612013c", "result": { - "stateRoot": "0xf682e4f8f820f44fb43a20c3db274bc3c9fcb9e52cc1af3c3ea7cb21fdb2250b", + "stateRoot": "0x69dc2041a0a7e71f9e5f916c26cb6b2d7d1e255509af793337d4010d19b4b87c", "txRoot": "0x8521df63211790726b6f1a437bb0fd4b27c00e13e7678d324c4cfddb8d834ad2", "receiptsRoot": "0x4bd8bd5580caf4ed45f873794ad7ff9d6fd2363ae529269b17b891b68d349d75", "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", @@ -252,82 +258,82 @@ "withdrawalRequests": [ { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", + "validatorPubkey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "amount": "0xfffffffffffffffe" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002", + "validatorPubkey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002", "amount": "0x0000000000000000" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003", + "validatorPubkey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003", "amount": "0xfffffffffffffffe" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004", + "validatorPubkey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004", "amount": "0x0000000000000000" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005", + "validatorPubkey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005", "amount": "0xfffffffffffffffe" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006", + "validatorPubkey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006", "amount": "0x0000000000000000" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007", + "validatorPubkey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007", "amount": "0xfffffffffffffffe" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008", + "validatorPubkey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008", "amount": "0x0000000000000000" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009", + "validatorPubkey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009", "amount": "0xfffffffffffffffe" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a", + "validatorPubkey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a", "amount": "0x0000000000000000" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b", + "validatorPubkey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b", "amount": "0xfffffffffffffffe" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c", + "validatorPubkey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c", "amount": "0x0000000000000000" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000d", + "validatorPubkey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000d", "amount": "0xfffffffffffffffe" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e", + "validatorPubkey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e", "amount": "0x0000000000000000" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f", + "validatorPubkey": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f", "amount": "0xfffffffffffffffe" }, { "sourceAddress": "0x0000000000000000000000000000000000000200", - "validatorPublicKey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010", + "validatorPubkey": "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010", "amount": "0x0000000000000000" } ] diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index fd7f3291b3b..dbbb5e4f2bf 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -70,7 +70,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'p8jZoKgvi9o8JpVLXTlwh9HoIot4A62hKFHwSOaTF+k=' + knownHash = 'Q6EK5By3BNKNa/JYqYjFw43VXWL0KVBUV3dGEQBjZ70=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/WithdrawalRequest.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/WithdrawalRequest.java index eeadd80418d..f07d0d08d03 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/WithdrawalRequest.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/WithdrawalRequest.java @@ -38,7 +38,7 @@ public interface WithdrawalRequest { * * @return public key of validator */ - PublicKey getValidatorPublicKey(); + PublicKey getValidatorPubkey(); /** * The amount for withdrawal From 08772b466ef4681bd91503805789b4af311ed9cd Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Mon, 1 Jul 2024 08:48:21 +0100 Subject: [PATCH 06/11] Add info-level diagnostic logs to aid with resolving stalled BFT chains (#7271) * Add info-level diagnostic logs to aid with resolving stalled BFT chains Signed-off-by: Matthew Whitehead * Add javadoc Signed-off-by: Matthew Whitehead --------- Signed-off-by: Matthew Whitehead Signed-off-by: Matt Whitehead --- CHANGELOG.md | 1 + .../besu/consensus/common/bft/RoundTimer.java | 16 +++++++++ .../statemachine/QbftBlockHeightManager.java | 13 +++++-- .../QbftBlockHeightManagerFactory.java | 3 +- .../qbft/statemachine/RoundChangeManager.java | 34 ++++++++++++++++++- 5 files changed, 63 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ba6b4dc5dd..819bb05a7cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - Refactored how code, initcode, and max stack size are configured in forks. [#7245](https://github.com/hyperledger/besu/pull/7245) - Nodes in a permissioned chain maintain (and retry) connections to bootnodes [#7257](https://github.com/hyperledger/besu/pull/7257) - Promote experimental `besu storage x-trie-log` subcommand to production-ready [#7278](https://github.com/hyperledger/besu/pull/7278) +- Enhanced BFT round-change diagnostics [#7271](https://github.com/hyperledger/besu/pull/7271) ### Bug fixes - Validation errors ignored in accounts-allowlist and empty list [#7138](https://github.com/hyperledger/besu/issues/7138) diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/RoundTimer.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/RoundTimer.java index a500a1ad7ee..6a8b02991d6 100644 --- a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/RoundTimer.java +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/RoundTimer.java @@ -20,8 +20,14 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** Class for starting and keeping organised round timers */ public class RoundTimer { + + private static final Logger LOG = LoggerFactory.getLogger(RoundTimer.class); + private final BftExecutors bftExecutors; private Optional> currentTimerTask; private final BftEventQueue queue; @@ -71,6 +77,16 @@ public synchronized void startTimer(final ConsensusRoundIdentifier round) { final ScheduledFuture newTimerTask = bftExecutors.scheduleTask(newTimerRunnable, expiryTime, TimeUnit.MILLISECONDS); + + // Once we are up to round 2 start logging round expiries + if (round.getRoundNumber() >= 2) { + LOG.info( + "QBFT round {} expired. Moved to round {} which will expire in {} seconds", + round.getRoundNumber() - 1, + round.getRoundNumber(), + (expiryTime / 1000)); + } + currentTimerTask = Optional.of(newTimerTask); } } diff --git a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManager.java b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManager.java index 0940d35df6b..f79537cc0f9 100644 --- a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManager.java +++ b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManager.java @@ -274,17 +274,26 @@ private

> void actionOrBufferMessage( @Override public void handleRoundChangePayload(final RoundChange message) { final ConsensusRoundIdentifier targetRound = message.getRoundIdentifier(); - LOG.trace("Received a RoundChange Payload for {}", targetRound); + + LOG.debug( + "Round change from {}: block {}, round {}", + message.getAuthor(), + message.getRoundIdentifier().getSequenceNumber(), + message.getRoundIdentifier().getRoundNumber()); + + // Diagnostic logging (only logs anything if the chain has stalled) + roundChangeManager.storeAndLogRoundChangeSummary(message); final MessageAge messageAge = determineAgeOfPayload(message.getRoundIdentifier().getRoundNumber()); if (messageAge == MessageAge.PRIOR_ROUND) { - LOG.trace("Received RoundChange Payload for a prior round. targetRound={}", targetRound); + LOG.debug("Received RoundChange Payload for a prior round. targetRound={}", targetRound); return; } final Optional> result = roundChangeManager.appendRoundChangeMessage(message); + if (result.isPresent()) { LOG.debug( "Received sufficient RoundChange messages to change round to targetRound={}", diff --git a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManagerFactory.java b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManagerFactory.java index 0d306b1fc9f..889f83f98a7 100644 --- a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManagerFactory.java +++ b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/QbftBlockHeightManagerFactory.java @@ -86,7 +86,8 @@ private BaseQbftBlockHeightManager createFullBlockHeightManager(final BlockHeade new RoundChangeManager( BftHelpers.calculateRequiredValidatorQuorum(finalState.getValidators().size()), messageValidatorFactory.createRoundChangeMessageValidator( - parentHeader.getNumber() + 1L, parentHeader)), + parentHeader.getNumber() + 1L, parentHeader), + finalState.getLocalAddress()), roundFactory, finalState.getClock(), messageValidatorFactory, diff --git a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/RoundChangeManager.java b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/RoundChangeManager.java index ca671a6f527..89e2888395c 100644 --- a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/RoundChangeManager.java +++ b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/statemachine/RoundChangeManager.java @@ -100,19 +100,51 @@ public Collection createRoundChangeCertificate() { @VisibleForTesting final Map roundChangeCache = Maps.newHashMap(); + /** A summary of the latest round each validator is on, for diagnostic purposes only */ + private final Map roundSummary = Maps.newHashMap(); + private final long quorum; private final RoundChangeMessageValidator roundChangeMessageValidator; + private final Address localAddress; /** * Instantiates a new Round change manager. * * @param quorum the quorum * @param roundChangeMessageValidator the round change message validator + * @param localAddress this node's address */ public RoundChangeManager( - final long quorum, final RoundChangeMessageValidator roundChangeMessageValidator) { + final long quorum, + final RoundChangeMessageValidator roundChangeMessageValidator, + final Address localAddress) { this.quorum = quorum; this.roundChangeMessageValidator = roundChangeMessageValidator; + this.localAddress = localAddress; + } + + /** + * Store the latest round for a node, and if chain is stalled log a summary of which round each + * address is on + * + * @param message the round-change message that has just been received + */ + public void storeAndLogRoundChangeSummary(final RoundChange message) { + roundSummary.put(message.getAuthor(), message.getRoundIdentifier()); + if (roundChangeCache.keySet().stream() + .findFirst() + .orElse(new ConsensusRoundIdentifier(0, 0)) + .getRoundNumber() + >= 2) { + LOG.info("BFT round summary (quorum = {})", quorum); + for (Map.Entry nextEntry : roundSummary.entrySet()) { + LOG.info( + "Address: {} Round: {} {}", + nextEntry.getKey(), + nextEntry.getValue().getRoundNumber(), + nextEntry.getKey().equals(localAddress) ? "(Local node)" : ""); + } + } } /** From c0efeef89a1befb23d53323f38f9c3f8b62ea785 Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Mon, 1 Jul 2024 09:58:40 -0600 Subject: [PATCH 07/11] Update EIP-2935 contract (#7281) Use the updated contract and address for EIP-2539. Signed-off-by: Danno Ferrin --- .../blockhash/PragueBlockHashProcessor.java | 2 +- .../besu/ethereum/chain/genesis_prague2.json | 4 ++-- .../besu/evmtool/t8n/prague-deposit.json | 15 ++++++++++++++- .../besu/evmtool/t8n/prague-withdrawal.json | 17 +++++++++-------- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockhash/PragueBlockHashProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockhash/PragueBlockHashProcessor.java index 634a7441596..b6b38b37d57 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockhash/PragueBlockHashProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/blockhash/PragueBlockHashProcessor.java @@ -36,7 +36,7 @@ public class PragueBlockHashProcessor extends CancunBlockHashProcessor { private static final Logger LOG = LoggerFactory.getLogger(PragueBlockHashProcessor.class); public static final Address HISTORY_STORAGE_ADDRESS = - Address.fromHexString("0x25a219378dad9b3503c8268c9ca836a52427a4fb"); + Address.fromHexString("0x0aae40965e6800cd9b1f4b05ff21581047e3f91e"); /** The HISTORY_SERVE_WINDOW */ public static final long HISTORY_SERVE_WINDOW = 8192; diff --git a/ethereum/core/src/test/resources/org/hyperledger/besu/ethereum/chain/genesis_prague2.json b/ethereum/core/src/test/resources/org/hyperledger/besu/ethereum/chain/genesis_prague2.json index e7bccadac0b..cdb755ace18 100644 --- a/ethereum/core/src/test/resources/org/hyperledger/besu/ethereum/chain/genesis_prague2.json +++ b/ethereum/core/src/test/resources/org/hyperledger/besu/ethereum/chain/genesis_prague2.json @@ -73,8 +73,8 @@ "balance": "0x0", "nonce": "0x1" }, - "25a219378dad9b3503c8268c9ca836a52427a4fb": { - "code": "0x60203611603157600143035f35116029575f356120000143116029576120005f3506545f5260205ff35b5f5f5260205ff35b5f5ffd00", + "0x0aae40965e6800cd9b1f4b05ff21581047e3f91e": { + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe1460575767ffffffffffffffff5f3511605357600143035f3511604b575f35612000014311604b57611fff5f3516545f5260205ff35b5f5f5260205ff35b5f5ffd5b5f35611fff60014303165500", "balance": "0x0", "nonce": "0x1" }, diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-deposit.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-deposit.json index 034325fbe86..d48fc66c704 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-deposit.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-deposit.json @@ -63,6 +63,11 @@ "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500", "storage": {} }, + "0x0aae40965e6800cd9b1f4b05ff21581047e3f91e": { + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe1460575767ffffffffffffffff5f3511605357600143035f3511604b575f35612000014311604b57611fff5f3516545f5260205ff35b5f5f5260205ff35b5f5ffd5b5f35611fff60014303165500", + "balance": "0x0", + "nonce": "0x1" + }, "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { "nonce": "0x00", "balance": "0xad78ebc5ac62000000", @@ -180,6 +185,14 @@ "balance": "0x0", "nonce": "0x1" }, + "0x0aae40965e6800cd9b1f4b05ff21581047e3f91e": { + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe1460575767ffffffffffffffff5f3511605357600143035f3511604b575f35612000014311604b57611fff5f3516545f5260205ff35b5f5f5260205ff35b5f5ffd5b5f35611fff60014303165500", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0xe4fb5d47f70d54b4f36777ea4c882cf767f93d8f8170285d97a1b8275dfe4dbb" + }, + "balance": "0x0", + "nonce": "0x1" + }, "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { "balance": "0xaa00be18c288efd690", "nonce": "0x2" @@ -187,7 +200,7 @@ }, "body": "0xf90404f901ff8007830f42409400000000219ab540356cbb839cbe05303d7705fa8901bc16d674ec800000b9019422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000d0000000000000000000000000000000000000000000000000000000000000011085acb6376c2707b118225da41825974c12b5924a05c6a53b988c9cbc33c55b05000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000325a0ffeb1d6e7ef8e9ee9b64dcdc3b056f9a1d2b94c1572f1949954e712364604575a03d0f42bad795205de84db8d4ab10b9abd0d081ffe560cbf45f6c281768112a69f901ff0107830f42409400000000219ab540356cbb839cbe05303d7705fa8901bc16d674ec800000b9019422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000d0000000000000000000000000000000000000000000000000000000000000011085acb6376c2707b118225da41825974c12b5924a05c6a53b988c9cbc33c55b05000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000326a05bb08e348c9c4b0a2e15d04f4a89d1a210d013205de8d3d93e38e5c1b0c8d8aba04300c0f575d9908d2cbc3413ab82895678bb8f3ef356224dd1e7cb972f2c4855", "result": { - "stateRoot": "0x83bc620ec3d5ff77109b241f561e233a5c4f35bfc3337594c12958f51ff2c1c8", + "stateRoot": "0x3aa7839837ee1564276a0a05554e35215353a97c4e255c3aacbcd7c7819daefa", "txRoot": "0x2b790bf82ef7259a0e4513d1b89a77d81e99672ba68758ef2ba3fde32851d023", "receiptsRoot": "0x9c8d7a917ecb3ff2566f264abbf39131e51b08b07eb2b69cb46989d79d985593", "logsHash": "0x43e31613bfefc1f55d8b3ca2b61f933f3838d523dc11cb5d7ffdd2ecf0ab5d49", diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-withdrawal.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-withdrawal.json index cf593cbb763..66dcf3e85e3 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-withdrawal.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/prague-withdrawal.json @@ -64,11 +64,10 @@ "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500", "storage": {} }, - "0x25a219378dad9b3503c8268c9ca836a52427a4fb": { - "nonce": "0x00", - "balance": "0x1", - "code": "0x", - "storage": {} + "0x0aae40965e6800cd9b1f4b05ff21581047e3f91e": { + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe1460575767ffffffffffffffff5f3511605357600143035f3511604b575f35612000014311604b57611fff5f3516545f5260205ff35b5f5f5260205ff35b5f5ffd5b5f35611fff60014303165500", + "balance": "0x0", + "nonce": "0x1" }, "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { "nonce": "0x00", @@ -215,11 +214,13 @@ "balance": "0x10", "nonce": "0x1" }, - "0x25a219378dad9b3503c8268c9ca836a52427a4fb": { + "0x0aae40965e6800cd9b1f4b05ff21581047e3f91e": { + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe1460575767ffffffffffffffff5f3511605357600143035f3511604b575f35612000014311604b57611fff5f3516545f5260205ff35b5f5f5260205ff35b5f5ffd5b5f35611fff60014303165500", "storage": { "0x0000000000000000000000000000000000000000000000000000000000000000": "0x10715cfbefdb8a0cb2f7d7ca5ee6d1ea65515ecb41cff0a22d1e11716a9d27fb" }, - "balance": "0x1" + "balance": "0x0", + "nonce": "0x1" }, "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { "balance": "0xad78ebc5ac619bbcea", @@ -228,7 +229,7 @@ }, "body": "0xf903e5f903e28007830f424094000000000000000000000000000000000000020080b90380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001fffffffffffffffe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fffffffffffffffe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005fffffffffffffffe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007fffffffffffffffe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009fffffffffffffffe00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000bfffffffffffffffe00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000dfffffffffffffffe00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ffffffffffffffffe000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000026a0963cb6620fe5828cbc93bb7139d3f4501067d76275dff648bf48c3c100ca8dd4a04ac396104a5be4643406718f59a6e74d62a32777f5f6135e55e805e87612013c", "result": { - "stateRoot": "0x69dc2041a0a7e71f9e5f916c26cb6b2d7d1e255509af793337d4010d19b4b87c", + "stateRoot": "0xdba3aee0733886dac565d2a654d8fac1953fd913261f44b3aa31f153e159e98a", "txRoot": "0x8521df63211790726b6f1a437bb0fd4b27c00e13e7678d324c4cfddb8d834ad2", "receiptsRoot": "0x4bd8bd5580caf4ed45f873794ad7ff9d6fd2363ae529269b17b891b68d349d75", "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", From 264e7d9facffe1c881d6de090ad99e807f4d836b Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Mon, 1 Jul 2024 16:32:12 -0600 Subject: [PATCH 08/11] Deeper tracing of self-destructed accounts (#7284) Consider previously self-destructed accounts when creating accounts. Signed-off-by: Danno Ferrin --- .../evm/worldstate/UpdateTrackingAccount.java | 1 - .../besu/evm/worldstate/WorldUpdater.java | 27 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/evm/src/main/java/org/hyperledger/besu/evm/worldstate/UpdateTrackingAccount.java b/evm/src/main/java/org/hyperledger/besu/evm/worldstate/UpdateTrackingAccount.java index f571ff2ff03..9f065aee98b 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/worldstate/UpdateTrackingAccount.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/worldstate/UpdateTrackingAccount.java @@ -128,7 +128,6 @@ public A getWrappedAccount() { public void setWrappedAccount(final A account) { if (this.account == null) { this.account = account; - storageWasCleared = false; } else { throw new IllegalStateException("Already tracking a wrapped account"); } diff --git a/evm/src/main/java/org/hyperledger/besu/evm/worldstate/WorldUpdater.java b/evm/src/main/java/org/hyperledger/besu/evm/worldstate/WorldUpdater.java index 9b57b6ab9e3..4cbda732798 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/worldstate/WorldUpdater.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/worldstate/WorldUpdater.java @@ -24,6 +24,8 @@ import java.util.Collection; import java.util.Optional; +import org.apache.tuweni.bytes.Bytes; + /** * An object that buffers updates made over a particular {@link WorldView}. * @@ -73,8 +75,29 @@ default MutableAccount createAccount(final Address address) { * #createAccount(Address)} (and thus all his fields will be zero/empty). */ default MutableAccount getOrCreate(final Address address) { - final MutableAccount account = getAccount(address); - return account == null ? createAccount(address) : account; + MutableAccount account = getAccount(address); + if (account == null) { + account = createAccount(address); + if (parentUpdater().isPresent() && parentUpdater().get().isDeleted(address)) { + account.clearStorage(); + account.setCode(Bytes.EMPTY); + } + } + return account; + } + + /** + * Check this and parent updaters to see if an address has been deleted since the last persist + * + * @param address address to check + * @return true if any updaters have marked the address as deleted. + */ + default boolean isDeleted(final Address address) { + if (getDeletedAccountAddresses().contains(address)) { + return true; + } else { + return parentUpdater().map(wu -> wu.isDeleted(address)).orElse(false); + } } /** From 3a73dccd619ed5ab1caf1548ec3a84aeb30a2d58 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Mon, 1 Jul 2024 16:06:55 -0700 Subject: [PATCH 09/11] next release version after 24.7.0 (#7285) Signed-off-by: garyschulte --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 819bb05a7cd..d14b3e0fc82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Changelog -## Next Release +## Next release + +### Breaking Changes + +### Additions and Improvements + +### Bug fixes + +## 24.7.0 ### Upcoming Breaking Changes - Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version. @@ -32,6 +40,8 @@ - Fix "Could not confirm best peer had pivot block" [#7109](https://github.com/hyperledger/besu/issues/7109) - Fix "Chain Download Halt" [#6884](https://github.com/hyperledger/besu/issues/6884) + + ## 24.6.0 ### Breaking Changes From 8ca7129b0be3ba26b60ceb8920c01f717d5f5aca Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Tue, 2 Jul 2024 09:39:59 +0100 Subject: [PATCH 10/11] Add experimental `--Xsnapsync-bft-enabled` which enables snap sync for BFT chains (#7140) * Create a BFT-specific pivot block handler Signed-off-by: Matthew Whitehead * Change visibility Signed-off-by: Matthew Whitehead * Refactor sync-peer-count internal variable to match name, add experimental flag to enabled snap + BFT Signed-off-by: Matthew Whitehead * Merge with main Signed-off-by: Matthew Whitehead * Fix uppercase Signed-off-by: Matthew Whitehead * Address synchronization issue with trie pruning. Create BFT-specific account range handler. Add pipeline name and logs Signed-off-by: Matthew Whitehead * Remove debug log Signed-off-by: Matthew Whitehead * fixing snapsync for empty state Signed-off-by: Karim Taam * Don't queue up events we can't handle Signed-off-by: Matthew Whitehead * Fix timing window where a validator with an empty data dir sometimes falls back to full sync if peer status isn't received quickly enough Signed-off-by: Matthew Whitehead * Remove BFT-specific account request class. Not needed Signed-off-by: Matthew Whitehead * Refactor some more 'fast' sync variables that are common to all pivot-based sync types Signed-off-by: Matthew Whitehead * In FULL sync mode, disable bonsai-limit-trie-logs-enabled instead of failing to start Signed-off-by: Matthew Whitehead * Add javadoc comments, clarify overriding bonsai-limit-trie-logs-enabled Signed-off-by: Matthew Whitehead * Add BFT pivot block selector tests Signed-off-by: Matthew Whitehead * Fix failure error message Signed-off-by: Matthew Whitehead * Remove the unnamed Pipe constructor and update tests to set a pipe name Signed-off-by: Matthew Whitehead * Revert some info logs back to debug given the feedback on noise in the logs syncing with holesky Signed-off-by: Matthew Whitehead * Refactor fastSyncPivotDistance to syncPivotDistance Signed-off-by: Matthew Whitehead * Incomplete refactoring Signed-off-by: Matthew Whitehead * Update BFT event queueing tests Signed-off-by: Matthew Whitehead * Event queue test fixes Signed-off-by: Matthew Whitehead * Remove automatic setting of bonsai-limit-trie-logs-enabled to false if sync-mode = FULL (moving to another PR) Signed-off-by: Matthew Whitehead --------- Signed-off-by: Matthew Whitehead Signed-off-by: Karim Taam Signed-off-by: Matt Whitehead Co-authored-by: Karim Taam --- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 16 +- .../cli/ConfigurationOverviewBuilder.java | 16 ++ .../options/unstable/SynchronizerOptions.java | 29 +++- .../controller/BesuControllerBuilder.java | 21 ++- .../java/org/hyperledger/besu/RunnerTest.java | 4 +- .../hyperledger/besu/cli/BesuCommandTest.java | 8 +- .../cli/CascadingDefaultProviderTest.java | 4 +- .../cli/options/SynchronizerOptionsTest.java | 6 +- .../consensus/common/bft/BftEventQueue.java | 27 ++- .../consensus/common/bft/BftProcessor.java | 3 + .../common/validator/ValidatorProvider.java | 12 ++ .../common/bft/BftEventQueueTest.java | 69 +++++++- .../common/bft/BftProcessorTest.java | 1 + .../consensus/common/bft/RoundTimerTest.java | 1 + .../BftMiningCoordinatorTest.java | 1 + .../qbft/BFTPivotSelectorFromPeers.java | 160 ++++++++++++++++++ .../qbft/sync/QbftPivotSelectorTest.java | 159 +++++++++++++++++ .../common/trielog/TrieLogPruner.java | 4 +- .../ethereum/eth/manager/snap/SnapServer.java | 18 +- .../eth/sync/DefaultSynchronizer.java | 29 ++-- .../eth/sync/SynchronizerConfiguration.java | 44 ++--- .../eth/sync/fastsync/FastSyncActions.java | 4 +- .../eth/sync/fastsync/FastSyncDownloader.java | 10 +- .../fastsync/NoSyncRequiredException.java | 17 ++ .../sync/fastsync/NoSyncRequiredState.java | 17 ++ .../sync/fastsync/PivotSelectorFromPeers.java | 16 +- .../sync/snapsync/SnapSyncConfiguration.java | 7 + .../request/AccountRangeDataRequest.java | 16 +- .../sync/fastsync/FastSyncActionsTest.java | 42 ++--- .../worldstate/LoadLocalDataStepTest.java | 2 +- .../sync/snapsync/LoadLocalDataStepTest.java | 2 +- .../trie/patricia/TrieNodeDecoder.java | 4 + .../pipeline/AsyncOperationProcessor.java | 2 +- .../besu/services/pipeline/Pipe.java | 19 ++- .../besu/services/pipeline/Pipeline.java | 13 ++ .../services/pipeline/PipelineBuilder.java | 3 +- .../pipeline/BatchingReadPipeTest.java | 3 +- .../services/pipeline/CompleterStageTest.java | 3 +- .../pipeline/FlatMapProcessorTest.java | 6 +- .../pipeline/IteratorSourceStageTest.java | 3 +- .../services/pipeline/MapProcessorTest.java | 6 +- .../besu/services/pipeline/PipeTest.java | 3 +- .../pipeline/ProcessingStageTest.java | 4 +- 44 files changed, 708 insertions(+), 127 deletions(-) create mode 100644 consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/BFTPivotSelectorFromPeers.java create mode 100644 consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/sync/QbftPivotSelectorTest.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/NoSyncRequiredException.java create mode 100644 ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/NoSyncRequiredState.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d14b3e0fc82..4d29b8f8af8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ - Nodes in a permissioned chain maintain (and retry) connections to bootnodes [#7257](https://github.com/hyperledger/besu/pull/7257) - Promote experimental `besu storage x-trie-log` subcommand to production-ready [#7278](https://github.com/hyperledger/besu/pull/7278) - Enhanced BFT round-change diagnostics [#7271](https://github.com/hyperledger/besu/pull/7271) +- `--Xsnapsync-bft-enabled` option enables experimental support for snap sync with IBFT/QBFT permissioned Bonsai-DB chains [#7140](https://github.com/hyperledger/besu/pull/7140) ### Bug fixes - Validation errors ignored in accounts-allowlist and empty list [#7138](https://github.com/hyperledger/besu/issues/7138) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 7e1be15fb65..12752f2e9a1 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1552,10 +1552,11 @@ private void validateOptions() { } private void validateConsensusSyncCompatibilityOptions() { - // snap and checkpoint can't be used with BFT but can for clique - if (genesisConfigOptionsSupplier.get().isIbftLegacy() - || genesisConfigOptionsSupplier.get().isIbft2() - || genesisConfigOptionsSupplier.get().isQbft()) { + // snap and checkpoint are experimental for BFT + if ((genesisConfigOptionsSupplier.get().isIbftLegacy() + || genesisConfigOptionsSupplier.get().isIbft2() + || genesisConfigOptionsSupplier.get().isQbft()) + && !unstableSynchronizerOptions.isSnapSyncBftEnabled()) { final String errorSuffix = "can't be used with BFT networks"; if (SyncMode.CHECKPOINT.equals(syncMode) || SyncMode.X_CHECKPOINT.equals(syncMode)) { throw new ParameterException( @@ -2181,7 +2182,7 @@ private SynchronizerConfiguration buildSyncConfig() { return unstableSynchronizerOptions .toDomainObject() .syncMode(syncMode) - .fastSyncMinimumPeerCount(syncMinPeerCount) + .syncMinimumPeerCount(syncMinPeerCount) .build(); } @@ -2194,14 +2195,14 @@ private TransactionPoolConfiguration buildTransactionPoolConfiguration() { .saveFile((dataPath.resolve(txPoolConf.getSaveFile().getPath()).toFile())); if (genesisConfigOptionsSupplier.get().isZeroBaseFee()) { - logger.info( + logger.warn( "Forcing price bump for transaction replacement to 0, since we are on a zero basefee network"); txPoolConfBuilder.priceBump(Percentage.ZERO); } if (miningParametersSupplier.get().getMinTransactionGasPrice().equals(Wei.ZERO) && !transactionPoolOptions.isPriceBumpSet(commandLine)) { - logger.info( + logger.warn( "Forcing price bump for transaction replacement to 0, since min-gas-price is set to 0"); txPoolConfBuilder.priceBump(Percentage.ZERO); } @@ -2810,6 +2811,7 @@ && getDataStorageConfiguration().getBonsaiLimitTrieLogsEnabled()) { } builder.setSnapServerEnabled(this.unstableSynchronizerOptions.isSnapsyncServerEnabled()); + builder.setSnapSyncBftEnabled(this.unstableSynchronizerOptions.isSnapSyncBftEnabled()); builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation()); builder.setWorldStateUpdateMode(unstableEvmOptions.toDomainObject().worldUpdaterMode()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java index 2d74ae8d26f..b2f89a349e2 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java @@ -56,6 +56,7 @@ public class ConfigurationOverviewBuilder { private long trieLogRetentionLimit = 0; private Integer trieLogsPruningWindowSize = null; private boolean isSnapServerEnabled = false; + private boolean isSnapSyncBftEnabled = false; private TransactionPoolConfiguration.Implementation txPoolImplementation; private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode; private Map environment; @@ -233,6 +234,17 @@ public ConfigurationOverviewBuilder setSnapServerEnabled(final boolean snapServe return this; } + /** + * Sets snap sync BFT enabled/disabled + * + * @param snapSyncBftEnabled bool to indicate if snap sync for BFT is enabled + * @return the builder + */ + public ConfigurationOverviewBuilder setSnapSyncBftEnabled(final boolean snapSyncBftEnabled) { + isSnapSyncBftEnabled = snapSyncBftEnabled; + return this; + } + /** * Sets trie logs pruning window size * @@ -357,6 +369,10 @@ public String build() { lines.add("Experimental Snap Sync server enabled"); } + if (isSnapSyncBftEnabled) { + lines.add("Experimental Snap Sync for BFT enabled"); + } + if (isBonsaiLimitTrieLogsEnabled) { final StringBuilder trieLogPruningString = new StringBuilder(); trieLogPruningString diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java index e52839cceb4..95bbe0a2b1f 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java @@ -85,6 +85,8 @@ public class SynchronizerOptions implements CLIOptions

validatorList = new ArrayList<>(); + validatorList.add(Address.ZERO); + validatorList.add(Address.ZERO); + validatorList.add(Address.ZERO); + + when(ethContext.getEthPeers()).thenReturn(ethPeers); + when(validatorProvider.nodeIsValidator(any())).thenReturn(true); + when(validatorProvider.getValidatorsAtHead()).thenReturn(validatorList); + BFTPivotSelectorFromPeers pivotSelector = + new BFTPivotSelectorFromPeers( + ethContext, + syncConfig, + syncState, + metricsSystem, + protocolContext, + nodeKey, + blockHeader); + Optional pivotState = pivotSelector.selectNewPivotBlock(); + assertThat(pivotState.isEmpty()).isTrue(); + } + + @Test + @SuppressWarnings("unused") + public void returnNoSyncRequiredIfOnlyValidatorAndNoPeers() { + + // We're the only validator so just put us in the list + List
validatorList = new ArrayList<>(); + validatorList.add(Address.ZERO); + + when(ethContext.getEthPeers()).thenReturn(ethPeers); + when(validatorProvider.nodeIsValidator(any())).thenReturn(true); + when(validatorProvider.getValidatorsAtHead()).thenReturn(validatorList); + BFTPivotSelectorFromPeers pivotSelector = + new BFTPivotSelectorFromPeers( + ethContext, + syncConfig, + syncState, + metricsSystem, + protocolContext, + nodeKey, + blockHeader); + + try { + Optional pivotState = pivotSelector.selectNewPivotBlock(); + fail("Expected NoSyncRequiredException but none thrown"); + } catch (NoSyncRequiredException e) { + // Ignore - just make sure we get here + } + } + + @Test + public void returnEmptySyncStateIfNonValidatorWithNoBestPeer() { + when(ethContext.getEthPeers()).thenReturn(ethPeers); + when(validatorProvider.nodeIsValidator(any())).thenReturn(false); + BFTPivotSelectorFromPeers pivotSelector = + new BFTPivotSelectorFromPeers( + ethContext, + syncConfig, + syncState, + metricsSystem, + protocolContext, + nodeKey, + blockHeader); + + Optional pivotState = pivotSelector.selectNewPivotBlock(); + assertThat(pivotState.isEmpty()).isTrue(); + } + + @Test + public void returnEmptySyncStateIfValidatorAndNotAtGenesisAndOtherValidators() { + when(ethContext.getEthPeers()).thenReturn(ethPeers); + when(validatorProvider.nodeIsValidator(any())).thenReturn(false); + when(blockHeader.getNumber()).thenReturn(10L); + BFTPivotSelectorFromPeers pivotSelector = + new BFTPivotSelectorFromPeers( + ethContext, + syncConfig, + syncState, + metricsSystem, + protocolContext, + nodeKey, + blockHeader); + + Optional pivotState = pivotSelector.selectNewPivotBlock(); + assertThat(pivotState.isEmpty()).isTrue(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java index 4a5cc875110..6f347be0334 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java @@ -99,7 +99,7 @@ private int preloadQueue() { } } - public void addToPruneQueue(final long blockNumber, final Hash blockHash) { + public synchronized void addToPruneQueue(final long blockNumber, final Hash blockHash) { LOG.atTrace() .setMessage("adding trie log to queue for later pruning blockNumber {}; blockHash {}") .addArgument(blockNumber) @@ -108,7 +108,7 @@ public void addToPruneQueue(final long blockNumber, final Hash blockHash) { trieLogBlocksAndForksByDescendingBlockNumber.put(blockNumber, blockHash); } - public int pruneFromQueue() { + public synchronized int pruneFromQueue() { final long retainAboveThisBlock = blockchain.getChainHeadBlockNumber() - numBlocksToRetain; final Optional finalized = blockchain.getFinalized(); if (requireFinalizedBlock && finalized.isEmpty()) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java index 49f725d825f..d6bfe900d9c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServer.java @@ -32,6 +32,7 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.trie.CompactEncoding; +import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; @@ -225,6 +226,9 @@ MessageData constructGetAccountRangeResponse(final MessageData message) { .addArgument(() -> asLogHash(range.endKeyHash())) .log(); try { + if (range.worldStateRootHash().equals(Hash.EMPTY_TRIE_HASH)) { + return AccountRangeMessage.create(new HashMap<>(), List.of(MerkleTrie.EMPTY_TRIE_NODE)); + } return worldStateStorageProvider .apply(range.worldStateRootHash()) .map( @@ -509,8 +513,11 @@ MessageData constructGetTrieNodesResponse(final MessageData message) { // first element in paths is account if (triePath.size() == 1) { // if there is only one path, presume it should be compact encoded account path - var optStorage = - storage.getTrieNodeUnsafe(CompactEncoding.decode(triePath.get(0))); + final Bytes location = CompactEncoding.decode(triePath.get(0)); + var optStorage = storage.getTrieNodeUnsafe(location); + if (optStorage.isEmpty() && location.isEmpty()) { + optStorage = Optional.of(MerkleTrie.EMPTY_TRIE_NODE); + } if (optStorage.isPresent()) { if (!trieNodes.isEmpty() && (sumListBytes(trieNodes) + optStorage.get().size() > maxResponseBytes @@ -534,9 +541,12 @@ MessageData constructGetTrieNodesResponse(final MessageData message) { List storagePaths = triePath.subList(1, triePath.size()); for (var path : storagePaths) { + final Bytes location = CompactEncoding.decode(path); var optStorage = - storage.getTrieNodeUnsafe( - Bytes.concatenate(accountPrefix, CompactEncoding.decode(path))); + storage.getTrieNodeUnsafe(Bytes.concatenate(accountPrefix, location)); + if (optStorage.isEmpty() && location.isEmpty()) { + optStorage = Optional.of(MerkleTrie.EMPTY_TRIE_NODE); + } if (optStorage.isPresent()) { if (!trieNodes.isEmpty() && sumListBytes(trieNodes) + optStorage.get().size() diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index af35898ac22..5e596576aa3 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.ethereum.eth.sync.checkpointsync.CheckpointDownloaderFactory; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncDownloader; import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState; +import org.hyperledger.besu.ethereum.eth.sync.fastsync.NoSyncRequiredState; import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.FastDownloaderFactory; import org.hyperledger.besu.ethereum.eth.sync.fullsync.FullSyncDownloader; import org.hyperledger.besu.ethereum.eth.sync.fullsync.SyncTerminationCondition; @@ -242,16 +243,24 @@ private CompletableFuture handleSyncResult(final FastSyncState result) { // We've been shutdown which will have triggered the fast sync future to complete return CompletableFuture.completedFuture(null); } - fastSyncDownloader.ifPresent(FastSyncDownloader::deleteFastSyncState); - result - .getPivotBlockHeader() - .ifPresent( - blockHeader -> protocolContext.getWorldStateArchive().resetArchiveStateTo(blockHeader)); - LOG.info( - "Sync completed successfully with pivot block {}", - result.getPivotBlockNumber().getAsLong()); - pivotBlockSelector.close(); - syncState.markInitialSyncPhaseAsDone(); + + if (result instanceof NoSyncRequiredState) { + LOG.info("Sync ended (no sync required)"); + syncState.markInitialSyncPhaseAsDone(); + } else { + fastSyncDownloader.ifPresent(FastSyncDownloader::deleteFastSyncState); + result + .getPivotBlockHeader() + .ifPresent( + blockHeader -> + protocolContext.getWorldStateArchive().resetArchiveStateTo(blockHeader)); + if (result.hasPivotBlockHash()) + LOG.info( + "Sync completed successfully with pivot block {}", + result.getPivotBlockNumber().getAsLong()); + pivotBlockSelector.close(); + syncState.markInitialSyncPhaseAsDone(); + } if (terminationCondition.shouldContinueDownload()) { return startFullSync(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SynchronizerConfiguration.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SynchronizerConfiguration.java index 28f8bee6db7..d46da85dc48 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SynchronizerConfiguration.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SynchronizerConfiguration.java @@ -29,7 +29,7 @@ public class SynchronizerConfiguration { public static final int DEFAULT_PIVOT_DISTANCE_FROM_HEAD = 50; public static final float DEFAULT_FULL_VALIDATION_RATE = .1f; - public static final int DEFAULT_FAST_SYNC_MINIMUM_PEERS = 5; + public static final int DEFAULT_SYNC_MINIMUM_PEERS = 5; public static final int DEFAULT_WORLD_STATE_HASH_COUNT_PER_REQUEST = 384; public static final int DEFAULT_WORLD_STATE_REQUEST_PARALLELISM = 10; public static final int DEFAULT_WORLD_STATE_MAX_REQUESTS_WITHOUT_PROGRESS = 1000; @@ -53,9 +53,9 @@ public class SynchronizerConfiguration { public static final boolean DEFAULT_CHECKPOINT_POST_MERGE_ENABLED = false; // Fast sync config - private final int fastSyncPivotDistance; + private final int syncPivotDistance; private final float fastSyncFullValidationRate; - private final int fastSyncMinimumPeerCount; + private final int syncMinimumPeerCount; private final int worldStateHashCountPerRequest; private final int worldStateRequestParallelism; private final int worldStateMaxRequestsWithoutProgress; @@ -87,9 +87,9 @@ public class SynchronizerConfiguration { private final long propagationManagerGetBlockTimeoutMillis; private SynchronizerConfiguration( - final int fastSyncPivotDistance, + final int syncPivotDistance, final float fastSyncFullValidationRate, - final int fastSyncMinimumPeerCount, + final int syncMinimumPeerCount, final int worldStateHashCountPerRequest, final int worldStateRequestParallelism, final int worldStateMaxRequestsWithoutProgress, @@ -109,9 +109,9 @@ private SynchronizerConfiguration( final int maxTrailingPeers, final long propagationManagerGetBlockTimeoutMillis, final boolean checkpointPostMergeEnabled) { - this.fastSyncPivotDistance = fastSyncPivotDistance; + this.syncPivotDistance = syncPivotDistance; this.fastSyncFullValidationRate = fastSyncFullValidationRate; - this.fastSyncMinimumPeerCount = fastSyncMinimumPeerCount; + this.syncMinimumPeerCount = syncMinimumPeerCount; this.worldStateHashCountPerRequest = worldStateHashCountPerRequest; this.worldStateRequestParallelism = worldStateRequestParallelism; this.worldStateMaxRequestsWithoutProgress = worldStateMaxRequestsWithoutProgress; @@ -171,12 +171,14 @@ public Range getBlockPropagationRange() { } /** - * The distance from the chain head at which we should switch from fast sync to full sync. + * The distance from the chain head at which we should switch from fast, snap, or checkpoint sync + * to full sync. * - * @return distance from the chain head at which we should switch from fast sync to full sync. + * @return distance from the chain head at which we should switch from fast, snap or checkpoint + * sync to full sync. */ - public int getFastSyncPivotDistance() { - return fastSyncPivotDistance; + public int getSyncPivotDistance() { + return syncPivotDistance; } public long getDownloaderChangeTargetThresholdByHeight() { @@ -222,8 +224,8 @@ public float getFastSyncFullValidationRate() { return fastSyncFullValidationRate; } - public int getFastSyncMinimumPeerCount() { - return fastSyncMinimumPeerCount; + public int getSyncMinimumPeerCount() { + return syncMinimumPeerCount; } public int getWorldStateHashCountPerRequest() { @@ -256,7 +258,7 @@ public long getPropagationManagerGetBlockTimeoutMillis() { public static class Builder { private SyncMode syncMode = SyncMode.FULL; - private int fastSyncMinimumPeerCount = DEFAULT_FAST_SYNC_MINIMUM_PEERS; + private int syncMinimumPeerCount = DEFAULT_SYNC_MINIMUM_PEERS; private int maxTrailingPeers = Integer.MAX_VALUE; private Range blockPropagationRange = DEFAULT_BLOCK_PROPAGATION_RANGE; private long downloaderChangeTargetThresholdByHeight = @@ -270,7 +272,7 @@ public static class Builder { private int downloaderParallelism = DEFAULT_DOWNLOADER_PARALLELISM; private int transactionsParallelism = DEFAULT_TRANSACTIONS_PARALLELISM; private int computationParallelism = DEFAULT_COMPUTATION_PARALLELISM; - private int fastSyncPivotDistance = DEFAULT_PIVOT_DISTANCE_FROM_HEAD; + private int syncPivotDistance = DEFAULT_PIVOT_DISTANCE_FROM_HEAD; private float fastSyncFullValidationRate = DEFAULT_FULL_VALIDATION_RATE; private int worldStateHashCountPerRequest = DEFAULT_WORLD_STATE_HASH_COUNT_PER_REQUEST; private int worldStateRequestParallelism = DEFAULT_WORLD_STATE_REQUEST_PARALLELISM; @@ -283,8 +285,8 @@ public static class Builder { DEFAULT_PROPAGATION_MANAGER_GET_BLOCK_TIMEOUT_MILLIS; private boolean checkpointPostMergeEnabled = DEFAULT_CHECKPOINT_POST_MERGE_ENABLED; - public Builder fastSyncPivotDistance(final int distance) { - fastSyncPivotDistance = distance; + public Builder syncPivotDistance(final int distance) { + syncPivotDistance = distance; return this; } @@ -357,8 +359,8 @@ public Builder computationParallelism(final int computationParallelism) { return this; } - public Builder fastSyncMinimumPeerCount(final int fastSyncMinimumPeerCount) { - this.fastSyncMinimumPeerCount = fastSyncMinimumPeerCount; + public Builder syncMinimumPeerCount(final int syncMinimumPeerCount) { + this.syncMinimumPeerCount = syncMinimumPeerCount; return this; } @@ -406,9 +408,9 @@ public Builder checkpointPostMergeEnabled(final boolean checkpointPostMergeEnabl public SynchronizerConfiguration build() { return new SynchronizerConfiguration( - fastSyncPivotDistance, + syncPivotDistance, fastSyncFullValidationRate, - fastSyncMinimumPeerCount, + syncMinimumPeerCount, worldStateHashCountPerRequest, worldStateRequestParallelism, worldStateMaxRequestsWithoutProgress, diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java index 52d384acf6e..06bf8aace6a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActions.java @@ -142,8 +142,8 @@ private CompletableFuture internalDownloadPivotBlockHeader( ethContext, metricsSystem, currentState.getPivotBlockNumber().getAsLong(), - syncConfig.getFastSyncMinimumPeerCount(), - syncConfig.getFastSyncPivotDistance()) + syncConfig.getSyncMinimumPeerCount(), + syncConfig.getSyncPivotDistance()) .downloadPivotBlockHeader())); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java index 7b64f885c29..565824a7808 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright contributors to Hyperledger Besu. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -80,7 +80,7 @@ public CompletableFuture start() { if (!running.compareAndSet(false, true)) { throw new IllegalStateException("SyncDownloader already running"); } - LOG.info("Starting sync"); + LOG.info("Starting pivot-based sync"); return start(initialFastSyncState); } @@ -94,7 +94,7 @@ protected CompletableFuture start(final FastSyncState fastSyncSta onBonsai.clearFlatDatabase(); onBonsai.clearTrieLog(); }); - LOG.debug("Start sync with initial sync state {}", fastSyncState); + LOG.debug("Start fast sync with initial sync state {}", fastSyncState); return findPivotBlock(fastSyncState, fss -> downloadChainAndWorldState(fastSyncActions, fss)); } @@ -114,7 +114,9 @@ public CompletableFuture findPivotBlock( protected CompletableFuture handleFailure(final Throwable error) { trailingPeerRequirements = Optional.empty(); Throwable rootCause = ExceptionUtils.rootCause(error); - if (rootCause instanceof SyncException) { + if (rootCause instanceof NoSyncRequiredException) { + return CompletableFuture.completedFuture(new NoSyncRequiredState()); + } else if (rootCause instanceof SyncException) { return CompletableFuture.failedFuture(error); } else if (rootCause instanceof StalledDownloadException) { LOG.debug("Stalled sync re-pivoting to newer block."); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/NoSyncRequiredException.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/NoSyncRequiredException.java new file mode 100644 index 00000000000..caa45abdc19 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/NoSyncRequiredException.java @@ -0,0 +1,17 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.sync.fastsync; + +public class NoSyncRequiredException extends RuntimeException {} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/NoSyncRequiredState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/NoSyncRequiredState.java new file mode 100644 index 00000000000..9cefe78ed15 --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/NoSyncRequiredState.java @@ -0,0 +1,17 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.eth.sync.fastsync; + +public class NoSyncRequiredState extends FastSyncState {} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotSelectorFromPeers.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotSelectorFromPeers.java index 6402b58d1fd..16f7e09d6d1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotSelectorFromPeers.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/PivotSelectorFromPeers.java @@ -35,8 +35,8 @@ public class PivotSelectorFromPeers implements PivotBlockSelector { private static final Logger LOG = LoggerFactory.getLogger(PivotSelectorFromPeers.class); - private final EthContext ethContext; - private final SynchronizerConfiguration syncConfig; + protected final EthContext ethContext; + protected final SynchronizerConfiguration syncConfig; private final SyncState syncState; private final MetricsSystem metricsSystem; @@ -66,7 +66,7 @@ public CompletableFuture prepareRetry() { conservativelyEstimatedPivotBlock(), syncConfig.getMaxTrailingPeers())); trailingPeerLimiter.enforceTrailingPeerLimit(); - return waitForPeers(syncConfig.getFastSyncMinimumPeerCount()); + return waitForPeers(syncConfig.getSyncMinimumPeerCount()); } @Override @@ -74,9 +74,9 @@ public long getBestChainHeight() { return syncState.bestChainHeight(); } - private Optional fromBestPeer(final EthPeer peer) { + protected Optional fromBestPeer(final EthPeer peer) { final long pivotBlockNumber = - peer.chainState().getEstimatedHeight() - syncConfig.getFastSyncPivotDistance(); + peer.chainState().getEstimatedHeight() - syncConfig.getSyncPivotDistance(); if (pivotBlockNumber <= BlockHeader.GENESIS_BLOCK_NUMBER) { // Peer's chain isn't long enough, return an empty value, so we can try again. LOG.info("Waiting for peers with sufficient chain height"); @@ -86,7 +86,7 @@ private Optional fromBestPeer(final EthPeer peer) { return Optional.of(new FastSyncState(pivotBlockNumber)); } - private Optional selectBestPeer() { + protected Optional selectBestPeer() { return ethContext .getEthPeers() .bestPeerMatchingCriteria(this::canPeerDeterminePivotBlock) @@ -96,7 +96,7 @@ private Optional selectBestPeer() { private boolean enoughFastSyncPeersArePresent() { final long peerCount = countPeersThatCanDeterminePivotBlock(); - final int minPeerCount = syncConfig.getFastSyncMinimumPeerCount(); + final int minPeerCount = syncConfig.getSyncMinimumPeerCount(); if (peerCount < minPeerCount) { LOG.info( "Waiting for valid peers with chain height information. {} / {} required peers currently available.", @@ -126,7 +126,7 @@ private boolean canPeerDeterminePivotBlock(final EthPeer peer) { private long conservativelyEstimatedPivotBlock() { final long estimatedNextPivot = - syncState.getLocalChainHeight() + syncConfig.getFastSyncPivotDistance(); + syncState.getLocalChainHeight() + syncConfig.getSyncPivotDistance(); return Math.min(syncState.bestChainHeight(), estimatedNextPivot); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncConfiguration.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncConfiguration.java index 669287cfef2..dbe043b92a0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncConfiguration.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncConfiguration.java @@ -38,6 +38,8 @@ public class SnapSyncConfiguration { public static final Boolean DEFAULT_SNAP_SERVER_ENABLED = Boolean.FALSE; + public static final Boolean DEFAULT_SNAP_SYNC_BFT_ENABLED = Boolean.FALSE; + public static SnapSyncConfiguration getDefault() { return ImmutableSnapSyncConfiguration.builder().build(); } @@ -81,4 +83,9 @@ public int getLocalFlatStorageCountToHealPerRequest() { public Boolean isSnapServerEnabled() { return DEFAULT_SNAP_SERVER_ENABLED; } + + @Value.Default + public Boolean isSnapSyncBftEnabled() { + return DEFAULT_SNAP_SYNC_BFT_ENABLED; + } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java index 213912a1ac3..21a707e9a63 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java @@ -58,12 +58,12 @@ public class AccountRangeDataRequest extends SnapDataRequest { private static final Logger LOG = LoggerFactory.getLogger(AccountRangeDataRequest.class); - private final Bytes32 startKeyHash; - private final Bytes32 endKeyHash; - private final Optional startStorageRange; - private final Optional endStorageRange; + protected final Bytes32 startKeyHash; + protected final Bytes32 endKeyHash; + protected final Optional startStorageRange; + protected final Optional endStorageRange; - private final StackTrie stackTrie; + protected final StackTrie stackTrie; private Optional isProofValid; protected AccountRangeDataRequest( @@ -170,6 +170,12 @@ startKeyHash, endKeyHash, getRootHash(), proofs, accounts)) { } else { stackTrie.addElement(startKeyHash, proofs, accounts); isProofValid = Optional.of(true); + LOG.atDebug() + .setMessage("{} accounts received during sync for account range {} {}") + .addArgument(accounts.size()) + .addArgument(startKeyHash) + .addArgument(endKeyHash) + .log(); } } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java index e060bab6fbe..d29bb7260e5 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncActionsTest.java @@ -62,7 +62,7 @@ public class FastSyncActionsTest { private final SynchronizerConfiguration.Builder syncConfigBuilder = - new SynchronizerConfiguration.Builder().syncMode(SyncMode.FAST).fastSyncPivotDistance(1000); + new SynchronizerConfiguration.Builder().syncMode(SyncMode.FAST).syncPivotDistance(1000); private final WorldStateStorageCoordinator worldStateStorageCoordinator = mock(WorldStateStorageCoordinator.class); @@ -113,9 +113,9 @@ public void setUp(final DataStorageFormat storageFormat) { public void waitForPeersShouldSucceedIfEnoughPeersAreFound( final DataStorageFormat storageFormat) { setUp(storageFormat); - for (int i = 0; i < syncConfig.getFastSyncMinimumPeerCount(); i++) { + for (int i = 0; i < syncConfig.getSyncMinimumPeerCount(); i++) { EthProtocolManagerTestUtil.createPeer( - ethProtocolManager, syncConfig.getFastSyncPivotDistance() + i + 1); + ethProtocolManager, syncConfig.getSyncPivotDistance() + i + 1); } final CompletableFuture result = fastSyncActions.selectPivotBlock(FastSyncState.EMPTY_SYNC_STATE); @@ -153,7 +153,7 @@ public void selectPivotBlockShouldSelectBlockPivotDistanceFromBestPeer( final DataStorageFormat storageFormat) { setUp(storageFormat); final int minPeers = 1; - syncConfigBuilder.fastSyncMinimumPeerCount(minPeers); + syncConfigBuilder.syncMinimumPeerCount(minPeers); syncConfig = syncConfigBuilder.build(); fastSyncActions = createFastSyncActions( @@ -174,7 +174,7 @@ public void selectPivotBlockShouldConsiderTotalDifficultyWhenSelectingBestPeer( final DataStorageFormat storageFormat) { setUp(storageFormat); final int minPeers = 1; - syncConfigBuilder.fastSyncMinimumPeerCount(minPeers); + syncConfigBuilder.syncMinimumPeerCount(minPeers); syncConfig = syncConfigBuilder.build(); fastSyncActions = createFastSyncActions( @@ -197,7 +197,7 @@ public void selectPivotBlockShouldWaitAndRetryUntilMinHeightEstimatesAreAvailabl setUp(storageFormat); EthProtocolManagerTestUtil.disableEthSchedulerAutoRun(ethProtocolManager); final int minPeers = 2; - syncConfigBuilder.fastSyncMinimumPeerCount(minPeers); + syncConfigBuilder.syncMinimumPeerCount(minPeers); syncConfig = syncConfigBuilder.build(); fastSyncActions = createFastSyncActions( @@ -228,13 +228,13 @@ public void selectPivotBlockShouldWaitAndRetryIfSufficientChainHeightEstimatesAr final DataStorageFormat storageFormat) { setUp(storageFormat); final int minPeers = 3; - syncConfigBuilder.fastSyncMinimumPeerCount(minPeers); + syncConfigBuilder.syncMinimumPeerCount(minPeers); syncConfig = syncConfigBuilder.build(); fastSyncActions = createFastSyncActions( syncConfig, new PivotSelectorFromPeers(ethContext, syncConfig, syncState, metricsSystem)); - final long minPivotHeight = syncConfig.getFastSyncPivotDistance() + 1L; + final long minPivotHeight = syncConfig.getSyncPivotDistance() + 1L; EthProtocolManagerTestUtil.disableEthSchedulerAutoRun(ethProtocolManager); // Create peers without chain height estimates @@ -267,7 +267,7 @@ public void selectPivotBlockShouldWaitAndRetryIfSufficientChainHeightEstimatesAr final long bestPeerHeight = minPivotHeight + 1; peers.get(minPeers - 1).getEthPeer().chainState().updateHeightEstimate(bestPeerHeight); final FastSyncState expected = - new FastSyncState(bestPeerHeight - syncConfig.getFastSyncPivotDistance()); + new FastSyncState(bestPeerHeight - syncConfig.getSyncPivotDistance()); EthProtocolManagerTestUtil.runPendingFutures(ethProtocolManager); assertThat(result).isCompletedWithValue(expected); } @@ -279,13 +279,13 @@ public void selectPivotBlockShouldWaitAndRetryIfSufficientValidatedPeersUnavaila setUp(storageFormat); final int minPeers = 3; final PeerValidator validator = mock(PeerValidator.class); - syncConfigBuilder.fastSyncMinimumPeerCount(minPeers); + syncConfigBuilder.syncMinimumPeerCount(minPeers); syncConfig = syncConfigBuilder.build(); fastSyncActions = createFastSyncActions( syncConfig, new PivotSelectorFromPeers(ethContext, syncConfig, syncState, metricsSystem)); - final long minPivotHeight = syncConfig.getFastSyncPivotDistance() + 1L; + final long minPivotHeight = syncConfig.getSyncPivotDistance() + 1L; EthProtocolManagerTestUtil.disableEthSchedulerAutoRun(ethProtocolManager); // Create peers that are not validated @@ -319,7 +319,7 @@ public void selectPivotBlockShouldWaitAndRetryIfSufficientValidatedPeersUnavaila bestPeer.chainState().updateHeightEstimate(bestPeerHeight); bestPeer.markValidated(validator); final FastSyncState expected = - new FastSyncState(bestPeerHeight - syncConfig.getFastSyncPivotDistance()); + new FastSyncState(bestPeerHeight - syncConfig.getSyncPivotDistance()); EthProtocolManagerTestUtil.runPendingFutures(ethProtocolManager); assertThat(result).isCompletedWithValue(expected); } @@ -351,13 +351,13 @@ private void selectPivotBlockUsesBestPeerMatchingRequiredCriteria( final boolean bestMissingHeight, final boolean bestNotValidated) { final int minPeers = 3; final int peerCount = minPeers + 1; - syncConfigBuilder.fastSyncMinimumPeerCount(minPeers); + syncConfigBuilder.syncMinimumPeerCount(minPeers); syncConfig = syncConfigBuilder.build(); fastSyncActions = createFastSyncActions( syncConfig, new PivotSelectorFromPeers(ethContext, syncConfig, syncState, metricsSystem)); - final long minPivotHeight = syncConfig.getFastSyncPivotDistance() + 1L; + final long minPivotHeight = syncConfig.getSyncPivotDistance() + 1L; EthProtocolManagerTestUtil.disableEthSchedulerAutoRun(ethProtocolManager); // Create peers without chain height estimates @@ -392,7 +392,7 @@ private void selectPivotBlockUsesBestPeerMatchingRequiredCriteria( final long expectedBestChainHeight = peers.get(1).getEthPeer().chainState().getEstimatedHeight(); final FastSyncState expected = - new FastSyncState(expectedBestChainHeight - syncConfig.getFastSyncPivotDistance()); + new FastSyncState(expectedBestChainHeight - syncConfig.getSyncPivotDistance()); EthProtocolManagerTestUtil.runPendingFutures(ethProtocolManager); assertThat(result).isCompletedWithValue(expected); } @@ -403,13 +403,13 @@ public void selectPivotBlockShouldWaitAndRetryIfBestPeerChainIsShorterThanPivotD final DataStorageFormat storageFormat) { setUp(storageFormat); final int minPeers = 1; - syncConfigBuilder.fastSyncMinimumPeerCount(minPeers); + syncConfigBuilder.syncMinimumPeerCount(minPeers); syncConfig = syncConfigBuilder.build(); fastSyncActions = createFastSyncActions( syncConfig, new PivotSelectorFromPeers(ethContext, syncConfig, syncState, metricsSystem)); - final long pivotDistance = syncConfig.getFastSyncPivotDistance(); + final long pivotDistance = syncConfig.getSyncPivotDistance(); EthProtocolManagerTestUtil.disableEthSchedulerAutoRun(ethProtocolManager); EthProtocolManagerTestUtil.createPeer(ethProtocolManager, pivotDistance - 1); @@ -432,10 +432,10 @@ public void selectPivotBlockShouldWaitAndRetryIfBestPeerChainIsShorterThanPivotD public void selectPivotBlockShouldRetryIfBestPeerChainIsEqualToPivotDistance( final DataStorageFormat storageFormat) { setUp(storageFormat); - final long pivotDistance = syncConfig.getFastSyncPivotDistance(); + final long pivotDistance = syncConfig.getSyncPivotDistance(); EthProtocolManagerTestUtil.disableEthSchedulerAutoRun(ethProtocolManager); // Create peers with chains that are too short - for (int i = 0; i < syncConfig.getFastSyncMinimumPeerCount(); i++) { + for (int i = 0; i < syncConfig.getSyncMinimumPeerCount(); i++) { EthProtocolManagerTestUtil.createPeer(ethProtocolManager, pivotDistance); } @@ -467,7 +467,7 @@ public void downloadPivotBlockHeaderShouldUseExistingPivotBlockHeaderIfPresent( public void downloadPivotBlockHeaderShouldRetrievePivotBlockHeader( final DataStorageFormat storageFormat) { setUp(storageFormat); - syncConfig = SynchronizerConfiguration.builder().fastSyncMinimumPeerCount(1).build(); + syncConfig = SynchronizerConfiguration.builder().syncMinimumPeerCount(1).build(); fastSyncActions = createFastSyncActions( syncConfig, @@ -489,7 +489,7 @@ public void downloadPivotBlockHeaderShouldRetrievePivotBlockHeader( public void downloadPivotBlockHeaderShouldRetrievePivotBlockHash( final DataStorageFormat storageFormat) { setUp(storageFormat); - syncConfig = SynchronizerConfiguration.builder().fastSyncMinimumPeerCount(1).build(); + syncConfig = SynchronizerConfiguration.builder().syncMinimumPeerCount(1).build(); GenesisConfigOptions genesisConfig = mock(GenesisConfigOptions.class); when(genesisConfig.getTerminalBlockNumber()).thenReturn(OptionalLong.of(10L)); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java index 6707ed97054..582bdff6335 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java @@ -51,7 +51,7 @@ public class LoadLocalDataStepTest { private final Task task = new StubTask(request); private final Pipe> completedTasks = - new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); + new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER, "test_pipe"); private final LoadLocalDataStep loadLocalDataStep = new LoadLocalDataStep( new WorldStateStorageCoordinator(worldStateKeyValueStorage), new NoOpMetricsSystem()); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStepTest.java index 9027538ce51..06f92460c27 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/LoadLocalDataStepTest.java @@ -55,7 +55,7 @@ public class LoadLocalDataStepTest { private final Task task = new StubTask(request); private final Pipe> completedTasks = - new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); + new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER, "test_pipe"); private final SnapSyncProcessState snapSyncState = mock(SnapSyncProcessState.class); private final SnapWorldDownloadState downloadState = mock(SnapWorldDownloadState.class); diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/TrieNodeDecoder.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/TrieNodeDecoder.java index b3476337a77..6179f45fc0a 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/TrieNodeDecoder.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/TrieNodeDecoder.java @@ -16,6 +16,7 @@ import static com.google.common.base.Preconditions.checkArgument; +import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.Node; import org.hyperledger.besu.ethereum.trie.NodeLoader; import org.hyperledger.besu.ethereum.trie.NullNode; @@ -59,6 +60,9 @@ public static Node decode(final Bytes location, final Bytes rlp) { * @return A list of nodes and node references embedded in the given rlp. */ public static List> decodeNodes(final Bytes location, final Bytes nodeRlp) { + if (nodeRlp.equals(MerkleTrie.EMPTY_TRIE_NODE)) { + return new ArrayList<>(); + } final Node node = decode(location, nodeRlp); final List> nodes = new ArrayList<>(); nodes.add(node); diff --git a/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/AsyncOperationProcessor.java b/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/AsyncOperationProcessor.java index f64454fa9f1..c8c7e5905db 100644 --- a/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/AsyncOperationProcessor.java +++ b/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/AsyncOperationProcessor.java @@ -83,7 +83,7 @@ private void outputNextCompletedTask(final WritePipe outputPipe) { waitForAnyFutureToComplete(); outputCompletedTasks(outputPipe); } catch (final InterruptedException e) { - LOG.trace("Interrupted while waiting for processing to complete", e); + LOG.trace("Interrupted while waiting for processing to complete", e.getMessage()); } catch (final ExecutionException e) { throw new AsyncOperationException("Async operation failed. " + e.getMessage(), e); } catch (final TimeoutException e) { diff --git a/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/Pipe.java b/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/Pipe.java index 3697e0e3ba2..63fb01bad13 100644 --- a/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/Pipe.java +++ b/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/Pipe.java @@ -44,6 +44,7 @@ public class Pipe implements ReadPipe, WritePipe { private final Counter abortedItemCounter; private final AtomicBoolean closed = new AtomicBoolean(); private final AtomicBoolean aborted = new AtomicBoolean(); + private String pipeName = ""; /** * Instantiates a new Pipe. @@ -52,16 +53,28 @@ public class Pipe implements ReadPipe, WritePipe { * @param inputCounter the input counter * @param outputCounter the output counter * @param abortedItemCounter the aborted item counter + * @param pipeName the name of the pipe */ public Pipe( final int capacity, final Counter inputCounter, final Counter outputCounter, - final Counter abortedItemCounter) { + final Counter abortedItemCounter, + final String pipeName) { queue = new ArrayBlockingQueue<>(capacity); this.inputCounter = inputCounter; this.outputCounter = outputCounter; this.abortedItemCounter = abortedItemCounter; + this.pipeName = pipeName; + } + + /** + * Get the name of this pipe + * + * @return the name of the pipe + */ + public String getPipeName() { + return pipeName; } @Override @@ -110,7 +123,7 @@ public T get() { } } } catch (final InterruptedException e) { - LOG.trace("Interrupted while waiting for next item", e); + LOG.trace("Interrupted while waiting for next item from pipe {}", pipeName); } return null; } @@ -140,7 +153,7 @@ public void put(final T value) { return; } } catch (final InterruptedException e) { - LOG.trace("Interrupted while waiting to add to output", e); + LOG.trace("Interrupted while waiting to add to output to pipe {}", pipeName); } } } diff --git a/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/Pipeline.java b/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/Pipeline.java index 7f37a4bac91..507e87a8c0c 100644 --- a/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/Pipeline.java +++ b/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/Pipeline.java @@ -89,6 +89,19 @@ public class Pipeline { this.stages = stages; this.pipes = pipes; this.completerStage = completerStage; + + if (LOG.isTraceEnabled()) { + StringBuilder sb = new StringBuilder(); + sb.append("Building pipeline "); + sb.append(name); + sb.append(". Stages: "); + for (Stage nextStage : stages) { + sb.append(nextStage.getName()); + sb.append(" -> "); + } + sb.append("END"); + LOG.trace("{}", sb.toString()); + } } /** diff --git a/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/PipelineBuilder.java b/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/PipelineBuilder.java index 3bc6656e700..aaf68f5e3f1 100644 --- a/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/PipelineBuilder.java +++ b/services/pipeline/src/main/java/org/hyperledger/besu/services/pipeline/PipelineBuilder.java @@ -421,6 +421,7 @@ private static Pipe createPipe( newBufferSize, outputCounter.labels(labelName, "added"), outputCounter.labels(labelName, "removed"), - outputCounter.labels(labelName, "aborted")); + outputCounter.labels(labelName, "aborted"), + stageName); } } diff --git a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/BatchingReadPipeTest.java b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/BatchingReadPipeTest.java index fb90128fbb3..da0908e8148 100644 --- a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/BatchingReadPipeTest.java +++ b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/BatchingReadPipeTest.java @@ -31,7 +31,8 @@ public class BatchingReadPipeTest { - private final Pipe source = new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); + private final Pipe source = + new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER, "source_pipe"); private final Counter batchCounter = mock(Counter.class); private final BatchingReadPipe batchingPipe = new BatchingReadPipe<>(source, 3, batchCounter); diff --git a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/CompleterStageTest.java b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/CompleterStageTest.java index ef3e29afebe..ea4a81a9d98 100644 --- a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/CompleterStageTest.java +++ b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/CompleterStageTest.java @@ -24,7 +24,8 @@ public class CompleterStageTest { - private final Pipe pipe = new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); + private final Pipe pipe = + new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER, "test_pipe"); private final List output = new ArrayList<>(); private final CompleterStage stage = new CompleterStage<>("name", pipe, output::add); diff --git a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/FlatMapProcessorTest.java b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/FlatMapProcessorTest.java index 1012d3b0123..4e3b8cfa513 100644 --- a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/FlatMapProcessorTest.java +++ b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/FlatMapProcessorTest.java @@ -28,8 +28,10 @@ public class FlatMapProcessorTest { - private final Pipe input = new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); - private final Pipe output = new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); + private final Pipe input = + new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER, "input_pipe"); + private final Pipe output = + new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER, "output_pipe"); @SuppressWarnings("unchecked") private final Function> mapper = mock(Function.class); diff --git a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/IteratorSourceStageTest.java b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/IteratorSourceStageTest.java index 02052615b75..915a90b1fd3 100644 --- a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/IteratorSourceStageTest.java +++ b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/IteratorSourceStageTest.java @@ -22,7 +22,8 @@ public class IteratorSourceStageTest { - private final Pipe output = new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); + private final Pipe output = + new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER, "output_pipe"); private final IteratorSourceStage stage = new IteratorSourceStage<>("name", Iterators.forArray("a", "b", "c", "d"), output); diff --git a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/MapProcessorTest.java b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/MapProcessorTest.java index 0d7a913c3e3..7698e8d3279 100644 --- a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/MapProcessorTest.java +++ b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/MapProcessorTest.java @@ -27,8 +27,10 @@ public class MapProcessorTest { - private final Pipe input = new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); - private final Pipe output = new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); + private final Pipe input = + new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER, "intput_pipe"); + private final Pipe output = + new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER, "output_pipe"); @SuppressWarnings("unchecked") private final Function processor = mock(Function.class); diff --git a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/PipeTest.java b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/PipeTest.java index c838bc887a3..c4a7a88b527 100644 --- a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/PipeTest.java +++ b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/PipeTest.java @@ -30,7 +30,8 @@ public class PipeTest { private final Counter inputCounter = mock(Counter.class); private final Counter outputCounter = mock(Counter.class); private final Counter abortedItemCounter = mock(Counter.class); - private final Pipe pipe = new Pipe<>(5, inputCounter, outputCounter, abortedItemCounter); + private final Pipe pipe = + new Pipe<>(5, inputCounter, outputCounter, abortedItemCounter, "test_pipe"); @Test public void shouldNotHaveMoreWhenEmptyAndClosed() { diff --git a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/ProcessingStageTest.java b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/ProcessingStageTest.java index efbe16da668..a07530f5497 100644 --- a/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/ProcessingStageTest.java +++ b/services/pipeline/src/test/java/org/hyperledger/besu/services/pipeline/ProcessingStageTest.java @@ -34,9 +34,9 @@ public class ProcessingStageTest { private final Pipe inputPipe = - new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); + new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER, "input_pipe"); private final Pipe outputPipe = - new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER); + new Pipe<>(10, NO_OP_COUNTER, NO_OP_COUNTER, NO_OP_COUNTER, "output_pipe"); @Mock private Processor singleStep; private ProcessingStage stage; From 8d8dbf05b51ed4c3b1a18a4956ac9bdeb2fa2325 Mon Sep 17 00:00:00 2001 From: Chaminda Divitotawela Date: Wed, 3 Jul 2024 11:49:26 +1000 Subject: [PATCH 11/11] Turn off CicleCI for Besu (#7291) All the CI jobs run in GitHub actions and Circle CI it no longer needed in Besu project Signed-off-by: Chaminda Divitotawela --- .circleci/config.yml | 502 ------------------------------------------- 1 file changed, 502 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 45aa3beb287..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,502 +0,0 @@ ---- -version: 2.1 -orbs: - win: circleci/windows@5.0 - -executors: - besu_executor_med: # 2cpu, 4G ram - docker: - - image: cimg/openjdk:21.0 - resource_class: medium - working_directory: ~/project - environment: - architecture: "amd64" - GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=2 - - besu_arm64_executor_med: # 2cpu, 8G ram - machine: #https://circleci.com/developer/machine/image/ubuntu-2204 - image: ubuntu-2204:2022.10.2 - resource_class: arm.medium - working_directory: ~/project - environment: - architecture: "arm64" - GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=2 - - besu_executor_xl: # 8cpu, 16G ram - docker: - - image: cimg/openjdk:21.0 - resource_class: xlarge - working_directory: ~/project - environment: - GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=4 - - xl_machine_executor: - machine: #https://circleci.com/developer/machine/image/ubuntu-2204 - image: ubuntu-2204:2022.10.2 - resource_class: xlarge - - trivy_executor: - docker: - - image: docker:stable-git - resource_class: small - working_directory: ~/project - -commands: - prepare: - description: "Prepare" - steps: - - checkout - - run: - name: Install Packages - LibSodium, nssdb - command: | - sudo apt-get update - sudo apt-get install -y libsodium23 libsodium-dev libjemalloc-dev apt-transport-https haveged libnss3-tools - sudo service haveged restart - java --version - - restore_gradle_cache - restore_gradle_cache: - description: "Restore Gradle cache" - steps: - - restore_cache: - name: Restore cached gradle dependencies - keys: - - deps-{{ checksum "gradle/versions.gradle" }}-{{ .Branch }}-{{ .Revision }} - - deps-{{ checksum "gradle/versions.gradle" }} - - deps- - - capture_test_results: - description: "Capture test results" - steps: - - run: - name: Jacoco - command: | - ./gradlew --no-daemon jacocoTestReport - - run: - name: Gather test results - when: always - command: | - FILES=`find . -name test-results` - for FILE in $FILES - do - MODULE=`echo "$FILE" | sed -e 's@./\(.*\)/build/test-results@\1@'` - TARGET="build/test-results/$MODULE" - mkdir -p "$TARGET" - cp -rf ${FILE}/*/* "$TARGET" - done - - store_test_results: - path: build/test-results - - store_artifacts: - path: besu/build/reports/jacoco - - capture_test_logs: - description: "Capture test logs" - steps: - - store_artifacts: - path: acceptance-tests/tests/build/acceptanceTestLogs - destination: acceptance-tests-logs - - store_artifacts: - path: acceptance-tests/tests/build/jvmErrorLogs - -jobs: - assemble: - executor: besu_executor_xl - steps: - - prepare - - run: - name: Assemble - command: | - ./gradlew --no-daemon clean compileJava compileTestJava assemble - - save_cache: - name: Caching gradle dependencies - key: deps-{{ checksum "gradle/versions.gradle" }}-{{ .Branch }}-{{ .Revision }} - paths: - - .gradle - - ~/.gradle - - persist_to_workspace: - root: ~/project - paths: - - ./ - - store_artifacts: - name: Distribution artifacts - path: build/distributions - destination: distributions - when: always - - testWindows: - executor: win/default - steps: - - attach_workspace: - at: ~/project - - run: - name: Unzip Windows build - no_output_timeout: 20m - command: | - cd build/distributions - unzip besu-*.zip -d besu-tmp - cd besu-tmp - mv besu-* ../besu - - run: - name: Test Besu Windows executable - no_output_timeout: 10m - command: | - build\distributions\besu\bin\besu.bat --help - build\distributions\besu\bin\besu.bat --version - - dockerScan: - executor: trivy_executor - steps: - - checkout - - restore_gradle_cache - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Install trivy - command: | - apk add --update-cache --upgrade curl bash - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin - - run: - name: Scan with trivy - shell: /bin/sh - command: | - for FILE in $(ls docker) - do - if [[ $FILE == "test.sh" || $FILE == "tests" ]]; then - continue - fi - docker pull -q "hyperledger/besu:develop-$FILE" - trivy -q image --exit-code 1 --no-progress --severity HIGH,CRITICAL "hyperledger/besu:develop-$FILE" - done - - unitTests: - executor: besu_executor_xl - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: Build - no_output_timeout: 20m - command: | - ./gradlew --no-daemon build - - capture_test_results - - integrationTests: - executor: xl_machine_executor # needs the machine executor since privacy test uses container tests (docker) - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: IntegrationTests - command: | - ./gradlew --no-daemon integrationTest - - run: - name: CompileJmh - command: | - ./gradlew --no-daemon compileJmh - - capture_test_results - - referenceTests: - executor: besu_executor_xl - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: ReferenceTests - no_output_timeout: 40m - command: | - git submodule update --init --recursive - ./gradlew --no-daemon referenceTest - - capture_test_results - - acceptanceTests: - parallelism: 4 - executor: xl_machine_executor # needs the machine executor since privacy test uses container tests (docker) - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: AcceptanceTests (Mainnet) - no_output_timeout: 20m - command: | - CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \ - | sed 's@.*/src/test/java/@@' \ - | sed 's@/@.@g' \ - | sed 's/.\{5\}$//' \ - | circleci tests split --split-by=timings --timings-type=classname) - # Format the arguments to "./gradlew test" - GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}') - ./gradlew --no-daemon acceptanceTestMainnet $GRADLE_ARGS - - capture_test_results - - capture_test_logs - - acceptanceTestsCliqueBft: - executor: besu_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: AcceptanceTests (Non-Mainnet) - no_output_timeout: 30m - command: | - ./gradlew --no-daemon --max-workers=1 acceptanceTestCliqueBft - - capture_test_results - - capture_test_logs - - acceptanceTestsPrivacy: - parallelism: 4 - executor: xl_machine_executor # needs the machine executor since it uses container tests (docker) - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: AcceptanceTests (Non-Mainnet) - no_output_timeout: 20m - command: | - CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \ - | sed 's@.*/src/test/java/@@' \ - | sed 's@/@.@g' \ - | sed 's/.\{5\}$//' \ - | circleci tests split --split-by=timings --timings-type=classname) - # Format the arguments to "./gradlew test" - GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}') - ./gradlew --no-daemon acceptanceTestPrivacy $GRADLE_ARGS - - capture_test_results - - capture_test_logs - - acceptanceTestsPermissioning: - executor: besu_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: AcceptanceTests (Non-Mainnet) - no_output_timeout: 20m - command: | - ./gradlew --no-daemon --max-workers=1 acceptanceTestPermissioning - - capture_test_results - - capture_test_logs - - buildDocker: - executor: besu_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - setup_remote_docker - - run: - name: hadoLint_openjdk_17 - command: | - docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile - - run: - name: hadoLint_openjdk_17_debug - command: | - docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile - - run: - name: hadoLint_openjdk_latest - command: | - docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile - - run: - name: hadoLint_graalvm - command: | - docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile - - run: - name: build image - command: | - ./gradlew --no-daemon distDocker - - run: - name: test image - command: | - mkdir -p docker/reports - curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-linux-amd64 -o ./docker/tests/goss-linux-amd64 - ./gradlew --no-daemon testDocker - - buildArm64Docker: - executor: besu_arm64_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: hadoLint_openjdk_17 - command: | - docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile - - run: - name: hadoLint_openjdk_latest - command: | - docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile - - run: - name: hadoLint_graalvm - command: | - docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile - - run: - name: Java_17 - command: | - sudo apt install -q --assume-yes openjdk-17-jre-headless openjdk-17-jdk-headless - sudo update-java-alternatives -a - - run: - name: build image - command: | - ./gradlew --no-daemon distDocker - - run: - name: test image - command: | - mkdir -p docker/reports - curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-linux-arm -o ./docker/tests/goss-linux-arm64 - ./gradlew --no-daemon testDocker - - publish: - executor: besu_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: Publish - command: | - ./gradlew --no-daemon artifactoryPublish - - publishDocker: - executor: besu_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - setup_remote_docker - - run: - name: Publish Docker - command: | - docker login --username "${DOCKER_USER_RW}" --password "${DOCKER_PASSWORD_RW}" - ./gradlew --no-daemon "-Pbranch=${CIRCLE_BRANCH}" dockerUpload - - publishArm64Docker: - executor: besu_arm64_executor_med - steps: - - prepare - - attach_workspace: - at: ~/project - - run: - name: Java_17 - command: | - sudo apt install -q --assume-yes openjdk-17-jre-headless openjdk-17-jdk-headless - sudo update-java-alternatives -a - - run: - name: Publish Docker - command: | - docker login --username "${DOCKER_USER_RW}" --password "${DOCKER_PASSWORD_RW}" - ./gradlew --no-daemon "-Pbranch=${CIRCLE_BRANCH}" dockerUpload - manifestDocker: - executor: besu_executor_med - steps: - - prepare - - setup_remote_docker - - run: - name: Create and publish docker manifest - command: | - docker login --username "${DOCKER_USER_RW}" --password "${DOCKER_PASSWORD_RW}" - ./gradlew --no-daemon "-Pbranch=${CIRCLE_BRANCH}" --parallel manifestDocker - -workflows: - version: 2 - default: - jobs: - - assemble - - unitTests: - requires: - - assemble - - testWindows: - requires: - - assemble - - referenceTests: - requires: - - assemble - - integrationTests: - requires: - - assemble - - acceptanceTests: - requires: - - assemble - - acceptanceTestsCliqueBft: - requires: - - assemble - - acceptanceTests - - acceptanceTestsPermissioning: - requires: - - assemble - - buildDocker: - requires: - - assemble - - buildArm64Docker: - requires: - - buildDocker - - assemble - - publish: - filters: - branches: - only: - - main - - /^release-.*/ - requires: - - assemble - - integrationTests - - unitTests - - acceptanceTests - - referenceTests - - buildDocker - - publishDocker: - filters: - branches: - only: - - main - - /^release-.*/ - requires: - - assemble - - integrationTests - - unitTests - - acceptanceTests - - referenceTests - - buildDocker - context: - - besu-dockerhub-rw - - publishArm64Docker: - filters: - branches: - only: - - main - - /^release-.*/ - requires: - - integrationTests - - unitTests - - acceptanceTests - - referenceTests - - buildArm64Docker - context: - - besu-dockerhub-rw - - manifestDocker: - filters: - branches: - only: - - main - - /^release-.*/ - requires: - - publishDocker - - publishArm64Docker - context: - - besu-dockerhub-rw - - nightly: - triggers: - - schedule: - cron: "0 19 * * *" - filters: - branches: - only: - - main - jobs: - - assemble - - dockerScan