diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 06920701cbf..9648122de3b 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2154,7 +2154,6 @@ private void initMiningParametersMetrics(final MiningParameters miningParameters new MiningParametersMetrics(getMetricsSystem(), miningParameters); } - private OptionalInt getGenesisBlockPeriodSeconds( final GenesisConfigOptions genesisConfigOptions) { if (genesisConfigOptions.isClique()) { diff --git a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/ForksScheduleFactoryTest.java b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/ForksScheduleFactoryTest.java index 1c82fae023f..fb50c2684b1 100644 --- a/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/ForksScheduleFactoryTest.java +++ b/consensus/common/src/test/java/org/hyperledger/besu/consensus/common/ForksScheduleFactoryTest.java @@ -63,15 +63,15 @@ public void throwsErrorIfHasForksWithDuplicateBlock() { @SuppressWarnings("unchecked") @Test - public void createsSchegit statusduleUsingSpecCreator() { + public void createsScheduleUsingSpecCreator() { final BftConfigOptions genesisConfigOptions = JsonBftConfigOptions.DEFAULT; final ForkSpec genesisForkSpec = new ForkSpec<>(0, genesisConfigOptions); - final BftFork fork1 = createFork(1, 10, 30); - final BftFork fork2 = createFork(2, 20, 60); + final BftFork fork1 = createFork(1, 10); + final BftFork fork2 = createFork(2, 20); final SpecCreator specCreator = Mockito.mock(SpecCreator.class); - final BftConfigOptions configOptions1 = createBftConfigOptions(10, 30); - final BftConfigOptions configOptions2 = createBftConfigOptions(20, 60); + final BftConfigOptions configOptions1 = createBftConfigOptions(10); + final BftConfigOptions configOptions2 = createBftConfigOptions(20); when(specCreator.create(genesisForkSpec, fork1)).thenReturn(configOptions1); when(specCreator.create(new ForkSpec<>(1, configOptions1), fork2)).thenReturn(configOptions2); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java index 7993f4ebec3..febce15fe13 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java @@ -170,7 +170,6 @@ public void registerPluginTransactionSelectorFactory( }; } - @Value.Derived public long getBlockTxsSelectionMaxTime() { final var maybeBlockPeriodSeconds = getMutableRuntimeValues().blockPeriodSeconds; if (maybeBlockPeriodSeconds.isPresent()) { diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java index 6c65fc2804e..c48d1cf1da3 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java @@ -56,7 +56,7 @@ @Command( name = COMMAND_NAME, aliases = {COMMAND_ALIAS}, - description = "Execute an Ethereum State Test.", + description = "Block Builder subcommand.", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class) public class B11rSubCommand implements Runnable { diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java index 6a8134e999e..59a72f28254 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder; import org.hyperledger.besu.ethereum.core.Difficulty; +import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; @@ -33,13 +34,11 @@ import org.hyperledger.besu.evm.Code; import org.hyperledger.besu.evm.EVM; import org.hyperledger.besu.evm.EvmSpecVersion; -import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.evm.code.CodeInvalid; import org.hyperledger.besu.evm.frame.MessageFrame; import org.hyperledger.besu.evm.log.LogsBloomFilter; import org.hyperledger.besu.evm.tracing.OperationTracer; import org.hyperledger.besu.evm.tracing.StandardJsonTracer; -import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.MetricsSystemModule; import org.hyperledger.besu.util.LogConfigurator; @@ -58,7 +57,7 @@ import java.util.Deque; import java.util.LinkedHashMap; import java.util.List; -import java.util.NavigableMap; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -474,8 +473,9 @@ public void run() { lastTime = stopwatch.elapsed().toNanos(); stopwatch.reset(); if (showJsonAlloc && lastLoop) { + initialMessageFrame.getSelfDestructs().forEach(updater::deleteAccount); updater.commit(); - WorldState worldState = component.getWorldState(); + MutableWorldState worldState = component.getWorldState(); dumpWorldState(worldState, out); } } while (remainingIters-- > 0); @@ -486,34 +486,39 @@ public void run() { } } - public static void dumpWorldState(final WorldState worldState, final PrintWriter out) { + public static void dumpWorldState(final MutableWorldState worldState, final PrintWriter out) { out.println("{"); worldState .streamAccounts(Bytes32.ZERO, Integer.MAX_VALUE) .sorted(Comparator.comparing(o -> o.getAddress().get().toHexString())) .forEach( - account -> { - out.println( - " \"" + account.getAddress().map(Address::toHexString).orElse("-") + "\": {"); + a -> { + var account = worldState.get(a.getAddress().get()); + out.println(" \"" + account.getAddress().toHexString() + "\": {"); if (account.getCode() != null && !account.getCode().isEmpty()) { out.println(" \"code\": \"" + account.getCode().toHexString() + "\","); } - NavigableMap storageEntries = - account.storageEntriesFrom(Bytes32.ZERO, Integer.MAX_VALUE); + var storageEntries = + account.storageEntriesFrom(Bytes32.ZERO, Integer.MAX_VALUE).values().stream() + .map( + e -> + Map.entry( + e.getKey().get(), + account.getStorageValue(UInt256.fromBytes(e.getKey().get())))) + .filter(e -> !e.getValue().isZero()) + .sorted(Map.Entry.comparingByKey()) + .toList(); if (!storageEntries.isEmpty()) { out.println(" \"storage\": {"); out.println( STORAGE_JOINER.join( - storageEntries.values().stream() + storageEntries.stream() .map( - accountStorageEntry -> + e -> " \"" - + accountStorageEntry - .getKey() - .map(UInt256::toQuantityHexString) - .orElse("-") + + e.getKey().toQuantityHexString() + "\": \"" - + accountStorageEntry.getValue().toQuantityHexString() + + e.getValue().toQuantityHexString() + "\"") .toList())); out.println(" },"); diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java index f45797f9643..66b0e26ab87 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java @@ -85,6 +85,11 @@ public class StateTestSubCommand implements Runnable { description = "Force the state tests to run on a specific fork.") private String fork = null; + @Option( + names = {"--test-name"}, + description = "Limit execution to one named test.") + private String testName = null; + @Option( names = {"--data-index"}, description = "Limit execution to one data variable.") @@ -173,10 +178,12 @@ public void run() { private void executeStateTest(final Map generalStateTests) { for (final Map.Entry generalStateTestEntry : generalStateTests.entrySet()) { - generalStateTestEntry - .getValue() - .finalStateSpecs() - .forEach((__, specs) -> traceTestSpecs(generalStateTestEntry.getKey(), specs)); + if (testName == null || testName.equals(generalStateTestEntry.getKey())) { + generalStateTestEntry + .getValue() + .finalStateSpecs() + .forEach((__, specs) -> traceTestSpecs(generalStateTestEntry.getKey(), specs)); + } } } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java index 683458bf7c0..3b0123b45cc 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java @@ -18,6 +18,7 @@ import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_PROTECTED_V_BASE; import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_PROTECTED_V_MIN; import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_UNPROTECTED_V_BASE; +import static org.hyperledger.besu.ethereum.mainnet.feemarket.ExcessBlobGasCalculator.calculateExcessBlobGasForParent; import static org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules.shouldClearEmptyAccounts; import org.hyperledger.besu.config.StubGenesisConfigOptions; @@ -26,7 +27,6 @@ import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; import org.hyperledger.besu.datatypes.AccessListEntry; import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.datatypes.BlobGas; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.TransactionType; import org.hyperledger.besu.datatypes.VersionedHash; @@ -37,6 +37,7 @@ import org.hyperledger.besu.ethereum.core.TransactionReceipt; import org.hyperledger.besu.ethereum.mainnet.BodyValidation; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; +import org.hyperledger.besu.ethereum.mainnet.ParentBeaconBlockRootHelper; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; @@ -50,7 +51,7 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.evm.account.Account; -import org.hyperledger.besu.evm.account.AccountStorageEntry; +import org.hyperledger.besu.evm.gascalculator.GasCalculator; import org.hyperledger.besu.evm.log.Log; import org.hyperledger.besu.evm.tracing.OperationTracer; import org.hyperledger.besu.evm.worldstate.WorldUpdater; @@ -63,10 +64,11 @@ import java.util.Comparator; import java.util.Iterator; import java.util.List; -import java.util.NavigableMap; +import java.util.Map; import java.util.Spliterator; import java.util.Spliterators; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.StreamSupport; import com.fasterxml.jackson.databind.JsonNode; @@ -252,20 +254,40 @@ static T8nResult runTest( final MainnetTransactionProcessor processor = protocolSpec.getTransactionProcessor(); final WorldUpdater worldStateUpdater = worldState.updater(); final ReferenceTestBlockchain blockchain = new ReferenceTestBlockchain(blockHeader.getNumber()); + final Wei blobGasPrice = protocolSpec .getFeeMarket() - .blobGasPricePerGas(blockHeader.getExcessBlobGas().orElse(BlobGas.ZERO)); + .blobGasPricePerGas(calculateExcessBlobGasForParent(protocolSpec, blockHeader)); + long blobGasLimit = protocolSpec.getGasLimitCalculator().currentBlobGasLimit(); + referenceTestEnv + .getParentBeaconBlockRoot() + .ifPresent( + bytes32 -> + ParentBeaconBlockRootHelper.storeParentBeaconBlockRoot( + worldStateUpdater.updater(), referenceTestEnv.getTimestamp(), bytes32)); List receipts = new ArrayList<>(); List invalidTransactions = new ArrayList<>(rejections); List validTransactions = new ArrayList<>(); ArrayNode receiptsArray = objectMapper.createArrayNode(); long gasUsed = 0; + long blobGasUsed = 0; for (int i = 0; i < transactions.size(); i++) { Transaction transaction = transactions.get(i); - final Stopwatch timer = Stopwatch.createStarted(); + + GasCalculator gasCalculator = protocolSpec.getGasCalculator(); + int blobCount = transaction.getBlobCount(); + blobGasUsed += gasCalculator.blobGasCost(blobCount); + if (blobGasUsed > blobGasLimit) { + invalidTransactions.add( + new RejectedTransaction( + i, + String.format( + "blob gas (%d) would exceed block maximum %d", blobGasUsed, blobGasLimit))); + continue; + } final OperationTracer tracer; // You should have picked Mercy. final TransactionProcessingResult result; @@ -304,52 +326,51 @@ static T8nResult runTest( if (result.isInvalid()) { invalidTransactions.add( new RejectedTransaction(i, result.getValidationResult().getErrorMessage())); + continue; + } + validTransactions.add(transaction); + + long transactionGasUsed = transaction.getGasLimit() - result.getGasRemaining(); + + gasUsed += transactionGasUsed; + long intrinsicGas = + gasCalculator.transactionIntrinsicGasCost( + transaction.getPayload(), transaction.getTo().isEmpty()); + TransactionReceipt receipt = + protocolSpec + .getTransactionReceiptFactory() + .create(transaction.getType(), result, worldState, gasUsed); + tracer.traceEndTransaction( + worldStateUpdater, + transaction, + result.isSuccessful(), + result.getOutput(), + result.getLogs(), + gasUsed - intrinsicGas, + timer.elapsed(TimeUnit.NANOSECONDS)); + Bytes gasUsedInTransaction = Bytes.ofUnsignedLong(transactionGasUsed); + receipts.add(receipt); + ObjectNode receiptObject = receiptsArray.addObject(); + receiptObject.put( + "root", receipt.getStateRoot() == null ? "0x" : receipt.getStateRoot().toHexString()); + int status = receipt.getStatus(); + receiptObject.put("status", "0x" + Math.max(status, 0)); + receiptObject.put("cumulativeGasUsed", Bytes.ofUnsignedLong(gasUsed).toQuantityHexString()); + receiptObject.put("logsBloom", receipt.getBloomFilter().toHexString()); + if (result.getLogs().isEmpty()) { + receiptObject.putNull("logs"); } else { - validTransactions.add(transaction); - - long transactionGasUsed = transaction.getGasLimit() - result.getGasRemaining(); - - gasUsed += transactionGasUsed; - long intrinsicGas = - protocolSpec - .getGasCalculator() - .transactionIntrinsicGasCost( - transaction.getPayload(), transaction.getTo().isEmpty()); - TransactionReceipt receipt = - protocolSpec - .getTransactionReceiptFactory() - .create(transaction.getType(), result, worldState, gasUsed); - tracer.traceEndTransaction( - worldStateUpdater, - transaction, - result.isSuccessful(), - result.getOutput(), - result.getLogs(), - gasUsed - intrinsicGas, - timer.elapsed(TimeUnit.NANOSECONDS)); - Bytes gasUsedInTransaction = Bytes.ofUnsignedLong(transactionGasUsed); - receipts.add(receipt); - ObjectNode receiptObject = receiptsArray.addObject(); - receiptObject.put( - "root", receipt.getStateRoot() == null ? "0x" : receipt.getStateRoot().toHexString()); - receiptObject.put("status", "0x" + receipt.getStatus()); - receiptObject.put("cumulativeGasUsed", Bytes.ofUnsignedLong(gasUsed).toQuantityHexString()); - receiptObject.put("logsBloom", receipt.getBloomFilter().toHexString()); - if (result.getLogs().isEmpty()) { - receiptObject.putNull("logs"); - } else { - ArrayNode logsArray = receiptObject.putArray("logs"); - for (Log log : result.getLogs()) { - logsArray.addPOJO(log); - } + ArrayNode logsArray = receiptObject.putArray("logs"); + for (Log log : result.getLogs()) { + logsArray.addPOJO(log); } - receiptObject.put("transactionHash", transaction.getHash().toHexString()); - receiptObject.put( - "contractAddress", transaction.contractAddress().orElse(Address.ZERO).toHexString()); - receiptObject.put("gasUsed", gasUsedInTransaction.toQuantityHexString()); - receiptObject.put("blockHash", Hash.ZERO.toHexString()); - receiptObject.put("transactionIndex", Bytes.ofUnsignedLong(i).toQuantityHexString()); } + receiptObject.put("transactionHash", transaction.getHash().toHexString()); + receiptObject.put( + "contractAddress", transaction.contractAddress().orElse(Address.ZERO).toHexString()); + receiptObject.put("gasUsed", gasUsedInTransaction.toQuantityHexString()); + receiptObject.put("blockHash", Hash.ZERO.toHexString()); + receiptObject.put("transactionIndex", Bytes.ofUnsignedLong(i).toQuantityHexString()); } final ObjectNode resultObject = objectMapper.createObjectNode(); @@ -366,19 +387,19 @@ static T8nResult runTest( .incrementBalance(reward); } + worldStateUpdater.commit(); // Invoke the withdrawal processor to handle CL withdrawals. if (!referenceTestEnv.getWithdrawals().isEmpty()) { try { protocolSpec .getWithdrawalsProcessor() .ifPresent( - p -> p.processWithdrawals(referenceTestEnv.getWithdrawals(), worldStateUpdater)); + p -> p.processWithdrawals(referenceTestEnv.getWithdrawals(), worldState.updater())); } catch (RuntimeException re) { resultObject.put("exception", re.getMessage()); } } - worldStateUpdater.commit(); worldState.persist(blockHeader); resultObject.put("stateRoot", worldState.rootHash().toHexString()); @@ -411,41 +432,48 @@ static T8nResult runTest( blockHeader .getWithdrawalsRoot() .ifPresent(wr -> resultObject.put("withdrawalsRoot", wr.toHexString())); - blockHeader - .getBlobGasUsed() - .ifPresentOrElse( - bgu -> resultObject.put("blobGasUsed", Bytes.ofUnsignedLong(bgu).toQuantityHexString()), - () -> - blockHeader - .getExcessBlobGas() - .ifPresent(ebg -> resultObject.put("blobGasUsed", "0x0"))); + AtomicLong bgHolder = new AtomicLong(blobGasUsed); blockHeader .getExcessBlobGas() - .ifPresent(ebg -> resultObject.put("currentExcessBlobGas", ebg.toShortHexString())); + .ifPresent( + ebg -> { + resultObject.put( + "currentExcessBlobGas", + calculateExcessBlobGasForParent(protocolSpec, blockHeader) + .toBytes() + .toQuantityHexString()); + resultObject.put( + "blobGasUsed", Bytes.ofUnsignedLong(bgHolder.longValue()).toQuantityHexString()); + }); ObjectNode allocObject = objectMapper.createObjectNode(); worldState .streamAccounts(Bytes32.ZERO, Integer.MAX_VALUE) .sorted(Comparator.comparing(o -> o.getAddress().get().toHexString())) .forEach( - account -> { - ObjectNode accountObject = - allocObject.putObject( - account.getAddress().map(Address::toHexString).orElse("0x")); + a -> { + var account = worldState.get(a.getAddress().get()); + ObjectNode accountObject = allocObject.putObject(account.getAddress().toHexString()); if (account.getCode() != null && !account.getCode().isEmpty()) { accountObject.put("code", account.getCode().toHexString()); } - NavigableMap storageEntries = - account.storageEntriesFrom(Bytes32.ZERO, Integer.MAX_VALUE); + var storageEntries = + account.storageEntriesFrom(Bytes32.ZERO, Integer.MAX_VALUE).values().stream() + .map( + e -> + Map.entry( + e.getKey().get(), + account.getStorageValue(UInt256.fromBytes(e.getKey().get())))) + .filter(e -> !e.getValue().isZero()) + .sorted(Map.Entry.comparingByKey()) + .toList(); if (!storageEntries.isEmpty()) { ObjectNode storageObject = accountObject.putObject("storage"); - storageEntries.values().stream() - .sorted(Comparator.comparing(a -> a.getKey().get())) - .forEach( - accountStorageEntry -> - storageObject.put( - accountStorageEntry.getKey().map(UInt256::toHexString).orElse("0x"), - accountStorageEntry.getValue().toHexString())); + storageEntries.forEach( + accountStorageEntry -> + storageObject.put( + accountStorageEntry.getKey().toHexString(), + accountStorageEntry.getValue().toHexString())); } accountObject.put("balance", account.getBalance().toShortHexString()); if (account.getNonce() != 0) { diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nServerSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nServerSubCommand.java index a0ef6384b08..afb12635afd 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nServerSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nServerSubCommand.java @@ -25,14 +25,19 @@ import org.hyperledger.besu.evm.EvmSpecVersion; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.tracing.OperationTracer; +import org.hyperledger.besu.evm.tracing.StandardJsonTracer; import org.hyperledger.besu.evmtool.T8nExecutor.RejectedTransaction; import org.hyperledger.besu.util.LogConfigurator; import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -49,7 +54,9 @@ import io.vertx.core.http.HttpServerOptions; import io.vertx.core.http.HttpServerRequest; import picocli.CommandLine; +import picocli.CommandLine.ParentCommand; +@SuppressWarnings("java:S106") // using standard output is the point of this class @CommandLine.Command( name = "t8n-server", description = "Run Ethereum State Test server", @@ -66,6 +73,24 @@ public class T8nServerSubCommand implements Runnable { description = "Port to bind to") private int port = 3000; + @CommandLine.Option( + names = {"--output.basedir"}, + paramLabel = "full path", + description = "The output ") + private final Path outDir = Path.of("."); + + @ParentCommand private final EvmToolCommand parentCommand; + + @SuppressWarnings("unused") + public T8nServerSubCommand() { + // PicoCLI requires this + this(null); + } + + T8nServerSubCommand(final EvmToolCommand parentCommand) { + this.parentCommand = parentCommand; + } + @Override public void run() { LogConfigurator.setLevel("", "OFF"); @@ -144,6 +169,57 @@ void handleT8nRequest( } } + T8nExecutor.TracerManager tracerManager; + if (parentCommand.showJsonResults) { + tracerManager = + new T8nExecutor.TracerManager() { + private final Map outputStreams = + new HashMap<>(); + + @Override + public OperationTracer getManagedTracer(final int txIndex, final Hash txHash) + throws Exception { + outDir.toFile().mkdirs(); + var traceDest = + new FileOutputStream( + outDir + .resolve( + String.format("trace-%d-%s.jsonl", txIndex, txHash.toHexString())) + .toFile()); + + var jsonTracer = + new StandardJsonTracer( + new PrintStream(traceDest), + parentCommand.showMemory, + !parentCommand.hideStack, + parentCommand.showReturnData, + parentCommand.showStorage); + outputStreams.put(jsonTracer, traceDest); + return jsonTracer; + } + + @Override + public void disposeTracer(final OperationTracer tracer) throws IOException { + if (outputStreams.containsKey(tracer)) { + outputStreams.remove(tracer).close(); + } + } + }; + } else { + tracerManager = + new T8nExecutor.TracerManager() { + @Override + public OperationTracer getManagedTracer(final int txIndex, final Hash txHash) { + return OperationTracer.NO_TRACING; + } + + @Override + public void disposeTracer(final OperationTracer tracer) { + // single-test mode doesn't need to track tracers + } + }; + } + result = T8nExecutor.runTest( chainId, @@ -154,17 +230,7 @@ void handleT8nRequest( initialWorldState, transactions, rejections, - new T8nExecutor.TracerManager() { - @Override - public OperationTracer getManagedTracer(final int txIndex, final Hash txHash) { - return OperationTracer.NO_TRACING; - } - - @Override - public void disposeTracer(final OperationTracer tracer) { - // No output streams to dispose of - } - }); + tracerManager); } ObjectNode outputObject = objectMapper.createObjectNode(); @@ -172,15 +238,11 @@ public void disposeTracer(final OperationTracer tracer) { outputObject.set("body", result.bodyBytes()); outputObject.set("result", result.resultObject()); - try { - String response = - objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(outputObject); - req.response().setChunked(true); - req.response().putHeader("Content-Type", "application/json").send(response); - } catch (JsonProcessingException e) { - req.response().setStatusCode(500).end(e.getMessage()); - } - } catch (Throwable t) { + String response = + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(outputObject); + req.response().setChunked(true); + req.response().putHeader("Content-Type", "application/json").send(response); + } catch (Exception t) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos, true, StandardCharsets.UTF_8); t.printStackTrace(ps); diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-blobs-per-tx.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-blobs-per-tx.json index 0912b52b81a..d5822aa7d71 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-blobs-per-tx.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-blobs-per-tx.json @@ -111,8 +111,8 @@ "gasUsed": "0x5208", "currentBaseFee": "0x7", "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "blobGasUsed": "0x0", - "currentExcessBlobGas": "0x0" + "currentExcessBlobGas": "0x0", + "blobGasUsed": "0x20000" } } } diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java index 0993d4425be..6cdbb3834e4 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.evm.worldstate.WorldState; import java.util.Comparator; +import java.util.Map; import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; @@ -53,7 +54,7 @@ public NavigableMap storageEntriesFrom( .stream() .collect( Collectors.toMap( - e -> e.getKey(), + Map.Entry::getKey, e -> AccountStorageEntry.create( UInt256.fromBytes(RLP.decodeValue(e.getValue())), @@ -80,6 +81,7 @@ public Stream streamAccounts( BonsaiAccount.fromRLP(context, address, entry.getValue(), false)))) .filter(Optional::isPresent) .map(Optional::get) + .filter(acct -> context.updater().getAccount(acct.getAddress().orElse(null)) != null) .sorted(Comparator.comparing(account -> account.getAddress().orElse(Address.ZERO))); } } diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java index 8e4b3d0f9b9..4fb35f9724f 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java @@ -107,10 +107,8 @@ public ReferenceTestEnv( @JsonProperty("currentBeaconRoot") final String currentBeaconRoot, @JsonProperty("currentBlobGasUsed") final String currentBlobGasUsed, @JsonProperty("currentCoinbase") final String coinbase, - @JsonProperty("currentDataGasUsed") final String currentDataGasUsed, @JsonProperty("currentDifficulty") final String difficulty, @JsonProperty("currentExcessBlobGas") final String currentExcessBlobGas, - @JsonProperty("currentExcessDataGas") final String currentExcessDataGas, @JsonProperty("currentGasLimit") final String gasLimit, @JsonProperty("currentNumber") final String number, @JsonProperty("currentRandom") final String random, @@ -119,10 +117,8 @@ public ReferenceTestEnv( @JsonProperty("currentWithdrawalsRoot") final String currentWithdrawalsRoot, @JsonProperty("parentBaseFee") final String parentBaseFee, @JsonProperty("parentBlobGasUsed") final String parentBlobGasUsed, - @JsonProperty("parentDataGasUsed") final String parentDataGasUsed, @JsonProperty("parentDifficulty") final String parentDifficulty, @JsonProperty("parentExcessBlobGas") final String parentExcessBlobGas, - @JsonProperty("parentExcessDataGas") final String parentExcessDataGas, @JsonProperty("parentGasLimit") final String parentGasLimit, @JsonProperty("parentGasUsed") final String parentGasUsed, @JsonProperty("parentTimestamp") final String parentTimestamp, @@ -145,12 +141,8 @@ public ReferenceTestEnv( Optional.ofNullable(random).map(Difficulty::fromHexString).orElse(Difficulty.ZERO), 0L, currentWithdrawalsRoot == null ? null : Hash.fromHexString(currentWithdrawalsRoot), - currentBlobGasUsed == null - ? currentDataGasUsed == null ? null : Long.decode(currentDataGasUsed) - : Long.decode(currentBlobGasUsed), - currentExcessBlobGas == null - ? currentExcessDataGas == null ? null : BlobGas.fromHexString(currentExcessDataGas) - : BlobGas.fromHexString(currentExcessBlobGas), + currentBlobGasUsed == null ? null : Long.decode(currentBlobGasUsed), + currentExcessBlobGas == null ? null : BlobGas.of(Long.decode(currentExcessBlobGas)), beaconRoot == null ? null : Bytes32.fromHexString(beaconRoot), null, // depositsRoot null, // exitsRoot @@ -160,9 +152,8 @@ public ReferenceTestEnv( this.parentGasUsed = parentGasUsed; this.parentGasLimit = parentGasLimit; this.parentTimestamp = parentTimestamp; - this.parentExcessBlobGas = - parentExcessBlobGas == null ? parentExcessDataGas : parentExcessBlobGas; - this.parentBlobGasUsed = parentBlobGasUsed == null ? parentDataGasUsed : parentBlobGasUsed; + this.parentExcessBlobGas = parentExcessBlobGas; + this.parentBlobGasUsed = parentBlobGasUsed; this.withdrawals = withdrawals == null ? List.of() @@ -230,12 +221,8 @@ public BlockHeader updateFromParentValues(final ProtocolSpec protocolSpec) { null))); } if (parentExcessBlobGas != null && parentBlobGasUsed != null) { - builder.excessBlobGas( - BlobGas.of( - protocolSpec - .getGasCalculator() - .computeExcessBlobGas( - Long.decode(parentExcessBlobGas), Long.decode(parentBlobGasUsed)))); + builder.excessBlobGas(BlobGas.of(Long.decode(parentExcessBlobGas))); + builder.blobGasUsed(Long.decode(parentBlobGasUsed)); } return builder.buildBlockHeader(); diff --git a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java index 69b3a300205..08d0fb6b6bc 100644 --- a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java +++ b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/BlockchainReferenceTestTools.java @@ -101,12 +101,13 @@ public static Collection generateTestParametersForConfig(final String[ return params.generate(filePath); } + @SuppressWarnings("java:S5960") // this is actually test code public static void executeTest(final BlockchainReferenceTestCaseSpec spec) { final BlockHeader genesisBlockHeader = spec.getGenesisBlockHeader(); final MutableWorldState worldState = spec.getWorldStateArchive() .getMutable(genesisBlockHeader.getStateRoot(), genesisBlockHeader.getHash()) - .get(); + .orElseThrow(); final ProtocolSchedule schedule = REFERENCE_TEST_PROTOCOL_SCHEDULES.getByName(spec.getNetwork()); @@ -126,18 +127,7 @@ public static void executeTest(final BlockchainReferenceTestCaseSpec spec) { final ProtocolSpec protocolSpec = schedule.getByBlockHeader(block.getHeader()); final BlockImporter blockImporter = protocolSpec.getBlockImporter(); - EVM evm = protocolSpec.getEvm(); - if (evm.getEvmConfiguration().worldUpdaterMode() == WorldUpdaterMode.JOURNALED) { - assumeThat( - worldState - .streamAccounts(Bytes32.ZERO, Integer.MAX_VALUE) - .anyMatch(AccountState::isEmpty)) - .withFailMessage("Journaled account configured and empty account detected") - .isFalse(); - assumeThat(EvmSpecVersion.SPURIOUS_DRAGON.compareTo(evm.getEvmVersion()) > 0) - .withFailMessage("Journaled account configured and fork prior to the merge specified") - .isFalse(); - } + verifyJournaledEVMAccountCompatability(worldState, protocolSpec); final HeaderValidationMode validationMode = "NoProof".equalsIgnoreCase(spec.getSealEngine()) @@ -154,4 +144,20 @@ public static void executeTest(final BlockchainReferenceTestCaseSpec spec) { Assertions.assertThat(blockchain.getChainHeadHash()).isEqualTo(spec.getLastBlockHash()); } + + static void verifyJournaledEVMAccountCompatability( + final MutableWorldState worldState, final ProtocolSpec protocolSpec) { + EVM evm = protocolSpec.getEvm(); + if (evm.getEvmConfiguration().worldUpdaterMode() == WorldUpdaterMode.JOURNALED) { + assumeThat( + worldState + .streamAccounts(Bytes32.ZERO, Integer.MAX_VALUE) + .anyMatch(AccountState::isEmpty)) + .withFailMessage("Journaled account configured and empty account detected") + .isFalse(); + assumeThat(EvmSpecVersion.SPURIOUS_DRAGON.compareTo(evm.getEvmVersion()) > 0) + .withFailMessage("Journaled account configured and fork prior to the merge specified") + .isFalse(); + } + } } diff --git a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java index 7d1915b9106..3dde0e6d078 100644 --- a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java +++ b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java @@ -15,14 +15,13 @@ package org.hyperledger.besu.ethereum.vm; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assumptions.assumeThat; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; -import org.apache.tuweni.bytes.Bytes32; + import org.hyperledger.besu.datatypes.BlobGas; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; @@ -39,11 +38,7 @@ import org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules; import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.evm.EVM; -import org.hyperledger.besu.evm.EvmSpecVersion; import org.hyperledger.besu.evm.account.Account; -import org.hyperledger.besu.evm.account.AccountState; -import org.hyperledger.besu.evm.internal.EvmConfiguration.WorldUpdaterMode; import org.hyperledger.besu.evm.log.Log; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.testutil.JsonTestParameters; @@ -121,24 +116,14 @@ public static Collection generateTestParametersForConfig(final String[ return params.generate(filePath); } + @SuppressWarnings("java:S5960") // this is actually test support code, not production code public static void executeTest(final GeneralStateTestCaseEipSpec spec) { final BlockHeader blockHeader = spec.getBlockHeader(); final ReferenceTestWorldState initialWorldState = spec.getInitialWorldState(); final Transaction transaction = spec.getTransaction(); ProtocolSpec protocolSpec = protocolSpec(spec.getFork()); - EVM evm = protocolSpec.getEvm(); - if (evm.getEvmConfiguration().worldUpdaterMode() == WorldUpdaterMode.JOURNALED) { - assumeThat( - initialWorldState - .streamAccounts(Bytes32.ZERO, Integer.MAX_VALUE) - .anyMatch(AccountState::isEmpty)) - .withFailMessage("Journaled account configured and empty account detected") - .isFalse(); - assumeThat(EvmSpecVersion.SPURIOUS_DRAGON.compareTo(evm.getEvmVersion()) > 0) - .withFailMessage("Journaled account configured and fork prior to the merge specified") - .isFalse(); - } + BlockchainReferenceTestTools.verifyJournaledEVMAccountCompatability(initialWorldState, protocolSpec); // Sometimes the tests ask us assemble an invalid transaction. If we have // no valid transaction then there is no test. GeneralBlockChain tests diff --git a/evm/src/main/java/org/hyperledger/besu/evm/frame/MessageFrame.java b/evm/src/main/java/org/hyperledger/besu/evm/frame/MessageFrame.java index e203649557e..d59e7e45b29 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/frame/MessageFrame.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/frame/MessageFrame.java @@ -18,6 +18,7 @@ import static java.util.Collections.emptySet; import org.hyperledger.besu.collections.trie.BytesTrieSet; +import org.hyperledger.besu.collections.undo.UndoScalar; import org.hyperledger.besu.collections.undo.UndoSet; import org.hyperledger.besu.collections.undo.UndoTable; import org.hyperledger.besu.datatypes.Address; @@ -220,7 +221,6 @@ public enum Type { // Transaction state fields. private final List logs = new ArrayList<>(); - private long gasRefund = 0L; private final Map refunds = new HashMap<>(); // Execution Environment fields. @@ -414,7 +414,8 @@ public void clearGasRemaining() { * @return the amount of gas available, after deductions. */ public long decrementRemainingGas(final long amount) { - return this.gasRemaining -= amount; + this.gasRemaining -= amount; + return this.gasRemaining; } /** @@ -892,12 +893,12 @@ public List getLogs() { * @param amount The amount to increment the refund */ public void incrementGasRefund(final long amount) { - this.gasRefund += amount; + this.txValues.gasRefunds().set(this.txValues.gasRefunds().get() + amount); } /** Clear the accumulated gas refund. */ public void clearGasRefund() { - gasRefund = 0L; + this.txValues.gasRefunds().set(0L); } /** @@ -906,7 +907,7 @@ public void clearGasRefund() { * @return accumulated gas refund */ public long getGasRefund() { - return gasRefund; + return txValues.gasRefunds().get(); } /** @@ -1731,7 +1732,8 @@ public MessageFrame build() { versionedHashes, UndoTable.of(HashBasedTable.create()), UndoSet.of(new BytesTrieSet<>(Address.SIZE)), - UndoSet.of(new BytesTrieSet<>(Address.SIZE))); + UndoSet.of(new BytesTrieSet<>(Address.SIZE)), + new UndoScalar<>(0L)); updater = worldUpdater; newStatic = isStatic; } else { diff --git a/evm/src/main/java/org/hyperledger/besu/evm/frame/TxValues.java b/evm/src/main/java/org/hyperledger/besu/evm/frame/TxValues.java index 6ef1143530f..eba208d5b5d 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/frame/TxValues.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/frame/TxValues.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.evm.frame; +import org.hyperledger.besu.collections.undo.UndoScalar; import org.hyperledger.besu.collections.undo.UndoSet; import org.hyperledger.besu.collections.undo.UndoTable; import org.hyperledger.besu.datatypes.Address; @@ -47,7 +48,8 @@ public record TxValues( Optional> versionedHashes, UndoTable transientStorage, UndoSet
creates, - UndoSet
selfDestructs) { + UndoSet
selfDestructs, + UndoScalar gasRefunds) { /** * For all data stored in this record, undo the changes since the mark. @@ -60,5 +62,6 @@ public void undoChanges(final long mark) { transientStorage.undo(mark); creates.undo(mark); selfDestructs.undo(mark); + gasRefunds.undo(mark); } } diff --git a/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCallOperation.java b/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCallOperation.java index 511578d10da..3e038de65ef 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCallOperation.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCallOperation.java @@ -289,7 +289,6 @@ public void complete(final MessageFrame frame, final MessageFrame childFrame) { frame.addLogs(childFrame.getLogs()); frame.addSelfDestructs(childFrame.getSelfDestructs()); frame.addCreates(childFrame.getCreates()); - frame.incrementGasRefund(childFrame.getGasRefund()); final long gasRemaining = childFrame.getRemainingGas(); frame.incrementRemainingGas(gasRemaining); diff --git a/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java b/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java index f18461e2c09..1cac2ceee86 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/operation/AbstractCreateOperation.java @@ -191,7 +191,6 @@ private void complete(final MessageFrame frame, final MessageFrame childFrame, f frame.addLogs(childFrame.getLogs()); frame.addSelfDestructs(childFrame.getSelfDestructs()); frame.addCreates(childFrame.getCreates()); - frame.incrementGasRefund(childFrame.getGasRefund()); if (childFrame.getState() == MessageFrame.State.COMPLETED_SUCCESS) { Address createdAddress = childFrame.getContractAddress();