Fix verkle pipeline (#7962)

Signed-off-by: Karim Taam <[email protected]>
hyperledger · Dec 2, 2024 · 6117995 · 6117995
1 parent f7535f3
commit 6117995
Show file tree

Hide file tree

Showing 14 changed files with 290 additions and 252 deletions.
diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/BlockHeader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/BlockHeader.java
@@ -183,19 +183,19 @@ public void writeTo(final RLPOutput out) {
 
       if (withdrawalsRoot == null) break;
       out.writeBytes(withdrawalsRoot);
-      /*
-        if (excessBlobGas == null || blobGasUsed == null) break;
-        out.writeLongScalar(blobGasUsed);
-        out.writeUInt64Scalar(excessBlobGas);
+
+      if (excessBlobGas == null || blobGasUsed == null) break;
+      out.writeLongScalar(blobGasUsed);
+      out.writeUInt64Scalar(excessBlobGas);
 
       if (parentBeaconBlockRoot == null) break;
-        out.writeBytes(parentBeaconBlockRoot);
+      out.writeBytes(parentBeaconBlockRoot);
 
-        if (requestsHash == null) break;
-        out.writeBytes(requestsHash);
+      if (requestsHash == null) break;
+      out.writeBytes(requestsHash);
 
-        if (targetBlobCount == null) break;
-        out.writeUInt64Scalar(targetBlobCount);*/
+      if (targetBlobCount == null) break;
+      out.writeUInt64Scalar(targetBlobCount);
     } while (false);
     out.endList();
   }
@@ -224,16 +224,13 @@ public static BlockHeader readFrom(
             ? Hash.wrap(input.readBytes32())
             : null;
 
-    // TODO REACTIVATE
-    /*
     final Long blobGasUsed = !input.isEndOfCurrentList() ? input.readLongScalar() : null;
     final BlobGas excessBlobGas =
         !input.isEndOfCurrentList() ? BlobGas.of(input.readUInt64Scalar()) : null;
     final Bytes32 parentBeaconBlockRoot = !input.isEndOfCurrentList() ? input.readBytes32() : null;
 
-     final Hash requestsHash = !input.isEndOfCurrentList() ? Hash.wrap(input.readBytes32()) : null;
+    final Hash requestsHash = !input.isEndOfCurrentList() ? Hash.wrap(input.readBytes32()) : null;
     final UInt64 targetBlobCount = !input.isEndOfCurrentList() ? input.readUInt64Scalar() : null;
-        */
 
     final ExecutionWitness executionWitness =
         !input.isEndOfCurrentList() ? ExecutionWitness.readFrom(input) : null;
@@ -257,11 +254,11 @@ public static BlockHeader readFrom(
         mixHashOrPrevRandao,
         nonce,
         withdrawalHashRoot,
-        null,
-        null,
-        null,
-        null,
-        null,
+        blobGasUsed,
+        excessBlobGas,
+        parentBeaconBlockRoot,
+        requestsHash,
+        targetBlobCount,
         executionWitness,
         blockHeaderFunctions);
   }

diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java
@@ -906,6 +906,7 @@ static ProtocolSpecBuilder verkleDefinition(
         .withdrawalsProcessor(new WithdrawalsProcessor(clearEmptyAccountStrategy))
         .executionWitnessValidator(new ExecutionWitnessValidator.AllowedExecutionWitness())
         .blockHashProcessor(new PragueBlockHashProcessor())
+        .blockHeaderFunctions(new VerkleDevnetBlockHeaderFunctions())
         .name("Verkle");
   }
 

diff --git a/...src/main/java/org/hyperledger/besu/ethereum/mainnet/VerkleDevnetBlockHeaderFunctions.java b/...src/main/java/org/hyperledger/besu/ethereum/mainnet/VerkleDevnetBlockHeaderFunctions.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.ethereum.mainnet;
+
+import org.hyperledger.besu.datatypes.Hash;
+import org.hyperledger.besu.ethereum.core.BlockHeader;
+import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions;
+import org.hyperledger.besu.ethereum.core.ParsedExtraData;
+import org.hyperledger.besu.ethereum.rlp.RLP;
+import org.hyperledger.besu.ethereum.rlp.RLPOutput;
+
+import org.apache.tuweni.bytes.Bytes;
+
+/** Implements the block hashing algorithm for MainNet as per the yellow paper. */
+public class VerkleDevnetBlockHeaderFunctions implements BlockHeaderFunctions {
+
+  @Override
+  public Hash hash(final BlockHeader header) {
+    return createHash(header);
+  }
+
+  public static Hash createHash(final BlockHeader header) {
+    final Bytes rlp = RLP.encode(rlpOutput -> serializeBlockHeader(header, rlpOutput));
+    return Hash.hash(rlp);
+  }
+
+  // TODO: Remove for mainnet, only needed for the current Verkle devnet.
+  protected static void serializeBlockHeader(final BlockHeader blockHeader, final RLPOutput out) {
+    out.startList();
+
+    out.writeBytes(blockHeader.getParentHash());
+    out.writeBytes(blockHeader.getOmmersHash());
+    out.writeBytes(blockHeader.getCoinbase());
+    out.writeBytes(blockHeader.getStateRoot());
+    out.writeBytes(blockHeader.getTransactionsRoot());
+    out.writeBytes(blockHeader.getReceiptsRoot());
+    out.writeBytes(blockHeader.getLogsBloom());
+    out.writeUInt256Scalar(blockHeader.getDifficulty());
+    out.writeLongScalar(blockHeader.getNumber());
+    out.writeLongScalar(blockHeader.getGasLimit());
+    out.writeLongScalar(blockHeader.getGasUsed());
+    out.writeLongScalar(blockHeader.getTimestamp());
+    out.writeBytes(blockHeader.getExtraData());
+    out.writeBytes(blockHeader.getMixHashOrPrevRandao());
+    out.writeLong(blockHeader.getNonce());
+    do {
+      if (blockHeader.getBaseFee().isEmpty()) break;
+      out.writeUInt256Scalar(blockHeader.getBaseFee().get());
+
+      if (blockHeader.getWithdrawalsRoot().isEmpty()) break;
+      out.writeBytes(blockHeader.getWithdrawalsRoot().get());
+    } while (false);
+    out.endList();
+  }
+
+  @Override
+  public ParsedExtraData parseExtraData(final BlockHeader header) {
+    return null;
+  }
+}
diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BlockchainTestSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/BlockchainTestSubCommand.java
@@ -29,6 +29,7 @@
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
 import org.hyperledger.besu.ethereum.referencetests.BlockchainReferenceTestCaseSpec;
+import org.hyperledger.besu.ethereum.referencetests.CandidateBlock;
 import org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules;
 import org.hyperledger.besu.ethereum.rlp.RLPException;
 import org.hyperledger.besu.evm.EVM;
@@ -177,13 +178,13 @@ private void traceTestSpecs(final String test, final BlockchainReferenceTestCase
     final MutableBlockchain blockchain = spec.getBlockchain();
     final ProtocolContext context = spec.getProtocolContext();
 
-    for (final Block block : spec.getBlocks()) {
-      if (!spec.isExecutable(block)) {
+    for (final CandidateBlock candidateBlock : spec.getCandidateBlocks()) {
+      if (!candidateBlock.isExecutable()) {
         return;
       }
 
       try {
-
+        final Block block = candidateBlock.getBlock();
         final ProtocolSpec protocolSpec = schedule.getByBlockHeader(block.getHeader());
         final BlockImporter blockImporter = protocolSpec.getBlockImporter();
 
@@ -196,7 +197,7 @@ private void traceTestSpecs(final String test, final BlockchainReferenceTestCase
         final BlockImportResult importResult =
             blockImporter.importBlock(context, block, validationMode, validationMode);
 
-        if (importResult.isImported() != spec.isValid(block)) {
+        if (importResult.isImported() != candidateBlock.isValid()) {
           parentCommand.out.printf(
               "Block %d (%s) %s%n",
               block.getHeader().getNumber(),
@@ -210,10 +211,12 @@ private void traceTestSpecs(final String test, final BlockchainReferenceTestCase
               importResult.isImported() ? "Imported" : "Rejected (correctly)");
         }
       } catch (final RLPException e) {
-        if (spec.isValid(block)) {
+        if (candidateBlock.isValid()) {
           parentCommand.out.printf(
               "Block %d (%s) should have imported but had an RLP exception %s%n",
-              block.getHeader().getNumber(), block.getHash(), e.getMessage());
+              candidateBlock.getBlock().getHeader().getNumber(),
+              candidateBlock.getBlock().getHash(),
+              e.getMessage());
         }
       }
     }

diff --git a/ethereum/referencetests/build.gradle b/ethereum/referencetests/build.gradle
@@ -119,7 +119,7 @@ def executionSpecTests = tasks.register("executionSpecTests") {
   generateTestFiles(
     fileTree(referenceTestsPath + "/fixtures/blockchain_tests"),
     file("src/reference-test/templates/BlockchainReferenceTest.java.template"),
-    "fixtures",
+    "all/fixtures",
     "$generatedTestsPath/org/hyperledger/besu/ethereum/vm/executionspec",
     "ExecutionSpecBlockchainTest",
     "org.hyperledger.besu.ethereum.vm.executionspec",
@@ -129,7 +129,7 @@ def executionSpecTests = tasks.register("executionSpecTests") {
   generateTestFiles(
     fileTree(referenceTestsPath + "/fixtures/state_tests"),
     file("src/reference-test/templates/GeneralStateReferenceTest.java.template"),
-    "fixtures",
+    "all/fixtures",
     "$generatedTestsPath/org/hyperledger/besu/ethereum/vm/executionspec",
     "ExecutionSpecStateTest",
     "org.hyperledger.besu.ethereum.vm.executionspec",

diff --git a/...c/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCase.java b/...c/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCase.java
@@ -16,7 +16,6 @@
 
 import org.hyperledger.besu.ethereum.ProtocolContext;
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
-import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 
@@ -31,15 +30,9 @@ public interface BlockchainReferenceTestCase {
 
   ProtocolContext getProtocolContext();
 
-  Iterable<Block> getBlocks();
+  CandidateBlock[] getCandidateBlocks();
 
   String getSealEngine();
 
   Object getLastBlockHash();
-
-  boolean isExecutable(Block candidateBlock);
-
-  boolean isValid(Block candidateBlock);
-
-  boolean areAllTransactionsValid(final Block block);
 }
diff --git a/...in/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java b/...in/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java
@@ -32,20 +32,12 @@
 import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider;
 import org.hyperledger.besu.ethereum.core.MutableWorldState;
 import org.hyperledger.besu.ethereum.core.ParsedExtraData;
-import org.hyperledger.besu.ethereum.core.Transaction;
-import org.hyperledger.besu.ethereum.core.Withdrawal;
-import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions;
-import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
-import org.hyperledger.besu.ethereum.rlp.RLPInput;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.evm.log.LogsBloomFilter;
 import org.hyperledger.besu.evm.worldstate.WorldUpdater;
 
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@@ -58,7 +50,7 @@ public class BlockchainReferenceTestCaseSpec implements BlockchainReferenceTestC
 
   private final String network;
 
-  private final CandidateBlock[] candidateBlocks;
+  private final DefaultCandidateBlock[] candidateBlocks;
 
   private final ReferenceTestBlockHeader genesisBlockHeader;
 
@@ -97,7 +89,7 @@ private static MutableBlockchain buildBlockchain(final BlockHeader genesisBlockH
   @JsonCreator
   public BlockchainReferenceTestCaseSpec(
       @JsonProperty("network") final String network,
-      @JsonProperty("blocks") final CandidateBlock[] candidateBlocks,
+      @JsonProperty("blocks") final DefaultCandidateBlock[] candidateBlocks,
       @JsonProperty("genesisBlockHeader") final ReferenceTestBlockHeader genesisBlockHeader,
       @SuppressWarnings("unused") @JsonProperty("genesisRLP") final String genesisRLP,
       @JsonProperty("pre") final Map<String, ReferenceTestWorldState.AccountMock> accounts,
@@ -124,35 +116,8 @@ public String getNetwork() {
   }
 
   @Override
-  public List<Block> getBlocks() {
-    return Arrays.stream(candidateBlocks).map(CandidateBlock::getBlock).toList();
-  }
-
-  @Override
-  public boolean isExecutable(final Block block) {
-    Optional<CandidateBlock> candidateBlock =
-        Arrays.stream(candidateBlocks)
-            .filter(cb -> Objects.equals(cb.getBlock(), block))
-            .findFirst();
-    return candidateBlock.isPresent() && candidateBlock.get().rlp != null;
-  }
-
-  @Override
-  public boolean isValid(final Block block) {
-    Optional<CandidateBlock> candidateBlock =
-        Arrays.stream(candidateBlocks)
-            .filter(cb -> Objects.equals(cb.getBlock(), block))
-            .findFirst();
-    return candidateBlock.isPresent() && candidateBlock.get().valid;
-  }
-
-  @Override
-  public boolean areAllTransactionsValid(final Block block) {
-    Optional<CandidateBlock> candidateBlock =
-        Arrays.stream(candidateBlocks)
-            .filter(cb -> Objects.equals(cb.getBlock(), block))
-            .findFirst();
-    return candidateBlock.isPresent() && candidateBlock.get().areAllTransactionsValid();
+  public CandidateBlock[] getCandidateBlocks() {
+    return candidateBlocks;
   }
 
   @Override
@@ -270,15 +235,12 @@ public ParsedExtraData parseExtraData(final BlockHeader header) {
     "hasBigInt",
     "rlp_decoded"
   })
-  public static class CandidateBlock {
+  public static class DefaultCandidateBlock extends CandidateBlock {
 
-    private final Bytes rlp;
-
-    private final Boolean valid;
     private final List<TransactionSequence> transactionSequence;
 
     @JsonCreator
-    public CandidateBlock(
+    public DefaultCandidateBlock(
         @JsonProperty("rlp") final String rlp,
         @JsonProperty("blockHeader") final Object blockHeader,
         @JsonProperty("transactions") final Object transactions,
@@ -288,53 +250,21 @@ public CandidateBlock(
         @JsonProperty("withdrawalRequests") final Object withdrawalRequests,
         @JsonProperty("consolidationRequests") final Object consolidationRequests,
         @JsonProperty("transactionSequence") final List<TransactionSequence> transactionSequence) {
-      boolean blockValid = true;
-      // The BLOCK__WrongCharAtRLP_0 test has an invalid character in its rlp string.
-      Bytes rlpAttempt = null;
-      try {
-        rlpAttempt = Bytes.fromHexString(rlp);
-      } catch (final IllegalArgumentException e) {
-        blockValid = false;
-      }
-      this.rlp = rlpAttempt;
+      super(rlp);
 
       if (blockHeader == null
           && transactions == null
           && uncleHeaders == null
           && withdrawals == null) {
-        blockValid = false;
+        this.valid = false;
       }
-
-      this.valid = blockValid;
       this.transactionSequence = transactionSequence;
     }
 
-    public boolean isValid() {
-      return valid;
-    }
-
+    @Override
     public boolean areAllTransactionsValid() {
       return transactionSequence == null
           || transactionSequence.stream().filter(t -> !t.valid()).count() == 0;
     }
-
-    public boolean isExecutable() {
-      return rlp != null;
-    }
-
-    public Block getBlock() {
-      final RLPInput input = new BytesValueRLPInput(rlp, false);
-      input.enterList();
-      final MainnetBlockHeaderFunctions blockHeaderFunctions = new MainnetBlockHeaderFunctions();
-      final BlockHeader header = BlockHeader.readFrom(input, blockHeaderFunctions);
-      final BlockBody body =
-          new BlockBody(
-              input.readList(Transaction::readFrom),
-              input.readList(inputData -> BlockHeader.readFrom(inputData, blockHeaderFunctions)),
-              input.isEndOfCurrentList()
-                  ? Optional.empty()
-                  : Optional.of(input.readList(Withdrawal::readFrom)));
-      return new Block(header, body);
-    }
   }
 }