From 98724d3cd58870d5cb6e3024a411da494fb453c4 Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Mon, 23 Oct 2023 14:18:28 +0100 Subject: [PATCH] Support Deneb fork choice reference tests (#7541) --- .../teku/reference/Eth2ReferenceTestCase.java | 6 - .../forkchoice/ForkChoiceTestExecutor.java | 73 ++++++-- .../forkchoice/StubBlobSidecarManager.java | 138 ++++++++++++++++ .../teku/ethtests/finder/TestDefinition.java | 38 ++++- .../java/tech/pegasys/teku/spec/Spec.java | 16 +- .../logic/common/helpers/MiscHelpers.java | 8 + .../deneb/helpers/MiscHelpersDeneb.java | 20 ++- .../spec/schemas/SchemaDefinitionsDeneb.java | 9 + .../pegasys/teku/spec/TestSpecFactory.java | 156 +++++++----------- 9 files changed, 340 insertions(+), 124 deletions(-) create mode 100644 eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java index 2ab76c2d7bf..971c0151fd2 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java @@ -106,12 +106,6 @@ private TestExecutor getExecutorFor(final TestDefinition testDefinition) { testExecutor = COMMON_TEST_TYPES.get(testDefinition.getTestType()); } - // TODO: https://github.com/Consensys/teku/issues/7539 - if (testDefinition.getFork().equals(TestFork.DENEB) - && testExecutor instanceof ForkChoiceTestExecutor) { - testExecutor = TestExecutor.IGNORE_TESTS; - } - if (testExecutor == null) { return Assertions.fail("Unsupported test type " + testDefinition.getTestType()); } diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/ForkChoiceTestExecutor.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/ForkChoiceTestExecutor.java index cd23af9c014..1adc17c8440 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/ForkChoiceTestExecutor.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/ForkChoiceTestExecutor.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.nio.ByteOrder; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -41,10 +42,15 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.async.eventthread.InlineEventThread; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.kzg.KZG; +import tech.pegasys.teku.kzg.KZGProof; import tech.pegasys.teku.reference.TestDataUtils; import tech.pegasys.teku.reference.TestExecutor; import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.SpecMilestone; +import tech.pegasys.teku.spec.SpecVersion; import tech.pegasys.teku.spec.datastructures.attestation.ValidatableAttestation; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState; @@ -60,8 +66,9 @@ import tech.pegasys.teku.spec.executionlayer.ExecutionLayerChannelStub; import tech.pegasys.teku.spec.executionlayer.ExecutionPayloadStatus; import tech.pegasys.teku.spec.executionlayer.PayloadStatus; +import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; import tech.pegasys.teku.spec.logic.common.statetransition.results.BlockImportResult; -import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager; +import tech.pegasys.teku.spec.logic.versions.deneb.helpers.MiscHelpersDeneb; import tech.pegasys.teku.statetransition.forkchoice.ForkChoice; import tech.pegasys.teku.statetransition.forkchoice.ForkChoiceStateProvider; import tech.pegasys.teku.statetransition.forkchoice.MergeTransitionBlockValidator; @@ -76,6 +83,7 @@ public class ForkChoiceTestExecutor implements TestExecutor { private static final Logger LOG = LogManager.getLogger(); + private static final String SSZ_SNAPPY_EXTENSION = ".ssz_snappy"; public static final ImmutableMap FORK_CHOICE_TEST_TYPES = ImmutableMap.builder() .put("fork_choice/get_head", new ForkChoiceTestExecutor()) @@ -102,9 +110,9 @@ public void runTest(final TestDefinition testDefinition) throws Throwable { // Note: The fork choice spec says there may be settings in a meta.yaml file but currently no // tests actually have one, so we currently don't bother trying to load it. + final Spec spec = testDefinition.getSpec(false); final BeaconState anchorState = - TestDataUtils.loadStateFromSsz(testDefinition, "anchor_state.ssz_snappy"); - final Spec spec = testDefinition.getSpec(); + TestDataUtils.loadStateFromSsz(testDefinition, "anchor_state" + SSZ_SNAPPY_EXTENSION); final SignedBeaconBlock anchorBlock = loadAnchorBlock(testDefinition); final StorageSystem storageSystem = @@ -118,12 +126,13 @@ spec, new SignedBlockAndState(anchorBlock, anchorState)), final MergeTransitionBlockValidator transitionBlockValidator = new MergeTransitionBlockValidator(spec, recentChainData, ExecutionLayerChannel.NOOP); final InlineEventThread eventThread = new InlineEventThread(); + final StubBlobSidecarManager blobSidecarManager = new StubBlobSidecarManager(spec); final ForkChoice forkChoice = new ForkChoice( spec, eventThread, recentChainData, - BlobSidecarManager.NOOP, + blobSidecarManager, new StubForkChoiceNotifier(), new ForkChoiceStateProvider(eventThread, recentChainData), new TickProcessor(spec, recentChainData), @@ -135,7 +144,8 @@ spec, new SignedBlockAndState(anchorBlock, anchorState)), new ExecutionLayerChannelStub(spec, false, Optional.empty()); try { - runSteps(testDefinition, spec, recentChainData, forkChoice, executionLayer); + runSteps( + testDefinition, spec, recentChainData, blobSidecarManager, forkChoice, executionLayer); } catch (final AssertionError e) { final String protoArrayData = recentChainData.getForkChoiceStrategy().orElseThrow().getBlockData().stream() @@ -151,6 +161,8 @@ spec, new SignedBlockAndState(anchorBlock, anchorState)), + "\nProtoarray data:\n" + protoArrayData, e); + } finally { + freeTrustedSetupIfRequired(spec); } } @@ -165,7 +177,7 @@ private SignedBeaconBlock loadAnchorBlock(final TestDefinition testDefinition) { final BeaconBlock anchorBlock = TestDataUtils.loadSsz( testDefinition, - "anchor_block.ssz_snappy", + "anchor_block" + SSZ_SNAPPY_EXTENSION, testDefinition.getSpec()::deserializeBeaconBlock); return SignedBeaconBlock.create(testDefinition.getSpec(), anchorBlock, BLSSignature.empty()); } @@ -174,6 +186,7 @@ private void runSteps( final TestDefinition testDefinition, final Spec spec, final RecentChainData recentChainData, + final StubBlobSidecarManager blobSidecarManager, final ForkChoice forkChoice, final ExecutionLayerChannelStub executionLayer) throws IOException { @@ -188,7 +201,14 @@ private void runSteps( final UInt64 currentSlot = recentChainData.getCurrentSlot().orElse(UInt64.ZERO); LOG.info("Current slot: {} Epoch: {}", currentSlot, spec.computeEpochAtSlot(currentSlot)); } else if (step.containsKey("block")) { - applyBlock(testDefinition, spec, recentChainData, forkChoice, step, executionLayer); + applyBlock( + testDefinition, + spec, + recentChainData, + blobSidecarManager, + forkChoice, + step, + executionLayer); } else if (step.containsKey("attestation")) { applyAttestation(testDefinition, forkChoice, step); @@ -214,7 +234,7 @@ private void applyPowBlock( final ExecutionLayerChannelStub executionLayer) { final String filename = (String) step.get("pow_block"); final PowBlock block = - TestDataUtils.loadSsz(testDefinition, filename + ".ssz_snappy", this::parsePowBlock); + TestDataUtils.loadSsz(testDefinition, filename + SSZ_SNAPPY_EXTENSION, this::parsePowBlock); executionLayer.addPowBlock(block); } @@ -262,7 +282,7 @@ private void applyAttestation( final Attestation attestation = TestDataUtils.loadSsz( testDefinition, - attestationName + ".ssz_snappy", + attestationName + SSZ_SNAPPY_EXTENSION, testDefinition.getSpec().getGenesisSchemaDefinitions().getAttestationSchema()); final Spec spec = testDefinition.getSpec(); assertThat(forkChoice.onAttestation(ValidatableAttestation.from(spec, attestation))) @@ -277,7 +297,7 @@ private void applyAttesterSlashing( final AttesterSlashing attesterSlashing = TestDataUtils.loadSsz( testDefinition, - slashingName + ".ssz_snappy", + slashingName + SSZ_SNAPPY_EXTENSION, testDefinition.getSpec().getGenesisSchemaDefinitions().getAttesterSlashingSchema()); assertDoesNotThrow( () -> @@ -288,6 +308,7 @@ private void applyBlock( final TestDefinition testDefinition, final Spec spec, final RecentChainData recentChainData, + final StubBlobSidecarManager blobSidecarManager, final ForkChoice forkChoice, final Map step, final ExecutionLayerChannelStub executionLayer) { @@ -295,7 +316,29 @@ private void applyBlock( final boolean valid = !step.containsKey("valid") || (boolean) step.get("valid"); final SignedBeaconBlock block = TestDataUtils.loadSsz( - testDefinition, blockName + ".ssz_snappy", spec::deserializeSignedBeaconBlock); + testDefinition, blockName + SSZ_SNAPPY_EXTENSION, spec::deserializeSignedBeaconBlock); + final List blobs = + getOptionally(step, "blobs") + .map( + blobsName -> + TestDataUtils.loadSsz( + testDefinition, + blobsName + SSZ_SNAPPY_EXTENSION, + sszBytes -> spec.deserializeBlobsInBlock(sszBytes, block.getSlot())) + .asList()) + .orElse(Collections.emptyList()); + @SuppressWarnings("unchecked") + final List proofs = + getOptionally(step, "proofs") + .map( + proofsArray -> + ((List) proofsArray).stream().map(KZGProof::fromHexString).toList()) + .orElse(Collections.emptyList()); + if (block.getMessage().getBody().toVersionDeneb().isPresent()) { + LOG.info( + "Preparing {} blobs with proofs {} for block {}", blobs.size(), proofs, block.getRoot()); + blobSidecarManager.prepareBlobsAndProofsForBlock(block, blobs, proofs); + } LOG.info( "Importing block {} at slot {} with parent {}", block.getRoot(), @@ -446,6 +489,14 @@ private void assertCheckpoint( .isEqualTo(new Checkpoint(expectedEpoch, expectedRoot)); } + private void freeTrustedSetupIfRequired(final Spec spec) { + Optional.ofNullable(spec.forMilestone(SpecMilestone.DENEB)) + .map(SpecVersion::miscHelpers) + .flatMap(MiscHelpers::toVersionDeneb) + .map(MiscHelpersDeneb::getKzg) + .ifPresent(KZG::freeTrustedSetup); + } + @SuppressWarnings({"unchecked", "TypeParameterUnusedInFormals"}) private static T get(final Map yamlData, final String key) { return (T) yamlData.get(key); diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java new file mode 100644 index 00000000000..240a1727bf5 --- /dev/null +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java @@ -0,0 +1,138 @@ +/* + * Copyright Consensys Software Inc., 2023 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.reference.phase0.forkchoice; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import tech.pegasys.teku.infrastructure.async.SafeFuture; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.kzg.KZGCommitment; +import tech.pegasys.teku.kzg.KZGProof; +import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.SignedBlobSidecar; +import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; +import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodyDeneb; +import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment; +import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; +import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; +import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; +import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager; +import tech.pegasys.teku.statetransition.validation.InternalValidationResult; + +/** Simplified version of {@link BlobSidecarManager} which is used in fork choice reference tests */ +class StubBlobSidecarManager implements BlobSidecarManager { + + private final Map blobsAndProofsByBlockRoot = new HashMap<>(); + + private final Spec spec; + + StubBlobSidecarManager(final Spec spec) { + this.spec = spec; + } + + /** Prepare the blobs and proofs for a block provided by the reference test * */ + public void prepareBlobsAndProofsForBlock( + final SignedBeaconBlock block, final List blobs, final List proofs) { + blobsAndProofsByBlockRoot.put(block.getRoot(), new BlobsAndProofs(blobs, proofs)); + } + + @Override + public SafeFuture validateAndPrepareForBlockImport( + final SignedBlobSidecar signedBlobSidecar) { + return SafeFuture.failedFuture( + new UnsupportedOperationException("Not available in fork choice reference tests")); + } + + @Override + public void prepareForBlockImport(final BlobSidecar blobSidecar) { + // NOOP + } + + @Override + public void subscribeToReceivedBlobSidecar( + final ReceivedBlobSidecarListener receivedBlobSidecarListener) { + // NOOP + } + + @Override + public boolean isAvailabilityRequiredAtSlot(final UInt64 slot) { + // NOOP + return false; + } + + /** + * Creates an implementation of {@link BlobSidecarsAvailabilityChecker} which uses the simplified + * {@link MiscHelpers#isDataAvailable(List, List, List)} method with the blobs and proofs that the + * reference test has provided + */ + @Override + public BlobSidecarsAvailabilityChecker createAvailabilityChecker(final SignedBeaconBlock block) { + return new BlobSidecarsAvailabilityChecker() { + @Override + public boolean initiateDataAvailabilityCheck() { + return true; + } + + @Override + public SafeFuture getAvailabilityCheckResult() { + final BlobsAndProofs blobsAndProofs = blobsAndProofsByBlockRoot.remove(block.getRoot()); + if (blobsAndProofs == null) { + return SafeFuture.completedFuture(BlobSidecarsAndValidationResult.NOT_REQUIRED); + } + return SafeFuture.completedFuture(validateImmediately(block, blobsAndProofs)); + } + + @Override + public BlobSidecarsAndValidationResult validateImmediately( + final List blobSidecars) { + throw new UnsupportedOperationException("Not available in fork choice reference tests"); + } + + private BlobSidecarsAndValidationResult validateImmediately( + final SignedBeaconBlock block, final BlobsAndProofs blobsAndProofs) { + final List kzgCommitments = + BeaconBlockBodyDeneb.required(block.getMessage().getBody()) + .getBlobKzgCommitments() + .stream() + .map(SszKZGCommitment::getKZGCommitment) + .toList(); + final List blobs = blobsAndProofs.blobs.stream().map(Blob::getBytes).toList(); + try { + if (!spec.atSlot(block.getSlot()) + .miscHelpers() + .isDataAvailable(blobs, kzgCommitments, blobsAndProofs.proofs)) { + return BlobSidecarsAndValidationResult.invalidResult(Collections.emptyList()); + } + } catch (final Exception ex) { + return BlobSidecarsAndValidationResult.invalidResult(Collections.emptyList(), ex); + } + return BlobSidecarsAndValidationResult.validResult(Collections.emptyList()); + } + }; + } + + @Override + public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( + final SignedBeaconBlock block, final List blobSidecars) { + throw new UnsupportedOperationException("Not available in fork choice reference tests"); + } + + private record BlobsAndProofs(List blobs, List proofs) {} +} diff --git a/eth-tests/src/main/java/tech/pegasys/teku/ethtests/finder/TestDefinition.java b/eth-tests/src/main/java/tech/pegasys/teku/ethtests/finder/TestDefinition.java index 72a8166d77e..1062d68522a 100644 --- a/eth-tests/src/main/java/tech/pegasys/teku/ethtests/finder/TestDefinition.java +++ b/eth-tests/src/main/java/tech/pegasys/teku/ethtests/finder/TestDefinition.java @@ -14,11 +14,15 @@ package tech.pegasys.teku.ethtests.finder; import java.nio.file.Path; +import java.util.Objects; +import java.util.function.Consumer; import tech.pegasys.teku.ethtests.TestFork; import tech.pegasys.teku.ethtests.TestSpecConfig; +import tech.pegasys.teku.networks.Eth2NetworkConfiguration; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.config.builder.DenebBuilder; import tech.pegasys.teku.spec.networks.Eth2Network; public class TestDefinition { @@ -52,21 +56,24 @@ public String getFork() { } public Spec getSpec() { + return getSpec(true); + } + + public Spec getSpec(final boolean kzgNoop) { if (spec == null) { - createSpec(); + createSpec(kzgNoop); } - return spec; } - private void createSpec() { + private void createSpec(final boolean kzgNoop) { final Eth2Network network = switch (configName) { case TestSpecConfig.MAINNET -> Eth2Network.MAINNET; case TestSpecConfig.MINIMAL -> Eth2Network.MINIMAL; default -> throw new IllegalArgumentException("Unknown configName: " + configName); }; - final SpecMilestone highestSupportedMilestone = + final SpecMilestone milestone = switch (fork) { case TestFork.PHASE0 -> SpecMilestone.PHASE0; case TestFork.ALTAIR -> SpecMilestone.ALTAIR; @@ -75,7 +82,28 @@ private void createSpec() { case TestFork.DENEB -> SpecMilestone.DENEB; default -> throw new IllegalArgumentException("Unknown fork: " + fork); }; - spec = TestSpecFactory.create(highestSupportedMilestone, network); + spec = + TestSpecFactory.create( + milestone, + network, + configBuilder -> { + if (!kzgNoop) { + configBuilder.denebBuilder(denebConfigKzgModifier(network)); + } + }); + } + + private Consumer denebConfigKzgModifier(final Eth2Network network) { + return builder -> { + final String trustedSetupFilename = + network.equals(Eth2Network.MAINNET) + ? "mainnet-trusted-setup.txt" + : "minimal-trusted-setup.txt"; + final String trustedSetupPath = + Objects.requireNonNull(Eth2NetworkConfiguration.class.getResource(trustedSetupFilename)) + .toExternalForm(); + builder.trustedSetupPath(trustedSetupPath).kzgNoop(false); + }; } public String getTestType() { diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/Spec.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/Spec.java index 5cf9e234bb9..25983ac1e1b 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/Spec.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/Spec.java @@ -39,6 +39,7 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.bytes.Bytes4; import tech.pegasys.teku.infrastructure.ssz.Merkleizable; +import tech.pegasys.teku.infrastructure.ssz.SszList; import tech.pegasys.teku.infrastructure.ssz.collections.SszBitlist; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.cache.IndexedAttestationCache; @@ -51,6 +52,7 @@ import tech.pegasys.teku.spec.datastructures.attestation.ValidatableAttestation; import tech.pegasys.teku.spec.datastructures.blobs.SignedBlobSidecarsUnblinder; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlindedBlobSidecar; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.SignedBlindedBlobSidecar; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.SignedBlobSidecar; @@ -377,11 +379,21 @@ public BeaconBlock deserializeBeaconBlock(final Bytes serializedBlock) { .sszDeserialize(serializedBlock); } + public SszList deserializeBlobsInBlock(final Bytes serializedBlobs, final UInt64 slot) { + return atSlot(slot) + .getSchemaDefinitions() + .toVersionDeneb() + .orElseThrow(() -> new RuntimeException("Deneb milestone is required to deserialize blobs")) + .getBlobsInBlockSchema() + .sszDeserialize(serializedBlobs); + } + public BlobSidecar deserializeBlobSidecar(final Bytes serializedBlobSidecar, final UInt64 slot) { return atSlot(slot) .getSchemaDefinitions() .toVersionDeneb() - .orElseThrow(() -> new RuntimeException("Deneb milestone is required to load blob sidecar")) + .orElseThrow( + () -> new RuntimeException("Deneb milestone is required to deserialize blob sidecar")) .getBlobSidecarSchema() .sszDeserialize(serializedBlobSidecar); } @@ -394,7 +406,7 @@ public ExecutionPayloadHeader deserializeJsonExecutionPayloadHeader( .orElseThrow( () -> new RuntimeException( - "Bellatrix milestone is required to load execution payload header")) + "Bellatrix milestone is required to deserialize execution payload header")) .getExecutionPayloadHeaderSchema() .jsonDeserialize(objectMapper.createParser(jsonFile)); } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java index cead36ba8a4..a66de6e487b 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java @@ -35,6 +35,7 @@ import tech.pegasys.teku.infrastructure.ssz.primitive.SszUInt64; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.kzg.KZGCommitment; +import tech.pegasys.teku.kzg.KZGProof; import tech.pegasys.teku.spec.config.SpecConfig; import tech.pegasys.teku.spec.constants.NetworkConstants; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; @@ -347,6 +348,13 @@ public boolean isDataAvailable(final List blobSidecars, final Beaco return false; } + public boolean isDataAvailable( + final List blobs, + final List kzgCommitments, + final List proofs) { + return false; + } + public boolean verifyBlobKzgProofBatch(final List blobSidecars) { return false; } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java index d62f637f27f..7af5a85cd83 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java @@ -60,6 +60,10 @@ private static KZG initKZG(final SpecConfigDeneb config) { return kzg; } + public KZG getKzg() { + return kzg; + } + /** * Performs complete data availability check is_data_available @@ -85,6 +89,18 @@ public boolean isDataAvailable(final List blobSidecars, final Beaco return true; } + /** + * Simplified version of {@link #isDataAvailable(List, BeaconBlock)} which accepts + * blobs,commitments and proofs directly instead of blob sidecars + */ + @Override + public boolean isDataAvailable( + final List blobs, + final List kzgCommitments, + final List proofs) { + return kzg.verifyBlobKzgProofBatch(blobs, kzgCommitments, proofs); + } + /** * Performs a verifyBlobKzgProofBatch on the given blob sidecars * @@ -94,14 +110,14 @@ public boolean isDataAvailable(final List blobSidecars, final Beaco @Override public boolean verifyBlobKzgProofBatch(final List blobSidecars) { final List blobs = new ArrayList<>(); - final List kzgProofs = new ArrayList<>(); final List kzgCommitments = new ArrayList<>(); + final List kzgProofs = new ArrayList<>(); blobSidecars.forEach( blobSidecar -> { blobs.add(blobSidecar.getBlob().getBytes()); - kzgProofs.add(blobSidecar.getKZGProof()); kzgCommitments.add(blobSidecar.getKZGCommitment()); + kzgProofs.add(blobSidecar.getKZGProof()); }); return kzg.verifyBlobKzgProofBatch(blobs, kzgCommitments, kzgProofs); diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/SchemaDefinitionsDeneb.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/SchemaDefinitionsDeneb.java index 5b64343e17d..b697a38d3bf 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/SchemaDefinitionsDeneb.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/SchemaDefinitionsDeneb.java @@ -16,8 +16,11 @@ import static com.google.common.base.Preconditions.checkArgument; import java.util.Optional; +import tech.pegasys.teku.infrastructure.ssz.SszList; +import tech.pegasys.teku.infrastructure.ssz.schema.SszListSchema; import tech.pegasys.teku.spec.config.SpecConfigDeneb; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlindedBlobSidecarSchema; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSchema; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarSchema; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.SignedBlindedBlobSidecarSchema; @@ -74,6 +77,7 @@ public class SchemaDefinitionsDeneb extends SchemaDefinitionsCapella { private final SignedBuilderBidSchema signedBuilderBidSchemaDeneb; private final BlobSchema blobSchema; + private final SszListSchema> blobsInBlockSchema; private final BlobSidecarSchema blobSidecarSchema; private final SignedBlobSidecarSchema signedBlobSidecarSchema; private final BlindedBlobSidecarSchema blindedBlobSidecarSchema; @@ -123,6 +127,7 @@ public SchemaDefinitionsDeneb(final SpecConfigDeneb specConfig) { new SignedBuilderBidSchema("SignedBuilderBidDeneb", builderBidSchemaDeneb); this.blobSchema = new BlobSchema(specConfig); + this.blobsInBlockSchema = SszListSchema.create(blobSchema, specConfig.getMaxBlobsPerBlock()); this.blobSidecarSchema = BlobSidecarSchema.create(blobSchema); this.signedBlobSidecarSchema = SignedBlobSidecarSchema.create(blobSidecarSchema); this.blindedBlobSidecarSchema = BlindedBlobSidecarSchema.create(); @@ -250,6 +255,10 @@ public BlobSchema getBlobSchema() { return blobSchema; } + public SszListSchema> getBlobsInBlockSchema() { + return blobsInBlockSchema; + } + public BlobSidecarSchema getBlobSidecarSchema() { return blobSidecarSchema; } diff --git a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/TestSpecFactory.java b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/TestSpecFactory.java index b163cf60a27..c20c946b801 100644 --- a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/TestSpecFactory.java +++ b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/TestSpecFactory.java @@ -38,37 +38,23 @@ public static Spec createDefault(final Consumer modifier) { } public static Spec createMinimal(final SpecMilestone specMilestone) { - switch (specMilestone) { - case PHASE0: - return createMinimalPhase0(); - case ALTAIR: - return createMinimalAltair(); - case BELLATRIX: - return createMinimalBellatrix(); - case CAPELLA: - return createMinimalCapella(); - case DENEB: - return createMinimalDeneb(); - default: - throw new IllegalStateException("unsupported milestone"); - } + return switch (specMilestone) { + case PHASE0 -> createMinimalPhase0(); + case ALTAIR -> createMinimalAltair(); + case BELLATRIX -> createMinimalBellatrix(); + case CAPELLA -> createMinimalCapella(); + case DENEB -> createMinimalDeneb(); + }; } public static Spec createMainnet(final SpecMilestone specMilestone) { - switch (specMilestone) { - case PHASE0: - return createMainnetPhase0(); - case ALTAIR: - return createMainnetAltair(); - case BELLATRIX: - return createMainnetBellatrix(); - case CAPELLA: - return createMainnetCapella(); - case DENEB: - return createMainnetDeneb(); - default: - throw new IllegalStateException("unsupported milestone"); - } + return switch (specMilestone) { + case PHASE0 -> createMainnetPhase0(); + case ALTAIR -> createMainnetAltair(); + case BELLATRIX -> createMainnetBellatrix(); + case CAPELLA -> createMainnetCapella(); + case DENEB -> createMainnetDeneb(); + }; } public static Spec createMinimalWithAltairAndBellatrixForkEpoch( @@ -145,8 +131,7 @@ public static Spec createMinimalWithBellatrixForkEpoch(final UInt64 bellatrixFor * epoch */ public static Spec createMinimalWithCapellaForkEpoch(final UInt64 capellaForkEpoch) { - final SpecConfigCapella config = - getCapellaSpecConfig(Eth2Network.MINIMAL, UInt64.ZERO, UInt64.ZERO, capellaForkEpoch); + final SpecConfigCapella config = getCapellaSpecConfig(Eth2Network.MINIMAL, capellaForkEpoch); return create(config, SpecMilestone.CAPELLA); } @@ -158,8 +143,7 @@ public static Spec createMinimalWithCapellaForkEpoch(final UInt64 capellaForkEpo */ public static Spec createMinimalWithDenebForkEpoch(final UInt64 denebForkEpoch) { final SpecConfigDeneb config = - getDenebSpecConfig( - Eth2Network.MINIMAL, UInt64.ZERO, UInt64.ZERO, UInt64.ZERO, denebForkEpoch); + getDenebSpecConfig(Eth2Network.MINIMAL, UInt64.ZERO, denebForkEpoch); return create(config, SpecMilestone.DENEB); } @@ -193,11 +177,6 @@ public static Spec createMainnetPhase0() { return create(specConfig, SpecMilestone.PHASE0); } - public static Spec createPhase0(final String configName) { - final SpecConfig specConfig = SpecConfigLoader.loadConfig(configName); - return createPhase0(specConfig); - } - public static Spec createPhase0(final SpecConfig config) { return create(config, SpecMilestone.PHASE0); } @@ -226,44 +205,29 @@ public static Spec create( final SpecMilestone specMilestone, final Eth2Network network, final Consumer configModifier) { - final Consumer actualModifier; - switch (specMilestone) { - case PHASE0: - actualModifier = configModifier; - break; - case ALTAIR: - actualModifier = - configModifier.andThen( - builder -> builder.altairBuilder(altair -> altair.altairForkEpoch(UInt64.ZERO))); - break; - case BELLATRIX: - actualModifier = - configModifier.andThen( - c -> - c.altairBuilder(a -> a.altairForkEpoch(UInt64.ZERO)) - .bellatrixBuilder(m -> m.bellatrixForkEpoch(UInt64.ZERO))); - break; - case CAPELLA: - actualModifier = - configModifier.andThen( - z -> - z.altairBuilder(a -> a.altairForkEpoch(UInt64.ZERO)) - .bellatrixBuilder(b -> b.bellatrixForkEpoch(UInt64.ZERO)) - .capellaBuilder(c -> c.capellaForkEpoch(UInt64.ZERO))); - break; - case DENEB: - actualModifier = - configModifier.andThen( - z -> - z.altairBuilder(a -> a.altairForkEpoch(UInt64.ZERO)) - .bellatrixBuilder(b -> b.bellatrixForkEpoch(UInt64.ZERO)) - .capellaBuilder(c -> c.capellaForkEpoch(UInt64.ZERO)) - .denebBuilder(d -> d.denebForkEpoch(UInt64.ZERO).kzgNoop(true))); - break; - default: - throw new IllegalStateException("unsupported milestone"); - } - return create(SpecConfigLoader.loadConfig(network.configName(), actualModifier), specMilestone); + final Consumer defaultModifier = + switch (specMilestone) { + case PHASE0 -> __ -> {}; + case ALTAIR -> builder -> builder.altairBuilder(a -> a.altairForkEpoch(UInt64.ZERO)); + case BELLATRIX -> builder -> + builder + .altairBuilder(a -> a.altairForkEpoch(UInt64.ZERO)) + .bellatrixBuilder(b -> b.bellatrixForkEpoch(UInt64.ZERO)); + case CAPELLA -> builder -> + builder + .altairBuilder(a -> a.altairForkEpoch(UInt64.ZERO)) + .bellatrixBuilder(b -> b.bellatrixForkEpoch(UInt64.ZERO)) + .capellaBuilder(c -> c.capellaForkEpoch(UInt64.ZERO)); + case DENEB -> builder -> + builder + .altairBuilder(a -> a.altairForkEpoch(UInt64.ZERO)) + .bellatrixBuilder(b -> b.bellatrixForkEpoch(UInt64.ZERO)) + .capellaBuilder(c -> c.capellaForkEpoch(UInt64.ZERO)) + .denebBuilder(d -> d.denebForkEpoch(UInt64.ZERO).kzgNoop(true)); + }; + return create( + SpecConfigLoader.loadConfig(network.configName(), defaultModifier.andThen(configModifier)), + specMilestone); } public static Spec create( @@ -279,7 +243,8 @@ private static SpecConfigAltair getAltairSpecConfig( final Eth2Network network, final UInt64 altairForkEpoch) { return SpecConfigAltair.required( SpecConfigLoader.loadConfig( - network.configName(), c -> c.altairBuilder(a -> a.altairForkEpoch(altairForkEpoch)))); + network.configName(), + builder -> builder.altairBuilder(a -> a.altairForkEpoch(altairForkEpoch)))); } private static SpecConfigBellatrix getBellatrixSpecConfig(final Eth2Network network) { @@ -290,9 +255,10 @@ private static SpecConfigBellatrix getBellatrixSpecConfig( final Eth2Network network, final UInt64 altairForkEpoch, UInt64 bellatrixForkEpoch) { return getBellatrixSpecConfig( network, - c -> - c.altairBuilder(a -> a.altairForkEpoch(altairForkEpoch)) - .bellatrixBuilder(m -> m.bellatrixForkEpoch(bellatrixForkEpoch))); + builder -> + builder + .altairBuilder(a -> a.altairForkEpoch(altairForkEpoch)) + .bellatrixBuilder(b -> b.bellatrixForkEpoch(bellatrixForkEpoch))); } private static SpecConfigBellatrix getBellatrixSpecConfig( @@ -309,19 +275,17 @@ private static SpecConfigBellatrix getBellatrixSpecConfig( } private static SpecConfigCapella getCapellaSpecConfig(final Eth2Network network) { - return getCapellaSpecConfig(network, UInt64.ZERO, UInt64.ZERO, UInt64.ZERO); + return getCapellaSpecConfig(network, UInt64.ZERO); } private static SpecConfigCapella getCapellaSpecConfig( - final Eth2Network network, - final UInt64 altairForkEpoch, - final UInt64 bellatrixForkEpoch, - final UInt64 capellaForkEpoch) { + final Eth2Network network, final UInt64 capellaForkEpoch) { return getCapellaSpecConfig( network, - z -> - z.altairBuilder(a -> a.altairForkEpoch(altairForkEpoch)) - .bellatrixBuilder(b -> b.bellatrixForkEpoch(bellatrixForkEpoch)) + builder -> + builder + .altairBuilder(a -> a.altairForkEpoch(UInt64.ZERO)) + .bellatrixBuilder(b -> b.bellatrixForkEpoch(UInt64.ZERO)) .capellaBuilder(c -> c.capellaForkEpoch(capellaForkEpoch))); } @@ -340,20 +304,17 @@ private static SpecConfigCapella getCapellaSpecConfig( } private static SpecConfigDeneb getDenebSpecConfig(final Eth2Network network) { - return getDenebSpecConfig(network, UInt64.ZERO, UInt64.ZERO, UInt64.ZERO, UInt64.ZERO); + return getDenebSpecConfig(network, UInt64.ZERO, UInt64.ZERO); } private static SpecConfigDeneb getDenebSpecConfig( - final Eth2Network network, - final UInt64 altairForkEpoch, - final UInt64 bellatrixForkEpoch, - final UInt64 capellaForkEpoch, - final UInt64 denebForkEpoch) { + final Eth2Network network, final UInt64 capellaForkEpoch, final UInt64 denebForkEpoch) { return getDenebSpecConfig( network, - z -> - z.altairBuilder(a -> a.altairForkEpoch(altairForkEpoch)) - .bellatrixBuilder(b -> b.bellatrixForkEpoch(bellatrixForkEpoch)) + builder -> + builder + .altairBuilder(a -> a.altairForkEpoch(UInt64.ZERO)) + .bellatrixBuilder(b -> b.bellatrixForkEpoch(UInt64.ZERO)) .capellaBuilder(c -> c.capellaForkEpoch(capellaForkEpoch)) .denebBuilder(d -> d.denebForkEpoch(denebForkEpoch).kzgNoop(true))); } @@ -376,8 +337,7 @@ private static SpecConfigDeneb getDenebSpecConfig( public static Spec createMinimalWithCapellaAndDenebForkEpoch( final UInt64 capellaForkEpoch, final UInt64 denebForkEpoch) { final SpecConfigBellatrix config = - getDenebSpecConfig( - Eth2Network.MINIMAL, UInt64.ZERO, UInt64.ZERO, capellaForkEpoch, denebForkEpoch); + getDenebSpecConfig(Eth2Network.MINIMAL, capellaForkEpoch, denebForkEpoch); return create(config, SpecMilestone.DENEB); } }