From 65112f6e0a0485a22cd61b32f1bf681f11265d53 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Mon, 18 Nov 2024 16:43:16 +0100 Subject: [PATCH 01/31] tmp --- .../historical/HistoricalBatchFetcher.java | 18 +- .../BlobSidecarsAvailabilityChecker.java | 9 - .../blobs/BlobSidecarManager.java | 9 - .../blobs/BlobSidecarManagerImpl.java | 27 +- ...ChoiceBlobSidecarsAvailabilityChecker.java | 302 ++---------------- .../blobs/BlobSidecarManagerTest.java | 4 - ...ceBlobSidecarsAvailabilityCheckerTest.java | 61 +--- .../beaconchain/BeaconChainController.java | 2 - 8 files changed, 27 insertions(+), 405 deletions(-) diff --git a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java index ee18ffbc5df..ab8474d9e6e 100644 --- a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java +++ b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java @@ -402,20 +402,14 @@ private void validateBlobSidecars(final SignedBeaconBlock block) { final List blobSidecars = blobSidecarsBySlotToImport.getOrDefault( block.getSlotAndBlockRoot(), Collections.emptyList()); - LOG.trace("Validating {} blob sidecars for block {}", blobSidecars.size(), block.getRoot()); - final BlobSidecarsAndValidationResult validationResult = - blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars); - - if (validationResult.isFailure()) { - final String causeMessage = - validationResult - .getCause() - .map(cause -> " (" + ExceptionUtil.getRootCauseMessage(cause) + ")") - .orElse(""); + + final boolean blobsBlockRootMatchBlockRoot = blobSidecars.stream().allMatch(blobSidecar -> blobSidecar.getBlockRoot().equals(block.getRoot())); + + if (!blobsBlockRootMatchBlockRoot) { throw new IllegalArgumentException( String.format( - "Blob sidecars validation for block %s failed: %s%s", - block.getRoot(), validationResult.getValidationResult(), causeMessage)); + "Blob sidecars' block root don't match block root %s", + block.getRoot())); } } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAvailabilityChecker.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAvailabilityChecker.java index d156f33c890..1ee646c9487 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAvailabilityChecker.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAvailabilityChecker.java @@ -35,12 +35,6 @@ public boolean initiateDataAvailabilityCheck() { public SafeFuture getAvailabilityCheckResult() { return NOT_REQUIRED_RESULT_FUTURE; } - - @Override - public BlobSidecarsAndValidationResult validateImmediately( - final List blobSidecars) { - return BlobSidecarsAndValidationResult.NOT_REQUIRED; - } }; BlobSidecarsAvailabilityChecker NOT_REQUIRED = NOOP; @@ -55,7 +49,4 @@ public BlobSidecarsAndValidationResult validateImmediately( boolean initiateDataAvailabilityCheck(); SafeFuture getAvailabilityCheckResult(); - - /** Perform the data availability check immediately on the provided blob sidecars */ - BlobSidecarsAndValidationResult validateImmediately(List blobSidecars); } diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java index 7bfea3926be..4b27e4463c6 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java @@ -51,12 +51,6 @@ public BlobSidecarsAvailabilityChecker createAvailabilityChecker( final SignedBeaconBlock block) { return BlobSidecarsAvailabilityChecker.NOOP; } - - @Override - public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( - final SignedBeaconBlock block, final List blobSidecars) { - return BlobSidecarsAndValidationResult.NOT_REQUIRED; - } }; SafeFuture validateAndPrepareForBlockImport( @@ -70,9 +64,6 @@ SafeFuture validateAndPrepareForBlockImport( BlobSidecarsAvailabilityChecker createAvailabilityChecker(SignedBeaconBlock block); - BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( - SignedBeaconBlock block, List blobSidecars); - interface ReceivedBlobSidecarListener { void onBlobSidecarReceived(BlobSidecar blobSidecar); } diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java index 7859aea03c5..d837ed0f2a7 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java @@ -37,10 +37,8 @@ public class BlobSidecarManagerImpl implements BlobSidecarManager, SlotEventsChannel { private final Spec spec; - private final AsyncRunner asyncRunner; private final RecentChainData recentChainData; private final BlobSidecarGossipValidator validator; - private final KZG kzg; private final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool; private final FutureItems futureBlobSidecars; private final Map invalidBlobSidecarRoots; @@ -50,18 +48,14 @@ public class BlobSidecarManagerImpl implements BlobSidecarManager, SlotEventsCha public BlobSidecarManagerImpl( final Spec spec, - final AsyncRunner asyncRunner, final RecentChainData recentChainData, final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool, final BlobSidecarGossipValidator validator, - final KZG kzg, final FutureItems futureBlobSidecars, final Map invalidBlobSidecarRoots) { this.spec = spec; - this.asyncRunner = asyncRunner; this.recentChainData = recentChainData; this.validator = validator; - this.kzg = kzg; this.blockBlobSidecarsTrackersPool = blockBlobSidecarsTrackersPool; this.futureBlobSidecars = futureBlobSidecars; this.invalidBlobSidecarRoots = invalidBlobSidecarRoots; @@ -129,26 +123,7 @@ public BlobSidecarsAvailabilityChecker createAvailabilityChecker(final SignedBea blockBlobSidecarsTrackersPool.getOrCreateBlockBlobSidecarsTracker(block); return new ForkChoiceBlobSidecarsAvailabilityChecker( - spec, asyncRunner, recentChainData, blockBlobSidecarsTracker, kzg); - } - - @Override - public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( - final SignedBeaconBlock block, final List blobSidecars) { - // Block is pre-Deneb, blobs are not supported yet - if (block.getMessage().getBody().toVersionDeneb().isEmpty()) { - return BlobSidecarsAndValidationResult.NOT_REQUIRED; - } - - // we don't care to set maxBlobsPerBlock since it isn't used with this immediate validation flow - final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(block.getSlotAndBlockRoot(), UInt64.ZERO); - - blockBlobSidecarsTracker.setBlock(block); - - return new ForkChoiceBlobSidecarsAvailabilityChecker( - spec, asyncRunner, recentChainData, blockBlobSidecarsTracker, kzg) - .validateImmediately(blobSidecars); + spec, recentChainData, blockBlobSidecarsTracker); } @Override diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java index fe6c43a345b..15ae439d6f1 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java @@ -53,35 +53,21 @@ * href="https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/fork-choice.md#is_data_available">is_data_available */ public class ForkChoiceBlobSidecarsAvailabilityChecker implements BlobSidecarsAvailabilityChecker { - private static final Logger LOG = LogManager.getLogger(); - - private final Spec spec; - private final AsyncRunner asyncRunner; +private final Spec spec; private final RecentChainData recentChainData; private final BlockBlobSidecarsTracker blockBlobSidecarsTracker; - private final KZG kzg; - - private final NavigableMap validatedBlobSidecars = - new ConcurrentSkipListMap<>(); private final SafeFuture validationResult = new SafeFuture<>(); - private final Supplier> kzgCommitmentsFromBlockSupplier = - createLazyKzgCommitmentsSupplier(this); - private final Duration waitForTrackerCompletionTimeout; public ForkChoiceBlobSidecarsAvailabilityChecker( final Spec spec, - final AsyncRunner asyncRunner, final RecentChainData recentChainData, - final BlockBlobSidecarsTracker blockBlobSidecarsTracker, - final KZG kzg) { + final BlockBlobSidecarsTracker blockBlobSidecarsTracker) { this.spec = spec; - this.asyncRunner = asyncRunner; this.recentChainData = recentChainData; this.blockBlobSidecarsTracker = blockBlobSidecarsTracker; - this.kzg = kzg; this.waitForTrackerCompletionTimeout = calculateCompletionTimeout(spec, blockBlobSidecarsTracker.getSlotAndBlockRoot().getSlot()); } @@ -89,27 +75,34 @@ public ForkChoiceBlobSidecarsAvailabilityChecker( @VisibleForTesting ForkChoiceBlobSidecarsAvailabilityChecker( final Spec spec, - final AsyncRunner asyncRunner, final RecentChainData recentChainData, final BlockBlobSidecarsTracker blockBlobSidecarsTracker, - final KZG kzg, final Duration waitForTrackerCompletionTimeout) { this.spec = spec; - this.asyncRunner = asyncRunner; this.recentChainData = recentChainData; this.blockBlobSidecarsTracker = blockBlobSidecarsTracker; - this.kzg = kzg; this.waitForTrackerCompletionTimeout = waitForTrackerCompletionTimeout; } @Override public boolean initiateDataAvailabilityCheck() { - asyncRunner - .runAsync(this::validateImmediatelyAvailable) - .thenCompose(this::completeValidation) - .thenApply(this::performCompletenessValidation) - .propagateTo(validationResult); - + blockBlobSidecarsTracker + .getCompletionFuture() + .orTimeout(waitForTrackerCompletionTimeout) + .thenApply(__ -> BlobSidecarsAndValidationResult.validResult(List.copyOf(blockBlobSidecarsTracker.getBlobSidecars().values()))) + .exceptionallyCompose( + error -> + ExceptionUtil.getCause(error, TimeoutException.class) + .map( + timeoutException -> { + if(isBlockOutsideDataAvailabilityWindow()) { + return SafeFuture.completedFuture(BlobSidecarsAndValidationResult.NOT_REQUIRED); + } + return SafeFuture.completedFuture( + BlobSidecarsAndValidationResult.notAvailable( + timeoutException));}) + .orElseGet(() -> SafeFuture.failedFuture(error))) + .propagateTo(validationResult); return true; } @@ -118,268 +111,11 @@ public SafeFuture getAvailabilityCheckResult() return validationResult; } - @Override - public BlobSidecarsAndValidationResult validateImmediately(final List blobSidecars) { - - final List kzgCommitmentsFromBlock = kzgCommitmentsFromBlockSupplier.get(); - - if (!blobSidecars.isEmpty()) { - final BlobSidecarsAndValidationResult blobSidecarsAndValidationResult = - validateBatch(blobSidecars); - - return performCompletenessValidation(blobSidecarsAndValidationResult); - } - - // no blob sidecars - - if (isBlockOutsideDataAvailabilityWindow()) { - return BlobSidecarsAndValidationResult.NOT_REQUIRED; - } - - if (kzgCommitmentsFromBlock.isEmpty()) { - return BlobSidecarsAndValidationResult.validResult(List.of()); - } - - return BlobSidecarsAndValidationResult.NOT_AVAILABLE; - } - - private BlobSidecarsAndValidationResult validateBatch(final List blobSidecars) { - final BeaconBlock block = blockBlobSidecarsTracker.getBlock().orElseThrow().getMessage(); - final SlotAndBlockRoot slotAndBlockRoot = blockBlobSidecarsTracker.getSlotAndBlockRoot(); - - final MiscHelpers miscHelpers = spec.atSlot(slotAndBlockRoot.getSlot()).miscHelpers(); - - try { - miscHelpers.validateBlobSidecarsBatchAgainstBlock( - blobSidecars, block, kzgCommitmentsFromBlockSupplier.get()); - - if (!miscHelpers.verifyBlobKzgProofBatch(kzg, blobSidecars)) { - return BlobSidecarsAndValidationResult.invalidResult(blobSidecars); - } - } catch (final Exception ex) { - return BlobSidecarsAndValidationResult.invalidResult(blobSidecars, ex); - } - - return BlobSidecarsAndValidationResult.validResult(blobSidecars); - } - - /** - * Step 1 of 2 - * - *

This function performs a partial validation with blobs that are currently available from the - * tracker. If all blobs are already available, it performs a full validation. - * - * @return a validation result only in case it is a definitive result. If the validation needs to - * be completed it returns empty - */ - private Optional validateImmediatelyAvailable() { - final List kzgCommitmentsInBlock = kzgCommitmentsFromBlockSupplier.get(); - - final List blobSidecarsToValidate; - - final boolean performCompleteValidation = blockBlobSidecarsTracker.isCompleted(); - - if (performCompleteValidation) { - // the tracker contains all required blobs for our block: we can perform a complete validation - blobSidecarsToValidate = List.copyOf(blockBlobSidecarsTracker.getBlobSidecars().values()); - LOG.debug( - "The expected {} BlobSidecars have already been received. Performing full validation.", - blobSidecarsToValidate.size()); - } else { - // prepare partial validation by matching the currently available blobs with the corresponding - // commitments from the block - blobSidecarsToValidate = - blockBlobSidecarsTracker.getBlobSidecars().values().stream().toList(); - - LOG.debug( - "{} out of {} BlobSidecars have been received so far. Performing partial validation.", - blobSidecarsToValidate.size(), - kzgCommitmentsInBlock.size()); - - if (blobSidecarsToValidate.isEmpty() - && !kzgCommitmentsInBlock.isEmpty() - && isBlockOutsideDataAvailabilityWindow()) { - // there are no available blobs so far, but we are outside the availability window. We can - // skip additional checks - return Optional.of(BlobSidecarsAndValidationResult.NOT_REQUIRED); - } - } - - // perform the actual validation - final BlobSidecarsAndValidationResult result = validateBatch(blobSidecarsToValidate); - - if (result.isFailure()) { - return Optional.of(result); - } - - if (performCompleteValidation) { - return Optional.of(result); - } - - // cache partially validated blobs - blobSidecarsToValidate.forEach( - blobSidecar -> validatedBlobSidecars.put(blobSidecar.getIndex(), blobSidecar)); - - return Optional.empty(); - } - - /** - * Step 2 of 2 - * - *

This function completes the validation if necessary. If there are still blobs to validate, - * it waits for the tracker to complete and then performs the final validation by computing and - * validating the non-yet validated blobs-commitments, and then computes the final validation - * result with the entire list of blobs - * - * @param maybeBlobSidecarsAndValidationResult is the validation result coming from Step 1. - * @return final validation result - */ - private SafeFuture completeValidation( - final Optional maybeBlobSidecarsAndValidationResult) { - - return maybeBlobSidecarsAndValidationResult - .map(SafeFuture::completedFuture) - .orElseGet( - () -> - blockBlobSidecarsTracker - .getCompletionFuture() - .orTimeout(waitForTrackerCompletionTimeout) - .thenApplyChecked(__ -> computeAndValidateRemaining()) - .thenApply(this::computeFinalValidationResult) - .exceptionallyCompose( - error -> - ExceptionUtil.getCause(error, TimeoutException.class) - .map( - timeoutException -> - SafeFuture.completedFuture( - BlobSidecarsAndValidationResult.notAvailable( - timeoutException))) - .orElseGet(() -> SafeFuture.failedFuture(error)))); - } - - /** - * Knowing that: - * - *

(1) `validatedBlobSidecars` contains already validated blobSidecars - * - *

(2) `blockBlobSidecarsTracker` is now completed - * - *

This function computes and validates a batch of BlobSidecars-kzgCommitments that needs to be - * additionally validated to verify the entire set - * - * @return validation result for the batch - */ - private BlobSidecarsAndValidationResult computeAndValidateRemaining() { - checkState( - blockBlobSidecarsTracker.isCompleted(), - "BlobSidecar tracker assumed to be completed but it is not."); - - final List additionalBlobSidecarsToBeValidated = new ArrayList<>(); - - final List kzgCommitmentsInBlock = kzgCommitmentsFromBlockSupplier.get(); - final SortedMap completeBlobSidecars = - blockBlobSidecarsTracker.getBlobSidecars(); - - UInt64.range(UInt64.ZERO, UInt64.valueOf(kzgCommitmentsInBlock.size())) - .forEachOrdered( - index -> { - if (!validatedBlobSidecars.containsKey(index)) { - - Optional.ofNullable(completeBlobSidecars.get(index)) - .ifPresentOrElse( - additionalBlobSidecarsToBeValidated::add, - () -> - LOG.error( - "Index {} not found in blob sidecar returned by tracker for block {}", - index, - blockBlobSidecarsTracker.getSlotAndBlockRoot().toLogString())); - } - }); - - LOG.debug( - "Remaining {} out of {} BlobSidecars have been received. Completing validation.", - additionalBlobSidecarsToBeValidated.size(), - kzgCommitmentsInBlock.size()); - - return validateBatch(additionalBlobSidecarsToBeValidated); - } - - /** - * Computes the final validation result combining the already validated blobs from - * `validatedBlobSidecars` - */ - private BlobSidecarsAndValidationResult computeFinalValidationResult( - final BlobSidecarsAndValidationResult additionalBlobSidecarsAndValidationResult) { - if (additionalBlobSidecarsAndValidationResult.isFailure()) { - return additionalBlobSidecarsAndValidationResult; - } - - // put all additional validated blobSidecar in validatedBlobSidecars which now should - // be complete - additionalBlobSidecarsAndValidationResult - .getBlobSidecars() - .forEach(blobSidecar -> validatedBlobSidecars.put(blobSidecar.getIndex(), blobSidecar)); - - final List completeValidatedBlobSidecars = - new ArrayList<>(validatedBlobSidecars.values()); - - return BlobSidecarsAndValidationResult.validResult( - Collections.unmodifiableList(completeValidatedBlobSidecars)); - } - - /** Makes sure that final blob sidecar list is complete */ - private BlobSidecarsAndValidationResult performCompletenessValidation( - final BlobSidecarsAndValidationResult blobSidecarsAndValidationResult) { - - if (blobSidecarsAndValidationResult.isFailure() - || blobSidecarsAndValidationResult.isNotRequired()) { - return blobSidecarsAndValidationResult; - } - - try { - spec.atSlot(blockBlobSidecarsTracker.getSlotAndBlockRoot().getSlot()) - .miscHelpers() - .verifyBlobSidecarCompleteness( - blobSidecarsAndValidationResult.getBlobSidecars(), - kzgCommitmentsFromBlockSupplier.get()); - } catch (final IllegalArgumentException ex) { - if (!isBlockOutsideDataAvailabilityWindow()) { - return BlobSidecarsAndValidationResult.invalidResult( - blobSidecarsAndValidationResult.getBlobSidecars(), - new IllegalArgumentException( - "Validated blobs are less than commitments present in block.", ex)); - } - - LOG.error( - "Inconsistent state detected: validated blobs is less then commitments in block. Since slot is outside availability the window we can consider blobs as NOT_REQUIRED."); - return BlobSidecarsAndValidationResult.NOT_REQUIRED; - } - - return blobSidecarsAndValidationResult; - } - private boolean isBlockOutsideDataAvailabilityWindow() { return !spec.isAvailabilityOfBlobSidecarsRequiredAtSlot( recentChainData.getStore(), blockBlobSidecarsTracker.getSlotAndBlockRoot().getSlot()); } - static Supplier> createLazyKzgCommitmentsSupplier( - final ForkChoiceBlobSidecarsAvailabilityChecker availabilityChecker) { - return Suppliers.memoize( - () -> - BeaconBlockBodyDeneb.required( - availabilityChecker - .blockBlobSidecarsTracker - .getBlock() - .flatMap(SignedBeaconBlock::getBeaconBlock) - .map(BeaconBlock::getBody) - .orElseThrow()) - .getBlobKzgCommitments() - .stream() - .map(SszKZGCommitment::getKZGCommitment) - .toList()); - } - static Duration calculateCompletionTimeout(final Spec spec, final UInt64 slot) { return Duration.ofMillis((spec.atSlot(slot).getConfig().getSecondsPerSlot() * 1000L) / 3); } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index 576c65827b2..cf414e75b77 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -54,11 +54,9 @@ public class BlobSidecarManagerTest { private final Spec spec = TestSpecFactory.createMinimalDeneb(); private final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec); - private final StubAsyncRunner asyncRunner = new StubAsyncRunner(); private final RecentChainData recentChainData = mock(RecentChainData.class); private final BlobSidecarGossipValidator blobSidecarValidator = mock(BlobSidecarGossipValidator.class); - private final KZG kzg = mock(KZG.class); private final BlockBlobSidecarsTrackersPoolImpl blockBlobSidecarsTrackersPool = mock(BlockBlobSidecarsTrackersPoolImpl.class); private final Map invalidBlobSidecarRoots = new HashMap<>(); @@ -69,11 +67,9 @@ public class BlobSidecarManagerTest { private final BlobSidecarManagerImpl blobSidecarManager = new BlobSidecarManagerImpl( spec, - asyncRunner, recentChainData, blockBlobSidecarsTrackersPool, blobSidecarValidator, - kzg, futureBlobSidecars, invalidBlobSidecarRoots); diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java index 10acbc68200..849c85869b8 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java @@ -339,63 +339,6 @@ void shouldReturnInvalidIfTrackerLiesWithCompletionButItIsNot() { "Validated blobs are less than commitments present in block.", cause))); } - @Test - void validateImmediately_shouldReturnAvailable() { - prepareInitialAvailability(Availability.FULL); - - whenDataAvailability(blobSidecarsComplete).thenReturn(true); - - assertAvailable( - SafeFuture.completedFuture( - blobSidecarsAvailabilityChecker.validateImmediately(blobSidecarsComplete))); - } - - @Test - void validateImmediately_shouldReturnInvalidIfCompletenessCheckFails() { - prepareInitialAvailability(Availability.FULL); - - whenDataAvailability(blobSidecarsComplete).thenReturn(true); - final Throwable cause = new IllegalArgumentException("oops"); - doThrow(cause).when(miscHelpers).verifyBlobSidecarCompleteness(any(), any()); - - final BlobSidecarsAndValidationResult availabilityCheckResult = - blobSidecarsAvailabilityChecker.validateImmediately(blobSidecarsComplete); - - assertInvalid( - SafeFuture.completedFuture(availabilityCheckResult), - blobSidecarsComplete, - Optional.of( - new IllegalArgumentException( - "Validated blobs are less than commitments present in block.", cause))); - } - - @Test - void validateImmediately_shouldReturnNotAvailableWithEmptyBlobsButRequired() { - prepareInitialAvailability(Availability.FULL); - - assertNotAvailable( - SafeFuture.completedFuture( - blobSidecarsAvailabilityChecker.validateImmediately(Collections.emptyList()))); - } - - @Test - void validateImmediately_shouldReturnAvailableOnEmptyBlobs() { - prepareInitialAvailabilityWithEmptyCommitmentsBlock(); - - assertAvailable( - SafeFuture.completedFuture( - blobSidecarsAvailabilityChecker.validateImmediately(Collections.emptyList()))); - } - - @Test - void validateImmediately_shouldReturnNotRequiredWhenBlockIsOutsideAvailabilityWindow() { - prepareBlockAndBlobSidecarsOutsideAvailabilityWindow(); - - assertNotRequired( - SafeFuture.completedFuture( - blobSidecarsAvailabilityChecker.validateImmediately(Collections.emptyList()))); - } - private void assertNotRequired( final SafeFuture availabilityOrValidityCheck) { assertThat(availabilityOrValidityCheck) @@ -531,7 +474,7 @@ private void prepareInitialAvailability( blobSidecarsAvailabilityChecker = new ForkChoiceBlobSidecarsAvailabilityChecker( - spec, asyncRunner, recentChainData, blockBlobSidecarsTracker, kzg, timeout); + spec, recentChainData, blockBlobSidecarsTracker, timeout); } private void completeTrackerWith(final List blobSidecars) { @@ -618,10 +561,8 @@ private void prepareBlockAndBlobSidecarsOutsideAvailabilityWindow() { blobSidecarsAvailabilityChecker = new ForkChoiceBlobSidecarsAvailabilityChecker( spec, - asyncRunner, recentChainData, blockBlobSidecarsTracker, - kzg, Duration.ofSeconds(30)); } diff --git a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java index 47e0d1540e4..fa8cdec79d5 100644 --- a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java +++ b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java @@ -586,11 +586,9 @@ protected void initBlobSidecarManager() { final BlobSidecarManagerImpl blobSidecarManagerImpl = new BlobSidecarManagerImpl( spec, - beaconAsyncRunner, recentChainData, blockBlobSidecarsTrackersPool, blobSidecarValidator, - kzg, futureBlobSidecars, invalidBlobSidecarRoots); eventChannels.subscribe(SlotEventsChannel.class, blobSidecarManagerImpl); From 2d807207d647e6a3410e3c734730b7d6cca9bfe7 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Mon, 18 Nov 2024 16:43:38 +0100 Subject: [PATCH 02/31] tmp --- .../historical/HistoricalBatchFetcher.java | 10 ++-- .../BlobSidecarsAvailabilityChecker.java | 2 - .../blobs/BlobSidecarManager.java | 2 - .../blobs/BlobSidecarManagerImpl.java | 4 -- ...ChoiceBlobSidecarsAvailabilityChecker.java | 60 +++++++------------ .../blobs/BlobSidecarManagerTest.java | 2 - ...ceBlobSidecarsAvailabilityCheckerTest.java | 6 +- 7 files changed, 26 insertions(+), 60 deletions(-) diff --git a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java index ab8474d9e6e..5fe5ef583c8 100644 --- a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java +++ b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java @@ -36,7 +36,6 @@ import tech.pegasys.teku.bls.BLSPublicKey; import tech.pegasys.teku.bls.BLSSignature; import tech.pegasys.teku.infrastructure.async.SafeFuture; -import tech.pegasys.teku.infrastructure.exceptions.ExceptionUtil; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.networking.eth2.peers.Eth2Peer; import tech.pegasys.teku.networking.eth2.rpc.core.InvalidResponseException; @@ -53,7 +52,6 @@ import tech.pegasys.teku.spec.datastructures.state.Fork; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; import tech.pegasys.teku.spec.logic.common.util.AsyncBLSSignatureVerifier; -import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager; import tech.pegasys.teku.storage.api.StorageUpdateChannel; import tech.pegasys.teku.storage.client.CombinedChainDataClient; @@ -403,13 +401,13 @@ private void validateBlobSidecars(final SignedBeaconBlock block) { blobSidecarsBySlotToImport.getOrDefault( block.getSlotAndBlockRoot(), Collections.emptyList()); - final boolean blobsBlockRootMatchBlockRoot = blobSidecars.stream().allMatch(blobSidecar -> blobSidecar.getBlockRoot().equals(block.getRoot())); + final boolean blobsBlockRootMatchBlockRoot = + blobSidecars.stream() + .allMatch(blobSidecar -> blobSidecar.getBlockRoot().equals(block.getRoot())); if (!blobsBlockRootMatchBlockRoot) { throw new IllegalArgumentException( - String.format( - "Blob sidecars' block root don't match block root %s", - block.getRoot())); + String.format("Blob sidecars' block root don't match block root %s", block.getRoot())); } } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAvailabilityChecker.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAvailabilityChecker.java index 1ee646c9487..7456a69cc73 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAvailabilityChecker.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAvailabilityChecker.java @@ -15,9 +15,7 @@ import static tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult.NOT_REQUIRED_RESULT_FUTURE; -import java.util.List; import tech.pegasys.teku.infrastructure.async.SafeFuture; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadHeader; import tech.pegasys.teku.spec.datastructures.execution.NewPayloadRequest; import tech.pegasys.teku.spec.logic.versions.bellatrix.block.OptimisticExecutionPayloadExecutor; diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java index 4b27e4463c6..a61a92db880 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java @@ -13,13 +13,11 @@ package tech.pegasys.teku.statetransition.blobs; -import java.util.List; import java.util.Optional; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; -import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; import tech.pegasys.teku.statetransition.validation.InternalValidationResult; diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java index d837ed0f2a7..610529ba125 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java @@ -13,20 +13,16 @@ package tech.pegasys.teku.statetransition.blobs; -import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.ethereum.events.SlotEventsChannel; -import tech.pegasys.teku.infrastructure.async.AsyncRunner; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.subscribers.Subscribers; import tech.pegasys.teku.infrastructure.unsigned.UInt64; -import tech.pegasys.teku.kzg.KZG; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; -import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; import tech.pegasys.teku.statetransition.forkchoice.ForkChoiceBlobSidecarsAvailabilityChecker; import tech.pegasys.teku.statetransition.util.FutureItems; diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java index 15ae439d6f1..0985b2227ae 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java @@ -13,36 +13,14 @@ package tech.pegasys.teku.statetransition.forkchoice; -import static com.google.common.base.Preconditions.checkState; - import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Suppliers; import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.NavigableMap; -import java.util.Optional; -import java.util.SortedMap; -import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.TimeoutException; -import java.util.function.Supplier; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import tech.pegasys.teku.infrastructure.async.AsyncRunner; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.exceptions.ExceptionUtil; import tech.pegasys.teku.infrastructure.unsigned.UInt64; -import tech.pegasys.teku.kzg.KZG; -import tech.pegasys.teku.kzg.KZGCommitment; import tech.pegasys.teku.spec.Spec; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock; -import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; -import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; -import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodyDeneb; -import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment; -import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; import tech.pegasys.teku.statetransition.blobs.BlockBlobSidecarsTracker; @@ -53,7 +31,7 @@ * href="https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/fork-choice.md#is_data_available">is_data_available */ public class ForkChoiceBlobSidecarsAvailabilityChecker implements BlobSidecarsAvailabilityChecker { -private final Spec spec; + private final Spec spec; private final RecentChainData recentChainData; private final BlockBlobSidecarsTracker blockBlobSidecarsTracker; @@ -87,22 +65,26 @@ public ForkChoiceBlobSidecarsAvailabilityChecker( @Override public boolean initiateDataAvailabilityCheck() { blockBlobSidecarsTracker - .getCompletionFuture() - .orTimeout(waitForTrackerCompletionTimeout) - .thenApply(__ -> BlobSidecarsAndValidationResult.validResult(List.copyOf(blockBlobSidecarsTracker.getBlobSidecars().values()))) - .exceptionallyCompose( - error -> - ExceptionUtil.getCause(error, TimeoutException.class) - .map( - timeoutException -> { - if(isBlockOutsideDataAvailabilityWindow()) { - return SafeFuture.completedFuture(BlobSidecarsAndValidationResult.NOT_REQUIRED); - } - return SafeFuture.completedFuture( - BlobSidecarsAndValidationResult.notAvailable( - timeoutException));}) - .orElseGet(() -> SafeFuture.failedFuture(error))) - .propagateTo(validationResult); + .getCompletionFuture() + .orTimeout(waitForTrackerCompletionTimeout) + .thenApply( + __ -> + BlobSidecarsAndValidationResult.validResult( + List.copyOf(blockBlobSidecarsTracker.getBlobSidecars().values()))) + .exceptionallyCompose( + error -> + ExceptionUtil.getCause(error, TimeoutException.class) + .map( + timeoutException -> { + if (isBlockOutsideDataAvailabilityWindow()) { + return SafeFuture.completedFuture( + BlobSidecarsAndValidationResult.NOT_REQUIRED); + } + return SafeFuture.completedFuture( + BlobSidecarsAndValidationResult.notAvailable(timeoutException)); + }) + .orElseGet(() -> SafeFuture.failedFuture(error))) + .propagateTo(validationResult); return true; } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index cf414e75b77..c279a6d5aa5 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -31,9 +31,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import tech.pegasys.teku.infrastructure.async.SafeFuture; -import tech.pegasys.teku.infrastructure.async.StubAsyncRunner; import tech.pegasys.teku.infrastructure.unsigned.UInt64; -import tech.pegasys.teku.kzg.KZG; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java index 849c85869b8..e740a93bf33 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java @@ -31,7 +31,6 @@ import com.google.common.collect.ImmutableSortedMap; import java.time.Duration; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -560,10 +559,7 @@ private void prepareBlockAndBlobSidecarsOutsideAvailabilityWindow() { blobSidecarsAvailabilityChecker = new ForkChoiceBlobSidecarsAvailabilityChecker( - spec, - recentChainData, - blockBlobSidecarsTracker, - Duration.ofSeconds(30)); + spec, recentChainData, blockBlobSidecarsTracker, Duration.ofSeconds(30)); } private enum Availability { From 37b61e32e1d63171c81397cd85315a661a40cea4 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Tue, 19 Nov 2024 13:57:25 +0100 Subject: [PATCH 03/31] tmp --- .../teku/beacon/sync/historical/HistoricalBatchFetcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java index 5fe5ef583c8..dfcfc70b1ff 100644 --- a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java +++ b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java @@ -403,11 +403,11 @@ private void validateBlobSidecars(final SignedBeaconBlock block) { final boolean blobsBlockRootMatchBlockRoot = blobSidecars.stream() - .allMatch(blobSidecar -> blobSidecar.getBlockRoot().equals(block.getRoot())); + .allMatch(blobSidecar -> blobSidecar.getSignedBeaconBlockHeader().hashTreeRoot().equals(block.hashTreeRoot())); if (!blobsBlockRootMatchBlockRoot) { throw new IllegalArgumentException( - String.format("Blob sidecars' block root don't match block root %s", block.getRoot())); + String.format("Blob sidecars' signed beacon block header don't match signed block for %s", block.toLogString())); } } From 1b243580f2ced4c81ba955c0f739357c6fb289cc Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 20 Nov 2024 17:50:59 +0100 Subject: [PATCH 04/31] update some tests --- .../HistoricalBatchFetcherTest.java | 12 +- .../blobs/BlobSidecarManagerTest.java | 31 +- ...ceBlobSidecarsAvailabilityCheckerTest.java | 338 ++---------------- 3 files changed, 32 insertions(+), 349 deletions(-) diff --git a/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java b/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java index 7fe92edf82a..a0ae8c33872 100644 --- a/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java +++ b/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java @@ -117,7 +117,7 @@ public void setup() { .map(SignedBlockAndState::getBlock) .collect(Collectors.toList()); lastBlockInBatch = chainBuilder.getLatestBlockAndState().getBlock(); - firstBlockInBatch = blockBatch.get(0); + firstBlockInBatch = blockBatch.getFirst(); blobSidecarsBatch = chainBuilder .streamBlobSidecars(10, 20) @@ -148,8 +148,6 @@ public void setup() { when(signatureVerifier.verify(any(), any(), anyList())) .thenReturn(SafeFuture.completedFuture(true)); - when(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(any(), anyList())) - .thenAnswer(i -> BlobSidecarsAndValidationResult.validResult(i.getArgument(1))); } @Test @@ -202,17 +200,13 @@ public void run_returnAllBlocksAndBlobSidecarsOnFirstRequest() { earliestBlobSidecarSlotCaptor.capture()); assertThat(blockCaptor.getValue()).containsExactlyElementsOf(blockBatch); assertThat(blobSidecarCaptor.getValue()).isEqualTo(blobSidecarsBatch); - assertThat(earliestBlobSidecarSlotCaptor.getValue()).contains(blockBatch.get(0).getSlot()); + assertThat(earliestBlobSidecarSlotCaptor.getValue()).contains(blockBatch.getFirst().getSlot()); } @Test public void run_failsOnBlobSidecarsValidationFailure() { when(blobSidecarManager.isAvailabilityRequiredAtSlot(any())).thenReturn(true); - when(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(any(), anyList())) - .thenAnswer( - i -> - BlobSidecarsAndValidationResult.invalidResult( - i.getArgument(1), new IllegalStateException("oopsy"))); + assertThat(peer.getOutstandingRequests()).isEqualTo(0); final SafeFuture future = fetcher.run(); diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index c279a6d5aa5..97797a90a26 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -184,8 +184,8 @@ void onSlot_shouldInteractWithPoolAndFutureBlobs() { verify(futureBlobSidecars).onSlot(UInt64.ONE); verify(blockBlobSidecarsTrackersPool) - .onNewBlobSidecar(futureBlobSidecarsList.get(0), RemoteOrigin.GOSSIP); - verify(receivedBlobSidecarListener).onBlobSidecarReceived(futureBlobSidecarsList.get(0)); + .onNewBlobSidecar(futureBlobSidecarsList.getFirst(), RemoteOrigin.GOSSIP); + verify(receivedBlobSidecarListener).onBlobSidecarReceived(futureBlobSidecarsList.getFirst()); } @Test @@ -198,17 +198,6 @@ void createAvailabilityChecker_shouldReturnANotRequiredAvailabilityCheckerWhenBl .isEqualTo(BlobSidecarsAvailabilityChecker.NOT_REQUIRED); } - @Test - void - createAvailabilityCheckerAndValidateImmediately_shouldReturnANotRequiredAvailabilityCheckerWhenBlockIsPreDeneb() { - final Spec spec = TestSpecFactory.createMainnetCapella(); - final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec); - final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(); - - assertThat(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, List.of())) - .isEqualTo(BlobSidecarsAndValidationResult.NOT_REQUIRED); - } - @Test void createAvailabilityChecker_shouldReturnAnAvailabilityChecker() { final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(UInt64.ONE); @@ -222,20 +211,4 @@ void createAvailabilityChecker_shouldReturnAnAvailabilityChecker() { assertThat(blobSidecarManager.createAvailabilityChecker(block)) .isInstanceOf(ForkChoiceBlobSidecarsAvailabilityChecker.class); } - - @Test - void - createAvailabilityCheckerAndValidateImmediately_shouldReturnABlobSidecarsAndValidationResult() { - final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(UInt64.ONE); - - final UpdatableStore store = mock(UpdatableStore.class); - when(recentChainData.getStore()).thenReturn(store); - when(store.getTimeSeconds()).thenReturn(UInt64.ONE); - when(store.getGenesisTime()).thenReturn(UInt64.ZERO); - - assertThat(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, List.of())) - .isInstanceOf(BlobSidecarsAndValidationResult.class); - - verifyNoInteractions(blockBlobSidecarsTrackersPool); - } } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java index e740a93bf33..e50320ba362 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java @@ -27,13 +27,16 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import static tech.pegasys.teku.infrastructure.async.SafeFutureAssert.assertThatSafeFuture; import com.google.common.collect.ImmutableSortedMap; import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -68,20 +71,13 @@ public class ForkChoiceBlobSidecarsAvailabilityCheckerTest { private final Spec spec = mock(Spec.class); private final SpecVersion specVersion = mock(SpecVersion.class); - private final KZG kzg = mock(KZG.class); - private final StubAsyncRunner asyncRunner = new StubAsyncRunner(); private final UpdatableStore store = mock(UpdatableStore.class); private final BlockBlobSidecarsTracker blockBlobSidecarsTracker = mock(BlockBlobSidecarsTracker.class); - private final MiscHelpers miscHelpers = mock(MiscHelpers.class); private final RecentChainData recentChainData = mock(RecentChainData.class); private SignedBeaconBlock block; private List blobSidecarsComplete; - private List kzgCommitmentsComplete; - - private List blobSidecarsInitial; - private List blobSidecarsAdditional; private final SafeFuture trackerCompletionFuture = new SafeFuture<>(); @@ -90,85 +86,53 @@ public class ForkChoiceBlobSidecarsAvailabilityCheckerTest { @BeforeEach void setUp() { when(spec.atSlot(any())).thenReturn(specVersion); - when(specVersion.miscHelpers()).thenReturn(miscHelpers); when(recentChainData.getStore()).thenReturn(store); when(blockBlobSidecarsTracker.getCompletionFuture()).thenReturn(trackerCompletionFuture); } - @ParameterizedTest - @EnumSource(Availability.class) - void shouldVerifyAvailableBlobsInTwoBatches(final Availability availability) throws Exception { - prepareInitialAvailability(availability); + @Test + void shouldVerifyAvailableBlobs() throws Exception { + prepareInitialAvailability(); final SafeFuture availabilityCheckResult = blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - SafeFutureAssert.assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); // initiate availability check assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - // all validation on a separate thread, so no interaction so far. - SafeFutureAssert.assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - verifyDataAvailabilityNeverCalled(); + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); - // mock kzg availability check to be OK for the initial set - whenDataAvailability(blobSidecarsInitial).thenReturn(true); - - // let availability check to be performed. - asyncRunner.executeDueActions(); - - // verify that kzg validation has been performed for the initial batch - verifyValidationAndDataAvailabilityCall(blobSidecarsInitial, false); - - // mock the additional check to be OK. - whenDataAvailability(blobSidecarsAdditional).thenReturn(true); // let the tracker complete with all blobSidecars completeTrackerWith(blobSidecarsComplete); Waiter.waitFor(availabilityCheckResult); - // verify that kzg validation has been performed for the additional batch - verifyValidationAndDataAvailabilityCall(blobSidecarsAdditional, true); - assertAvailable(availabilityCheckResult); - - // no interaction since last verify - verifyNoInteractions(miscHelpers); } @Test - void shouldVerifyAvailableBlobsInOneBatch() throws Exception { - prepareInitialAvailability(Availability.FULL); + void shouldFailIfTrackerCompletesWithFailure() { + prepareInitialAvailability(); final SafeFuture availabilityCheckResult = - blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); + blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - SafeFutureAssert.assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); // initiate availability check assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - // mock kzg availability check to be OK for the initial set - whenDataAvailability(blobSidecarsComplete).thenReturn(true); - - // tracker is completed in advance - completeTrackerWith(blobSidecarsComplete); - - // let availability check to be performed. - asyncRunner.executeDueActions(); - - // verify that kzg validation has been performed for the initial batch - verifyValidationAndDataAvailabilityCall(blobSidecarsComplete, true); - - Waiter.waitFor(availabilityCheckResult); + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); - assertAvailable(availabilityCheckResult); + // let the tracker complete with all blobSidecars + trackerCompletionFuture.completeExceptionally(new RuntimeException("oops")); - // no interaction since last verify - verifyNoInteractions(miscHelpers); + assertThatSafeFuture(availabilityCheckResult).isCompletedExceptionallyWithMessage("oops"); } @Test @@ -180,162 +144,26 @@ void shouldReturnNotAvailableOnTimeout() throws Exception { assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - SafeFutureAssert.assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - verifyDataAvailabilityNeverCalled(); verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); - whenDataAvailability(blobSidecarsInitial).thenReturn(true); - - asyncRunner.executeDueActions(); - Waiter.waitFor(availabilityCheckResult); assertNotAvailableDueToTimeout(availabilityCheckResult); } @Test - void shouldReturnNotRequiredWhenBlockIsOutsideAvailabilityWindow() { - prepareBlockAndBlobSidecarsOutsideAvailabilityWindow(); - - final SafeFuture availabilityCheckResult = - blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - - SafeFutureAssert.assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - - assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - SafeFutureAssert.assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - asyncRunner.executeDueActions(); - assertNotRequired(availabilityCheckResult); - } - - @ParameterizedTest - @EnumSource(value = BatchFailure.class) - void shouldReturnNotAvailableIfFirstBatchFails(final BatchFailure batchFailure) { - prepareInitialAvailability(Availability.PARTIAL); - - final Optional cause = - switch (batchFailure) { - case BLOB_SIDECAR_VALIDATION_EXCEPTION, IS_DATA_AVAILABLE_EXCEPTION -> - Optional.of(new RuntimeException("oops")); - default -> Optional.empty(); - }; - - final SafeFuture availabilityCheckResult = - blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - - SafeFutureAssert.assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - - // initiate availability check - assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - - switch (batchFailure) { - // blobsidecar validation check failure for the initial set - case BLOB_SIDECAR_VALIDATION_EXCEPTION -> - throwWhenValidatingBlobSidecarsBatchAgainstBlock(blobSidecarsInitial, cause.get()); - // mock kzg availability check failure for the initial set - case IS_DATA_AVAILABLE_EXCEPTION -> - whenDataAvailability(blobSidecarsInitial).thenThrow(cause.get()); - case IS_DATA_AVAILABLE_RETURN_FALSE -> - whenDataAvailability(blobSidecarsInitial).thenReturn(false); - } - - asyncRunner.executeDueActions(); - - assertInvalid(availabilityCheckResult, blobSidecarsInitial, cause); - } - - @ParameterizedTest - @EnumSource(value = BatchFailure.class) - void shouldReturnNotAvailableIfSecondBatchFails(final BatchFailure batchFailure) { - prepareInitialAvailability(Availability.PARTIAL); - - final Optional cause = - switch (batchFailure) { - case BLOB_SIDECAR_VALIDATION_EXCEPTION, IS_DATA_AVAILABLE_EXCEPTION -> - Optional.of(new RuntimeException("oops")); - default -> Optional.empty(); - }; - - final SafeFuture availabilityCheckResult = - blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - - SafeFutureAssert.assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - - // initiate availability check - assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - - // mock kzg availability check to be OK for the initial set - whenDataAvailability(blobSidecarsInitial).thenReturn(true); - - // let availability check to be performed. - asyncRunner.executeDueActions(); - - switch (batchFailure) { - // blobsidecar validation check failure for the additional set - case BLOB_SIDECAR_VALIDATION_EXCEPTION -> - throwWhenValidatingBlobSidecarsBatchAgainstBlock(blobSidecarsAdditional, cause.get()); - // mock kzg availability check failure for the additional set - case IS_DATA_AVAILABLE_EXCEPTION -> - whenDataAvailability(blobSidecarsAdditional).thenThrow(cause.get()); - case IS_DATA_AVAILABLE_RETURN_FALSE -> - whenDataAvailability(blobSidecarsAdditional).thenReturn(false); - } - - // let the tracker complete with all blobSidecars - completeTrackerWith(blobSidecarsComplete); - - assertInvalid(availabilityCheckResult, blobSidecarsAdditional, cause); - } - - @Test - void shouldReturnInvalidIfTrackerLiesWithCompletionButItIsNot() { - prepareInitialAvailability(Availability.PARTIAL); + void shouldReturnNotRequiredWhenBlockIsOutsideAvailabilityWindow() throws Exception { + prepareForImmediateTimeoutWithBlockAndBlobSidecarsOutsideAvailabilityWindow(); final SafeFuture availabilityCheckResult = blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - SafeFutureAssert.assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - // initiate availability check assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - // all validation on a separate thread, so no interaction so far. - SafeFutureAssert.assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - verifyDataAvailabilityNeverCalled(); - verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); - - // mock kzg availability check to be OK for the initial set - whenDataAvailability(blobSidecarsInitial).thenReturn(true); - - // let availability check to be performed. - asyncRunner.executeDueActions(); - - // verify that kzg validation has been performed for the initial batch - verifyValidationAndDataAvailabilityCall(blobSidecarsInitial, false); - - // we complete the blobs without index 3 - final List partialBlobs = blobSidecarsComplete.subList(1, 2); - // we lie on availability check too (not actually possible) - whenDataAvailability(partialBlobs).thenReturn(true); - - final List expectedIncompleteBlobSidecar = new ArrayList<>(); - expectedIncompleteBlobSidecar.add(blobSidecarsComplete.get(0)); // blob 0 - expectedIncompleteBlobSidecar.add(blobSidecarsComplete.get(1)); // blob 1 - expectedIncompleteBlobSidecar.add(blobSidecarsComplete.get(2)); // blob 2 - - final Throwable cause = new IllegalArgumentException("oops"); - - throwWhenVerifyingBlobSidecarCompleteness(expectedIncompleteBlobSidecar, cause); + Waiter.waitFor(availabilityCheckResult); - // let the tracker complete with all blobSidecars - completeTrackerWith(partialBlobs); - - assertInvalid( - availabilityCheckResult, - expectedIncompleteBlobSidecar, - Optional.of( - new IllegalArgumentException( - "Validated blobs are less than commitments present in block.", cause))); + assertNotRequired(availabilityCheckResult); } private void assertNotRequired( @@ -409,66 +237,24 @@ private void assertAvailable( result -> result.getBlobSidecars().equals(blobSidecarsComplete), "has blob sidecars"); } - private void prepareInitialAvailabilityWithEmptyCommitmentsBlock() { - prepareInitialAvailability( - Availability.FULL, - Optional.of(dataStructureUtil.randomSignedBeaconBlockWithEmptyCommitments())); - } - private void prepareForImmediateTimeout() { - prepareInitialAvailability(Availability.PARTIAL, Optional.empty(), Duration.ZERO); - } - - private void prepareInitialAvailability(final Availability blobsAvailability) { - prepareInitialAvailability(blobsAvailability, Optional.empty(), Duration.ofSeconds(30)); + prepareInitialAvailability(Optional.empty(), Duration.ZERO); } - private void prepareInitialAvailability( - final Availability blobsAvailability, final Optional providedBlock) { - prepareInitialAvailability(blobsAvailability, providedBlock, Duration.ofSeconds(30)); + private void prepareInitialAvailability() { + prepareInitialAvailability(Optional.empty(), Duration.ofSeconds(30)); } private void prepareInitialAvailability( - final Availability blobsAvailability, final Optional providedBlock, final Duration timeout) { block = providedBlock.orElse(dataStructureUtil.randomSignedBeaconBlockWithCommitments(4)); blobSidecarsComplete = dataStructureUtil.randomBlobSidecarsForBlock(block); - kzgCommitmentsComplete = - block - .getBeaconBlock() - .orElseThrow() - .getBody() - .toVersionDeneb() - .orElseThrow() - .getBlobKzgCommitments() - .stream() - .map(SszKZGCommitment::getKZGCommitment) - .toList(); when(spec.isAvailabilityOfBlobSidecarsRequiredAtSlot(store, block.getSlot())).thenReturn(true); - switch (blobsAvailability) { - case FULL -> { - blobSidecarsInitial = blobSidecarsComplete; - blobSidecarsAdditional = List.of(); - } - case EMPTY -> { - blobSidecarsInitial = List.of(); - blobSidecarsAdditional = blobSidecarsComplete; - } - case PARTIAL -> { - blobSidecarsInitial = List.of(blobSidecarsComplete.get(0), blobSidecarsComplete.get(2)); - blobSidecarsAdditional = List.of(blobSidecarsComplete.get(1), blobSidecarsComplete.get(3)); - } - } - - final ImmutableSortedMap.Builder mapBuilder = - ImmutableSortedMap.naturalOrder(); - blobSidecarsInitial.forEach(blobSidecar -> mapBuilder.put(blobSidecar.getIndex(), blobSidecar)); - when(blockBlobSidecarsTracker.getBlock()).thenReturn(Optional.of(block)); - when(blockBlobSidecarsTracker.getBlobSidecars()).thenReturn(mapBuilder.build()); + when(blockBlobSidecarsTracker.getBlobSidecars()).thenReturn(ImmutableSortedMap.of()); when(blockBlobSidecarsTracker.getSlotAndBlockRoot()).thenReturn(block.getSlotAndBlockRoot()); blobSidecarsAvailabilityChecker = @@ -485,64 +271,7 @@ private void completeTrackerWith(final List blobSidecars) { trackerCompletionFuture.complete(null); } - private OngoingStubbing whenDataAvailability(final List blobSidecars) { - return when(miscHelpers.verifyBlobKzgProofBatch(kzg, blobSidecars)); - } - - private void throwWhenValidatingBlobSidecarsBatchAgainstBlock( - final List blobSidecars, final Throwable cause) { - doThrow(cause) - .when(miscHelpers) - .validateBlobSidecarsBatchAgainstBlock( - eq(blobSidecars), - argThat(block -> block.equals(this.block.getBeaconBlock().orElseThrow())), - assertArg( - kzgCommitmentsArg -> - assertThat(kzgCommitmentsArg).isEqualTo(kzgCommitmentsComplete))); - } - - private void throwWhenVerifyingBlobSidecarCompleteness( - final List blobSidecars, final Throwable cause) { - doThrow(cause) - .when(miscHelpers) - .verifyBlobSidecarCompleteness( - eq(blobSidecars), - assertArg( - kzgCommitmentsArg -> - assertThat(kzgCommitmentsArg).isEqualTo(kzgCommitmentsComplete))); - } - - private void verifyValidationAndDataAvailabilityCall( - final List blobSidecars, final boolean isFinalValidation) { - verify(miscHelpers, times(1)) - .validateBlobSidecarsBatchAgainstBlock( - eq(blobSidecars), - argThat(block -> block.equals(this.block.getBeaconBlock().orElseThrow())), - assertArg( - kzgCommitmentsArg -> - assertThat(kzgCommitmentsArg).isEqualTo(kzgCommitmentsComplete))); - - verify(miscHelpers, times(1)).verifyBlobKzgProofBatch(kzg, blobSidecars); - - if (isFinalValidation) { - verify(miscHelpers, times(1)) - .verifyBlobSidecarCompleteness( - eq(blobSidecarsComplete), - assertArg( - kzgCommitmentsArg -> - assertThat(kzgCommitmentsArg).isEqualTo(kzgCommitmentsComplete))); - } - - // assume we verified all interaction before resetting - verifyNoMoreInteractions(miscHelpers); - reset(miscHelpers); - } - - private void verifyDataAvailabilityNeverCalled() { - verify(miscHelpers, never()).verifyBlobKzgProofBatch(eq(kzg), any()); - } - - private void prepareBlockAndBlobSidecarsOutsideAvailabilityWindow() { + private void prepareForImmediateTimeoutWithBlockAndBlobSidecarsOutsideAvailabilityWindow() { block = dataStructureUtil.randomSignedBeaconBlock(); blobSidecarsComplete = dataStructureUtil.randomBlobSidecarsForBlock(block); @@ -553,24 +282,11 @@ private void prepareBlockAndBlobSidecarsOutsideAvailabilityWindow() { when(spec.isAvailabilityOfBlobSidecarsRequiredAtSlot(store, block.getSlot())).thenReturn(false); when(blockBlobSidecarsTracker.getBlock()).thenReturn(Optional.of(block)); - when(blockBlobSidecarsTracker.getCompletionFuture()).thenReturn(SafeFuture.COMPLETE); when(blockBlobSidecarsTracker.getBlobSidecars()).thenReturn(ImmutableSortedMap.of()); when(blockBlobSidecarsTracker.getSlotAndBlockRoot()).thenReturn(block.getSlotAndBlockRoot()); blobSidecarsAvailabilityChecker = new ForkChoiceBlobSidecarsAvailabilityChecker( - spec, recentChainData, blockBlobSidecarsTracker, Duration.ofSeconds(30)); - } - - private enum Availability { - EMPTY, - PARTIAL, - FULL - } - - private enum BatchFailure { - BLOB_SIDECAR_VALIDATION_EXCEPTION, - IS_DATA_AVAILABLE_EXCEPTION, - IS_DATA_AVAILABLE_RETURN_FALSE, + spec, recentChainData, blockBlobSidecarsTracker, Duration.ZERO); } } From 36460978043b40a65613875a6a15199dc8ead4b3 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 20 Nov 2024 17:51:30 +0100 Subject: [PATCH 05/31] update some tests --- .../historical/HistoricalBatchFetcher.java | 11 ++++++-- .../HistoricalBatchFetcherTest.java | 2 -- .../blobs/BlobSidecarManagerTest.java | 3 --- ...ceBlobSidecarsAvailabilityCheckerTest.java | 27 ++----------------- 4 files changed, 11 insertions(+), 32 deletions(-) diff --git a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java index dfcfc70b1ff..c8118b548d3 100644 --- a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java +++ b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java @@ -403,11 +403,18 @@ private void validateBlobSidecars(final SignedBeaconBlock block) { final boolean blobsBlockRootMatchBlockRoot = blobSidecars.stream() - .allMatch(blobSidecar -> blobSidecar.getSignedBeaconBlockHeader().hashTreeRoot().equals(block.hashTreeRoot())); + .allMatch( + blobSidecar -> + blobSidecar + .getSignedBeaconBlockHeader() + .hashTreeRoot() + .equals(block.hashTreeRoot())); if (!blobsBlockRootMatchBlockRoot) { throw new IllegalArgumentException( - String.format("Blob sidecars' signed beacon block header don't match signed block for %s", block.toLogString())); + String.format( + "Blob sidecars' signed beacon block header don't match signed block for %s", + block.toLogString())); } } diff --git a/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java b/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java index a0ae8c33872..d50226e7476 100644 --- a/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java +++ b/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java @@ -48,7 +48,6 @@ import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.generator.ChainBuilder; import tech.pegasys.teku.spec.logic.common.util.AsyncBLSSignatureVerifier; -import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager; import tech.pegasys.teku.storage.api.StorageQueryChannel; import tech.pegasys.teku.storage.api.StorageUpdateChannel; @@ -207,7 +206,6 @@ public void run_returnAllBlocksAndBlobSidecarsOnFirstRequest() { public void run_failsOnBlobSidecarsValidationFailure() { when(blobSidecarManager.isAvailabilityRequiredAtSlot(any())).thenReturn(true); - assertThat(peer.getOutstandingRequests()).isEqualTo(0); final SafeFuture future = fetcher.run(); peer.completePendingRequests(); diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index 97797a90a26..9207150b1ad 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -19,7 +19,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import static tech.pegasys.teku.infrastructure.async.SafeFutureAssert.assertThatSafeFuture; @@ -36,7 +35,6 @@ import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; -import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; import tech.pegasys.teku.spec.util.DataStructureUtil; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager.ReceivedBlobSidecarListener; @@ -47,7 +45,6 @@ import tech.pegasys.teku.statetransition.validation.BlobSidecarGossipValidator; import tech.pegasys.teku.statetransition.validation.InternalValidationResult; import tech.pegasys.teku.storage.client.RecentChainData; -import tech.pegasys.teku.storage.store.UpdatableStore; public class BlobSidecarManagerTest { private final Spec spec = TestSpecFactory.createMinimalDeneb(); diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java index e50320ba362..38564e74d87 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java @@ -15,48 +15,28 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.assertArg; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import static tech.pegasys.teku.infrastructure.async.SafeFutureAssert.assertThatSafeFuture; import com.google.common.collect.ImmutableSortedMap; import java.time.Duration; -import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import org.mockito.stubbing.OngoingStubbing; import tech.pegasys.teku.infrastructure.async.SafeFuture; -import tech.pegasys.teku.infrastructure.async.SafeFutureAssert; -import tech.pegasys.teku.infrastructure.async.StubAsyncRunner; import tech.pegasys.teku.infrastructure.async.Waiter; import tech.pegasys.teku.infrastructure.unsigned.UInt64; -import tech.pegasys.teku.kzg.KZG; -import tech.pegasys.teku.kzg.KZGCommitment; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.SpecVersion; import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; -import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment; -import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsValidationResult; import tech.pegasys.teku.spec.util.DataStructureUtil; @@ -105,7 +85,6 @@ void shouldVerifyAvailableBlobs() throws Exception { assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); - // let the tracker complete with all blobSidecars completeTrackerWith(blobSidecarsComplete); @@ -119,7 +98,7 @@ void shouldFailIfTrackerCompletesWithFailure() { prepareInitialAvailability(); final SafeFuture availabilityCheckResult = - blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); + blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); @@ -158,7 +137,6 @@ void shouldReturnNotRequiredWhenBlockIsOutsideAvailabilityWindow() throws Except final SafeFuture availabilityCheckResult = blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); Waiter.waitFor(availabilityCheckResult); @@ -246,8 +224,7 @@ private void prepareInitialAvailability() { } private void prepareInitialAvailability( - final Optional providedBlock, - final Duration timeout) { + final Optional providedBlock, final Duration timeout) { block = providedBlock.orElse(dataStructureUtil.randomSignedBeaconBlockWithCommitments(4)); blobSidecarsComplete = dataStructureUtil.randomBlobSidecarsForBlock(block); From b8339d4f50ed622b7dec2e65899e7843068a6171 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 20 Nov 2024 18:49:48 +0100 Subject: [PATCH 06/31] other fixes --- .../BlobSidecarsAndValidationResult.java | 24 ------- .../blobs/BlobSidecarsValidationResult.java | 1 - .../forkchoice/ForkChoice.java | 18 ++---- .../block/BlockManagerTest.java | 63 ------------------- ...ceBlobSidecarsAvailabilityCheckerTest.java | 32 ---------- .../forkchoice/ForkChoiceTest.java | 25 -------- 6 files changed, 5 insertions(+), 158 deletions(-) diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAndValidationResult.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAndValidationResult.java index 09bdb79ef8e..2eebaa18e12 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAndValidationResult.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAndValidationResult.java @@ -43,18 +43,6 @@ public static BlobSidecarsAndValidationResult validResult(final List blobSidecars) { - return new BlobSidecarsAndValidationResult( - BlobSidecarsValidationResult.INVALID, blobSidecars, Optional.empty()); - } - - public static BlobSidecarsAndValidationResult invalidResult( - final List blobSidecars, final Throwable cause) { - return new BlobSidecarsAndValidationResult( - BlobSidecarsValidationResult.INVALID, blobSidecars, Optional.of(cause)); - } - public static BlobSidecarsAndValidationResult notAvailable(final Throwable cause) { return new BlobSidecarsAndValidationResult( BlobSidecarsValidationResult.NOT_AVAILABLE, Collections.emptyList(), Optional.of(cause)); @@ -89,18 +77,6 @@ public boolean isNotRequired() { return validationResult.equals(BlobSidecarsValidationResult.NOT_REQUIRED); } - public boolean isInvalid() { - return validationResult.equals(BlobSidecarsValidationResult.INVALID); - } - - public boolean isNotAvailable() { - return validationResult.equals(BlobSidecarsValidationResult.NOT_AVAILABLE); - } - - public boolean isFailure() { - return isInvalid() || isNotAvailable(); - } - public boolean isSuccess() { return isValid() || isNotRequired(); } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsValidationResult.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsValidationResult.java index e40f5734d30..72257425801 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsValidationResult.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsValidationResult.java @@ -16,6 +16,5 @@ public enum BlobSidecarsValidationResult { NOT_REQUIRED, NOT_AVAILABLE, - INVALID, VALID } diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java index d8778c223b5..05660d23cb5 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java @@ -17,7 +17,6 @@ import static tech.pegasys.teku.infrastructure.logging.P2PLogger.P2P_LOG; import static tech.pegasys.teku.infrastructure.time.TimeUtilities.secondsToMillis; import static tech.pegasys.teku.spec.constants.NetworkConstants.INTERVALS_PER_SLOT; -import static tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsValidationResult.INVALID; import static tech.pegasys.teku.statetransition.forkchoice.StateRootCollector.addParentStateRoots; import com.google.common.annotations.VisibleForTesting; @@ -72,6 +71,7 @@ import tech.pegasys.teku.spec.logic.common.util.ForkChoiceUtil; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; +import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsValidationResult; import tech.pegasys.teku.statetransition.attestation.DeferredAttestations; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager; import tech.pegasys.teku.statetransition.block.BlockImportPerformance; @@ -543,18 +543,10 @@ private BlockImportResult importBlockAndState( LOG.debug("blobSidecars validation result: {}", blobSidecarsAndValidationResult::toLogString); - switch (blobSidecarsAndValidationResult.getValidationResult()) { - case NOT_AVAILABLE -> { - return BlockImportResult.failedDataAvailabilityCheckNotAvailable( - blobSidecarsAndValidationResult.getCause()); - } - case INVALID -> { - debugDataDumper.saveInvalidBlobSidecars( - blobSidecarsAndValidationResult.getBlobSidecars(), block); - return BlockImportResult.failedDataAvailabilityCheckInvalid( - blobSidecarsAndValidationResult.getCause()); - } - default -> {} + if (blobSidecarsAndValidationResult.getValidationResult() + == BlobSidecarsValidationResult.NOT_AVAILABLE) { + return BlockImportResult.failedDataAvailabilityCheckNotAvailable( + blobSidecarsAndValidationResult.getCause()); } final ForkChoiceStrategy forkChoiceStrategy = getForkChoiceStrategy(); diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/block/BlockManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/block/BlockManagerTest.java index 1095051d219..daf13b3633f 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/block/BlockManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/block/BlockManagerTest.java @@ -32,9 +32,7 @@ import static tech.pegasys.teku.infrastructure.async.SafeFutureAssert.assertThatSafeFuture; import static tech.pegasys.teku.spec.config.SpecConfig.GENESIS_SLOT; import static tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel.GOSSIP; -import static tech.pegasys.teku.spec.logic.common.statetransition.results.BlockImportResult.FailureReason.FAILED_DATA_AVAILABILITY_CHECK_INVALID; import static tech.pegasys.teku.spec.logic.common.statetransition.results.BlockImportResult.FailureReason.FAILED_DATA_AVAILABILITY_CHECK_NOT_AVAILABLE; -import static tech.pegasys.teku.spec.logic.common.statetransition.results.BlockImportResult.FailureReason.UNKNOWN_PARENT; import static tech.pegasys.teku.statetransition.block.BlockImportPerformance.ARRIVAL_EVENT_LABEL; import static tech.pegasys.teku.statetransition.block.BlockImportPerformance.BEGIN_IMPORTING_LABEL; import static tech.pegasys.teku.statetransition.block.BlockImportPerformance.COMPLETED_EVENT_LABEL; @@ -969,52 +967,6 @@ void onDeneb_shouldStoreBlockWhenBlobSidecarsNotRequired() { .isCompletedWithValueMatching(Optional::isEmpty); } - @Test - void onDeneb_shouldNotStoreBlockWhenBlobSidecarsIsInvalid() { - // If we start genesis with Deneb, 0 will be earliestBlobSidecarSlot, so started on epoch 1 - setupWithSpec(TestSpecFactory.createMinimalWithDenebForkEpoch(UInt64.valueOf(1))); - final UInt64 slotsPerEpoch = UInt64.valueOf(spec.slotsPerEpoch(UInt64.ZERO)); - incrementSlotTo(slotsPerEpoch); - - final SignedBlockAndState signedBlockAndState1 = - localChain - .chainBuilder() - .generateBlockAtSlot(currentSlot, BlockOptions.create().setGenerateRandomBlobs(true)); - final SignedBlockAndState signedBlockAndState2 = - localChain - .chainBuilder() - .generateBlockAtSlot( - currentSlot.increment(), BlockOptions.create().setGenerateRandomBlobs(true)); - - final List blobSidecars1 = - localChain.chainBuilder().getBlobSidecars(signedBlockAndState1.getRoot()); - assertThatNothingStoredForSlotRoot(signedBlockAndState1.getSlotAndBlockRoot()); - assertThat(blobSidecars1).isNotEmpty(); - assertThat(localRecentChainData.retrieveEarliestBlobSidecarSlot()) - .isCompletedWithValueMatching(Optional::isEmpty); - - final SignedBeaconBlock block1 = signedBlockAndState1.getBlock(); - final BlobSidecarsAvailabilityChecker blobSidecarsAvailabilityChecker1 = - createAvailabilityCheckerWithInvalidBlobSidecars(block1, blobSidecars1); - - assertThatBlockImport(block1) - .isCompletedWithValueMatching( - cause -> cause.getFailureReason().equals(FAILED_DATA_AVAILABILITY_CHECK_INVALID)); - verify(blobSidecarsAvailabilityChecker1).getAvailabilityCheckResult(); - assertThat(localRecentChainData.retrieveBlockByRoot(block1.getRoot())) - .isCompletedWithValue(Optional.empty()); - assertThat(localRecentChainData.getBlobSidecars(block1.getSlotAndBlockRoot())).isEmpty(); - assertThat(localRecentChainData.retrieveEarliestBlobSidecarSlot()) - .isCompletedWithValueMatching(Optional::isEmpty); - - verify(blockBlobSidecarsTrackersPool).removeAllForBlock(block1.getRoot()); - assertThat(invalidBlockRoots).doesNotContainKeys(block1.getRoot()); - - // if we receive a block building on top of block1, we should trigger unknown parent flow - assertThatBlockImport(signedBlockAndState2.getBlock()) - .isCompletedWithValueMatching(cause -> cause.getFailureReason().equals(UNKNOWN_PARENT)); - } - @Test void onDeneb_shouldNotStoreBlockWhenBlobSidecarsIsNotAvailable() { // If we start genesis with Deneb, 0 will be earliestBlobSidecarSlot, so started on epoch 1 @@ -1115,21 +1067,6 @@ private BlobSidecarsAvailabilityChecker createAvailabilityCheckerWithNotAvailabl return blobSidecarsAvailabilityChecker; } - private BlobSidecarsAvailabilityChecker createAvailabilityCheckerWithInvalidBlobSidecars( - final SignedBeaconBlock block, final List blobSidecars) { - reset(blobSidecarManager); - final BlobSidecarsAvailabilityChecker blobSidecarsAvailabilityChecker = - mock(BlobSidecarsAvailabilityChecker.class); - when(blobSidecarManager.createAvailabilityChecker(eq(block))) - .thenReturn(blobSidecarsAvailabilityChecker); - when(blobSidecarsAvailabilityChecker.getAvailabilityCheckResult()) - .thenReturn( - SafeFuture.completedFuture( - BlobSidecarsAndValidationResult.invalidResult( - blobSidecars, new RuntimeException("ouch")))); - return blobSidecarsAvailabilityChecker; - } - private void assertThatStored( final BeaconBlock beaconBlock, final List blobSidecars) { assertThat(localRecentChainData.retrieveBlockByRoot(beaconBlock.getRoot())) diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java index 38564e74d87..714c6683c1a 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java @@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableSortedMap; import java.time.Duration; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.BeforeEach; @@ -147,7 +146,6 @@ void shouldReturnNotRequiredWhenBlockIsOutsideAvailabilityWindow() throws Except private void assertNotRequired( final SafeFuture availabilityOrValidityCheck) { assertThat(availabilityOrValidityCheck) - .isCompletedWithValueMatching(result -> !result.isFailure(), "is not failure") .isCompletedWithValueMatching(result -> !result.isValid(), "is not valid") .isCompletedWithValueMatching( BlobSidecarsAndValidationResult::isNotRequired, "is not required") @@ -155,34 +153,6 @@ private void assertNotRequired( result -> result.getBlobSidecars().isEmpty(), "doesn't have blob sidecars"); } - private void assertInvalid( - final SafeFuture availabilityOrValidityCheck, - final List invalidBlobs, - final Optional cause) { - assertThat(availabilityOrValidityCheck) - .isCompletedWithValueMatching(result -> !result.isValid(), "is not valid") - .isCompletedWithValueMatching( - result -> result.getValidationResult() == BlobSidecarsValidationResult.INVALID, - "is not available") - .isCompletedWithValueMatching( - result -> result.getBlobSidecars().equals(invalidBlobs), "doesn't have blob sidecars") - .isCompletedWithValueMatching( - result -> { - if (cause.isEmpty() != result.getCause().isEmpty()) { - return false; - } - return result - .getCause() - .map( - resultCause -> - resultCause.getClass().equals(cause.get().getClass()) - && Objects.equals(resultCause.getMessage(), cause.get().getMessage()) - && Objects.equals(resultCause.getCause(), cause.get().getCause())) - .orElse(true); - }, - "matches the cause"); - } - private void assertNotAvailableDueToTimeout( final SafeFuture availabilityOrValidityCheck) { assertNotAvailable(availabilityOrValidityCheck); @@ -194,7 +164,6 @@ private void assertNotAvailableDueToTimeout( private void assertNotAvailable( final SafeFuture availabilityOrValidityCheck) { assertThat(availabilityOrValidityCheck) - .isCompletedWithValueMatching(BlobSidecarsAndValidationResult::isFailure, "is failure") .isCompletedWithValueMatching(result -> !result.isValid(), "is not valid") .isCompletedWithValueMatching( result -> result.getValidationResult() == BlobSidecarsValidationResult.NOT_AVAILABLE, @@ -206,7 +175,6 @@ private void assertNotAvailable( private void assertAvailable( final SafeFuture availabilityOrValidityCheck) { assertThat(availabilityOrValidityCheck) - .isCompletedWithValueMatching(result -> !result.isFailure(), "is not failure") .isCompletedWithValueMatching(BlobSidecarsAndValidationResult::isValid, "is valid") .isCompletedWithValueMatching( result -> result.getValidationResult() == BlobSidecarsValidationResult.VALID, diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceTest.java index fda3089f451..589b3b0e151 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceTest.java @@ -247,31 +247,6 @@ void onBlock_shouldFailIfBlobsAreNotAvailable() { verify(blobSidecarsAvailabilityChecker).getAvailabilityCheckResult(); } - @Test - void onBlock_shouldFailIfBlobsAreInvalid() { - setupWithSpec(TestSpecFactory.createMinimalDeneb()); - final SignedBlockAndState blockAndState = chainBuilder.generateBlockAtSlot(ONE); - storageSystem.chainUpdater().advanceCurrentSlotToAtLeast(blockAndState.getSlot()); - final List blobSidecars = - storageSystem - .chainStorage() - .getBlobSidecarsBySlotAndBlockRoot(blockAndState.getSlotAndBlockRoot()) - .join(); - - when(blobSidecarsAvailabilityChecker.getAvailabilityCheckResult()) - .thenReturn( - SafeFuture.completedFuture( - BlobSidecarsAndValidationResult.invalidResult(blobSidecars))); - - importBlockAndAssertFailure( - blockAndState, FailureReason.FAILED_DATA_AVAILABILITY_CHECK_INVALID); - - verify(blobSidecarManager).createAvailabilityChecker(blockAndState.getBlock()); - verify(blobSidecarsAvailabilityChecker).initiateDataAvailabilityCheck(); - verify(blobSidecarsAvailabilityChecker).getAvailabilityCheckResult(); - verify(debugDataDumper).saveInvalidBlobSidecars(blobSidecars, blockAndState.getBlock()); - } - @Test void onBlock_consensusValidationShouldNotResolveWhenDataAvailabilityFails() { setupWithSpec(TestSpecFactory.createMinimalDeneb()); From 9fb1da181148a9512bcca7b75648dc402664e33c Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 20 Nov 2024 19:17:37 +0100 Subject: [PATCH 07/31] fix stub for reference tests --- .../HistoricalBatchFetcherTest.java | 2 ++ .../forkchoice/StubBlobSidecarManager.java | 20 ++----------------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java b/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java index d50226e7476..8a32f6ad74a 100644 --- a/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java +++ b/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java @@ -210,6 +210,8 @@ public void run_failsOnBlobSidecarsValidationFailure() { final SafeFuture future = fetcher.run(); peer.completePendingRequests(); + // TODO: find a way to provoke blob sidecar validation failure + assertThat(future) .failsWithin(Duration.ZERO) .withThrowableThat() diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java index 5210b3353ad..6555c7e6d06 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java @@ -98,12 +98,6 @@ public SafeFuture getAvailabilityCheckResult() return SafeFuture.completedFuture(validateImmediately(block, blobsAndProofs)); } - @Override - public BlobSidecarsAndValidationResult validateImmediately( - final List blobSidecars) { - throw new UnsupportedOperationException("Not available in fork choice reference tests"); - } - private BlobSidecarsAndValidationResult validateImmediately( final SignedBeaconBlock block, final BlobsAndProofs blobsAndProofs) { final List kzgCommitments = @@ -113,23 +107,13 @@ private BlobSidecarsAndValidationResult validateImmediately( .map(SszKZGCommitment::getKZGCommitment) .toList(); final List blobs = blobsAndProofs.blobs.stream().map(Blob::getBytes).toList(); - try { - if (!kzg.verifyBlobKzgProofBatch(blobs, kzgCommitments, blobsAndProofs.proofs)) { - return BlobSidecarsAndValidationResult.invalidResult(Collections.emptyList()); - } - } catch (final Exception ex) { - return BlobSidecarsAndValidationResult.invalidResult(Collections.emptyList(), ex); + if (!kzg.verifyBlobKzgProofBatch(blobs, kzgCommitments, blobsAndProofs.proofs)) { + throw new RuntimeException("Invalid KZG proof"); } return BlobSidecarsAndValidationResult.validResult(Collections.emptyList()); } }; } - @Override - public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( - final SignedBeaconBlock block, final List blobSidecars) { - throw new UnsupportedOperationException("Not available in fork choice reference tests"); - } - private record BlobsAndProofs(List blobs, List proofs) {} } From fa8f4496ad8c1fd611a0b3b0f4b1b8631d609a7d Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 21 Nov 2024 11:08:13 +0100 Subject: [PATCH 08/31] fix stub for reference tests --- .../pegasys/teku/reference/ManualReferenceTestRunner.java | 2 +- .../reference/phase0/forkchoice/StubBlobSidecarManager.java | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/ManualReferenceTestRunner.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/ManualReferenceTestRunner.java index abdbc308224..fd87b4c497a 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/ManualReferenceTestRunner.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/ManualReferenceTestRunner.java @@ -63,7 +63,7 @@ public class ManualReferenceTestRunner extends Eth2ReferenceTestCase { * *

May be overridden by the ENV_DISPLAY_NAME environment variable. */ - private static final String DISPLAY_NAME = ""; + private static final String DISPLAY_NAME = "invalid_incorrect_proof"; @ParameterizedTest(name = "{0}") @MethodSource("loadReferenceTests") diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java index 6555c7e6d06..a8608be6960 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java @@ -95,7 +95,11 @@ public SafeFuture getAvailabilityCheckResult() if (blobsAndProofs == null) { return SafeFuture.completedFuture(BlobSidecarsAndValidationResult.NOT_REQUIRED); } - return SafeFuture.completedFuture(validateImmediately(block, blobsAndProofs)); + try { + return SafeFuture.completedFuture(validateImmediately(block, blobsAndProofs)); + } catch (RuntimeException e) { + return SafeFuture.completedFuture(BlobSidecarsAndValidationResult.notAvailable(e)); + } } private BlobSidecarsAndValidationResult validateImmediately( From aecce4583d4744570ef51f809bc96eece5cc40e4 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 21 Nov 2024 12:16:13 +0100 Subject: [PATCH 09/31] improvement --- .../historical/HistoricalBatchFetcher.java | 25 +++++++++++------ .../blobs/versions/deneb/BlobSidecar.java | 19 +++++++++++++ .../blobs/BlockBlobSidecarsTracker.java | 4 +++ ...ChoiceBlobSidecarsAvailabilityChecker.java | 27 ++++++++++++++++--- .../BlobSidecarGossipValidator.java | 7 +++++ .../methods/BlobSidecarsByRootValidator.java | 2 ++ 6 files changed, 72 insertions(+), 12 deletions(-) diff --git a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java index c8118b548d3..1330aa2e97d 100644 --- a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java +++ b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java @@ -401,16 +401,25 @@ private void validateBlobSidecars(final SignedBeaconBlock block) { blobSidecarsBySlotToImport.getOrDefault( block.getSlotAndBlockRoot(), Collections.emptyList()); - final boolean blobsBlockRootMatchBlockRoot = + final boolean blobsBlockHeaderRootMatchBlockRoot = blobSidecars.stream() .allMatch( - blobSidecar -> - blobSidecar - .getSignedBeaconBlockHeader() - .hashTreeRoot() - .equals(block.hashTreeRoot())); - - if (!blobsBlockRootMatchBlockRoot) { + blobSidecar -> { + final boolean signedHeaderMatchesBlock = + blobSidecar + .getSignedBeaconBlockHeader() + .hashTreeRoot() + .equals(block.hashTreeRoot()); + + if (signedHeaderMatchesBlock) { + // since we already validated block's signature, we can mark the blob sidecar's + // signature as validated + blobSidecar.markSignatureAsValidated(); + } + return signedHeaderMatchesBlock; + }); + + if (!blobsBlockHeaderRootMatchBlockRoot) { throw new IllegalArgumentException( String.format( "Blob sidecars' signed beacon block header don't match signed block for %s", diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/deneb/BlobSidecar.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/deneb/BlobSidecar.java index 812cfeb2c63..0693ce6e016 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/deneb/BlobSidecar.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/deneb/BlobSidecar.java @@ -39,6 +39,9 @@ public class BlobSidecar SignedBeaconBlockHeader, SszBytes32Vector> { + private volatile boolean kzgAndInclusionProofValidated = false; + private volatile boolean signatureValidated = false; + BlobSidecar(final BlobSidecarSchema blobSidecarSchema, final TreeNode backingTreeNode) { super(blobSidecarSchema, backingTreeNode); } @@ -139,6 +142,22 @@ public String toLogString() { getKZGProof().toAbbreviatedString()); } + public boolean isKzgAndInclusionProofValidated() { + return kzgAndInclusionProofValidated; + } + + public void markKzgAndInclusionProofAsValidated() { + kzgAndInclusionProofValidated = true; + } + + public boolean isSignatureValidated() { + return signatureValidated; + } + + public void markSignatureAsValidated() { + signatureValidated = true; + } + @Override public BlobSidecarSchema getSchema() { return (BlobSidecarSchema) super.getSchema(); diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java index 721211524f3..a9dc3c9cadc 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java @@ -142,6 +142,10 @@ public boolean add(final BlobSidecar blobSidecar) { blobSidecar.getBlockRoot().equals(slotAndBlockRoot.getBlockRoot()), "Wrong blobSidecar block root"); + checkArgument( + blobSidecar.isKzgAndInclusionProofValidated(), + "BlobSidecar must be validated before adding"); + if (blobSidecarsComplete.isDone()) { // already completed return false; diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java index 0985b2227ae..ac08c648bed 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java @@ -21,6 +21,8 @@ import tech.pegasys.teku.infrastructure.exceptions.ExceptionUtil; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; import tech.pegasys.teku.statetransition.blobs.BlockBlobSidecarsTracker; @@ -67,10 +69,7 @@ public boolean initiateDataAvailabilityCheck() { blockBlobSidecarsTracker .getCompletionFuture() .orTimeout(waitForTrackerCompletionTimeout) - .thenApply( - __ -> - BlobSidecarsAndValidationResult.validResult( - List.copyOf(blockBlobSidecarsTracker.getBlobSidecars().values()))) + .thenApply(__ -> validateCompletedBlobSidecars()) .exceptionallyCompose( error -> ExceptionUtil.getCause(error, TimeoutException.class) @@ -88,6 +87,26 @@ public boolean initiateDataAvailabilityCheck() { return true; } + private BlobSidecarsAndValidationResult validateCompletedBlobSidecars() { + final List blobSidecars = + List.copyOf(blockBlobSidecarsTracker.getBlobSidecars().values()); + final SignedBeaconBlock block = blockBlobSidecarsTracker.getBlock().orElseThrow(); + + for (final BlobSidecar blobSidecar : blobSidecars) { + if (!blobSidecar.isKzgAndInclusionProofValidated()) { + return BlobSidecarsAndValidationResult.notAvailable( + new IllegalStateException("Blob sidecars are not validated")); + } + if (!blobSidecar.isSignatureValidated() + && blobSidecar.getSignedBeaconBlockHeader().hashTreeRoot() != block.hashTreeRoot()) { + return BlobSidecarsAndValidationResult.notAvailable( + new IllegalStateException("Blob sidecars block header does not match signed block")); + } + } + + return BlobSidecarsAndValidationResult.validResult(blobSidecars); + } + @Override public SafeFuture getAvailabilityCheckResult() { return validationResult; diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java index 9baad8da958..b13094e11d1 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java @@ -159,6 +159,7 @@ public SafeFuture validate(final BlobSidecar blobSidec // Optimization: If we have already completely verified BlobSidecar with the same // SignedBlockHeader, we can skip most steps and jump to shortened validation if (validSignedBlockHeaders.contains(blobSidecar.getSignedBeaconBlockHeader().hashTreeRoot())) { + blobSidecar.markSignatureAsValidated(); return validateBlobSidecarWithKnownValidHeader(blobSidecar, blockHeader); } @@ -225,6 +226,8 @@ public SafeFuture validate(final BlobSidecar blobSidec return completedFuture(reject("BlobSidecar does not pass kzg validation")); } + blobSidecar.markKzgAndInclusionProofAsValidated(); + return gossipValidationHelper .getParentStateInBlockEpoch( parentBlockSlot, blockHeader.getParentRoot(), blockHeader.getSlot()) @@ -260,6 +263,8 @@ public SafeFuture validate(final BlobSidecar blobSidec return reject("BlobSidecar block header signature is invalid."); } + blobSidecar.markSignatureAsValidated(); + /* * Checking it again at the very end because whole method is not synchronized * @@ -317,6 +322,8 @@ private SafeFuture validateBlobSidecarWithKnownValidHe reject("BlobSidecar block header does not descend from finalized checkpoint")); } + blobSidecar.markKzgAndInclusionProofAsValidated(); + /* * [IGNORE] The sidecar is the first sidecar for the tuple (block_header.slot, block_header.proposer_index, blob_sidecar.index) * with valid header signature, sidecar inclusion proof, and kzg proof. diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootValidator.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootValidator.java index ea4e96fe2b0..71032550086 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootValidator.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootValidator.java @@ -47,5 +47,7 @@ public void validate(final BlobSidecar blobSidecar) { verifyInclusionProof(blobSidecar); verifyKzg(blobSidecar); + + blobSidecar.markKzgAndInclusionProofAsValidated(); } } From 8a1ddff3baa53c0c41c5cd8b356038d73d1e3866 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 21 Nov 2024 13:29:53 +0100 Subject: [PATCH 10/31] fix comparison --- .../ForkChoiceBlobSidecarsAvailabilityChecker.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java index ac08c648bed..cdff49d3963 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java @@ -98,7 +98,10 @@ private BlobSidecarsAndValidationResult validateCompletedBlobSidecars() { new IllegalStateException("Blob sidecars are not validated")); } if (!blobSidecar.isSignatureValidated() - && blobSidecar.getSignedBeaconBlockHeader().hashTreeRoot() != block.hashTreeRoot()) { + && !blobSidecar + .getSignedBeaconBlockHeader() + .hashTreeRoot() + .equals(block.hashTreeRoot())) { return BlobSidecarsAndValidationResult.notAvailable( new IllegalStateException("Blob sidecars block header does not match signed block")); } From b9d599bfef3ec1b5d9eb9af0ad7e217c595794d6 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 22 Nov 2024 11:31:48 +0100 Subject: [PATCH 11/31] mark locally fetched blobs as validated --- .../BlockBlobSidecarsTrackersPoolImpl.java | 4 ++ ...BlockBlobSidecarsTrackersPoolImplTest.java | 62 ++++++++++++++++++- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java index e056f36c1d6..92d12c6cf39 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java @@ -722,6 +722,10 @@ private synchronized SafeFuture fetchMissingContentFromLocalEL( beaconBlockBodyDeneb, signedBeaconBlockHeader); + blobSidecar.markSignatureAsValidated(); + // assume kzg validation done by local EL + blobSidecar.markKzgAndInclusionProofAsValidated(); + onNewBlobSidecar(blobSidecar, LOCAL_EL); } }); diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java index 2dad121a9eb..317cb65c07a 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java @@ -14,7 +14,9 @@ package tech.pegasys.teku.statetransition.util; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -154,6 +156,19 @@ public void onNewBlock_addTrackerWithBlock() { assertBlobSidecarsTrackersCount(1); } + @Test + public void onNewBlobSidecar_shouldThrowIfNotMarkedKzgAndInclusionProofAsValidated() { + final BlobSidecar blobSidecar = + dataStructureUtil + .createRandomBlobSidecarBuilder() + .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) + .build(); + + assertThrows( + IllegalArgumentException.class, + () -> blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP)); + } + @Test public void onNewBlobSidecar_addTrackerWithBlobSidecarIgnoringDuplicates() { final BlobSidecar blobSidecar = @@ -162,6 +177,8 @@ public void onNewBlobSidecar_addTrackerWithBlobSidecarIgnoringDuplicates() { .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); + blobSidecar.markKzgAndInclusionProofAsValidated(); + blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); assertThat( @@ -190,6 +207,8 @@ public void onNewBlobSidecar_shouldIgnoreDuplicates() { .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); + blobSidecar.markKzgAndInclusionProofAsValidated(); + blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); @@ -225,6 +244,10 @@ public void onNewBlobSidecar_shouldMarkForEquivocationAndPublishWhenOriginIsLoca .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); + blobSidecar1.markKzgAndInclusionProofAsValidated(); + blobSidecar2.markKzgAndInclusionProofAsValidated(); + blobSidecar3.markKzgAndInclusionProofAsValidated(); + when(blobSidecarGossipValidator.markForEquivocation(blobSidecar1)).thenReturn(true); blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar1, RemoteOrigin.LOCAL_EL); @@ -246,6 +269,8 @@ public void onNewBlobSidecar_shouldPublishWhenOriginIsLocalELAndEquivocating() { .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); + blobSidecar1.markKzgAndInclusionProofAsValidated(); + when(blobSidecarGossipValidator.markForEquivocation(blobSidecar1)).thenReturn(false); blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar1, RemoteOrigin.LOCAL_EL); @@ -265,6 +290,8 @@ public void onNewBlobSidecar_shouldNotPublishWhenOriginIsLocalELIsNotCurrentSlot .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); + blobSidecar1.markKzgAndInclusionProofAsValidated(); + when(blobSidecarGossipValidator.markForEquivocation(blobSidecar1)).thenReturn(false); blockBlobSidecarsTrackersPool.onSlot(currentSlot.plus(1)); @@ -352,6 +379,8 @@ public void onNewBlobSidecarOnNewBlock_addTrackerWithBothBlockAndBlobSidecar() { final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(currentSlot); final BlobSidecar blobSidecar = dataStructureUtil.randomBlobSidecarsForBlock(block).get(0); + blobSidecar.markKzgAndInclusionProofAsValidated(); + blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); blockBlobSidecarsTrackersPool.onNewBlock(block, Optional.empty()); @@ -394,6 +423,10 @@ public void twoOnNewBlobSidecar_addTrackerWithBothBlobSidecars() { .index(UInt64.ONE) .build(); + blobSidecar0.markKzgAndInclusionProofAsValidated(); + blobSidecar1.markKzgAndInclusionProofAsValidated(); + blobSidecar1bis.markKzgAndInclusionProofAsValidated(); + blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar0, RemoteOrigin.GOSSIP); blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar1, RemoteOrigin.GOSSIP); blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar1bis, RemoteOrigin.GOSSIP); @@ -447,6 +480,8 @@ public void onCompletedBlockAndBlobSidecars_shouldCreateTrackerIgnoringHistorica final List blobSidecars = dataStructureUtil.randomBlobSidecarsForBlock(block); + blobSidecars.forEach(BlobSidecar::markKzgAndInclusionProofAsValidated); + blockBlobSidecarsTrackersPool.onCompletedBlockAndBlobSidecars(block, blobSidecars); assertThat(asyncRunner.hasDelayedActions()).isFalse(); @@ -714,8 +749,21 @@ void shouldFetchMissingBlobSidecarsFromLocalELFirst() { engineGetBlobsResponse.complete(blobAndProofsFromEL); - verify(tracker).add(missingBlobSidecars.getFirst()); // 0 - verify(tracker).add(missingBlobSidecars.getLast()); // 2 + verify(tracker) + .add( + argThat( + b -> + b.equals(missingBlobSidecars.getFirst()) + && b.isKzgAndInclusionProofValidated() + && b.isSignatureValidated())); // 0 + verify(tracker) + .add( + argThat( + b -> + b.equals(missingBlobSidecars.getLast()) + && b.isKzgAndInclusionProofValidated() + && b.isSignatureValidated())); // 2) + verify(tracker, times(2)).add(any()); assertStats("blob_sidecar", "local_el_fetch", 3); @@ -927,6 +975,8 @@ public void shouldFetchContentWhenBlobSidecarIsNotForCurrentSlot() { .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(slot)) .build(); + blobSidecar.markKzgAndInclusionProofAsValidated(); + blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); assertThat(asyncRunner.hasDelayedActions()).isTrue(); @@ -1245,6 +1295,8 @@ void stats_onNewBlobSidecar() { .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); + blobSidecar.markKzgAndInclusionProofAsValidated(); + // new from gossip blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); @@ -1270,6 +1322,8 @@ void stats_onNewBlobSidecar() { dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot.increment())) .build(); + blobSidecar2.markKzgAndInclusionProofAsValidated(); + // new from RPC blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar2, RemoteOrigin.RPC); @@ -1285,6 +1339,8 @@ void stats_onNewBlobSidecar() { dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot.increment())) .build(); + blobSidecar3.markKzgAndInclusionProofAsValidated(); + // new from LOCAL_EL blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar3, RemoteOrigin.LOCAL_EL); assertStats("blob_sidecar", "local_el", 1); @@ -1349,6 +1405,8 @@ void stats_onNewBlock() { .signedBeaconBlockHeader(block4.asHeader()) .build(); + blobSidecar4.markKzgAndInclusionProofAsValidated(); + blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar4, RemoteOrigin.RPC); blockBlobSidecarsTrackersPool.onNewBlock(block4, Optional.of(RemoteOrigin.GOSSIP)); From 8a76ce83ec0ed81b197db3788f718562cd6a9866 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 22 Nov 2024 12:22:13 +0100 Subject: [PATCH 12/31] update BlockBlobSidecarsTrackerTest --- .../blobs/BlockBlobSidecarsTrackerTest.java | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java index d5d702acbc9..791d6201f30 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.async.SafeFutureAssert; @@ -62,6 +63,11 @@ public class BlockBlobSidecarsTrackerTest { blobSidecar -> new BlobIdentifier(blobSidecar.getBlockRoot(), blobSidecar.getIndex())) .collect(Collectors.toList()); + @BeforeEach + void setUp() { + blobSidecarsForBlock.forEach(BlobSidecar::markKzgAndInclusionProofAsValidated); + } + @Test void isNotCompletedJustAfterCreation() { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = @@ -165,6 +171,21 @@ void getCompletionFuture_returnsIndependentFutures() { SafeFutureAssert.assertThatSafeFuture(completionFuture3).isCompleted(); } + @Test + void add_shouldFailIfBlobsIsNotMarkedAsKzgAndInclusionProofValidated() { + final BlockBlobSidecarsTracker blockBlobSidecarsTracker = + new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + final BlobSidecar toAdd = + dataStructureUtil + .createRandomBlobSidecarBuilder() + .signedBeaconBlockHeader(block.asHeader()) + .build(); + + assertThatThrownBy(() -> blockBlobSidecarsTracker.add(toAdd)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("BlobSidecar must be validated before adding"); + } + @Test void add_shouldWorkTillCompletionWhenAddingBlobsBeforeBlockIsSet() { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = @@ -346,6 +367,8 @@ void getMissingBlobSidecars_shouldRespectMaxBlobsPerBlock() { .index(UInt64.valueOf(100)) .build(); + toAdd.markKzgAndInclusionProofAsValidated(); + blockBlobSidecarsTracker.add(toAdd); final List knownMissing = @@ -422,10 +445,13 @@ void enableBlockImportOnCompletion_shouldImportOnlyOnceWhenCalled() { } private BlobSidecar createBlobSidecar(final UInt64 index) { - return dataStructureUtil - .createRandomBlobSidecarBuilder() - .signedBeaconBlockHeader(block.asHeader()) - .index(index) - .build(); + final BlobSidecar blobSidecar = + dataStructureUtil + .createRandomBlobSidecarBuilder() + .signedBeaconBlockHeader(block.asHeader()) + .index(index) + .build(); + blobSidecar.markKzgAndInclusionProofAsValidated(); + return blobSidecar; } } From 2a4c123a9303d6eeba1ea67bd973b08391cb6c42 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 22 Nov 2024 14:23:56 +0100 Subject: [PATCH 13/31] improve test --- .../tech/pegasys/teku/reference/ManualReferenceTestRunner.java | 2 +- .../util/BlockBlobSidecarsTrackersPoolImplTest.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/ManualReferenceTestRunner.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/ManualReferenceTestRunner.java index fd87b4c497a..abdbc308224 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/ManualReferenceTestRunner.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/ManualReferenceTestRunner.java @@ -63,7 +63,7 @@ public class ManualReferenceTestRunner extends Eth2ReferenceTestCase { * *

May be overridden by the ENV_DISPLAY_NAME environment variable. */ - private static final String DISPLAY_NAME = "invalid_incorrect_proof"; + private static final String DISPLAY_NAME = ""; @ParameterizedTest(name = "{0}") @MethodSource("loadReferenceTests") diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java index 317cb65c07a..ee8b0a6906b 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java @@ -164,6 +164,8 @@ public void onNewBlobSidecar_shouldThrowIfNotMarkedKzgAndInclusionProofAsValidat .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); + assertThat(blobSidecar.isKzgAndInclusionProofValidated()).isFalse(); + assertThrows( IllegalArgumentException.class, () -> blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP)); From f1d5186be026ef3fcadb1fc715084e3a268d2b82 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 22 Nov 2024 18:54:04 +0100 Subject: [PATCH 14/31] change approach --- .../BlockOperationSelectorFactoryTest.java | 6 +- .../handlers/v1/events/EventSubscriber.java | 2 +- .../forkchoice/StubBlobSidecarManager.java | 14 +- .../blobs/versions/deneb/BlobSidecar.java | 19 ++- .../logic/common/helpers/MiscHelpers.java | 14 -- .../BlobSidecarsAndValidationResult.java | 24 +++ .../blobs/BlobSidecarsValidationResult.java | 1 + .../deneb/helpers/MiscHelpersDeneb.java | 143 ++++++------------ .../helpers/MiscHelpersDenebPropertyTest.java | 2 +- .../deneb/helpers/MiscHelpersDenebTest.java | 130 +--------------- .../blobs/BlobSidecarManagerImpl.java | 6 +- .../blobs/BlockBlobSidecarsTracker.java | 4 - .../forkchoice/ForkChoice.java | 17 ++- ...ChoiceBlobSidecarsAvailabilityChecker.java | 20 ++- .../BlockBlobSidecarsTrackersPoolImpl.java | 29 ++-- .../BlobSidecarGossipValidator.java | 11 +- .../blobs/BlobSidecarManagerTest.java | 3 + .../blobs/BlockBlobSidecarsTrackerTest.java | 21 +-- ...ceBlobSidecarsAvailabilityCheckerTest.java | 92 ++++++++++- ...BlockBlobSidecarsTrackersPoolImplTest.java | 64 +------- .../BlobSidecarGossipValidatorTest.java | 14 +- .../AbstractBlobSidecarsValidator.java | 2 +- .../methods/BlobSidecarsByRootValidator.java | 2 - .../beaconchain/BeaconChainController.java | 1 + 24 files changed, 267 insertions(+), 374 deletions(-) diff --git a/beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactoryTest.java b/beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactoryTest.java index cd18566109b..35c58e2a331 100644 --- a/beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactoryTest.java +++ b/beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactoryTest.java @@ -801,7 +801,8 @@ void shouldCreateBlobSidecarsForBlockContents() { assertThat(blobSidecar.getSszKZGCommitment()) .isEqualTo(expectedCommitments.get(index)); // verify the merkle proof - assertThat(miscHelpersDeneb.verifyBlobSidecarMerkleProof(blobSidecar)).isTrue(); + assertThat(miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(blobSidecar)) + .isTrue(); }); } @@ -921,7 +922,8 @@ void shouldCreateBlobSidecarsForBlindedBlock(final boolean useLocalFallback) { assertThat(blobSidecar.getSszKZGCommitment()) .isEqualTo(expectedCommitments.get(index)); // verify the merkle proof - assertThat(miscHelpersDeneb.verifyBlobSidecarMerkleProof(blobSidecar)).isTrue(); + assertThat(miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(blobSidecar)) + .isTrue(); }); } diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriber.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriber.java index 420dee5349c..4be2966528c 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriber.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriber.java @@ -45,7 +45,7 @@ public class EventSubscriber { private final AtomicBoolean processingQueue; private final AsyncRunner asyncRunner; private final AtomicLong excessiveQueueingDisconnectionTime = new AtomicLong(Long.MAX_VALUE); - private volatile AtomicInteger successiveFailureCounter = new AtomicInteger(0); + private final AtomicInteger successiveFailureCounter = new AtomicInteger(0); public EventSubscriber( final List eventTypes, diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java index a8608be6960..f2d520a1fb4 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java @@ -95,11 +95,7 @@ public SafeFuture getAvailabilityCheckResult() if (blobsAndProofs == null) { return SafeFuture.completedFuture(BlobSidecarsAndValidationResult.NOT_REQUIRED); } - try { - return SafeFuture.completedFuture(validateImmediately(block, blobsAndProofs)); - } catch (RuntimeException e) { - return SafeFuture.completedFuture(BlobSidecarsAndValidationResult.notAvailable(e)); - } + return SafeFuture.completedFuture(validateImmediately(block, blobsAndProofs)); } private BlobSidecarsAndValidationResult validateImmediately( @@ -111,8 +107,12 @@ private BlobSidecarsAndValidationResult validateImmediately( .map(SszKZGCommitment::getKZGCommitment) .toList(); final List blobs = blobsAndProofs.blobs.stream().map(Blob::getBytes).toList(); - if (!kzg.verifyBlobKzgProofBatch(blobs, kzgCommitments, blobsAndProofs.proofs)) { - throw new RuntimeException("Invalid KZG proof"); + try { + if (!kzg.verifyBlobKzgProofBatch(blobs, kzgCommitments, blobsAndProofs.proofs)) { + return BlobSidecarsAndValidationResult.invalidResult(Collections.emptyList()); + } + } catch (final Exception ex) { + return BlobSidecarsAndValidationResult.invalidResult(Collections.emptyList(), ex); } return BlobSidecarsAndValidationResult.validResult(Collections.emptyList()); } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/deneb/BlobSidecar.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/deneb/BlobSidecar.java index 0693ce6e016..aae359ffd10 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/deneb/BlobSidecar.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/deneb/BlobSidecar.java @@ -39,7 +39,8 @@ public class BlobSidecar SignedBeaconBlockHeader, SszBytes32Vector> { - private volatile boolean kzgAndInclusionProofValidated = false; + private volatile boolean kzgValidated = false; + private volatile boolean kzgCommitmentInclusionProofValidated = false; private volatile boolean signatureValidated = false; BlobSidecar(final BlobSidecarSchema blobSidecarSchema, final TreeNode backingTreeNode) { @@ -142,18 +143,26 @@ public String toLogString() { getKZGProof().toAbbreviatedString()); } - public boolean isKzgAndInclusionProofValidated() { - return kzgAndInclusionProofValidated; + public boolean isKzgValidated() { + return kzgValidated; } - public void markKzgAndInclusionProofAsValidated() { - kzgAndInclusionProofValidated = true; + public boolean isKzgCommitmentInclusionProofValidated() { + return kzgCommitmentInclusionProofValidated; } public boolean isSignatureValidated() { return signatureValidated; } + public void markKzgAsValidated() { + kzgValidated = true; + } + + public void markKzgCommitmentInclusionProofAsValidated() { + kzgCommitmentInclusionProofValidated = true; + } + public void markSignatureAsValidated() { signatureValidated = true; } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java index 559f3a01c87..a3d452c791e 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java @@ -380,20 +380,6 @@ public boolean verifyBlobKzgProofBatch(final KZG kzg, final List bl return false; } - public void validateBlobSidecarsBatchAgainstBlock( - final List blobSidecars, - final BeaconBlock block, - final List kzgCommitmentsFromBlock) { - throw new UnsupportedOperationException("No Blob Sidecars before Deneb"); - } - - public void verifyBlobSidecarCompleteness( - final List verifiedBlobSidecars, - final List kzgCommitmentsFromBlock) - throws IllegalArgumentException { - throw new UnsupportedOperationException("No Blob Sidecars before Deneb"); - } - public VersionedHash kzgCommitmentToVersionedHash(final KZGCommitment kzgCommitment) { throw new UnsupportedOperationException("No KZGCommitments before Deneb"); } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAndValidationResult.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAndValidationResult.java index 2eebaa18e12..09bdb79ef8e 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAndValidationResult.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAndValidationResult.java @@ -43,6 +43,18 @@ public static BlobSidecarsAndValidationResult validResult(final List blobSidecars) { + return new BlobSidecarsAndValidationResult( + BlobSidecarsValidationResult.INVALID, blobSidecars, Optional.empty()); + } + + public static BlobSidecarsAndValidationResult invalidResult( + final List blobSidecars, final Throwable cause) { + return new BlobSidecarsAndValidationResult( + BlobSidecarsValidationResult.INVALID, blobSidecars, Optional.of(cause)); + } + public static BlobSidecarsAndValidationResult notAvailable(final Throwable cause) { return new BlobSidecarsAndValidationResult( BlobSidecarsValidationResult.NOT_AVAILABLE, Collections.emptyList(), Optional.of(cause)); @@ -77,6 +89,18 @@ public boolean isNotRequired() { return validationResult.equals(BlobSidecarsValidationResult.NOT_REQUIRED); } + public boolean isInvalid() { + return validationResult.equals(BlobSidecarsValidationResult.INVALID); + } + + public boolean isNotAvailable() { + return validationResult.equals(BlobSidecarsValidationResult.NOT_AVAILABLE); + } + + public boolean isFailure() { + return isInvalid() || isNotAvailable(); + } + public boolean isSuccess() { return isValid() || isNotRequired(); } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsValidationResult.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsValidationResult.java index 72257425801..e40f5734d30 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsValidationResult.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsValidationResult.java @@ -16,5 +16,6 @@ public enum BlobSidecarsValidationResult { NOT_REQUIRED, NOT_AVAILABLE, + INVALID, VALID } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java index c1a09477d72..e7a7592e050 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java @@ -13,14 +13,12 @@ package tech.pegasys.teku.spec.logic.versions.deneb.helpers; -import static com.google.common.base.Preconditions.checkArgument; import static tech.pegasys.teku.spec.config.SpecConfigDeneb.VERSIONED_HASH_VERSION_KZG; import java.util.ArrayList; import java.util.List; import java.util.NoSuchElementException; import java.util.Optional; -import java.util.stream.IntStream; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.crypto.Hash; @@ -35,7 +33,6 @@ import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarSchema; -import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBody; import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodySchemaDeneb; @@ -84,10 +81,20 @@ public MiscHelpersDeneb( */ @Override public boolean verifyBlobKzgProof(final KZG kzg, final BlobSidecar blobSidecar) { - return kzg.verifyBlobKzgProof( - blobSidecar.getBlob().getBytes(), - blobSidecar.getKZGCommitment(), - blobSidecar.getKZGProof()); + if (blobSidecar.isKzgValidated()) { + return true; + } + final boolean result = + kzg.verifyBlobKzgProof( + blobSidecar.getBlob().getBytes(), + blobSidecar.getKZGCommitment(), + blobSidecar.getKZGProof()); + + if (result) { + blobSidecar.markKzgAsValidated(); + } + + return result; } /** @@ -105,90 +112,28 @@ public boolean verifyBlobKzgProofBatch(final KZG kzg, final List bl final List kzgCommitments = new ArrayList<>(); final List kzgProofs = new ArrayList<>(); - blobSidecars.forEach( - blobSidecar -> { - blobs.add(blobSidecar.getBlob().getBytes()); - kzgCommitments.add(blobSidecar.getKZGCommitment()); - kzgProofs.add(blobSidecar.getKZGProof()); - }); - - return kzg.verifyBlobKzgProofBatch(blobs, kzgCommitments, kzgProofs); - } - - /** - * Validates blob sidecars against block. We need to check block root and kzg commitment, it's - * enough to guarantee BlobSidecars belong to block - * - * @param blobSidecars blob sidecars to validate - * @param block block to validate blob sidecar against - * @param kzgCommitmentsFromBlock kzg commitments from block. They could be extracted from block - * but since we already have them we can avoid extracting them again. - */ - @Override - public void validateBlobSidecarsBatchAgainstBlock( - final List blobSidecars, - final BeaconBlock block, - final List kzgCommitmentsFromBlock) { - - final String slotAndBlockRoot = block.getSlotAndBlockRoot().toLogString(); - - blobSidecars.forEach( - blobSidecar -> { - final UInt64 blobIndex = blobSidecar.getIndex(); - - checkArgument( - blobSidecar.getBlockRoot().equals(block.getRoot()), - "Block and blob sidecar root mismatch for %s, blob index %s", - slotAndBlockRoot, - blobIndex); - - final KZGCommitment kzgCommitmentFromBlock; - - try { - kzgCommitmentFromBlock = kzgCommitmentsFromBlock.get(blobIndex.intValue()); - } catch (IndexOutOfBoundsException e) { - throw new IllegalArgumentException( - String.format( - "Blob sidecar index out of bound with respect to block %s, blob index %s", - slotAndBlockRoot, blobIndex)); - } + blobSidecars.stream() + .filter(blobSidecar -> !blobSidecar.isKzgValidated()) + .forEach( + blobSidecar -> { + blobs.add(blobSidecar.getBlob().getBytes()); + kzgCommitments.add(blobSidecar.getKZGCommitment()); + kzgProofs.add(blobSidecar.getKZGProof()); + }); - checkArgument( - blobSidecar.getKZGCommitment().equals(kzgCommitmentFromBlock), - "Block and blob sidecar kzg commitments mismatch for %s, blob index %s", - slotAndBlockRoot, - blobIndex); - }); - } + if (blobs.isEmpty()) { + return true; + } - /** - * Verifies that blob sidecars are complete and with expected indexes - * - * @param completeVerifiedBlobSidecars blob sidecars to verify, It is assumed that it is an - * ordered list based on BlobSidecar index - * @param kzgCommitmentsFromBlock kzg commitments from block. - */ - @Override - public void verifyBlobSidecarCompleteness( - final List completeVerifiedBlobSidecars, - final List kzgCommitmentsFromBlock) - throws IllegalArgumentException { - checkArgument( - completeVerifiedBlobSidecars.size() == kzgCommitmentsFromBlock.size(), - "Blob sidecars are not complete"); + final boolean result = kzg.verifyBlobKzgProofBatch(blobs, kzgCommitments, kzgProofs); - IntStream.range(0, completeVerifiedBlobSidecars.size()) - .forEach( - index -> { - final BlobSidecar blobSidecar = completeVerifiedBlobSidecars.get(index); - final UInt64 blobIndex = blobSidecar.getIndex(); + if (result) { + blobSidecars.stream() + .filter(blobSidecar -> !blobSidecar.isKzgValidated()) + .forEach(BlobSidecar::markKzgAsValidated); + } - checkArgument( - blobIndex.longValue() == index, - "Blob sidecar index mismatch, expected %s, got %s", - index, - blobIndex); - }); + return result; } /** @@ -262,12 +207,22 @@ public BlobSidecar constructBlobSidecar( index, blob, commitment, proof, signedBeaconBlock.asHeader(), kzgCommitmentInclusionProof); } - public boolean verifyBlobSidecarMerkleProof(final BlobSidecar blobSidecar) { - return predicates.isValidMerkleBranch( - blobSidecar.getSszKZGCommitment().hashTreeRoot(), - blobSidecar.getKzgCommitmentInclusionProof(), - SpecConfigDeneb.required(specConfig).getKzgCommitmentInclusionProofDepth(), - getBlobSidecarKzgCommitmentGeneralizedIndex(blobSidecar.getIndex()), - blobSidecar.getBlockBodyRoot()); + public boolean verifyBlobKzgCommitmentInclusionProof(final BlobSidecar blobSidecar) { + if (blobSidecar.isKzgCommitmentInclusionProofValidated()) { + return true; + } + final boolean result = + predicates.isValidMerkleBranch( + blobSidecar.getSszKZGCommitment().hashTreeRoot(), + blobSidecar.getKzgCommitmentInclusionProof(), + SpecConfigDeneb.required(specConfig).getKzgCommitmentInclusionProofDepth(), + getBlobSidecarKzgCommitmentGeneralizedIndex(blobSidecar.getIndex()), + blobSidecar.getBlockBodyRoot()); + + if (result) { + blobSidecar.markKzgCommitmentInclusionProofAsValidated(); + } + + return result; } } diff --git a/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java b/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java index f29ee89cd4b..746c802a9d6 100644 --- a/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java +++ b/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java @@ -119,7 +119,7 @@ void fuzzConstructBlobSidecarAndVerifyMerkleProof( try { final BlobSidecar blobSidecar = miscHelpers.constructBlobSidecar(signedBeaconBlock, index, blob, proof); - miscHelpers.verifyBlobSidecarMerkleProof(blobSidecar); + miscHelpers.verifyBlobKzgCommitmentInclusionProof(blobSidecar); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalArgumentException.class); } diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java index d6712b89470..6fab0c8710d 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java @@ -18,7 +18,6 @@ import static org.assertj.core.api.Assumptions.assumeThat; import static tech.pegasys.teku.spec.config.SpecConfigDeneb.VERSIONED_HASH_VERSION_KZG; -import java.util.ArrayList; import java.util.List; import org.apache.tuweni.bytes.Bytes32; import org.junit.jupiter.api.Test; @@ -69,124 +68,6 @@ public void versionedHash() { assertThat(actual).isEqualTo(VERSIONED_HASH); } - @Test - void validateBlobSidecarsAgainstBlock_shouldNotThrowOnValidBlobSidecar() { - final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(); - final List blobSidecars = dataStructureUtil.randomBlobSidecarsForBlock(block); - - // make sure we are testing something - assertThat(blobSidecars).isNotEmpty(); - - miscHelpersDeneb.validateBlobSidecarsBatchAgainstBlock( - blobSidecars, - block.getMessage(), - BeaconBlockBodyDeneb.required(block.getMessage().getBody()).getBlobKzgCommitments().stream() - .map(SszKZGCommitment::getKZGCommitment) - .toList()); - } - - @Test - void validateBlobSidecarsAgainstBlock_shouldThrowOnNotMatchingBlockRoot() { - - final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(); - - final List kzgCommitments = - BeaconBlockBodyDeneb.required(block.getMessage().getBody()).getBlobKzgCommitments().stream() - .map(SszKZGCommitment::getKZGCommitment) - .toList(); - - // make sure we are testing something - assertThat(kzgCommitments).isNotEmpty(); - - final int indexToBeAltered = - Math.toIntExact(dataStructureUtil.randomPositiveLong(kzgCommitments.size())); - - // let's create blobs with only one altered with the given alteration - final List blobSidecars = - dataStructureUtil.randomBlobSidecarsForBlock( - block, - (index, randomBlobSidecarBuilder) -> { - if (!index.equals(indexToBeAltered)) { - return randomBlobSidecarBuilder; - } - - return randomBlobSidecarBuilder.signedBeaconBlockHeader( - dataStructureUtil.randomSignedBeaconBlockHeader( - block.getSlot(), block.getProposerIndex())); - }); - - assertThatThrownBy( - () -> - miscHelpersDeneb.validateBlobSidecarsBatchAgainstBlock( - blobSidecars, block.getMessage(), kzgCommitments)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageStartingWith("Block and blob sidecar root mismatch for") - .hasMessageEndingWith("blob index %s", indexToBeAltered); - } - - @Test - void validateBlobSidecarsAgainstBlock_shouldThrowOnNotMatchingKzgCommitments() { - - final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(); - - final List kzgCommitments = - BeaconBlockBodyDeneb.required(block.getMessage().getBody()).getBlobKzgCommitments().stream() - .map(SszKZGCommitment::getKZGCommitment) - .toList(); - - // make sure we are testing something - assertThat(kzgCommitments).isNotEmpty(); - - final int indexToBeAltered = - Math.toIntExact(dataStructureUtil.randomPositiveLong(kzgCommitments.size())); - final List alteredKzgCommitments = new ArrayList<>(kzgCommitments); - alteredKzgCommitments.remove(indexToBeAltered); - alteredKzgCommitments.add(indexToBeAltered, dataStructureUtil.randomKZGCommitment()); - - final List blobSidecars = dataStructureUtil.randomBlobSidecarsForBlock(block); - - assertThatThrownBy( - () -> - miscHelpersDeneb.validateBlobSidecarsBatchAgainstBlock( - blobSidecars, block.getMessage(), List.of())) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageStartingWith("Blob sidecar index out of bound with respect to block") - .hasMessageEndingWith("blob index %s", 0); - - assertThatThrownBy( - () -> - miscHelpersDeneb.validateBlobSidecarsBatchAgainstBlock( - blobSidecars, block.getMessage(), alteredKzgCommitments)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageStartingWith("Block and blob sidecar kzg commitments mismatch for") - .hasMessageEndingWith("blob index %s", indexToBeAltered); - } - - @Test - void verifyBlobSidecarCompleteness_shouldThrowWhenSizesDoNotMatch() { - assertThatThrownBy( - () -> - miscHelpersDeneb.verifyBlobSidecarCompleteness( - dataStructureUtil.randomBlobSidecars(1), - List.of( - dataStructureUtil.randomKZGCommitment(), - dataStructureUtil.randomKZGCommitment()))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Blob sidecars are not complete"); - } - - @Test - void verifyBlobSidecarCompleteness_shouldThrowWhenBlobSidecarIndexIsWrong() { - final List blobSidecars = dataStructureUtil.randomBlobSidecars(1); - assertThatThrownBy( - () -> - miscHelpersDeneb.verifyBlobSidecarCompleteness( - blobSidecars, List.of(dataStructureUtil.randomKZGCommitment()))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Blob sidecar index mismatch, expected 0, got %s", blobSidecars.get(0).getIndex()); - } - @Test void shouldConstructValidBlobSidecar() { final SignedBeaconBlock signedBeaconBlock = @@ -207,7 +88,7 @@ void shouldConstructValidBlobSidecar() { assertThat(blobSidecar.getSszKZGCommitment()).isEqualTo(expectedCommitment); assertThat(blobSidecar.getSignedBeaconBlockHeader()).isEqualTo(signedBeaconBlock.asHeader()); // verify the merkle proof - assertThat(miscHelpersDeneb.verifyBlobSidecarMerkleProof(blobSidecar)).isTrue(); + assertThat(miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(blobSidecar)).isTrue(); } @Test @@ -227,7 +108,7 @@ void shouldThrowWhenConstructingBlobSidecarWithInvalidIndex() { } @Test - void verifyBlobSidecarMerkleProofShouldValidate() { + void verifyBlobKzgCommitmentInclusionProofShouldValidate() { final int numberOfCommitments = 4; final BeaconBlockBodyDeneb beaconBlockBody = BeaconBlockBodyDeneb.required( @@ -266,7 +147,7 @@ void verifyBlobSidecarMerkleProofShouldValidate() { .getBytes()) .kzgCommitmentInclusionProof(merkleProof) .build(); - assertThat(miscHelpersDeneb.verifyBlobSidecarMerkleProof(blobSidecar)).isTrue(); + assertThat(miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(blobSidecar)).isTrue(); // And the same blobSidecar but with wrong merkle proof for (int j = 0; j < numberOfCommitments; ++j) { @@ -295,8 +176,11 @@ void verifyBlobSidecarMerkleProofShouldValidate() { .getBytes()) .kzgCommitmentInclusionProof(merkleProofWrong) .build(); - assertThat(miscHelpersDeneb.verifyBlobSidecarMerkleProof(blobSidecarWrong)).isFalse(); + assertThat(miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(blobSidecarWrong)) + .isFalse(); } } } + + // TODO test mark as validated } diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java index 610529ba125..8cd875f4aea 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java @@ -20,6 +20,7 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.subscribers.Subscribers; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.kzg.KZG; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; @@ -35,6 +36,7 @@ public class BlobSidecarManagerImpl implements BlobSidecarManager, SlotEventsCha private final Spec spec; private final RecentChainData recentChainData; private final BlobSidecarGossipValidator validator; + private final KZG kzg; private final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool; private final FutureItems futureBlobSidecars; private final Map invalidBlobSidecarRoots; @@ -47,11 +49,13 @@ public BlobSidecarManagerImpl( final RecentChainData recentChainData, final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool, final BlobSidecarGossipValidator validator, + final KZG kzg, final FutureItems futureBlobSidecars, final Map invalidBlobSidecarRoots) { this.spec = spec; this.recentChainData = recentChainData; this.validator = validator; + this.kzg = kzg; this.blockBlobSidecarsTrackersPool = blockBlobSidecarsTrackersPool; this.futureBlobSidecars = futureBlobSidecars; this.invalidBlobSidecarRoots = invalidBlobSidecarRoots; @@ -119,7 +123,7 @@ public BlobSidecarsAvailabilityChecker createAvailabilityChecker(final SignedBea blockBlobSidecarsTrackersPool.getOrCreateBlockBlobSidecarsTracker(block); return new ForkChoiceBlobSidecarsAvailabilityChecker( - spec, recentChainData, blockBlobSidecarsTracker); + spec, recentChainData, blockBlobSidecarsTracker, kzg); } @Override diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java index a9dc3c9cadc..721211524f3 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java @@ -142,10 +142,6 @@ public boolean add(final BlobSidecar blobSidecar) { blobSidecar.getBlockRoot().equals(slotAndBlockRoot.getBlockRoot()), "Wrong blobSidecar block root"); - checkArgument( - blobSidecar.isKzgAndInclusionProofValidated(), - "BlobSidecar must be validated before adding"); - if (blobSidecarsComplete.isDone()) { // already completed return false; diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java index 05660d23cb5..bb31b9e2205 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java @@ -71,7 +71,6 @@ import tech.pegasys.teku.spec.logic.common.util.ForkChoiceUtil; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; -import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsValidationResult; import tech.pegasys.teku.statetransition.attestation.DeferredAttestations; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager; import tech.pegasys.teku.statetransition.block.BlockImportPerformance; @@ -543,10 +542,18 @@ private BlockImportResult importBlockAndState( LOG.debug("blobSidecars validation result: {}", blobSidecarsAndValidationResult::toLogString); - if (blobSidecarsAndValidationResult.getValidationResult() - == BlobSidecarsValidationResult.NOT_AVAILABLE) { - return BlockImportResult.failedDataAvailabilityCheckNotAvailable( - blobSidecarsAndValidationResult.getCause()); + switch (blobSidecarsAndValidationResult.getValidationResult()) { + case NOT_AVAILABLE -> { + return BlockImportResult.failedDataAvailabilityCheckNotAvailable( + blobSidecarsAndValidationResult.getCause()); + } + case INVALID -> { + debugDataDumper.saveInvalidBlobSidecars( + blobSidecarsAndValidationResult.getBlobSidecars(), block); + return BlockImportResult.failedDataAvailabilityCheckInvalid( + blobSidecarsAndValidationResult.getCause()); + } + default -> {} } final ForkChoiceStrategy forkChoiceStrategy = getForkChoiceStrategy(); diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java index cdff49d3963..f6f8f8713b1 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java @@ -20,6 +20,7 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.exceptions.ExceptionUtil; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.kzg.KZG; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; @@ -36,6 +37,7 @@ public class ForkChoiceBlobSidecarsAvailabilityChecker implements BlobSidecarsAv private final Spec spec; private final RecentChainData recentChainData; private final BlockBlobSidecarsTracker blockBlobSidecarsTracker; + private final KZG kzg; private final SafeFuture validationResult = new SafeFuture<>(); @@ -44,10 +46,12 @@ public class ForkChoiceBlobSidecarsAvailabilityChecker implements BlobSidecarsAv public ForkChoiceBlobSidecarsAvailabilityChecker( final Spec spec, final RecentChainData recentChainData, - final BlockBlobSidecarsTracker blockBlobSidecarsTracker) { + final BlockBlobSidecarsTracker blockBlobSidecarsTracker, + final KZG kzg) { this.spec = spec; this.recentChainData = recentChainData; this.blockBlobSidecarsTracker = blockBlobSidecarsTracker; + this.kzg = kzg; this.waitForTrackerCompletionTimeout = calculateCompletionTimeout(spec, blockBlobSidecarsTracker.getSlotAndBlockRoot().getSlot()); } @@ -57,10 +61,12 @@ public ForkChoiceBlobSidecarsAvailabilityChecker( final Spec spec, final RecentChainData recentChainData, final BlockBlobSidecarsTracker blockBlobSidecarsTracker, + final KZG kzg, final Duration waitForTrackerCompletionTimeout) { this.spec = spec; this.recentChainData = recentChainData; this.blockBlobSidecarsTracker = blockBlobSidecarsTracker; + this.kzg = kzg; this.waitForTrackerCompletionTimeout = waitForTrackerCompletionTimeout; } @@ -92,11 +98,15 @@ private BlobSidecarsAndValidationResult validateCompletedBlobSidecars() { List.copyOf(blockBlobSidecarsTracker.getBlobSidecars().values()); final SignedBeaconBlock block = blockBlobSidecarsTracker.getBlock().orElseThrow(); - for (final BlobSidecar blobSidecar : blobSidecars) { - if (!blobSidecar.isKzgAndInclusionProofValidated()) { - return BlobSidecarsAndValidationResult.notAvailable( - new IllegalStateException("Blob sidecars are not validated")); + try { + if (!spec.atSlot(block.getSlot()).miscHelpers().verifyBlobKzgProofBatch(kzg, blobSidecars)) { + return BlobSidecarsAndValidationResult.invalidResult(blobSidecars); } + } catch (final Exception ex) { + return BlobSidecarsAndValidationResult.invalidResult(blobSidecars, ex); + } + + for (final BlobSidecar blobSidecar : blobSidecars) { if (!blobSidecar.isSignatureValidated() && !blobSidecar .getSignedBeaconBlockHeader() diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java index 92d12c6cf39..c44933f14fd 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java @@ -721,11 +721,6 @@ private synchronized SafeFuture fetchMissingContentFromLocalEL( blobAndProof.get(), beaconBlockBodyDeneb, signedBeaconBlockHeader); - - blobSidecar.markSignatureAsValidated(); - // assume kzg validation done by local EL - blobSidecar.markKzgAndInclusionProofAsValidated(); - onNewBlobSidecar(blobSidecar, LOCAL_EL); } }); @@ -742,14 +737,22 @@ private BlobSidecar createBlobSidecarFromBlobAndProof( final SszKZGCommitment sszKZGCommitment = beaconBlockBodyDeneb.getBlobKzgCommitments().get(blobIdentifier.getIndex().intValue()); - return blobSidecarSchema.create( - blobIdentifier.getIndex(), - blobAndProof.blob(), - sszKZGCommitment, - new SszKZGProof(blobAndProof.proof()), - signedBeaconBlockHeader, - miscHelpersDeneb.computeKzgCommitmentInclusionProof( - blobIdentifier.getIndex(), beaconBlockBodyDeneb)); + final BlobSidecar blobSidecar = + blobSidecarSchema.create( + blobIdentifier.getIndex(), + blobAndProof.blob(), + sszKZGCommitment, + new SszKZGProof(blobAndProof.proof()), + signedBeaconBlockHeader, + miscHelpersDeneb.computeKzgCommitmentInclusionProof( + blobIdentifier.getIndex(), beaconBlockBodyDeneb)); + + blobSidecar.markSignatureAsValidated(); + blobSidecar.markKzgCommitmentInclusionProofAsValidated(); + // assume kzg validation done by local EL + blobSidecar.markKzgAsValidated(); + + return blobSidecar; } private synchronized void fetchMissingContentFromRemotePeers( diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java index b13094e11d1..f058d8e2375 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java @@ -159,7 +159,6 @@ public SafeFuture validate(final BlobSidecar blobSidec // Optimization: If we have already completely verified BlobSidecar with the same // SignedBlockHeader, we can skip most steps and jump to shortened validation if (validSignedBlockHeaders.contains(blobSidecar.getSignedBeaconBlockHeader().hashTreeRoot())) { - blobSidecar.markSignatureAsValidated(); return validateBlobSidecarWithKnownValidHeader(blobSidecar, blockHeader); } @@ -214,7 +213,7 @@ public SafeFuture validate(final BlobSidecar blobSidec /* * [REJECT] The sidecar's inclusion proof is valid as verified by `verify_blob_sidecar_inclusion_proof(blob_sidecar)`. */ - if (!miscHelpersDeneb.verifyBlobSidecarMerkleProof(blobSidecar)) { + if (!miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(blobSidecar)) { return completedFuture(reject("BlobSidecar inclusion proof validation failed")); } @@ -226,8 +225,6 @@ public SafeFuture validate(final BlobSidecar blobSidec return completedFuture(reject("BlobSidecar does not pass kzg validation")); } - blobSidecar.markKzgAndInclusionProofAsValidated(); - return gossipValidationHelper .getParentStateInBlockEpoch( parentBlockSlot, blockHeader.getParentRoot(), blockHeader.getSlot()) @@ -263,8 +260,6 @@ public SafeFuture validate(final BlobSidecar blobSidec return reject("BlobSidecar block header signature is invalid."); } - blobSidecar.markSignatureAsValidated(); - /* * Checking it again at the very end because whole method is not synchronized * @@ -299,7 +294,7 @@ private SafeFuture validateBlobSidecarWithKnownValidHe /* * [REJECT] The sidecar's inclusion proof is valid as verified by `verify_blob_sidecar_inclusion_proof(blob_sidecar)`. */ - if (!miscHelpersDeneb.verifyBlobSidecarMerkleProof(blobSidecar)) { + if (!miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(blobSidecar)) { return completedFuture(reject("BlobSidecar inclusion proof validation failed")); } @@ -322,8 +317,6 @@ private SafeFuture validateBlobSidecarWithKnownValidHe reject("BlobSidecar block header does not descend from finalized checkpoint")); } - blobSidecar.markKzgAndInclusionProofAsValidated(); - /* * [IGNORE] The sidecar is the first sidecar for the tuple (block_header.slot, block_header.proposer_index, blob_sidecar.index) * with valid header signature, sidecar inclusion proof, and kzg proof. diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index 9207150b1ad..276c16254df 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -31,6 +31,7 @@ import org.junit.jupiter.api.Test; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.kzg.KZG; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; @@ -52,6 +53,7 @@ public class BlobSidecarManagerTest { private final RecentChainData recentChainData = mock(RecentChainData.class); private final BlobSidecarGossipValidator blobSidecarValidator = mock(BlobSidecarGossipValidator.class); + private final KZG kzg = mock(KZG.class); private final BlockBlobSidecarsTrackersPoolImpl blockBlobSidecarsTrackersPool = mock(BlockBlobSidecarsTrackersPoolImpl.class); private final Map invalidBlobSidecarRoots = new HashMap<>(); @@ -65,6 +67,7 @@ public class BlobSidecarManagerTest { recentChainData, blockBlobSidecarsTrackersPool, blobSidecarValidator, + kzg, futureBlobSidecars, invalidBlobSidecarRoots); diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java index 791d6201f30..c3f5bab0133 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java @@ -29,7 +29,6 @@ import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.async.SafeFutureAssert; @@ -63,11 +62,6 @@ public class BlockBlobSidecarsTrackerTest { blobSidecar -> new BlobIdentifier(blobSidecar.getBlockRoot(), blobSidecar.getIndex())) .collect(Collectors.toList()); - @BeforeEach - void setUp() { - blobSidecarsForBlock.forEach(BlobSidecar::markKzgAndInclusionProofAsValidated); - } - @Test void isNotCompletedJustAfterCreation() { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = @@ -367,8 +361,6 @@ void getMissingBlobSidecars_shouldRespectMaxBlobsPerBlock() { .index(UInt64.valueOf(100)) .build(); - toAdd.markKzgAndInclusionProofAsValidated(); - blockBlobSidecarsTracker.add(toAdd); final List knownMissing = @@ -445,13 +437,10 @@ void enableBlockImportOnCompletion_shouldImportOnlyOnceWhenCalled() { } private BlobSidecar createBlobSidecar(final UInt64 index) { - final BlobSidecar blobSidecar = - dataStructureUtil - .createRandomBlobSidecarBuilder() - .signedBeaconBlockHeader(block.asHeader()) - .index(index) - .build(); - blobSidecar.markKzgAndInclusionProofAsValidated(); - return blobSidecar; + return dataStructureUtil + .createRandomBlobSidecarBuilder() + .signedBeaconBlockHeader(block.asHeader()) + .index(index) + .build(); } } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java index 714c6683c1a..7a07ba72121 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSortedMap; import java.time.Duration; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.BeforeEach; @@ -31,11 +32,13 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.async.Waiter; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.kzg.KZG; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.SpecVersion; import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; +import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsValidationResult; import tech.pegasys.teku.spec.util.DataStructureUtil; @@ -50,6 +53,8 @@ public class ForkChoiceBlobSidecarsAvailabilityCheckerTest { private final Spec spec = mock(Spec.class); private final SpecVersion specVersion = mock(SpecVersion.class); + private final MiscHelpers miscHelpers = mock(MiscHelpers.class); + private final KZG kzg = mock(KZG.class); private final UpdatableStore store = mock(UpdatableStore.class); private final BlockBlobSidecarsTracker blockBlobSidecarsTracker = mock(BlockBlobSidecarsTracker.class); @@ -65,6 +70,7 @@ public class ForkChoiceBlobSidecarsAvailabilityCheckerTest { @BeforeEach void setUp() { when(spec.atSlot(any())).thenReturn(specVersion); + when(specVersion.miscHelpers()).thenReturn(miscHelpers); when(recentChainData.getStore()).thenReturn(store); when(blockBlobSidecarsTracker.getCompletionFuture()).thenReturn(trackerCompletionFuture); } @@ -73,6 +79,8 @@ void setUp() { void shouldVerifyAvailableBlobs() throws Exception { prepareInitialAvailability(); + when(miscHelpers.verifyBlobKzgProofBatch(any(), any())).thenReturn(true); + final SafeFuture availabilityCheckResult = blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); @@ -92,6 +100,58 @@ void shouldVerifyAvailableBlobs() throws Exception { assertAvailable(availabilityCheckResult); } + @Test + void shouldVerifyInvalidBlobs() throws Exception { + prepareInitialAvailability(); + + when(miscHelpers.verifyBlobKzgProofBatch(any(), any())).thenReturn(false); + + final SafeFuture availabilityCheckResult = + blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); + + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + + // initiate availability check + assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); + + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); + + // let the tracker complete with all blobSidecars + completeTrackerWith(blobSidecarsComplete); + + Waiter.waitFor(availabilityCheckResult); + + assertInvalid(availabilityCheckResult, blobSidecarsComplete, Optional.empty()); + } + + @Test + void shouldVerifyInvalidBlobsWhenValidationThrows() throws Exception { + prepareInitialAvailability(); + + final RuntimeException error = new RuntimeException("oops"); + + when(miscHelpers.verifyBlobKzgProofBatch(any(), any())).thenThrow(error); + + final SafeFuture availabilityCheckResult = + blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); + + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + + // initiate availability check + assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); + + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); + + // let the tracker complete with all blobSidecars + completeTrackerWith(blobSidecarsComplete); + + Waiter.waitFor(availabilityCheckResult); + + assertInvalid(availabilityCheckResult, blobSidecarsComplete, Optional.of(error)); + } + @Test void shouldFailIfTrackerCompletesWithFailure() { prepareInitialAvailability(); @@ -143,6 +203,34 @@ void shouldReturnNotRequiredWhenBlockIsOutsideAvailabilityWindow() throws Except assertNotRequired(availabilityCheckResult); } + private void assertInvalid( + final SafeFuture availabilityOrValidityCheck, + final List invalidBlobs, + final Optional cause) { + assertThat(availabilityOrValidityCheck) + .isCompletedWithValueMatching(result -> !result.isValid(), "is not valid") + .isCompletedWithValueMatching( + result -> result.getValidationResult() == BlobSidecarsValidationResult.INVALID, + "is not available") + .isCompletedWithValueMatching( + result -> result.getBlobSidecars().equals(invalidBlobs), "doesn't have blob sidecars") + .isCompletedWithValueMatching( + result -> { + if (cause.isEmpty() != result.getCause().isEmpty()) { + return false; + } + return result + .getCause() + .map( + resultCause -> + resultCause.getClass().equals(cause.get().getClass()) + && Objects.equals(resultCause.getMessage(), cause.get().getMessage()) + && Objects.equals(resultCause.getCause(), cause.get().getCause())) + .orElse(true); + }, + "matches the cause"); + } + private void assertNotRequired( final SafeFuture availabilityOrValidityCheck) { assertThat(availabilityOrValidityCheck) @@ -204,7 +292,7 @@ private void prepareInitialAvailability( blobSidecarsAvailabilityChecker = new ForkChoiceBlobSidecarsAvailabilityChecker( - spec, recentChainData, blockBlobSidecarsTracker, timeout); + spec, recentChainData, blockBlobSidecarsTracker, kzg, timeout); } private void completeTrackerWith(final List blobSidecars) { @@ -232,6 +320,6 @@ private void prepareForImmediateTimeoutWithBlockAndBlobSidecarsOutsideAvailabili blobSidecarsAvailabilityChecker = new ForkChoiceBlobSidecarsAvailabilityChecker( - spec, recentChainData, blockBlobSidecarsTracker, Duration.ZERO); + spec, recentChainData, blockBlobSidecarsTracker, kzg, Duration.ZERO); } } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java index ee8b0a6906b..2dad121a9eb 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java @@ -14,9 +14,7 @@ package tech.pegasys.teku.statetransition.util; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -156,21 +154,6 @@ public void onNewBlock_addTrackerWithBlock() { assertBlobSidecarsTrackersCount(1); } - @Test - public void onNewBlobSidecar_shouldThrowIfNotMarkedKzgAndInclusionProofAsValidated() { - final BlobSidecar blobSidecar = - dataStructureUtil - .createRandomBlobSidecarBuilder() - .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) - .build(); - - assertThat(blobSidecar.isKzgAndInclusionProofValidated()).isFalse(); - - assertThrows( - IllegalArgumentException.class, - () -> blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP)); - } - @Test public void onNewBlobSidecar_addTrackerWithBlobSidecarIgnoringDuplicates() { final BlobSidecar blobSidecar = @@ -179,8 +162,6 @@ public void onNewBlobSidecar_addTrackerWithBlobSidecarIgnoringDuplicates() { .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); - blobSidecar.markKzgAndInclusionProofAsValidated(); - blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); assertThat( @@ -209,8 +190,6 @@ public void onNewBlobSidecar_shouldIgnoreDuplicates() { .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); - blobSidecar.markKzgAndInclusionProofAsValidated(); - blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); @@ -246,10 +225,6 @@ public void onNewBlobSidecar_shouldMarkForEquivocationAndPublishWhenOriginIsLoca .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); - blobSidecar1.markKzgAndInclusionProofAsValidated(); - blobSidecar2.markKzgAndInclusionProofAsValidated(); - blobSidecar3.markKzgAndInclusionProofAsValidated(); - when(blobSidecarGossipValidator.markForEquivocation(blobSidecar1)).thenReturn(true); blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar1, RemoteOrigin.LOCAL_EL); @@ -271,8 +246,6 @@ public void onNewBlobSidecar_shouldPublishWhenOriginIsLocalELAndEquivocating() { .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); - blobSidecar1.markKzgAndInclusionProofAsValidated(); - when(blobSidecarGossipValidator.markForEquivocation(blobSidecar1)).thenReturn(false); blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar1, RemoteOrigin.LOCAL_EL); @@ -292,8 +265,6 @@ public void onNewBlobSidecar_shouldNotPublishWhenOriginIsLocalELIsNotCurrentSlot .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); - blobSidecar1.markKzgAndInclusionProofAsValidated(); - when(blobSidecarGossipValidator.markForEquivocation(blobSidecar1)).thenReturn(false); blockBlobSidecarsTrackersPool.onSlot(currentSlot.plus(1)); @@ -381,8 +352,6 @@ public void onNewBlobSidecarOnNewBlock_addTrackerWithBothBlockAndBlobSidecar() { final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(currentSlot); final BlobSidecar blobSidecar = dataStructureUtil.randomBlobSidecarsForBlock(block).get(0); - blobSidecar.markKzgAndInclusionProofAsValidated(); - blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); blockBlobSidecarsTrackersPool.onNewBlock(block, Optional.empty()); @@ -425,10 +394,6 @@ public void twoOnNewBlobSidecar_addTrackerWithBothBlobSidecars() { .index(UInt64.ONE) .build(); - blobSidecar0.markKzgAndInclusionProofAsValidated(); - blobSidecar1.markKzgAndInclusionProofAsValidated(); - blobSidecar1bis.markKzgAndInclusionProofAsValidated(); - blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar0, RemoteOrigin.GOSSIP); blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar1, RemoteOrigin.GOSSIP); blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar1bis, RemoteOrigin.GOSSIP); @@ -482,8 +447,6 @@ public void onCompletedBlockAndBlobSidecars_shouldCreateTrackerIgnoringHistorica final List blobSidecars = dataStructureUtil.randomBlobSidecarsForBlock(block); - blobSidecars.forEach(BlobSidecar::markKzgAndInclusionProofAsValidated); - blockBlobSidecarsTrackersPool.onCompletedBlockAndBlobSidecars(block, blobSidecars); assertThat(asyncRunner.hasDelayedActions()).isFalse(); @@ -751,21 +714,8 @@ void shouldFetchMissingBlobSidecarsFromLocalELFirst() { engineGetBlobsResponse.complete(blobAndProofsFromEL); - verify(tracker) - .add( - argThat( - b -> - b.equals(missingBlobSidecars.getFirst()) - && b.isKzgAndInclusionProofValidated() - && b.isSignatureValidated())); // 0 - verify(tracker) - .add( - argThat( - b -> - b.equals(missingBlobSidecars.getLast()) - && b.isKzgAndInclusionProofValidated() - && b.isSignatureValidated())); // 2) - + verify(tracker).add(missingBlobSidecars.getFirst()); // 0 + verify(tracker).add(missingBlobSidecars.getLast()); // 2 verify(tracker, times(2)).add(any()); assertStats("blob_sidecar", "local_el_fetch", 3); @@ -977,8 +927,6 @@ public void shouldFetchContentWhenBlobSidecarIsNotForCurrentSlot() { .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(slot)) .build(); - blobSidecar.markKzgAndInclusionProofAsValidated(); - blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); assertThat(asyncRunner.hasDelayedActions()).isTrue(); @@ -1297,8 +1245,6 @@ void stats_onNewBlobSidecar() { .signedBeaconBlockHeader(dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot)) .build(); - blobSidecar.markKzgAndInclusionProofAsValidated(); - // new from gossip blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); @@ -1324,8 +1270,6 @@ void stats_onNewBlobSidecar() { dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot.increment())) .build(); - blobSidecar2.markKzgAndInclusionProofAsValidated(); - // new from RPC blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar2, RemoteOrigin.RPC); @@ -1341,8 +1285,6 @@ void stats_onNewBlobSidecar() { dataStructureUtil.randomSignedBeaconBlockHeader(currentSlot.increment())) .build(); - blobSidecar3.markKzgAndInclusionProofAsValidated(); - // new from LOCAL_EL blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar3, RemoteOrigin.LOCAL_EL); assertStats("blob_sidecar", "local_el", 1); @@ -1407,8 +1349,6 @@ void stats_onNewBlock() { .signedBeaconBlockHeader(block4.asHeader()) .build(); - blobSidecar4.markKzgAndInclusionProofAsValidated(); - blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar4, RemoteOrigin.RPC); blockBlobSidecarsTrackersPool.onNewBlock(block4, Optional.of(RemoteOrigin.GOSSIP)); diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidatorTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidatorTest.java index 4abe8e6db2c..6e021da1f08 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidatorTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidatorTest.java @@ -111,7 +111,7 @@ void setup(final SpecContext specContext) { any(), eq(proposerIndex), any(), eq(postState))) .thenReturn(true); when(miscHelpersDeneb.verifyBlobKzgProof(any(), any(BlobSidecar.class))).thenReturn(true); - when(miscHelpersDeneb.verifyBlobSidecarMerkleProof(any())).thenReturn(true); + when(miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(any())).thenReturn(true); } @TestTemplate @@ -224,7 +224,7 @@ void shouldRejectIfFinalizedCheckpointIsNotAnAncestorOfBlobSidecarsBlock() { @TestTemplate void shouldRejectWhenInclusionProofFailsValidation() { - when(miscHelpersDeneb.verifyBlobSidecarMerkleProof(any())).thenReturn(false); + when(miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(any())).thenReturn(false); SafeFutureAssert.assertThatSafeFuture(blobSidecarValidator.validate(blobSidecar)) .isCompletedWithValueMatching(InternalValidationResult::isReject); @@ -292,7 +292,7 @@ void shouldIgnoreImmediatelyWhenBlobFromValidInfoSet() { SafeFutureAssert.assertThatSafeFuture(blobSidecarValidator.validate(blobSidecar)) .isCompletedWithValueMatching(InternalValidationResult::isAccept); - verify(miscHelpersDeneb).verifyBlobSidecarMerkleProof(blobSidecar); + verify(miscHelpersDeneb).verifyBlobKzgCommitmentInclusionProof(blobSidecar); verify(miscHelpersDeneb).verifyBlobKzgProof(kzg, blobSidecar); verify(gossipValidationHelper).getParentStateInBlockEpoch(any(), any(), any()); verify(gossipValidationHelper).isProposerTheExpectedProposer(any(), any(), any()); @@ -304,7 +304,7 @@ void shouldIgnoreImmediatelyWhenBlobFromValidInfoSet() { SafeFutureAssert.assertThatSafeFuture(blobSidecarValidator.validate(blobSidecar)) .isCompletedWithValueMatching(InternalValidationResult::isIgnore); - verify(miscHelpersDeneb, never()).verifyBlobSidecarMerkleProof(blobSidecar); + verify(miscHelpersDeneb, never()).verifyBlobKzgCommitmentInclusionProof(blobSidecar); verify(miscHelpersDeneb, never()).verifyBlobKzgProof(kzg, blobSidecar); verify(gossipValidationHelper, never()).getParentStateInBlockEpoch(any(), any(), any()); verify(gossipValidationHelper, never()).isProposerTheExpectedProposer(any(), any(), any()); @@ -317,7 +317,7 @@ void shouldNotVerifyKnownValidSignedHeader() { SafeFutureAssert.assertThatSafeFuture(blobSidecarValidator.validate(blobSidecar)) .isCompletedWithValueMatching(InternalValidationResult::isAccept); - verify(miscHelpersDeneb).verifyBlobSidecarMerkleProof(blobSidecar); + verify(miscHelpersDeneb).verifyBlobKzgCommitmentInclusionProof(blobSidecar); verify(miscHelpersDeneb).verifyBlobKzgProof(kzg, blobSidecar); verify(gossipValidationHelper).getParentStateInBlockEpoch(any(), any(), any()); verify(gossipValidationHelper).isProposerTheExpectedProposer(any(), any(), any()); @@ -336,7 +336,7 @@ void shouldNotVerifyKnownValidSignedHeader() { SafeFutureAssert.assertThatSafeFuture(blobSidecarValidator.validate(blobSidecar0)) .isCompletedWithValueMatching(InternalValidationResult::isAccept); - verify(miscHelpersDeneb).verifyBlobSidecarMerkleProof(blobSidecar0); + verify(miscHelpersDeneb).verifyBlobKzgCommitmentInclusionProof(blobSidecar0); verify(miscHelpersDeneb).verifyBlobKzgProof(kzg, blobSidecar0); verify(gossipValidationHelper, never()).getParentStateInBlockEpoch(any(), any(), any()); verify(gossipValidationHelper, never()).isProposerTheExpectedProposer(any(), any(), any()); @@ -365,7 +365,7 @@ void shouldNotVerifyKnownValidSignedHeader() { SafeFutureAssert.assertThatSafeFuture(blobSidecarValidator.validate(blobSidecarNew)) .isCompletedWithValueMatching(InternalValidationResult::isIgnore); - verify(miscHelpersDeneb).verifyBlobSidecarMerkleProof(blobSidecarNew); + verify(miscHelpersDeneb).verifyBlobKzgCommitmentInclusionProof(blobSidecarNew); verify(miscHelpersDeneb).verifyBlobKzgProof(kzg, blobSidecarNew); verify(gossipValidationHelper).getParentStateInBlockEpoch(any(), any(), any()); } diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/AbstractBlobSidecarsValidator.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/AbstractBlobSidecarsValidator.java index 368f50d5e32..41d49539313 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/AbstractBlobSidecarsValidator.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/AbstractBlobSidecarsValidator.java @@ -65,7 +65,7 @@ private boolean verifyBlobKzgProof(final BlobSidecar blobSidecar) { private boolean verifyBlobSidecarInclusionProof(final BlobSidecar blobSidecar) { try { return MiscHelpersDeneb.required(spec.atSlot(blobSidecar.getSlot()).miscHelpers()) - .verifyBlobSidecarMerkleProof(blobSidecar); + .verifyBlobKzgCommitmentInclusionProof(blobSidecar); } catch (final Exception ex) { LOG.debug( "Block inclusion proof verification failed for BlobSidecar {}", diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootValidator.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootValidator.java index 71032550086..ea4e96fe2b0 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootValidator.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/rpc/beaconchain/methods/BlobSidecarsByRootValidator.java @@ -47,7 +47,5 @@ public void validate(final BlobSidecar blobSidecar) { verifyInclusionProof(blobSidecar); verifyKzg(blobSidecar); - - blobSidecar.markKzgAndInclusionProofAsValidated(); } } diff --git a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java index fa8cdec79d5..5fc563a9f4b 100644 --- a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java +++ b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java @@ -589,6 +589,7 @@ protected void initBlobSidecarManager() { recentChainData, blockBlobSidecarsTrackersPool, blobSidecarValidator, + kzg, futureBlobSidecars, invalidBlobSidecarRoots); eventChannels.subscribe(SlotEventsChannel.class, blobSidecarManagerImpl); From 2218284474f83d52e573cc1e1583b27f45734e3f Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Mon, 9 Dec 2024 17:51:12 +0100 Subject: [PATCH 15/31] fix property test --- .../helpers/MiscHelpersDenebPropertyTest.java | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java b/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java index 746c802a9d6..a55f2952b4f 100644 --- a/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java +++ b/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java @@ -30,7 +30,6 @@ import tech.pegasys.teku.spec.config.SpecConfigDeneb; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.type.SszKZGProof; import tech.pegasys.teku.spec.logic.common.helpers.Predicates; @@ -38,7 +37,6 @@ import tech.pegasys.teku.spec.propertytest.suppliers.blobs.versions.deneb.BlobSidecarIndexSupplier; import tech.pegasys.teku.spec.propertytest.suppliers.blobs.versions.deneb.BlobSidecarSupplier; import tech.pegasys.teku.spec.propertytest.suppliers.blobs.versions.deneb.BlobSupplier; -import tech.pegasys.teku.spec.propertytest.suppliers.blocks.versions.deneb.BeaconBlockSupplier; import tech.pegasys.teku.spec.propertytest.suppliers.blocks.versions.deneb.SignedBeaconBlockSupplier; import tech.pegasys.teku.spec.propertytest.suppliers.type.KZGCommitmentSupplier; import tech.pegasys.teku.spec.propertytest.suppliers.type.SszKZGProofSupplier; @@ -85,31 +83,6 @@ void fuzzVerifyBlobKzgProofBatch( } } - @Property(tries = 100) - void fuzzValidateBlobSidecarsBatchAgainstBlock( - @ForAll final List<@From(supplier = BlobSidecarSupplier.class) BlobSidecar> blobSidecars, - @ForAll(supplier = BeaconBlockSupplier.class) final BeaconBlock block, - @ForAll - final List<@From(supplier = KZGCommitmentSupplier.class) KZGCommitment> kzgCommitments) { - try { - miscHelpers.validateBlobSidecarsBatchAgainstBlock(blobSidecars, block, kzgCommitments); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalArgumentException.class); - } - } - - @Property(tries = 100) - void fuzzVerifyBlobSidecarCompleteness( - @ForAll final List<@From(supplier = BlobSidecarSupplier.class) BlobSidecar> blobSidecars, - @ForAll - final List<@From(supplier = KZGCommitmentSupplier.class) KZGCommitment> kzgCommitments) { - try { - miscHelpers.verifyBlobSidecarCompleteness(blobSidecars, kzgCommitments); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalArgumentException.class); - } - } - @Property(tries = 100) void fuzzConstructBlobSidecarAndVerifyMerkleProof( @ForAll(supplier = SignedBeaconBlockSupplier.class) final SignedBeaconBlock signedBeaconBlock, From 36cd7d959c75d9bbed62f574fcb55a6e76346892 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Mon, 9 Dec 2024 18:05:54 +0100 Subject: [PATCH 16/31] remove an unneeded test --- .../blobs/BlockBlobSidecarsTrackerTest.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java index c3f5bab0133..d5d702acbc9 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java @@ -165,21 +165,6 @@ void getCompletionFuture_returnsIndependentFutures() { SafeFutureAssert.assertThatSafeFuture(completionFuture3).isCompleted(); } - @Test - void add_shouldFailIfBlobsIsNotMarkedAsKzgAndInclusionProofValidated() { - final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); - final BlobSidecar toAdd = - dataStructureUtil - .createRandomBlobSidecarBuilder() - .signedBeaconBlockHeader(block.asHeader()) - .build(); - - assertThatThrownBy(() -> blockBlobSidecarsTracker.add(toAdd)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("BlobSidecar must be validated before adding"); - } - @Test void add_shouldWorkTillCompletionWhenAddingBlobsBeforeBlockIsSet() { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = From 46267fa5a0884cbbf6ab0203dccf5bd209b5f20d Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 11 Dec 2024 12:05:38 +0100 Subject: [PATCH 17/31] tmp --- .../historical/HistoricalBatchFetcher.java | 22 +++----- .../logic/common/helpers/MiscHelpers.java | 15 ++++++ .../deneb/helpers/MiscHelpersDeneb.java | 53 +++++++++++++++++++ ...ChoiceBlobSidecarsAvailabilityChecker.java | 17 +++--- 4 files changed, 83 insertions(+), 24 deletions(-) diff --git a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java index 1330aa2e97d..955d96baac5 100644 --- a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java +++ b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java @@ -48,9 +48,12 @@ import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockSummary; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; +import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBody; import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.BlobIdentifier; import tech.pegasys.teku.spec.datastructures.state.Fork; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; +import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment; +import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; import tech.pegasys.teku.spec.logic.common.util.AsyncBLSSignatureVerifier; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager; import tech.pegasys.teku.storage.api.StorageUpdateChannel; @@ -401,23 +404,12 @@ private void validateBlobSidecars(final SignedBeaconBlock block) { blobSidecarsBySlotToImport.getOrDefault( block.getSlotAndBlockRoot(), Collections.emptyList()); + final MiscHelpers miscHelpers = spec.atSlot(block.getSlot()).miscHelpers(); + final boolean blobsBlockHeaderRootMatchBlockRoot = blobSidecars.stream() .allMatch( - blobSidecar -> { - final boolean signedHeaderMatchesBlock = - blobSidecar - .getSignedBeaconBlockHeader() - .hashTreeRoot() - .equals(block.hashTreeRoot()); - - if (signedHeaderMatchesBlock) { - // since we already validated block's signature, we can mark the blob sidecar's - // signature as validated - blobSidecar.markSignatureAsValidated(); - } - return signedHeaderMatchesBlock; - }); + blobSidecar -> miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecar, block)); if (!blobsBlockHeaderRootMatchBlockRoot) { throw new IllegalArgumentException( @@ -425,6 +417,8 @@ private void validateBlobSidecars(final SignedBeaconBlock block) { "Blob sidecars' signed beacon block header don't match signed block for %s", block.toLogString())); } + + miscHelpers.verifyBlobSidecarCompleteness(blobSidecars, block); } private RequestParameters calculateRequestParams() { diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java index a3d452c791e..db2f0c78d0d 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java @@ -380,6 +380,21 @@ public boolean verifyBlobKzgProofBatch(final KZG kzg, final List bl return false; } + public boolean verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(final List blobSidecars, final SignedBeaconBlock signedBeaconBlock) { + return blobSidecars.stream().allMatch(blobSidecar -> verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecar, signedBeaconBlock)); + } + + public boolean verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(final BlobSidecar blobSidecar, final SignedBeaconBlock signedBeaconBlock) { + throw new UnsupportedOperationException("No Blob Sidecars before Deneb"); + } + + public void verifyBlobSidecarCompleteness( + final List verifiedBlobSidecars, + final SignedBeaconBlock signedBeaconBlock) + throws IllegalArgumentException { + throw new UnsupportedOperationException("No Blob Sidecars before Deneb"); + } + public VersionedHash kzgCommitmentToVersionedHash(final KZGCommitment kzgCommitment) { throw new UnsupportedOperationException("No KZGCommitments before Deneb"); } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java index e7a7592e050..8fe87cde15c 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java @@ -13,12 +13,15 @@ package tech.pegasys.teku.spec.logic.versions.deneb.helpers; +import static com.google.common.base.Preconditions.checkArgument; import static tech.pegasys.teku.spec.config.SpecConfigDeneb.VERSIONED_HASH_VERSION_KZG; import java.util.ArrayList; import java.util.List; import java.util.NoSuchElementException; import java.util.Optional; +import java.util.stream.IntStream; + import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.crypto.Hash; @@ -33,6 +36,7 @@ import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarSchema; +import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBody; import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodySchemaDeneb; @@ -136,6 +140,55 @@ public boolean verifyBlobKzgProofBatch(final KZG kzg, final List bl return result; } + @Override + public boolean verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(final BlobSidecar blobSidecar, final SignedBeaconBlock signedBeaconBlock) { + if (blobSidecar.isSignatureValidated()) { + return true; + } + + final boolean result = blobSidecar + .getSignedBeaconBlockHeader() + .hashTreeRoot() + .equals(signedBeaconBlock.hashTreeRoot()); + + if(result) { + blobSidecar.markSignatureAsValidated(); + } + + return result; + } + + /** + * Verifies that blob sidecars are complete and with expected indexes + * + * @param completeVerifiedBlobSidecars blob sidecars to verify, It is assumed that it is an + * ordered list based on BlobSidecar index + * @param signedBeaconBlock block with commitments + */ + @Override + public void verifyBlobSidecarCompleteness( + final List completeVerifiedBlobSidecars, + final SignedBeaconBlock signedBeaconBlock) + throws IllegalArgumentException { + int commitmentCount = signedBeaconBlock.getBeaconBlock().map(BeaconBlock::getBody).flatMap(BeaconBlockBody::getOptionalBlobKzgCommitments).orElseThrow().size(); + checkArgument( + completeVerifiedBlobSidecars.size() == commitmentCount, + "Blob sidecars are not complete"); + + IntStream.range(0, completeVerifiedBlobSidecars.size()) + .forEach( + index -> { + final BlobSidecar blobSidecar = completeVerifiedBlobSidecars.get(index); + final UInt64 blobIndex = blobSidecar.getIndex(); + + checkArgument( + blobIndex.longValue() == index, + "Blob sidecar index mismatch, expected %s, got %s", + index, + blobIndex); + }); + } + /** * kzg_commitment_to_versioned_hash diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java index f6f8f8713b1..ee8eda2699a 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java @@ -24,6 +24,7 @@ import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; +import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; import tech.pegasys.teku.statetransition.blobs.BlockBlobSidecarsTracker; @@ -94,29 +95,25 @@ public boolean initiateDataAvailabilityCheck() { } private BlobSidecarsAndValidationResult validateCompletedBlobSidecars() { + final MiscHelpers miscHelpers = spec.atSlot(blockBlobSidecarsTracker.getSlotAndBlockRoot().getSlot()).miscHelpers(); final List blobSidecars = List.copyOf(blockBlobSidecarsTracker.getBlobSidecars().values()); final SignedBeaconBlock block = blockBlobSidecarsTracker.getBlock().orElseThrow(); try { - if (!spec.atSlot(block.getSlot()).miscHelpers().verifyBlobKzgProofBatch(kzg, blobSidecars)) { + if (!miscHelpers.verifyBlobKzgProofBatch(kzg, blobSidecars)) { return BlobSidecarsAndValidationResult.invalidResult(blobSidecars); } } catch (final Exception ex) { return BlobSidecarsAndValidationResult.invalidResult(blobSidecars, ex); } - for (final BlobSidecar blobSidecar : blobSidecars) { - if (!blobSidecar.isSignatureValidated() - && !blobSidecar - .getSignedBeaconBlockHeader() - .hashTreeRoot() - .equals(block.hashTreeRoot())) { - return BlobSidecarsAndValidationResult.notAvailable( - new IllegalStateException("Blob sidecars block header does not match signed block")); - } + if (!miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecars, block)) { + return BlobSidecarsAndValidationResult.invalidResult(blobSidecars, new IllegalStateException("Blob sidecars block header does not match signed block")); } + miscHelpers.verifyBlobSidecarCompleteness(blobSidecars, block); + return BlobSidecarsAndValidationResult.validResult(blobSidecars); } From 6a589391fc7ee5b6c185f07d926a0f368b0c27b5 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 11 Dec 2024 16:57:30 +0100 Subject: [PATCH 18/31] tmp --- .../historical/HistoricalBatchFetcher.java | 26 +++++++------- .../blobs/BlobSidecarManager.java | 11 ++++++ .../blobs/BlobSidecarManagerImpl.java | 34 +++++++++++++++++++ 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java index 955d96baac5..25a094cd4fa 100644 --- a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java +++ b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java @@ -36,6 +36,7 @@ import tech.pegasys.teku.bls.BLSPublicKey; import tech.pegasys.teku.bls.BLSSignature; import tech.pegasys.teku.infrastructure.async.SafeFuture; +import tech.pegasys.teku.infrastructure.exceptions.ExceptionUtil; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.networking.eth2.peers.Eth2Peer; import tech.pegasys.teku.networking.eth2.rpc.core.InvalidResponseException; @@ -55,6 +56,7 @@ import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment; import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; import tech.pegasys.teku.spec.logic.common.util.AsyncBLSSignatureVerifier; +import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager; import tech.pegasys.teku.storage.api.StorageUpdateChannel; import tech.pegasys.teku.storage.client.CombinedChainDataClient; @@ -404,21 +406,21 @@ private void validateBlobSidecars(final SignedBeaconBlock block) { blobSidecarsBySlotToImport.getOrDefault( block.getSlotAndBlockRoot(), Collections.emptyList()); - final MiscHelpers miscHelpers = spec.atSlot(block.getSlot()).miscHelpers(); + LOG.trace("Validating {} blob sidecars for block {}", blobSidecars.size(), block.getRoot()); + final BlobSidecarsAndValidationResult validationResult = + blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars); - final boolean blobsBlockHeaderRootMatchBlockRoot = - blobSidecars.stream() - .allMatch( - blobSidecar -> miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecar, block)); - - if (!blobsBlockHeaderRootMatchBlockRoot) { + if (validationResult.isFailure()) { + final String causeMessage = + validationResult + .getCause() + .map(cause -> " (" + ExceptionUtil.getRootCauseMessage(cause) + ")") + .orElse(""); throw new IllegalArgumentException( - String.format( - "Blob sidecars' signed beacon block header don't match signed block for %s", - block.toLogString())); + String.format( + "Blob sidecars validation for block %s failed: %s%s", + block.getRoot(), validationResult.getValidationResult(), causeMessage)); } - - miscHelpers.verifyBlobSidecarCompleteness(blobSidecars, block); } private RequestParameters calculateRequestParams() { diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java index a61a92db880..4758fc8bd3d 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java @@ -13,11 +13,13 @@ package tech.pegasys.teku.statetransition.blobs; +import java.util.List; import java.util.Optional; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; +import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; import tech.pegasys.teku.statetransition.validation.InternalValidationResult; @@ -49,6 +51,12 @@ public BlobSidecarsAvailabilityChecker createAvailabilityChecker( final SignedBeaconBlock block) { return BlobSidecarsAvailabilityChecker.NOOP; } + + @Override + public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( + final SignedBeaconBlock block, final List blobSidecars) { + return BlobSidecarsAndValidationResult.NOT_REQUIRED; + } }; SafeFuture validateAndPrepareForBlockImport( @@ -62,6 +70,9 @@ SafeFuture validateAndPrepareForBlockImport( BlobSidecarsAvailabilityChecker createAvailabilityChecker(SignedBeaconBlock block); + BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( + SignedBeaconBlock block, List blobSidecars); + interface ReceivedBlobSidecarListener { void onBlobSidecarReceived(BlobSidecar blobSidecar); } diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java index 8cd875f4aea..bc3f1d5001a 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java @@ -13,6 +13,7 @@ package tech.pegasys.teku.statetransition.blobs; +import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.tuweni.bytes.Bytes32; @@ -24,6 +25,7 @@ import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; +import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; import tech.pegasys.teku.statetransition.forkchoice.ForkChoiceBlobSidecarsAvailabilityChecker; import tech.pegasys.teku.statetransition.util.FutureItems; @@ -126,6 +128,38 @@ public BlobSidecarsAvailabilityChecker createAvailabilityChecker(final SignedBea spec, recentChainData, blockBlobSidecarsTracker, kzg); } + @Override + public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( + final SignedBeaconBlock block, final List blobSidecars) { + // Block is pre-Deneb, blobs are not supported yet + if (block.getMessage().getBody().toVersionDeneb().isEmpty()) { + return BlobSidecarsAndValidationResult.NOT_REQUIRED; + } + + // we don't care to set maxBlobsPerBlock since it isn't used with this immediate validation flow + final BlockBlobSidecarsTracker blockBlobSidecarsTracker = + new BlockBlobSidecarsTracker(block.getSlotAndBlockRoot(), UInt64.ZERO); + + blockBlobSidecarsTracker.setBlock(block); + boolean allAdded = blobSidecars.stream().allMatch(blockBlobSidecarsTracker::add); + if(!allAdded) { + return BlobSidecarsAndValidationResult.invalidResult(blobSidecars, new IllegalStateException("Failed to add all blobs to tracker, possible blobs with same index or index out of blocks commitment range")); + } + + if(!blockBlobSidecarsTracker.isCompleted()) { + return BlobSidecarsAndValidationResult.NOT_AVAILABLE; + } + + final SafeFuture availabilityCheckResult = new ForkChoiceBlobSidecarsAvailabilityChecker( + spec, recentChainData, blockBlobSidecarsTracker, kzg).getAvailabilityCheckResult(); + + if(availabilityCheckResult.isDone()) { + return availabilityCheckResult.join(); + } else { + throw new IllegalStateException("Availability check should be done immediately"); + } + } + @Override public void onSlot(final UInt64 slot) { blockBlobSidecarsTrackersPool.onSlot(slot); From d4f11442e16b64b241b1301d4c8588b6f5c37a96 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 12 Dec 2024 11:35:39 +0100 Subject: [PATCH 19/31] tmp --- .../historical/HistoricalBatchFetcher.java | 19 ++++---- .../HistoricalBatchFetcherTest.java | 8 +++- .../logic/common/helpers/MiscHelpers.java | 17 ++++--- .../deneb/helpers/MiscHelpersDeneb.java | 48 +++++++++++-------- .../helpers/MiscHelpersDenebPropertyTest.java | 19 ++++++++ .../blobs/BlobSidecarManager.java | 4 +- .../blobs/BlobSidecarManagerImpl.java | 25 ++++++---- ...ChoiceBlobSidecarsAvailabilityChecker.java | 12 +++-- .../blobs/BlobSidecarManagerTest.java | 38 +++++++++++++++ 9 files changed, 136 insertions(+), 54 deletions(-) diff --git a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java index 25a094cd4fa..c0d24c06eec 100644 --- a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java +++ b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcher.java @@ -49,12 +49,9 @@ import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockSummary; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; -import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBody; import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.BlobIdentifier; import tech.pegasys.teku.spec.datastructures.state.Fork; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; -import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment; -import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; import tech.pegasys.teku.spec.logic.common.util.AsyncBLSSignatureVerifier; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager; @@ -408,18 +405,18 @@ private void validateBlobSidecars(final SignedBeaconBlock block) { LOG.trace("Validating {} blob sidecars for block {}", blobSidecars.size(), block.getRoot()); final BlobSidecarsAndValidationResult validationResult = - blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars); + blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars); if (validationResult.isFailure()) { final String causeMessage = - validationResult - .getCause() - .map(cause -> " (" + ExceptionUtil.getRootCauseMessage(cause) + ")") - .orElse(""); + validationResult + .getCause() + .map(cause -> " (" + ExceptionUtil.getRootCauseMessage(cause) + ")") + .orElse(""); throw new IllegalArgumentException( - String.format( - "Blob sidecars validation for block %s failed: %s%s", - block.getRoot(), validationResult.getValidationResult(), causeMessage)); + String.format( + "Blob sidecars validation for block %s failed: %s%s", + block.getRoot(), validationResult.getValidationResult(), causeMessage)); } } diff --git a/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java b/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java index 8a32f6ad74a..b87708cae8a 100644 --- a/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java +++ b/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java @@ -48,6 +48,7 @@ import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.generator.ChainBuilder; import tech.pegasys.teku.spec.logic.common.util.AsyncBLSSignatureVerifier; +import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager; import tech.pegasys.teku.storage.api.StorageQueryChannel; import tech.pegasys.teku.storage.api.StorageUpdateChannel; @@ -205,13 +206,16 @@ public void run_returnAllBlocksAndBlobSidecarsOnFirstRequest() { @Test public void run_failsOnBlobSidecarsValidationFailure() { when(blobSidecarManager.isAvailabilityRequiredAtSlot(any())).thenReturn(true); + when(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(any(), anyList())) + .thenAnswer( + i -> + BlobSidecarsAndValidationResult.invalidResult( + i.getArgument(1), new IllegalStateException("oopsy"))); assertThat(peer.getOutstandingRequests()).isEqualTo(0); final SafeFuture future = fetcher.run(); peer.completePendingRequests(); - // TODO: find a way to provoke blob sidecar validation failure - assertThat(future) .failsWithin(Duration.ZERO) .withThrowableThat() diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java index db2f0c78d0d..58ddc35e480 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java @@ -380,18 +380,23 @@ public boolean verifyBlobKzgProofBatch(final KZG kzg, final List bl return false; } - public boolean verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(final List blobSidecars, final SignedBeaconBlock signedBeaconBlock) { - return blobSidecars.stream().allMatch(blobSidecar -> verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecar, signedBeaconBlock)); + public boolean verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock( + final List blobSidecars, final SignedBeaconBlock signedBeaconBlock) { + return blobSidecars.stream() + .allMatch( + blobSidecar -> + verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock( + blobSidecar, signedBeaconBlock)); } - public boolean verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(final BlobSidecar blobSidecar, final SignedBeaconBlock signedBeaconBlock) { + public boolean verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock( + final BlobSidecar blobSidecar, final SignedBeaconBlock signedBeaconBlock) { throw new UnsupportedOperationException("No Blob Sidecars before Deneb"); } public void verifyBlobSidecarCompleteness( - final List verifiedBlobSidecars, - final SignedBeaconBlock signedBeaconBlock) - throws IllegalArgumentException { + final List verifiedBlobSidecars, final SignedBeaconBlock signedBeaconBlock) + throws IllegalArgumentException { throw new UnsupportedOperationException("No Blob Sidecars before Deneb"); } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java index 8fe87cde15c..ebf9e188b45 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java @@ -21,7 +21,6 @@ import java.util.NoSuchElementException; import java.util.Optional; import java.util.stream.IntStream; - import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.crypto.Hash; @@ -141,17 +140,19 @@ public boolean verifyBlobKzgProofBatch(final KZG kzg, final List bl } @Override - public boolean verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(final BlobSidecar blobSidecar, final SignedBeaconBlock signedBeaconBlock) { + public boolean verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock( + final BlobSidecar blobSidecar, final SignedBeaconBlock signedBeaconBlock) { if (blobSidecar.isSignatureValidated()) { return true; } - final boolean result = blobSidecar + final boolean result = + blobSidecar .getSignedBeaconBlockHeader() .hashTreeRoot() .equals(signedBeaconBlock.hashTreeRoot()); - if(result) { + if (result) { blobSidecar.markSignatureAsValidated(); } @@ -167,26 +168,31 @@ public boolean verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(fina */ @Override public void verifyBlobSidecarCompleteness( - final List completeVerifiedBlobSidecars, - final SignedBeaconBlock signedBeaconBlock) - throws IllegalArgumentException { - int commitmentCount = signedBeaconBlock.getBeaconBlock().map(BeaconBlock::getBody).flatMap(BeaconBlockBody::getOptionalBlobKzgCommitments).orElseThrow().size(); + final List completeVerifiedBlobSidecars, + final SignedBeaconBlock signedBeaconBlock) + throws IllegalArgumentException { + int commitmentCount = + signedBeaconBlock + .getBeaconBlock() + .map(BeaconBlock::getBody) + .flatMap(BeaconBlockBody::getOptionalBlobKzgCommitments) + .orElseThrow() + .size(); checkArgument( - completeVerifiedBlobSidecars.size() == commitmentCount, - "Blob sidecars are not complete"); + completeVerifiedBlobSidecars.size() == commitmentCount, "Blob sidecars are not complete"); IntStream.range(0, completeVerifiedBlobSidecars.size()) - .forEach( - index -> { - final BlobSidecar blobSidecar = completeVerifiedBlobSidecars.get(index); - final UInt64 blobIndex = blobSidecar.getIndex(); - - checkArgument( - blobIndex.longValue() == index, - "Blob sidecar index mismatch, expected %s, got %s", - index, - blobIndex); - }); + .forEach( + index -> { + final BlobSidecar blobSidecar = completeVerifiedBlobSidecars.get(index); + final UInt64 blobIndex = blobSidecar.getIndex(); + + checkArgument( + blobIndex.longValue() == index, + "Blob sidecar index mismatch, expected %s, got %s", + index, + blobIndex); + }); } /** diff --git a/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java b/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java index a55f2952b4f..304ed824bde 100644 --- a/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java +++ b/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java @@ -83,6 +83,25 @@ void fuzzVerifyBlobKzgProofBatch( } } + @Property(tries = 100) + void fuzzVerifyBlobSidecarCompleteness( + @ForAll final List<@From(supplier = BlobSidecarSupplier.class) BlobSidecar> blobSidecars, + @ForAll(supplier = SignedBeaconBlockSupplier.class) final SignedBeaconBlock signedBeaconBlock) { + try { + miscHelpers.verifyBlobSidecarCompleteness(blobSidecars, signedBeaconBlock); + } catch (Exception e) { + assertThat(e).isInstanceOf(IllegalArgumentException.class); + } + } + + @Property(tries = 100) + void fuzzVerifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock( + @ForAll final List<@From(supplier = BlobSidecarSupplier.class) BlobSidecar> blobSidecars, + @ForAll(supplier = SignedBeaconBlockSupplier.class) final SignedBeaconBlock signedBeaconBlock) { + + miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecars, signedBeaconBlock); + } + @Property(tries = 100) void fuzzConstructBlobSidecarAndVerifyMerkleProof( @ForAll(supplier = SignedBeaconBlockSupplier.class) final SignedBeaconBlock signedBeaconBlock, diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java index 4758fc8bd3d..7bfea3926be 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManager.java @@ -54,7 +54,7 @@ public BlobSidecarsAvailabilityChecker createAvailabilityChecker( @Override public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( - final SignedBeaconBlock block, final List blobSidecars) { + final SignedBeaconBlock block, final List blobSidecars) { return BlobSidecarsAndValidationResult.NOT_REQUIRED; } }; @@ -71,7 +71,7 @@ SafeFuture validateAndPrepareForBlockImport( BlobSidecarsAvailabilityChecker createAvailabilityChecker(SignedBeaconBlock block); BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( - SignedBeaconBlock block, List blobSidecars); + SignedBeaconBlock block, List blobSidecars); interface ReceivedBlobSidecarListener { void onBlobSidecarReceived(BlobSidecar blobSidecar); diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java index bc3f1d5001a..c97b3c2a246 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java @@ -130,7 +130,7 @@ public BlobSidecarsAvailabilityChecker createAvailabilityChecker(final SignedBea @Override public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( - final SignedBeaconBlock block, final List blobSidecars) { + final SignedBeaconBlock block, final List blobSidecars) { // Block is pre-Deneb, blobs are not supported yet if (block.getMessage().getBody().toVersionDeneb().isEmpty()) { return BlobSidecarsAndValidationResult.NOT_REQUIRED; @@ -138,22 +138,31 @@ public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmed // we don't care to set maxBlobsPerBlock since it isn't used with this immediate validation flow final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(block.getSlotAndBlockRoot(), UInt64.ZERO); + new BlockBlobSidecarsTracker(block.getSlotAndBlockRoot(), UInt64.ZERO); blockBlobSidecarsTracker.setBlock(block); boolean allAdded = blobSidecars.stream().allMatch(blockBlobSidecarsTracker::add); - if(!allAdded) { - return BlobSidecarsAndValidationResult.invalidResult(blobSidecars, new IllegalStateException("Failed to add all blobs to tracker, possible blobs with same index or index out of blocks commitment range")); + if (!allAdded) { + return BlobSidecarsAndValidationResult.invalidResult( + blobSidecars, + new IllegalStateException( + "Failed to add all blobs to tracker, possible blobs with same index or index out of blocks commitment range")); } - if(!blockBlobSidecarsTracker.isCompleted()) { + if (!blockBlobSidecarsTracker.isCompleted()) { return BlobSidecarsAndValidationResult.NOT_AVAILABLE; } - final SafeFuture availabilityCheckResult = new ForkChoiceBlobSidecarsAvailabilityChecker( - spec, recentChainData, blockBlobSidecarsTracker, kzg).getAvailabilityCheckResult(); + final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker =new ForkChoiceBlobSidecarsAvailabilityChecker( + spec, recentChainData, blockBlobSidecarsTracker, kzg); - if(availabilityCheckResult.isDone()) { + forkChoiceBlobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck(); + + final SafeFuture availabilityCheckResult = + forkChoiceBlobSidecarsAvailabilityChecker + .getAvailabilityCheckResult(); + + if (availabilityCheckResult.isDone()) { return availabilityCheckResult.join(); } else { throw new IllegalStateException("Availability check should be done immediately"); diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java index ee8eda2699a..b5c3f9006f7 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java @@ -95,7 +95,8 @@ public boolean initiateDataAvailabilityCheck() { } private BlobSidecarsAndValidationResult validateCompletedBlobSidecars() { - final MiscHelpers miscHelpers = spec.atSlot(blockBlobSidecarsTracker.getSlotAndBlockRoot().getSlot()).miscHelpers(); + final MiscHelpers miscHelpers = + spec.atSlot(blockBlobSidecarsTracker.getSlotAndBlockRoot().getSlot()).miscHelpers(); final List blobSidecars = List.copyOf(blockBlobSidecarsTracker.getBlobSidecars().values()); final SignedBeaconBlock block = blockBlobSidecarsTracker.getBlock().orElseThrow(); @@ -108,11 +109,14 @@ private BlobSidecarsAndValidationResult validateCompletedBlobSidecars() { return BlobSidecarsAndValidationResult.invalidResult(blobSidecars, ex); } - if (!miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecars, block)) { - return BlobSidecarsAndValidationResult.invalidResult(blobSidecars, new IllegalStateException("Blob sidecars block header does not match signed block")); + if (!miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock( + blobSidecars, block)) { + return BlobSidecarsAndValidationResult.invalidResult( + blobSidecars, + new IllegalStateException("Blob sidecars block header does not match signed block")); } - miscHelpers.verifyBlobSidecarCompleteness(blobSidecars, block); + miscHelpers.verifyBlobSidecarCompleteness(blobSidecars, block); return BlobSidecarsAndValidationResult.validResult(blobSidecars); } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index 276c16254df..55dfdaefcf0 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -19,6 +19,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import static tech.pegasys.teku.infrastructure.async.SafeFutureAssert.assertThatSafeFuture; @@ -26,6 +27,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.IntStream; + import org.apache.tuweni.bytes.Bytes32; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -36,6 +39,7 @@ import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; +import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAndValidationResult; import tech.pegasys.teku.spec.logic.versions.deneb.blobs.BlobSidecarsAvailabilityChecker; import tech.pegasys.teku.spec.util.DataStructureUtil; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager.ReceivedBlobSidecarListener; @@ -46,6 +50,7 @@ import tech.pegasys.teku.statetransition.validation.BlobSidecarGossipValidator; import tech.pegasys.teku.statetransition.validation.InternalValidationResult; import tech.pegasys.teku.storage.client.RecentChainData; +import tech.pegasys.teku.storage.store.UpdatableStore; public class BlobSidecarManagerTest { private final Spec spec = TestSpecFactory.createMinimalDeneb(); @@ -198,6 +203,17 @@ void createAvailabilityChecker_shouldReturnANotRequiredAvailabilityCheckerWhenBl .isEqualTo(BlobSidecarsAvailabilityChecker.NOT_REQUIRED); } + @Test + void + createAvailabilityCheckerAndValidateImmediately_shouldReturnANotRequiredAvailabilityCheckerWhenBlockIsPreDeneb() { + final Spec spec = TestSpecFactory.createMainnetCapella(); + final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec); + final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(); + + assertThat(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, List.of())) + .isEqualTo(BlobSidecarsAndValidationResult.NOT_REQUIRED); + } + @Test void createAvailabilityChecker_shouldReturnAnAvailabilityChecker() { final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(UInt64.ONE); @@ -211,4 +227,26 @@ void createAvailabilityChecker_shouldReturnAnAvailabilityChecker() { assertThat(blobSidecarManager.createAvailabilityChecker(block)) .isInstanceOf(ForkChoiceBlobSidecarsAvailabilityChecker.class); } + + @Test + void + createAvailabilityCheckerAndValidateImmediately_shouldReturnValidWhenComplete() { + final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(UInt64.ONE); + final int blobsCount = block.getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow().size(); + + final List blobSidecars = IntStream.range(0,blobsCount).mapToObj(index -> dataStructureUtil.randomBlobSidecarForBlock(block, index)).toList(); + + final UpdatableStore store = mock(UpdatableStore.class); +// when(recentChainData.getStore()).thenReturn(store); +// when(store.getTimeSeconds()).thenReturn(UInt64.ONE); +// when(store.getGenesisTime()).thenReturn(UInt64.ZERO); + + assertThat(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars)) + .matches(BlobSidecarsAndValidationResult::isValid).matches(result -> { + assertThat(result.getBlobSidecars()).containsExactlyElementsOf(blobSidecars); + return true; + } ); + + verifyNoInteractions(blockBlobSidecarsTrackersPool); + } } From e483c254bd0851f6883825a08ae7e62c9aa50d3b Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 12 Dec 2024 21:06:47 +0100 Subject: [PATCH 20/31] improvements --- .../blobs/BlobSidecarManagerImpl.java | 58 +++++++++++++++---- .../blobs/BlockBlobSidecarsTracker.java | 4 +- .../BlockBlobSidecarsTrackersPoolImpl.java | 8 +-- .../blobs/BlobSidecarManagerTest.java | 34 +++++++---- .../blobs/BlockBlobSidecarsTrackerTest.java | 10 ++-- ...ceBlobSidecarsAvailabilityCheckerTest.java | 2 +- 6 files changed, 82 insertions(+), 34 deletions(-) diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java index c97b3c2a246..c1268931834 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java @@ -16,6 +16,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; + +import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.ethereum.events.SlotEventsChannel; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -38,10 +40,12 @@ public class BlobSidecarManagerImpl implements BlobSidecarManager, SlotEventsCha private final Spec spec; private final RecentChainData recentChainData; private final BlobSidecarGossipValidator validator; - private final KZG kzg; private final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool; private final FutureItems futureBlobSidecars; private final Map invalidBlobSidecarRoots; +private final ForkChoiceBlobSidecarsAvailabilityCheckerProvider forkChoiceBlobSidecarsAvailabilityCheckerProvider; +private final UnpooledBlockBlobSidecarsTrackerProvider unpooledBlockBlobSidecarsTrackerProvider; + private final Subscribers receivedBlobSidecarSubscribers = Subscribers.create(true); @@ -54,13 +58,36 @@ public BlobSidecarManagerImpl( final KZG kzg, final FutureItems futureBlobSidecars, final Map invalidBlobSidecarRoots) { + this( + spec, + recentChainData, + blockBlobSidecarsTrackersPool, + validator, + futureBlobSidecars, + invalidBlobSidecarRoots, + (tracker) -> new ForkChoiceBlobSidecarsAvailabilityChecker(spec, recentChainData, tracker, kzg), + // we don't care to set maxBlobsPerBlock since it isn't used with this immediate validation flow + (block) -> new BlockBlobSidecarsTracker(block.getSlotAndBlockRoot(), UInt64.ZERO)); + } + + @VisibleForTesting + BlobSidecarManagerImpl( + final Spec spec, + final RecentChainData recentChainData, + final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool, + final BlobSidecarGossipValidator validator, + final FutureItems futureBlobSidecars, + final Map invalidBlobSidecarRoots, + final ForkChoiceBlobSidecarsAvailabilityCheckerProvider forkChoiceBlobSidecarsAvailabilityCheckerProvider, + final UnpooledBlockBlobSidecarsTrackerProvider unpooledBlockBlobSidecarsTrackerProvider) { this.spec = spec; this.recentChainData = recentChainData; this.validator = validator; - this.kzg = kzg; this.blockBlobSidecarsTrackersPool = blockBlobSidecarsTrackersPool; this.futureBlobSidecars = futureBlobSidecars; this.invalidBlobSidecarRoots = invalidBlobSidecarRoots; + this.forkChoiceBlobSidecarsAvailabilityCheckerProvider = forkChoiceBlobSidecarsAvailabilityCheckerProvider; + this.unpooledBlockBlobSidecarsTrackerProvider = unpooledBlockBlobSidecarsTrackerProvider; } @Override @@ -124,8 +151,7 @@ public BlobSidecarsAvailabilityChecker createAvailabilityChecker(final SignedBea final BlockBlobSidecarsTracker blockBlobSidecarsTracker = blockBlobSidecarsTrackersPool.getOrCreateBlockBlobSidecarsTracker(block); - return new ForkChoiceBlobSidecarsAvailabilityChecker( - spec, recentChainData, blockBlobSidecarsTracker, kzg); + return forkChoiceBlobSidecarsAvailabilityCheckerProvider.create(blockBlobSidecarsTracker); } @Override @@ -136,11 +162,9 @@ public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmed return BlobSidecarsAndValidationResult.NOT_REQUIRED; } - // we don't care to set maxBlobsPerBlock since it isn't used with this immediate validation flow - final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(block.getSlotAndBlockRoot(), UInt64.ZERO); - + final BlockBlobSidecarsTracker blockBlobSidecarsTracker = unpooledBlockBlobSidecarsTrackerProvider.create(block); blockBlobSidecarsTracker.setBlock(block); + boolean allAdded = blobSidecars.stream().allMatch(blockBlobSidecarsTracker::add); if (!allAdded) { return BlobSidecarsAndValidationResult.invalidResult( @@ -149,12 +173,11 @@ public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmed "Failed to add all blobs to tracker, possible blobs with same index or index out of blocks commitment range")); } - if (!blockBlobSidecarsTracker.isCompleted()) { + if (!blockBlobSidecarsTracker.isComplete()) { return BlobSidecarsAndValidationResult.NOT_AVAILABLE; } - final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker =new ForkChoiceBlobSidecarsAvailabilityChecker( - spec, recentChainData, blockBlobSidecarsTracker, kzg); + final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = forkChoiceBlobSidecarsAvailabilityCheckerProvider.create(blockBlobSidecarsTracker); forkChoiceBlobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck(); @@ -181,4 +204,17 @@ public void onSlot(final UInt64 slot) { validateAndPrepareForBlockImport(blobSidecar, Optional.empty()) .ifExceptionGetsHereRaiseABug()); } + + + @VisibleForTesting + @FunctionalInterface + interface ForkChoiceBlobSidecarsAvailabilityCheckerProvider { + ForkChoiceBlobSidecarsAvailabilityChecker create(final BlockBlobSidecarsTracker blockBlobSidecarsTracker); + } + + @VisibleForTesting + @FunctionalInterface + interface UnpooledBlockBlobSidecarsTrackerProvider { + BlockBlobSidecarsTracker create(final SignedBeaconBlock block); + } } diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java index 721211524f3..0f1dc3de7dd 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java @@ -243,7 +243,7 @@ private boolean isExcessiveBlobSidecar(final BlobSidecar blobSidecar) { .orElse(false); } - public boolean isCompleted() { + public boolean isComplete() { return blobSidecarsComplete.isDone(); } @@ -341,7 +341,7 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("slotAndBlockRoot", slotAndBlockRoot) .add("isBlockPresent", block.get().isPresent()) - .add("isCompleted", isCompleted()) + .add("isCompleted", isComplete()) .add("rpcFetchTriggered", rpcFetchTriggered) .add("localElFetchTriggered", localElFetchTriggered) .add("blockImportOnCompletionEnabled", blockImportOnCompletionEnabled.get()) diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java index c44933f14fd..0bfb291d0e2 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java @@ -340,7 +340,7 @@ public synchronized void onCompletedBlockAndBlobSidecars( totalBlobSidecars += (int) addedBlobs; sizeGauge.set(totalBlobSidecars, GAUGE_BLOB_SIDECARS_LABEL); - if (!blobSidecarsTracker.isCompleted()) { + if (!blobSidecarsTracker.isComplete()) { LOG.error( "Tracker for block {} is supposed to be completed but it is not. Missing blob sidecars: {}", block.toLogString(), @@ -510,7 +510,7 @@ private BlockBlobSidecarsTracker internalOnNewBlock( // if we attempted to fetch via RPC, we missed the opportunity to complete the // tracker via local EL (local El fetch requires the block to be known) // Let's try now - if (!existingTracker.isLocalElFetchTriggered() && !existingTracker.isCompleted()) { + if (!existingTracker.isLocalElFetchTriggered() && !existingTracker.isComplete()) { fetchMissingContentFromLocalEL(slotAndBlockRoot) .finish(this::logLocalElBlobsLookupFailure); } @@ -650,7 +650,7 @@ private synchronized SafeFuture fetchMissingContentFromLocalEL( return SafeFuture.COMPLETE; } - if (blockBlobSidecarsTracker.isCompleted()) { + if (blockBlobSidecarsTracker.isComplete()) { return SafeFuture.COMPLETE; } @@ -764,7 +764,7 @@ private synchronized void fetchMissingContentFromRemotePeers( return; } - if (blockBlobSidecarsTracker.isCompleted()) { + if (blockBlobSidecarsTracker.isComplete()) { return; } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index 55dfdaefcf0..82d5b41a463 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -44,6 +44,8 @@ import tech.pegasys.teku.spec.util.DataStructureUtil; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager.ReceivedBlobSidecarListener; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager.RemoteOrigin; +import tech.pegasys.teku.statetransition.blobs.BlobSidecarManagerImpl.ForkChoiceBlobSidecarsAvailabilityCheckerProvider; +import tech.pegasys.teku.statetransition.blobs.BlobSidecarManagerImpl.UnpooledBlockBlobSidecarsTrackerProvider; import tech.pegasys.teku.statetransition.forkchoice.ForkChoiceBlobSidecarsAvailabilityChecker; import tech.pegasys.teku.statetransition.util.BlockBlobSidecarsTrackersPoolImpl; import tech.pegasys.teku.statetransition.util.FutureItems; @@ -58,7 +60,6 @@ public class BlobSidecarManagerTest { private final RecentChainData recentChainData = mock(RecentChainData.class); private final BlobSidecarGossipValidator blobSidecarValidator = mock(BlobSidecarGossipValidator.class); - private final KZG kzg = mock(KZG.class); private final BlockBlobSidecarsTrackersPoolImpl blockBlobSidecarsTrackersPool = mock(BlockBlobSidecarsTrackersPoolImpl.class); private final Map invalidBlobSidecarRoots = new HashMap<>(); @@ -66,15 +67,20 @@ public class BlobSidecarManagerTest { @SuppressWarnings("unchecked") private final FutureItems futureBlobSidecars = mock(FutureItems.class); + private final ForkChoiceBlobSidecarsAvailabilityCheckerProvider forkChoiceBlobSidecarsAvailabilityCheckerProvider = mock(ForkChoiceBlobSidecarsAvailabilityCheckerProvider.class); + private final UnpooledBlockBlobSidecarsTrackerProvider unpooledBlockBlobSidecarsTrackerProvider = mock(UnpooledBlockBlobSidecarsTrackerProvider.class); + + private final BlobSidecarManagerImpl blobSidecarManager = new BlobSidecarManagerImpl( spec, recentChainData, blockBlobSidecarsTrackersPool, blobSidecarValidator, - kzg, futureBlobSidecars, - invalidBlobSidecarRoots); + invalidBlobSidecarRoots, + forkChoiceBlobSidecarsAvailabilityCheckerProvider, + unpooledBlockBlobSidecarsTrackerProvider); private final ReceivedBlobSidecarListener receivedBlobSidecarListener = mock(ReceivedBlobSidecarListener.class); @@ -217,15 +223,16 @@ void createAvailabilityChecker_shouldReturnANotRequiredAvailabilityCheckerWhenBl @Test void createAvailabilityChecker_shouldReturnAnAvailabilityChecker() { final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(UInt64.ONE); - + final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = mock(ForkChoiceBlobSidecarsAvailabilityChecker.class); final BlockBlobSidecarsTracker blockBlobSidecarsTracker = mock(BlockBlobSidecarsTracker.class); - when(blockBlobSidecarsTracker.getSlotAndBlockRoot()).thenReturn(block.getSlotAndBlockRoot()); + when(forkChoiceBlobSidecarsAvailabilityCheckerProvider.create(blockBlobSidecarsTracker)) + .thenReturn(forkChoiceBlobSidecarsAvailabilityChecker); when(blockBlobSidecarsTrackersPool.getOrCreateBlockBlobSidecarsTracker(block)) .thenReturn(blockBlobSidecarsTracker); assertThat(blobSidecarManager.createAvailabilityChecker(block)) - .isInstanceOf(ForkChoiceBlobSidecarsAvailabilityChecker.class); + .isSameAs(forkChoiceBlobSidecarsAvailabilityChecker); } @Test @@ -233,13 +240,16 @@ void createAvailabilityChecker_shouldReturnAnAvailabilityChecker() { createAvailabilityCheckerAndValidateImmediately_shouldReturnValidWhenComplete() { final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(UInt64.ONE); final int blobsCount = block.getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow().size(); - final List blobSidecars = IntStream.range(0,blobsCount).mapToObj(index -> dataStructureUtil.randomBlobSidecarForBlock(block, index)).toList(); + final BlockBlobSidecarsTracker blockBlobSidecarsTracker = mock(BlockBlobSidecarsTracker.class); + final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = mock(ForkChoiceBlobSidecarsAvailabilityChecker.class); - final UpdatableStore store = mock(UpdatableStore.class); -// when(recentChainData.getStore()).thenReturn(store); -// when(store.getTimeSeconds()).thenReturn(UInt64.ONE); -// when(store.getGenesisTime()).thenReturn(UInt64.ZERO); + when(forkChoiceBlobSidecarsAvailabilityCheckerProvider.create(blockBlobSidecarsTracker)) + .thenReturn(forkChoiceBlobSidecarsAvailabilityChecker); + when(unpooledBlockBlobSidecarsTrackerProvider.create(block)).thenReturn(blockBlobSidecarsTracker); + when(blockBlobSidecarsTracker.add(any())).thenReturn(true); + when(blockBlobSidecarsTracker.isComplete()).thenReturn(true); + when(forkChoiceBlobSidecarsAvailabilityChecker.getAvailabilityCheckResult()).thenReturn(SafeFuture.completedFuture(BlobSidecarsAndValidationResult.validResult(blobSidecars))); assertThat(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars)) .matches(BlobSidecarsAndValidationResult::isValid).matches(result -> { @@ -249,4 +259,6 @@ void createAvailabilityChecker_shouldReturnAnAvailabilityChecker() { verifyNoInteractions(blockBlobSidecarsTrackersPool); } + + //TODO complete tests for createAvailabilityCheckerAndValidateImmediately } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java index d5d702acbc9..275fb2fcada 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java @@ -123,7 +123,7 @@ void setBlock_immediatelyCompletesWithBlockWithoutBlobs() { blockBlobSidecarsTracker.setBlock(block); SafeFutureAssert.assertThatSafeFuture(completionFuture).isCompleted(); - assertThat(blockBlobSidecarsTracker.isCompleted()).isTrue(); + assertThat(blockBlobSidecarsTracker.isComplete()).isTrue(); assertThat(blockBlobSidecarsTracker.getMissingBlobSidecars()).isEmpty(); assertThat(blockBlobSidecarsTracker.getBlobSidecars()).isEmpty(); @@ -210,15 +210,15 @@ void add_shouldWorkTillCompletionWhenAddingBlobsBeforeBlockIsSet() { if (idx == blobIdentifiersForBlock.size() - 1) { SafeFutureAssert.assertThatSafeFuture(completionFuture).isCompleted(); - assertThat(blockBlobSidecarsTracker.isCompleted()).isTrue(); + assertThat(blockBlobSidecarsTracker.isComplete()).isTrue(); } else { SafeFutureAssert.assertThatSafeFuture(completionFuture).isNotCompleted(); - assertThat(blockBlobSidecarsTracker.isCompleted()).isFalse(); + assertThat(blockBlobSidecarsTracker.isComplete()).isFalse(); } } SafeFutureAssert.assertThatSafeFuture(completionFuture).isCompleted(); - assertThat(blockBlobSidecarsTracker.isCompleted()).isTrue(); + assertThat(blockBlobSidecarsTracker.isComplete()).isTrue(); } @Test @@ -416,7 +416,7 @@ void enableBlockImportOnCompletion_shouldImportOnlyOnceWhenCalled() { blobSidecarsForBlock.forEach(blockBlobSidecarsTracker::add); - assertThat(blockBlobSidecarsTracker.isCompleted()).isTrue(); + assertThat(blockBlobSidecarsTracker.isComplete()).isTrue(); verify(blockImportChannel, times(1)).importBlock(block); } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java index 7a07ba72121..c1c708e07e9 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java @@ -300,7 +300,7 @@ private void completeTrackerWith(final List blobSidecars) { ImmutableSortedMap.naturalOrder(); blobSidecars.forEach(blobSidecar -> mapBuilder.put(blobSidecar.getIndex(), blobSidecar)); when(blockBlobSidecarsTracker.getBlobSidecars()).thenReturn(mapBuilder.build()); - when(blockBlobSidecarsTracker.isCompleted()).thenReturn(true); + when(blockBlobSidecarsTracker.isComplete()).thenReturn(true); trackerCompletionFuture.complete(null); } From 6bfc4b10f3adbdeaac6c12152805e14d24231d62 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 12 Dec 2024 21:29:01 +0100 Subject: [PATCH 21/31] fix and spotless --- .../HistoricalBatchFetcherTest.java | 10 +-- .../helpers/MiscHelpersDenebPropertyTest.java | 13 ++-- .../blobs/BlobSidecarManagerImpl.java | 62 ++++++++++--------- .../blobs/BlobSidecarManagerTest.java | 56 ++++++++++------- 4 files changed, 80 insertions(+), 61 deletions(-) diff --git a/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java b/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java index b87708cae8a..0121cc26555 100644 --- a/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java +++ b/beacon/sync/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java @@ -148,6 +148,8 @@ public void setup() { when(signatureVerifier.verify(any(), any(), anyList())) .thenReturn(SafeFuture.completedFuture(true)); + when(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(any(), anyList())) + .thenAnswer(i -> BlobSidecarsAndValidationResult.validResult(i.getArgument(1))); } @Test @@ -207,10 +209,10 @@ public void run_returnAllBlocksAndBlobSidecarsOnFirstRequest() { public void run_failsOnBlobSidecarsValidationFailure() { when(blobSidecarManager.isAvailabilityRequiredAtSlot(any())).thenReturn(true); when(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(any(), anyList())) - .thenAnswer( - i -> - BlobSidecarsAndValidationResult.invalidResult( - i.getArgument(1), new IllegalStateException("oopsy"))); + .thenAnswer( + i -> + BlobSidecarsAndValidationResult.invalidResult( + i.getArgument(1), new IllegalStateException("oopsy"))); assertThat(peer.getOutstandingRequests()).isEqualTo(0); final SafeFuture future = fetcher.run(); diff --git a/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java b/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java index 304ed824bde..c47ec6e9546 100644 --- a/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java +++ b/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebPropertyTest.java @@ -85,8 +85,9 @@ void fuzzVerifyBlobKzgProofBatch( @Property(tries = 100) void fuzzVerifyBlobSidecarCompleteness( - @ForAll final List<@From(supplier = BlobSidecarSupplier.class) BlobSidecar> blobSidecars, - @ForAll(supplier = SignedBeaconBlockSupplier.class) final SignedBeaconBlock signedBeaconBlock) { + @ForAll final List<@From(supplier = BlobSidecarSupplier.class) BlobSidecar> blobSidecars, + @ForAll(supplier = SignedBeaconBlockSupplier.class) + final SignedBeaconBlock signedBeaconBlock) { try { miscHelpers.verifyBlobSidecarCompleteness(blobSidecars, signedBeaconBlock); } catch (Exception e) { @@ -96,10 +97,12 @@ void fuzzVerifyBlobSidecarCompleteness( @Property(tries = 100) void fuzzVerifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock( - @ForAll final List<@From(supplier = BlobSidecarSupplier.class) BlobSidecar> blobSidecars, - @ForAll(supplier = SignedBeaconBlockSupplier.class) final SignedBeaconBlock signedBeaconBlock) { + @ForAll final List<@From(supplier = BlobSidecarSupplier.class) BlobSidecar> blobSidecars, + @ForAll(supplier = SignedBeaconBlockSupplier.class) + final SignedBeaconBlock signedBeaconBlock) { - miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecars, signedBeaconBlock); + miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock( + blobSidecars, signedBeaconBlock); } @Property(tries = 100) diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java index c1268931834..c8929735eba 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java @@ -13,11 +13,10 @@ package tech.pegasys.teku.statetransition.blobs; +import com.google.common.annotations.VisibleForTesting; import java.util.List; import java.util.Map; import java.util.Optional; - -import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.ethereum.events.SlotEventsChannel; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -43,9 +42,9 @@ public class BlobSidecarManagerImpl implements BlobSidecarManager, SlotEventsCha private final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool; private final FutureItems futureBlobSidecars; private final Map invalidBlobSidecarRoots; -private final ForkChoiceBlobSidecarsAvailabilityCheckerProvider forkChoiceBlobSidecarsAvailabilityCheckerProvider; -private final UnpooledBlockBlobSidecarsTrackerProvider unpooledBlockBlobSidecarsTrackerProvider; - + private final ForkChoiceBlobSidecarsAvailabilityCheckerProvider + forkChoiceBlobSidecarsAvailabilityCheckerProvider; + private final UnpooledBlockBlobSidecarsTrackerProvider unpooledBlockBlobSidecarsTrackerProvider; private final Subscribers receivedBlobSidecarSubscribers = Subscribers.create(true); @@ -59,34 +58,38 @@ public BlobSidecarManagerImpl( final FutureItems futureBlobSidecars, final Map invalidBlobSidecarRoots) { this( - spec, - recentChainData, - blockBlobSidecarsTrackersPool, - validator, - futureBlobSidecars, - invalidBlobSidecarRoots, - (tracker) -> new ForkChoiceBlobSidecarsAvailabilityChecker(spec, recentChainData, tracker, kzg), - // we don't care to set maxBlobsPerBlock since it isn't used with this immediate validation flow - (block) -> new BlockBlobSidecarsTracker(block.getSlotAndBlockRoot(), UInt64.ZERO)); + spec, + recentChainData, + blockBlobSidecarsTrackersPool, + validator, + futureBlobSidecars, + invalidBlobSidecarRoots, + (tracker) -> + new ForkChoiceBlobSidecarsAvailabilityChecker(spec, recentChainData, tracker, kzg), + // we don't care to set maxBlobsPerBlock since it isn't used with this immediate validation + // flow + (block) -> new BlockBlobSidecarsTracker(block.getSlotAndBlockRoot(), UInt64.ZERO)); } @VisibleForTesting BlobSidecarManagerImpl( - final Spec spec, - final RecentChainData recentChainData, - final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool, - final BlobSidecarGossipValidator validator, - final FutureItems futureBlobSidecars, - final Map invalidBlobSidecarRoots, - final ForkChoiceBlobSidecarsAvailabilityCheckerProvider forkChoiceBlobSidecarsAvailabilityCheckerProvider, - final UnpooledBlockBlobSidecarsTrackerProvider unpooledBlockBlobSidecarsTrackerProvider) { + final Spec spec, + final RecentChainData recentChainData, + final BlockBlobSidecarsTrackersPool blockBlobSidecarsTrackersPool, + final BlobSidecarGossipValidator validator, + final FutureItems futureBlobSidecars, + final Map invalidBlobSidecarRoots, + final ForkChoiceBlobSidecarsAvailabilityCheckerProvider + forkChoiceBlobSidecarsAvailabilityCheckerProvider, + final UnpooledBlockBlobSidecarsTrackerProvider unpooledBlockBlobSidecarsTrackerProvider) { this.spec = spec; this.recentChainData = recentChainData; this.validator = validator; this.blockBlobSidecarsTrackersPool = blockBlobSidecarsTrackersPool; this.futureBlobSidecars = futureBlobSidecars; this.invalidBlobSidecarRoots = invalidBlobSidecarRoots; - this.forkChoiceBlobSidecarsAvailabilityCheckerProvider = forkChoiceBlobSidecarsAvailabilityCheckerProvider; + this.forkChoiceBlobSidecarsAvailabilityCheckerProvider = + forkChoiceBlobSidecarsAvailabilityCheckerProvider; this.unpooledBlockBlobSidecarsTrackerProvider = unpooledBlockBlobSidecarsTrackerProvider; } @@ -162,7 +165,8 @@ public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmed return BlobSidecarsAndValidationResult.NOT_REQUIRED; } - final BlockBlobSidecarsTracker blockBlobSidecarsTracker = unpooledBlockBlobSidecarsTrackerProvider.create(block); + final BlockBlobSidecarsTracker blockBlobSidecarsTracker = + unpooledBlockBlobSidecarsTrackerProvider.create(block); blockBlobSidecarsTracker.setBlock(block); boolean allAdded = blobSidecars.stream().allMatch(blockBlobSidecarsTracker::add); @@ -177,13 +181,13 @@ public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmed return BlobSidecarsAndValidationResult.NOT_AVAILABLE; } - final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = forkChoiceBlobSidecarsAvailabilityCheckerProvider.create(blockBlobSidecarsTracker); + final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = + forkChoiceBlobSidecarsAvailabilityCheckerProvider.create(blockBlobSidecarsTracker); forkChoiceBlobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck(); final SafeFuture availabilityCheckResult = - forkChoiceBlobSidecarsAvailabilityChecker - .getAvailabilityCheckResult(); + forkChoiceBlobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); if (availabilityCheckResult.isDone()) { return availabilityCheckResult.join(); @@ -205,11 +209,11 @@ public void onSlot(final UInt64 slot) { .ifExceptionGetsHereRaiseABug()); } - @VisibleForTesting @FunctionalInterface interface ForkChoiceBlobSidecarsAvailabilityCheckerProvider { - ForkChoiceBlobSidecarsAvailabilityChecker create(final BlockBlobSidecarsTracker blockBlobSidecarsTracker); + ForkChoiceBlobSidecarsAvailabilityChecker create( + final BlockBlobSidecarsTracker blockBlobSidecarsTracker); } @VisibleForTesting diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index 82d5b41a463..51cae360d3b 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -28,13 +28,11 @@ import java.util.Map; import java.util.Optional; import java.util.stream.IntStream; - import org.apache.tuweni.bytes.Bytes32; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; -import tech.pegasys.teku.kzg.KZG; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; @@ -52,7 +50,6 @@ import tech.pegasys.teku.statetransition.validation.BlobSidecarGossipValidator; import tech.pegasys.teku.statetransition.validation.InternalValidationResult; import tech.pegasys.teku.storage.client.RecentChainData; -import tech.pegasys.teku.storage.store.UpdatableStore; public class BlobSidecarManagerTest { private final Spec spec = TestSpecFactory.createMinimalDeneb(); @@ -67,9 +64,11 @@ public class BlobSidecarManagerTest { @SuppressWarnings("unchecked") private final FutureItems futureBlobSidecars = mock(FutureItems.class); - private final ForkChoiceBlobSidecarsAvailabilityCheckerProvider forkChoiceBlobSidecarsAvailabilityCheckerProvider = mock(ForkChoiceBlobSidecarsAvailabilityCheckerProvider.class); - private final UnpooledBlockBlobSidecarsTrackerProvider unpooledBlockBlobSidecarsTrackerProvider = mock(UnpooledBlockBlobSidecarsTrackerProvider.class); - + private final ForkChoiceBlobSidecarsAvailabilityCheckerProvider + forkChoiceBlobSidecarsAvailabilityCheckerProvider = + mock(ForkChoiceBlobSidecarsAvailabilityCheckerProvider.class); + private final UnpooledBlockBlobSidecarsTrackerProvider unpooledBlockBlobSidecarsTrackerProvider = + mock(UnpooledBlockBlobSidecarsTrackerProvider.class); private final BlobSidecarManagerImpl blobSidecarManager = new BlobSidecarManagerImpl( @@ -79,8 +78,8 @@ public class BlobSidecarManagerTest { blobSidecarValidator, futureBlobSidecars, invalidBlobSidecarRoots, - forkChoiceBlobSidecarsAvailabilityCheckerProvider, - unpooledBlockBlobSidecarsTrackerProvider); + forkChoiceBlobSidecarsAvailabilityCheckerProvider, + unpooledBlockBlobSidecarsTrackerProvider); private final ReceivedBlobSidecarListener receivedBlobSidecarListener = mock(ReceivedBlobSidecarListener.class); @@ -211,19 +210,20 @@ void createAvailabilityChecker_shouldReturnANotRequiredAvailabilityCheckerWhenBl @Test void - createAvailabilityCheckerAndValidateImmediately_shouldReturnANotRequiredAvailabilityCheckerWhenBlockIsPreDeneb() { + createAvailabilityCheckerAndValidateImmediately_shouldReturnANotRequiredAvailabilityCheckerWhenBlockIsPreDeneb() { final Spec spec = TestSpecFactory.createMainnetCapella(); final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec); final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(); assertThat(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, List.of())) - .isEqualTo(BlobSidecarsAndValidationResult.NOT_REQUIRED); + .isEqualTo(BlobSidecarsAndValidationResult.NOT_REQUIRED); } @Test void createAvailabilityChecker_shouldReturnAnAvailabilityChecker() { final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(UInt64.ONE); - final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = mock(ForkChoiceBlobSidecarsAvailabilityChecker.class); + final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = + mock(ForkChoiceBlobSidecarsAvailabilityChecker.class); final BlockBlobSidecarsTracker blockBlobSidecarsTracker = mock(BlockBlobSidecarsTracker.class); when(forkChoiceBlobSidecarsAvailabilityCheckerProvider.create(blockBlobSidecarsTracker)) @@ -236,29 +236,39 @@ void createAvailabilityChecker_shouldReturnAnAvailabilityChecker() { } @Test - void - createAvailabilityCheckerAndValidateImmediately_shouldReturnValidWhenComplete() { + void createAvailabilityCheckerAndValidateImmediately_shouldReturnValidWhenComplete() { final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(UInt64.ONE); - final int blobsCount = block.getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow().size(); - final List blobSidecars = IntStream.range(0,blobsCount).mapToObj(index -> dataStructureUtil.randomBlobSidecarForBlock(block, index)).toList(); + final int blobsCount = + block.getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow().size(); + final List blobSidecars = + IntStream.range(0, blobsCount) + .mapToObj(index -> dataStructureUtil.randomBlobSidecarForBlock(block, index)) + .toList(); final BlockBlobSidecarsTracker blockBlobSidecarsTracker = mock(BlockBlobSidecarsTracker.class); - final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = mock(ForkChoiceBlobSidecarsAvailabilityChecker.class); + final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = + mock(ForkChoiceBlobSidecarsAvailabilityChecker.class); when(forkChoiceBlobSidecarsAvailabilityCheckerProvider.create(blockBlobSidecarsTracker)) .thenReturn(forkChoiceBlobSidecarsAvailabilityChecker); - when(unpooledBlockBlobSidecarsTrackerProvider.create(block)).thenReturn(blockBlobSidecarsTracker); + when(unpooledBlockBlobSidecarsTrackerProvider.create(block)) + .thenReturn(blockBlobSidecarsTracker); when(blockBlobSidecarsTracker.add(any())).thenReturn(true); when(blockBlobSidecarsTracker.isComplete()).thenReturn(true); - when(forkChoiceBlobSidecarsAvailabilityChecker.getAvailabilityCheckResult()).thenReturn(SafeFuture.completedFuture(BlobSidecarsAndValidationResult.validResult(blobSidecars))); - - assertThat(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars)) - .matches(BlobSidecarsAndValidationResult::isValid).matches(result -> { + when(forkChoiceBlobSidecarsAvailabilityChecker.getAvailabilityCheckResult()) + .thenReturn( + SafeFuture.completedFuture(BlobSidecarsAndValidationResult.validResult(blobSidecars))); + + assertThat( + blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars)) + .matches(BlobSidecarsAndValidationResult::isValid) + .matches( + result -> { assertThat(result.getBlobSidecars()).containsExactlyElementsOf(blobSidecars); return true; - } ); + }); verifyNoInteractions(blockBlobSidecarsTrackersPool); } - //TODO complete tests for createAvailabilityCheckerAndValidateImmediately + // TODO complete tests for createAvailabilityCheckerAndValidateImmediately } From 0a7596c49eeeada3be2ba9e6bba857cabbd38e3c Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 12 Dec 2024 21:31:06 +0100 Subject: [PATCH 22/31] fix --- .../reference/phase0/forkchoice/StubBlobSidecarManager.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java index f2d520a1fb4..5ffe6cdec0b 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/StubBlobSidecarManager.java @@ -119,5 +119,11 @@ private BlobSidecarsAndValidationResult validateImmediately( }; } + @Override + public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmediately( + final SignedBeaconBlock block, final List blobSidecars) { + throw new UnsupportedOperationException("Not available in fork choice reference tests"); + } + private record BlobsAndProofs(List blobs, List proofs) {} } From 4d4483256655ac39789271bdeb9907417699fcc1 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 12 Dec 2024 21:37:24 +0100 Subject: [PATCH 23/31] tiny --- .../spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java | 2 +- .../teku/statetransition/blobs/BlockBlobSidecarsTracker.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java index 6fab0c8710d..8b1c0fff9af 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java @@ -182,5 +182,5 @@ void verifyBlobKzgCommitmentInclusionProofShouldValidate() { } } - // TODO test mark as validated + // TODO test mark as validated and verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock } diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java index 0f1dc3de7dd..64c89d91ba1 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java @@ -341,7 +341,7 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("slotAndBlockRoot", slotAndBlockRoot) .add("isBlockPresent", block.get().isPresent()) - .add("isCompleted", isComplete()) + .add("isComplete", isComplete()) .add("rpcFetchTriggered", rpcFetchTriggered) .add("localElFetchTriggered", localElFetchTriggered) .add("blockImportOnCompletionEnabled", blockImportOnCompletionEnabled.get()) From 8649d62b1ba97c0c6a9810193183ef4a4d721d84 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 12 Dec 2024 22:01:11 +0100 Subject: [PATCH 24/31] tests and other tiny things --- ...ChoiceBlobSidecarsAvailabilityChecker.java | 4 +- ...ceBlobSidecarsAvailabilityCheckerTest.java | 73 ++++++++++++++++++- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java index b5c3f9006f7..4cba1390a8c 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java @@ -105,6 +105,8 @@ private BlobSidecarsAndValidationResult validateCompletedBlobSidecars() { if (!miscHelpers.verifyBlobKzgProofBatch(kzg, blobSidecars)) { return BlobSidecarsAndValidationResult.invalidResult(blobSidecars); } + + miscHelpers.verifyBlobSidecarCompleteness(blobSidecars, block); } catch (final Exception ex) { return BlobSidecarsAndValidationResult.invalidResult(blobSidecars, ex); } @@ -116,8 +118,6 @@ private BlobSidecarsAndValidationResult validateCompletedBlobSidecars() { new IllegalStateException("Blob sidecars block header does not match signed block")); } - miscHelpers.verifyBlobSidecarCompleteness(blobSidecars, block); - return BlobSidecarsAndValidationResult.validResult(blobSidecars); } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java index c1c708e07e9..7b0d489bfb3 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java @@ -15,6 +15,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -76,10 +78,12 @@ void setUp() { } @Test - void shouldVerifyAvailableBlobs() throws Exception { + void shouldVerifyValidAvailableBlobs() throws Exception { prepareInitialAvailability(); when(miscHelpers.verifyBlobKzgProofBatch(any(), any())).thenReturn(true); + when(miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(anyList(), any())) + .thenReturn(true); final SafeFuture availabilityCheckResult = blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); @@ -101,10 +105,43 @@ void shouldVerifyAvailableBlobs() throws Exception { } @Test - void shouldVerifyInvalidBlobs() throws Exception { + void shouldVerifyInvalidBlobsDueToWrongBlockHeader() throws Exception { + prepareInitialAvailability(); + + when(miscHelpers.verifyBlobKzgProofBatch(any(), any())).thenReturn(true); + when(miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(anyList(), any())) + .thenReturn(false); + + final SafeFuture availabilityCheckResult = + blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); + + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + + // initiate availability check + assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); + + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); + + // let the tracker complete with all blobSidecars + completeTrackerWith(blobSidecarsComplete); + + Waiter.waitFor(availabilityCheckResult); + + assertInvalid( + availabilityCheckResult, + blobSidecarsComplete, + Optional.of( + new IllegalStateException("Blob sidecars block header does not match signed block"))); + } + + @Test + void shouldVerifyInvalidBlobsDueToWrongKzg() throws Exception { prepareInitialAvailability(); when(miscHelpers.verifyBlobKzgProofBatch(any(), any())).thenReturn(false); + when(miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(anyList(), any())) + .thenReturn(true); final SafeFuture availabilityCheckResult = blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); @@ -126,7 +163,7 @@ void shouldVerifyInvalidBlobs() throws Exception { } @Test - void shouldVerifyInvalidBlobsWhenValidationThrows() throws Exception { + void shouldVerifyInvalidBlobsWhenKzgValidationThrows() throws Exception { prepareInitialAvailability(); final RuntimeException error = new RuntimeException("oops"); @@ -152,6 +189,36 @@ void shouldVerifyInvalidBlobsWhenValidationThrows() throws Exception { assertInvalid(availabilityCheckResult, blobSidecarsComplete, Optional.of(error)); } + @Test + void shouldVerifyInvalidBlobsWhenCompletenessValidationThrows() throws Exception { + prepareInitialAvailability(); + + final RuntimeException error = new RuntimeException("oops"); + + when(miscHelpers.verifyBlobKzgProofBatch(any(), any())).thenReturn(true); + when(miscHelpers.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(anyList(), any())) + .thenReturn(true); + doThrow(error).when(miscHelpers).verifyBlobSidecarCompleteness(anyList(), any()); + + final SafeFuture availabilityCheckResult = + blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); + + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + + // initiate availability check + assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); + + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); + + // let the tracker complete with all blobSidecars + completeTrackerWith(blobSidecarsComplete); + + Waiter.waitFor(availabilityCheckResult); + + assertInvalid(availabilityCheckResult, blobSidecarsComplete, Optional.of(error)); + } + @Test void shouldFailIfTrackerCompletesWithFailure() { prepareInitialAvailability(); From ee8e99b9e10561c063ffadad899b8ab03936ab85 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 13 Dec 2024 12:32:11 +0100 Subject: [PATCH 25/31] simplify tests --- ...ceBlobSidecarsAvailabilityCheckerTest.java | 95 +++++-------------- 1 file changed, 25 insertions(+), 70 deletions(-) diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java index 7b0d489bfb3..b2406f0e9da 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityCheckerTest.java @@ -86,20 +86,7 @@ void shouldVerifyValidAvailableBlobs() throws Exception { .thenReturn(true); final SafeFuture availabilityCheckResult = - blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - - assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - - // initiate availability check - assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - - assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); - - // let the tracker complete with all blobSidecars - completeTrackerWith(blobSidecarsComplete); - - Waiter.waitFor(availabilityCheckResult); + runAvailabilityCheck(); assertAvailable(availabilityCheckResult); } @@ -113,20 +100,7 @@ void shouldVerifyInvalidBlobsDueToWrongBlockHeader() throws Exception { .thenReturn(false); final SafeFuture availabilityCheckResult = - blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - - assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - - // initiate availability check - assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - - assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); - - // let the tracker complete with all blobSidecars - completeTrackerWith(blobSidecarsComplete); - - Waiter.waitFor(availabilityCheckResult); + runAvailabilityCheck(); assertInvalid( availabilityCheckResult, @@ -144,20 +118,7 @@ void shouldVerifyInvalidBlobsDueToWrongKzg() throws Exception { .thenReturn(true); final SafeFuture availabilityCheckResult = - blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - - assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - - // initiate availability check - assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - - assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); - - // let the tracker complete with all blobSidecars - completeTrackerWith(blobSidecarsComplete); - - Waiter.waitFor(availabilityCheckResult); + runAvailabilityCheck(); assertInvalid(availabilityCheckResult, blobSidecarsComplete, Optional.empty()); } @@ -171,20 +132,7 @@ void shouldVerifyInvalidBlobsWhenKzgValidationThrows() throws Exception { when(miscHelpers.verifyBlobKzgProofBatch(any(), any())).thenThrow(error); final SafeFuture availabilityCheckResult = - blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - - assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - - // initiate availability check - assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - - assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); - - // let the tracker complete with all blobSidecars - completeTrackerWith(blobSidecarsComplete); - - Waiter.waitFor(availabilityCheckResult); + runAvailabilityCheck(); assertInvalid(availabilityCheckResult, blobSidecarsComplete, Optional.of(error)); } @@ -201,20 +149,7 @@ void shouldVerifyInvalidBlobsWhenCompletenessValidationThrows() throws Exception doThrow(error).when(miscHelpers).verifyBlobSidecarCompleteness(anyList(), any()); final SafeFuture availabilityCheckResult = - blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); - - assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - - // initiate availability check - assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); - - assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); - verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); - - // let the tracker complete with all blobSidecars - completeTrackerWith(blobSidecarsComplete); - - Waiter.waitFor(availabilityCheckResult); + runAvailabilityCheck(); assertInvalid(availabilityCheckResult, blobSidecarsComplete, Optional.of(error)); } @@ -270,6 +205,26 @@ void shouldReturnNotRequiredWhenBlockIsOutsideAvailabilityWindow() throws Except assertNotRequired(availabilityCheckResult); } + private SafeFuture runAvailabilityCheck() throws Exception { + final SafeFuture availabilityCheckResult = + blobSidecarsAvailabilityChecker.getAvailabilityCheckResult(); + + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + + // initiate availability check + assertThat(blobSidecarsAvailabilityChecker.initiateDataAvailabilityCheck()).isTrue(); + + assertThatSafeFuture(availabilityCheckResult).isNotCompleted(); + verify(blockBlobSidecarsTracker, never()).getBlobSidecars(); + + // let the tracker complete with all blobSidecars + completeTrackerWith(blobSidecarsComplete); + + Waiter.waitFor(availabilityCheckResult); + + return availabilityCheckResult; + } + private void assertInvalid( final SafeFuture availabilityOrValidityCheck, final List invalidBlobs, From 0c6da5afaa4b3af37e69f3dd8f3091144864b495 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 13 Dec 2024 13:02:04 +0100 Subject: [PATCH 26/31] complete deneb mischelper tests --- .../deneb/helpers/MiscHelpersDenebTest.java | 59 ++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java index 8b1c0fff9af..1793b9d8dd1 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java @@ -16,6 +16,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assumptions.assumeThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static tech.pegasys.teku.spec.config.SpecConfigDeneb.VERSIONED_HASH_VERSION_KZG; import java.util.List; @@ -68,6 +69,39 @@ public void versionedHash() { assertThat(actual).isEqualTo(VERSIONED_HASH); } + @Test + void verifyBlobSidecarCompleteness_shouldThrowWhenSizesDoNotMatch() { + assertThatThrownBy( + () -> + miscHelpersDeneb.verifyBlobSidecarCompleteness( + dataStructureUtil.randomBlobSidecars(1), + dataStructureUtil.randomSignedBeaconBlockWithCommitments(2))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Blob sidecars are not complete"); + } + + @Test + void verifyBlobSidecarCompleteness_shouldThrowWhenBlobSidecarIndexIsWrong() { + final List blobSidecars = dataStructureUtil.randomBlobSidecars(1); + assertThatThrownBy( + () -> + miscHelpersDeneb.verifyBlobSidecarCompleteness( + blobSidecars, dataStructureUtil.randomSignedBeaconBlockWithCommitments(1))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Blob sidecar index mismatch, expected 0, got %s", blobSidecars.getFirst().getIndex()); + } + + @Test + void verifyBlobSidecarCompleteness_shouldNotThrow() { + final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlockWithCommitments(2); + final List blobSidecars = List.of(dataStructureUtil.randomBlobSidecarForBlock(block, 0), dataStructureUtil.randomBlobSidecarForBlock(block, 1)); + assertDoesNotThrow( + () -> + miscHelpersDeneb.verifyBlobSidecarCompleteness( + blobSidecars, block)); + } + @Test void shouldConstructValidBlobSidecar() { final SignedBeaconBlock signedBeaconBlock = @@ -89,6 +123,7 @@ void shouldConstructValidBlobSidecar() { assertThat(blobSidecar.getSignedBeaconBlockHeader()).isEqualTo(signedBeaconBlock.asHeader()); // verify the merkle proof assertThat(miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(blobSidecar)).isTrue(); + assertThat(blobSidecar.isKzgCommitmentInclusionProofValidated()).isTrue(); } @Test @@ -148,6 +183,8 @@ void verifyBlobKzgCommitmentInclusionProofShouldValidate() { .kzgCommitmentInclusionProof(merkleProof) .build(); assertThat(miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(blobSidecar)).isTrue(); +assertThat(blobSidecar.isKzgCommitmentInclusionProofValidated()).isTrue(); + // And the same blobSidecar but with wrong merkle proof for (int j = 0; j < numberOfCommitments; ++j) { @@ -178,9 +215,29 @@ void verifyBlobKzgCommitmentInclusionProofShouldValidate() { .build(); assertThat(miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(blobSidecarWrong)) .isFalse(); + assertThat(blobSidecarWrong.isKzgCommitmentInclusionProofValidated()).isFalse(); } } } - // TODO test mark as validated and verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock + + @Test + void verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock_returnsTrue() { + final SignedBeaconBlock signedBeaconBlock = + dataStructureUtil.randomSignedBeaconBlockWithCommitments(1); + final BlobSidecar blobSidecar = + dataStructureUtil.randomBlobSidecarForBlock(signedBeaconBlock, 0); + assertThat(miscHelpersDeneb.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecar, signedBeaconBlock)).isTrue(); + assertThat(blobSidecar.isSignatureValidated()).isTrue(); + } + + @Test + void verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock_returnsFalse() { + final SignedBeaconBlock signedBeaconBlock = + dataStructureUtil.randomSignedBeaconBlockWithCommitments(1); + final BlobSidecar blobSidecar = + dataStructureUtil.randomBlobSidecar(); + assertThat(miscHelpersDeneb.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecar, signedBeaconBlock)).isFalse(); + assertThat(blobSidecar.isSignatureValidated()).isFalse(); + } } From e6e7d277f23ef254ace677c3ad46eca7f2debd80 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 13 Dec 2024 13:02:38 +0100 Subject: [PATCH 27/31] complete deneb mischelper tests --- .../deneb/helpers/MiscHelpersDenebTest.java | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java index 1793b9d8dd1..78ddb7fd975 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDenebTest.java @@ -73,11 +73,11 @@ public void versionedHash() { void verifyBlobSidecarCompleteness_shouldThrowWhenSizesDoNotMatch() { assertThatThrownBy( () -> - miscHelpersDeneb.verifyBlobSidecarCompleteness( - dataStructureUtil.randomBlobSidecars(1), - dataStructureUtil.randomSignedBeaconBlockWithCommitments(2))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Blob sidecars are not complete"); + miscHelpersDeneb.verifyBlobSidecarCompleteness( + dataStructureUtil.randomBlobSidecars(1), + dataStructureUtil.randomSignedBeaconBlockWithCommitments(2))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Blob sidecars are not complete"); } @Test @@ -85,21 +85,21 @@ void verifyBlobSidecarCompleteness_shouldThrowWhenBlobSidecarIndexIsWrong() { final List blobSidecars = dataStructureUtil.randomBlobSidecars(1); assertThatThrownBy( () -> - miscHelpersDeneb.verifyBlobSidecarCompleteness( - blobSidecars, dataStructureUtil.randomSignedBeaconBlockWithCommitments(1))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Blob sidecar index mismatch, expected 0, got %s", blobSidecars.getFirst().getIndex()); + miscHelpersDeneb.verifyBlobSidecarCompleteness( + blobSidecars, dataStructureUtil.randomSignedBeaconBlockWithCommitments(1))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Blob sidecar index mismatch, expected 0, got %s", blobSidecars.getFirst().getIndex()); } @Test void verifyBlobSidecarCompleteness_shouldNotThrow() { final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlockWithCommitments(2); - final List blobSidecars = List.of(dataStructureUtil.randomBlobSidecarForBlock(block, 0), dataStructureUtil.randomBlobSidecarForBlock(block, 1)); - assertDoesNotThrow( - () -> - miscHelpersDeneb.verifyBlobSidecarCompleteness( - blobSidecars, block)); + final List blobSidecars = + List.of( + dataStructureUtil.randomBlobSidecarForBlock(block, 0), + dataStructureUtil.randomBlobSidecarForBlock(block, 1)); + assertDoesNotThrow(() -> miscHelpersDeneb.verifyBlobSidecarCompleteness(blobSidecars, block)); } @Test @@ -183,8 +183,7 @@ void verifyBlobKzgCommitmentInclusionProofShouldValidate() { .kzgCommitmentInclusionProof(merkleProof) .build(); assertThat(miscHelpersDeneb.verifyBlobKzgCommitmentInclusionProof(blobSidecar)).isTrue(); -assertThat(blobSidecar.isKzgCommitmentInclusionProofValidated()).isTrue(); - + assertThat(blobSidecar.isKzgCommitmentInclusionProofValidated()).isTrue(); // And the same blobSidecar but with wrong merkle proof for (int j = 0; j < numberOfCommitments; ++j) { @@ -220,24 +219,28 @@ void verifyBlobKzgCommitmentInclusionProofShouldValidate() { } } - @Test void verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock_returnsTrue() { final SignedBeaconBlock signedBeaconBlock = dataStructureUtil.randomSignedBeaconBlockWithCommitments(1); final BlobSidecar blobSidecar = dataStructureUtil.randomBlobSidecarForBlock(signedBeaconBlock, 0); - assertThat(miscHelpersDeneb.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecar, signedBeaconBlock)).isTrue(); + assertThat( + miscHelpersDeneb.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock( + blobSidecar, signedBeaconBlock)) + .isTrue(); assertThat(blobSidecar.isSignatureValidated()).isTrue(); } @Test void verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock_returnsFalse() { final SignedBeaconBlock signedBeaconBlock = - dataStructureUtil.randomSignedBeaconBlockWithCommitments(1); - final BlobSidecar blobSidecar = - dataStructureUtil.randomBlobSidecar(); - assertThat(miscHelpersDeneb.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock(blobSidecar, signedBeaconBlock)).isFalse(); + dataStructureUtil.randomSignedBeaconBlockWithCommitments(1); + final BlobSidecar blobSidecar = dataStructureUtil.randomBlobSidecar(); + assertThat( + miscHelpersDeneb.verifyBlobSidecarBlockHeaderSignatureViaValidatedSignedBlock( + blobSidecar, signedBeaconBlock)) + .isFalse(); assertThat(blobSidecar.isSignatureValidated()).isFalse(); } } From e42508b38aaf8bc4b7e322317d95b4f8107359bf Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 13 Dec 2024 14:44:35 +0100 Subject: [PATCH 28/31] complete tests --- .../blobs/BlobSidecarManagerTest.java | 102 ++++++++++++++++-- 1 file changed, 91 insertions(+), 11 deletions(-) diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index 51cae360d3b..317843eda82 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.IntStream; import org.apache.tuweni.bytes.Bytes32; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -85,6 +84,8 @@ public class BlobSidecarManagerTest { mock(ReceivedBlobSidecarListener.class); private final BlobSidecar blobSidecar = dataStructureUtil.randomBlobSidecar(); + private final List blobSidecars = List.of(blobSidecar); + private final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(UInt64.ONE); @BeforeEach void setUp() { @@ -175,7 +176,6 @@ void validateAndPrepareForBlockImport_shouldRejectKnownInvalidBlobs() { @Test void prepareForBlockImport_shouldAddToPoolAndNotify() { - blobSidecarManager.prepareForBlockImport(blobSidecar, RemoteOrigin.GOSSIP); verify(receivedBlobSidecarListener).onBlobSidecarReceived(blobSidecar); @@ -221,7 +221,6 @@ void createAvailabilityChecker_shouldReturnANotRequiredAvailabilityCheckerWhenBl @Test void createAvailabilityChecker_shouldReturnAnAvailabilityChecker() { - final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(UInt64.ONE); final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = mock(ForkChoiceBlobSidecarsAvailabilityChecker.class); final BlockBlobSidecarsTracker blockBlobSidecarsTracker = mock(BlockBlobSidecarsTracker.class); @@ -237,13 +236,6 @@ void createAvailabilityChecker_shouldReturnAnAvailabilityChecker() { @Test void createAvailabilityCheckerAndValidateImmediately_shouldReturnValidWhenComplete() { - final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(UInt64.ONE); - final int blobsCount = - block.getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow().size(); - final List blobSidecars = - IntStream.range(0, blobsCount) - .mapToObj(index -> dataStructureUtil.randomBlobSidecarForBlock(block, index)) - .toList(); final BlockBlobSidecarsTracker blockBlobSidecarsTracker = mock(BlockBlobSidecarsTracker.class); final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = mock(ForkChoiceBlobSidecarsAvailabilityChecker.class); @@ -270,5 +262,93 @@ void createAvailabilityCheckerAndValidateImmediately_shouldReturnValidWhenComple verifyNoInteractions(blockBlobSidecarsTrackersPool); } - // TODO complete tests for createAvailabilityCheckerAndValidateImmediately + @Test + void createAvailabilityCheckerAndValidateImmediately_shouldReturnNotRequiredWhenPreDeneb() { + final SignedBeaconBlock preDenebBlock = + new DataStructureUtil(TestSpecFactory.createMinimalCapella()).randomSignedBeaconBlock(); + + assertThat( + blobSidecarManager.createAvailabilityCheckerAndValidateImmediately( + preDenebBlock, blobSidecars)) + .matches(BlobSidecarsAndValidationResult::isNotRequired); + + verifyNoInteractions(blockBlobSidecarsTrackersPool); + } + + @Test + void + createAvailabilityCheckerAndValidateImmediately_shouldReturnInvalidWhenBlobsHaveDuplicatedIndices() { + final BlockBlobSidecarsTracker blockBlobSidecarsTracker = mock(BlockBlobSidecarsTracker.class); + final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = + mock(ForkChoiceBlobSidecarsAvailabilityChecker.class); + + when(forkChoiceBlobSidecarsAvailabilityCheckerProvider.create(blockBlobSidecarsTracker)) + .thenReturn(forkChoiceBlobSidecarsAvailabilityChecker); + when(unpooledBlockBlobSidecarsTrackerProvider.create(block)) + .thenReturn(blockBlobSidecarsTracker); + when(blockBlobSidecarsTracker.add(any())).thenReturn(false); + when(blockBlobSidecarsTracker.isComplete()).thenReturn(true); + + assertThat( + blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars)) + .matches(BlobSidecarsAndValidationResult::isInvalid) + .matches( + result -> { + assertThat(result.getCause()).isPresent(); + assertThat(result.getCause().orElseThrow()) + .matches(cause -> cause instanceof IllegalStateException) + .hasMessage( + "Failed to add all blobs to tracker, possible blobs with same index or index out of blocks commitment range"); + return true; + }); + + verifyNoInteractions(blockBlobSidecarsTrackersPool); + } + + @Test + void + createAvailabilityCheckerAndValidateImmediately_shouldReturnNotAvailableWhenBlobsAreIncomplete() { + final BlockBlobSidecarsTracker blockBlobSidecarsTracker = mock(BlockBlobSidecarsTracker.class); + final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = + mock(ForkChoiceBlobSidecarsAvailabilityChecker.class); + + when(forkChoiceBlobSidecarsAvailabilityCheckerProvider.create(blockBlobSidecarsTracker)) + .thenReturn(forkChoiceBlobSidecarsAvailabilityChecker); + when(unpooledBlockBlobSidecarsTrackerProvider.create(block)) + .thenReturn(blockBlobSidecarsTracker); + when(blockBlobSidecarsTracker.add(any())).thenReturn(true); + when(blockBlobSidecarsTracker.isComplete()).thenReturn(false); + + assertThat( + blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars)) + .matches(BlobSidecarsAndValidationResult::isNotAvailable); + + verifyNoInteractions(blockBlobSidecarsTrackersPool); + } + + @Test + void + createAvailabilityCheckerAndValidateImmediately_shouldReturnTheAvailabilityCheckValidationResult() { + final BlockBlobSidecarsTracker blockBlobSidecarsTracker = mock(BlockBlobSidecarsTracker.class); + final ForkChoiceBlobSidecarsAvailabilityChecker forkChoiceBlobSidecarsAvailabilityChecker = + mock(ForkChoiceBlobSidecarsAvailabilityChecker.class); + + when(forkChoiceBlobSidecarsAvailabilityCheckerProvider.create(blockBlobSidecarsTracker)) + .thenReturn(forkChoiceBlobSidecarsAvailabilityChecker); + when(unpooledBlockBlobSidecarsTrackerProvider.create(block)) + .thenReturn(blockBlobSidecarsTracker); + when(blockBlobSidecarsTracker.add(any())).thenReturn(true); + when(blockBlobSidecarsTracker.isComplete()).thenReturn(true); + + final SafeFuture result = + SafeFuture.completedFuture(BlobSidecarsAndValidationResult.validResult(blobSidecars)); + + when(forkChoiceBlobSidecarsAvailabilityChecker.getAvailabilityCheckResult()).thenReturn(result); + + assertThat( + blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars)) + .isSameAs(result.join()); + + verifyNoInteractions(blockBlobSidecarsTrackersPool); + } } From ba4f460b6c02bf91f7d4c7a2ec2f90067af988b8 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 13 Dec 2024 14:48:11 +0100 Subject: [PATCH 29/31] complete tests --- .../blobs/BlobSidecarManagerTest.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index 317843eda82..eb10048dfc6 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -14,6 +14,7 @@ package tech.pegasys.teku.statetransition.blobs; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -340,11 +341,19 @@ void createAvailabilityCheckerAndValidateImmediately_shouldReturnNotRequiredWhen when(blockBlobSidecarsTracker.add(any())).thenReturn(true); when(blockBlobSidecarsTracker.isComplete()).thenReturn(true); - final SafeFuture result = - SafeFuture.completedFuture(BlobSidecarsAndValidationResult.validResult(blobSidecars)); + final SafeFuture result = new SafeFuture<>(); when(forkChoiceBlobSidecarsAvailabilityChecker.getAvailabilityCheckResult()).thenReturn(result); + assertThatThrownBy( + () -> + blobSidecarManager.createAvailabilityCheckerAndValidateImmediately( + block, blobSidecars)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Availability check should be done immediately"); + + result.complete(BlobSidecarsAndValidationResult.validResult(blobSidecars)); + assertThat( blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(block, blobSidecars)) .isSameAs(result.join()); From ea176d14dd539c73f87df42920aa65fe2de59cb7 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 13 Dec 2024 16:57:09 +0100 Subject: [PATCH 30/31] complete tests --- CHANGELOG.md | 1 + .../teku/statetransition/blobs/BlobSidecarManagerImpl.java | 3 ++- .../teku/statetransition/blobs/BlobSidecarManagerTest.java | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6558b740bfb..13a25dd2276 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Breaking Changes ### Additions and Improvements +- Optimized blobs validation pipeline ### Bug Fixes - Fixed an issue with the `/eth/v1/config/spec` API not returning all previously included configuration parameters. \ No newline at end of file diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java index c8929735eba..fc20d0c60e8 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java @@ -192,7 +192,8 @@ public BlobSidecarsAndValidationResult createAvailabilityCheckerAndValidateImmed if (availabilityCheckResult.isDone()) { return availabilityCheckResult.join(); } else { - throw new IllegalStateException("Availability check should be done immediately"); + throw new IllegalStateException( + "Availability check expected to be done synchronously when providing immediate blobs"); } } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java index eb10048dfc6..cac650a982e 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerTest.java @@ -350,7 +350,8 @@ void createAvailabilityCheckerAndValidateImmediately_shouldReturnNotRequiredWhen blobSidecarManager.createAvailabilityCheckerAndValidateImmediately( block, blobSidecars)) .isInstanceOf(IllegalStateException.class) - .hasMessage("Availability check should be done immediately"); + .hasMessage( + "Availability check expected to be done synchronously when providing immediate blobs"); result.complete(BlobSidecarsAndValidationResult.validResult(blobSidecars)); From 0b4524e451097c20650212417b44b255c00a8985 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 13 Dec 2024 17:38:28 +0100 Subject: [PATCH 31/31] mark signature validated during gossip --- .../BlobSidecarGossipValidator.java | 25 +++++++++++++------ .../BlobSidecarGossipValidatorTest.java | 6 +++++ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java index f058d8e2375..fa573f7cb17 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java @@ -255,8 +255,7 @@ public SafeFuture validate(final BlobSidecar blobSidec * [REJECT] The proposer signature of `blob_sidecar.signed_block_header`, is valid * with respect to the `block_header.proposer_index` pubkey. */ - if (!verifyBlockHeaderSignature( - postState, blobSidecar.getSignedBeaconBlockHeader())) { + if (!verifyBlockHeaderSignature(postState, blobSidecar)) { return reject("BlobSidecar block header signature is invalid."); } @@ -291,6 +290,8 @@ public boolean markForEquivocation(final BlobSidecar blobSidecar) { private SafeFuture validateBlobSidecarWithKnownValidHeader( final BlobSidecar blobSidecar, final BeaconBlockHeader blockHeader) { + blobSidecar.markSignatureAsValidated(); + /* * [REJECT] The sidecar's inclusion proof is valid as verified by `verify_blob_sidecar_inclusion_proof(blob_sidecar)`. */ @@ -330,7 +331,8 @@ private SafeFuture validateBlobSidecarWithKnownValidHe } private boolean verifyBlockHeaderSignature( - final BeaconState state, final SignedBeaconBlockHeader signedBlockHeader) { + final BeaconState state, final BlobSidecar blobSidecar) { + final SignedBeaconBlockHeader signedBlockHeader = blobSidecar.getSignedBeaconBlockHeader(); final Bytes32 domain = spec.getDomain( Domain.BEACON_PROPOSER, @@ -339,11 +341,18 @@ private boolean verifyBlockHeaderSignature( state.getGenesisValidatorsRoot()); final Bytes signingRoot = spec.computeSigningRoot(signedBlockHeader.getMessage(), domain); - return gossipValidationHelper.isSignatureValidWithRespectToProposerIndex( - signingRoot, - signedBlockHeader.getMessage().getProposerIndex(), - signedBlockHeader.getSignature(), - state); + final boolean result = + gossipValidationHelper.isSignatureValidWithRespectToProposerIndex( + signingRoot, + signedBlockHeader.getMessage().getProposerIndex(), + signedBlockHeader.getSignature(), + state); + + if (result) { + blobSidecar.markSignatureAsValidated(); + } + + return result; } private boolean isFirstValidForSlotProposerIndexAndBlobIndex( diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidatorTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidatorTest.java index 6e021da1f08..1a92bd8b42e 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidatorTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidatorTest.java @@ -118,6 +118,7 @@ void setup(final SpecContext specContext) { void shouldAccept() { SafeFutureAssert.assertThatSafeFuture(blobSidecarValidator.validate(blobSidecar)) .isCompletedWithValueMatching(InternalValidationResult::isAccept); + assertThat(blobSidecar.isSignatureValidated()).isTrue(); } @TestTemplate @@ -184,6 +185,8 @@ void shouldRejectIfSignatureIsInvalid() { SafeFutureAssert.assertThatSafeFuture(blobSidecarValidator.validate(blobSidecar)) .isCompletedWithValueMatching(InternalValidationResult::isReject); + + assertThat(blobSidecar.isSignatureValidated()).isFalse(); } @TestTemplate @@ -342,6 +345,9 @@ void shouldNotVerifyKnownValidSignedHeader() { verify(gossipValidationHelper, never()).isProposerTheExpectedProposer(any(), any(), any()); verify(gossipValidationHelper, never()) .isSignatureValidWithRespectToProposerIndex(any(), any(), any(), any()); + + assertThat(blobSidecar.isSignatureValidated()).isTrue(); + clearInvocations(gossipValidationHelper); // BlobSidecar from the new block