From 28213dea78bbce547a859da600e98dd13cf53c92 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko <zilm@zilm.net> Date: Fri, 13 Oct 2023 17:23:41 +0400 Subject: [PATCH] Add blob sidecars cache in Store + usage --- .../GetBlobSidecarsIntegrationTest.java | 4 +- .../forkchoice/MutableStore.java | 7 ++- .../forkchoice/ReadOnlyStore.java | 4 +- .../logic/common/util/ForkChoiceUtil.java | 2 +- .../forkchoice/TestStoreImpl.java | 13 ++---- .../forkchoice/ForkChoice.java | 7 ++- .../block/BlockManagerTest.java | 19 +++----- .../AbstractRpcMethodIntegrationTest.java | 6 +-- .../client/CombinedChainDataClient.java | 21 +++++++++ .../teku/storage/client/RecentChainData.java | 18 ++------ .../client/StorageBackedRecentChainData.java | 5 -- .../teku/storage/store/CacheableStore.java | 4 ++ .../teku/storage/store/EmptyStoreResults.java | 6 --- .../pegasys/teku/storage/store/Store.java | 46 +++++++++++++------ .../teku/storage/store/StoreBuilder.java | 9 ---- .../teku/storage/store/StoreTransaction.java | 13 ++---- .../store/StoreTransactionUpdates.java | 1 + .../storage/client/RecentChainDataTest.java | 12 ++--- .../StorageBackedRecentChainDataTest.java | 6 +-- .../teku/storage/store/AbstractStoreTest.java | 7 --- .../pegasys/teku/storage/store/StoreTest.java | 2 - .../teku/storage/client/ChainUpdater.java | 7 ++- .../client/MemoryOnlyRecentChainData.java | 2 - .../teku/storage/store/StoreAssertions.java | 4 +- 24 files changed, 108 insertions(+), 117 deletions(-) diff --git a/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v1/beacon/GetBlobSidecarsIntegrationTest.java b/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v1/beacon/GetBlobSidecarsIntegrationTest.java index 88ea97aefbb..e29afafeb2d 100644 --- a/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v1/beacon/GetBlobSidecarsIntegrationTest.java +++ b/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v1/beacon/GetBlobSidecarsIntegrationTest.java @@ -63,7 +63,7 @@ public void shouldGetBlobSidecars() throws Exception { final SignedBlockAndState lastBlock = chainUpdater.advanceChainUntil(targetSlot); chainUpdater.updateBestBlock(lastBlock); final List<BlobSidecar> expected = - recentChainData.retrieveBlobSidecars(lastBlock.getSlotAndBlockRoot()).get(); + recentChainData.getBlobSidecars(lastBlock.getSlotAndBlockRoot()).get(); final Response responseAll = get("head"); assertThat(responseAll.code()).isEqualTo(SC_OK); @@ -116,7 +116,7 @@ public void shouldGetBlobSidecarsAsSsz() throws Exception { final SignedBlockAndState lastBlock = chainUpdater.advanceChainUntil(targetSlot); chainUpdater.updateBestBlock(lastBlock); final List<BlobSidecar> expected = - recentChainData.retrieveBlobSidecars(lastBlock.getSlotAndBlockRoot()).get(); + recentChainData.getBlobSidecars(lastBlock.getSlotAndBlockRoot()).get(); final Response response = get("head", OCTET_STREAM); assertThat(response.code()).isEqualTo(SC_OK); diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/MutableStore.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/MutableStore.java index 65216daa8e0..85842ff39f2 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/MutableStore.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/MutableStore.java @@ -13,7 +13,6 @@ package tech.pegasys.teku.spec.datastructures.forkchoice; -import java.util.Collections; import java.util.List; import java.util.Optional; import org.apache.tuweni.bytes.Bytes32; @@ -43,7 +42,7 @@ void putBlockAndState( SignedBeaconBlock block, BeaconState state, BlockCheckpoints checkpoints, - List<BlobSidecar> blobSidecars, + Optional<List<BlobSidecar>> blobSidecars, Optional<UInt64> earliestBlobSidecarSlot); default void putBlockAndState( @@ -52,7 +51,7 @@ default void putBlockAndState( blockAndState.getBlock(), blockAndState.getState(), checkpoints, - Collections.emptyList(), + Optional.empty(), Optional.empty()); } @@ -64,7 +63,7 @@ default void putBlockAndState( blockAndState.getBlock(), blockAndState.getState(), checkpoints, - blobSidecars, + Optional.of(blobSidecars), Optional.empty()); } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/ReadOnlyStore.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/ReadOnlyStore.java index 159d0cf7f6c..9861de85179 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/ReadOnlyStore.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/ReadOnlyStore.java @@ -108,6 +108,8 @@ default UInt64 getGenesisTimeMillis() { */ Optional<SignedBeaconBlock> getBlockIfAvailable(final Bytes32 blockRoot); + Optional<List<BlobSidecar>> getBlobSidecarsIfAvailable(SlotAndBlockRoot slotAndBlockRoot); + default SafeFuture<Optional<BeaconBlock>> retrieveBlock(Bytes32 blockRoot) { return retrieveSignedBlock(blockRoot).thenApply(res -> res.map(SignedBeaconBlock::getMessage)); } @@ -124,8 +126,6 @@ default SafeFuture<Optional<BeaconBlock>> retrieveBlock(Bytes32 blockRoot) { SafeFuture<Optional<BeaconState>> retrieveStateAtSlot(SlotAndBlockRoot checkpoint); - SafeFuture<List<BlobSidecar>> retrieveBlobSidecars(SlotAndBlockRoot slotAndBlockRoot); - SafeFuture<Optional<UInt64>> retrieveEarliestBlobSidecarSlot(); SafeFuture<CheckpointState> retrieveFinalizedCheckpointAndState(); diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/ForkChoiceUtil.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/ForkChoiceUtil.java index 3ae6b4881fb..805d3fd9259 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/ForkChoiceUtil.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/ForkChoiceUtil.java @@ -384,7 +384,7 @@ public void applyBlockToStore( final SignedBeaconBlock signedBlock, final BeaconState postState, final boolean isBlockOptimistic, - final List<BlobSidecar> blobSidecars, + final Optional<List<BlobSidecar>> blobSidecars, final Optional<UInt64> earliestBlobSidecarsSlot) { BlockCheckpoints blockCheckpoints = epochProcessor.calculateBlockCheckpoints(postState); diff --git a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/forkchoice/TestStoreImpl.java b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/forkchoice/TestStoreImpl.java index 11aac05a4db..7a47f4a3be1 100644 --- a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/forkchoice/TestStoreImpl.java +++ b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/forkchoice/TestStoreImpl.java @@ -17,7 +17,6 @@ import static tech.pegasys.teku.infrastructure.time.TimeUtilities.secondsToMillis; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -248,10 +247,9 @@ public SafeFuture<Optional<BeaconState>> retrieveCheckpointState( } @Override - public SafeFuture<List<BlobSidecar>> retrieveBlobSidecars( + public Optional<List<BlobSidecar>> getBlobSidecarsIfAvailable( final SlotAndBlockRoot slotAndBlockRoot) { - return SafeFuture.completedFuture( - Optional.ofNullable(blobSidecars.get(slotAndBlockRoot)).orElse(Collections.emptyList())); + return Optional.ofNullable(blobSidecars.get(slotAndBlockRoot)); } @Override @@ -265,14 +263,13 @@ public void putBlockAndState( final SignedBeaconBlock block, final BeaconState state, final BlockCheckpoints checkpoints, - final List<BlobSidecar> blobSidecars, + final Optional<List<BlobSidecar>> blobSidecars, final Optional<UInt64> maybeEarliestBlobSidecarSlot) { blocks.put(block.getRoot(), block); blockStates.put(block.getRoot(), state); blockCheckpoints.put(block.getRoot(), checkpoints); - if (!blobSidecars.isEmpty()) { - this.blobSidecars.put(block.getSlotAndBlockRoot(), blobSidecars); - } + blobSidecars.ifPresent( + sidecars -> this.blobSidecars.put(block.getSlotAndBlockRoot(), sidecars)); if (earliestBlobSidecarSlot.isEmpty()) { earliestBlobSidecarSlot = maybeEarliestBlobSidecarSlot; } diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java index 46fa4629a66..9194e00918f 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoice.java @@ -22,7 +22,6 @@ import com.google.common.base.Throwables; import java.net.ConnectException; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Optional; import org.apache.logging.log4j.LogManager; @@ -540,12 +539,12 @@ private BlockImportResult importBlockAndState( final StoreTransaction transaction = recentChainData.startStoreTransaction(); addParentStateRoots(spec, blockSlotState, transaction); - final List<BlobSidecar> blobSidecars; + final Optional<List<BlobSidecar>> blobSidecars; if (blobSidecarsAndValidationResult.isNotRequired()) { // Outside availability window or pre-Deneb - blobSidecars = Collections.emptyList(); + blobSidecars = Optional.empty(); } else if (blobSidecarsAndValidationResult.isValid()) { - blobSidecars = blobSidecarsAndValidationResult.getBlobSidecars(); + blobSidecars = Optional.of(blobSidecarsAndValidationResult.getBlobSidecars()); } else { throw new IllegalStateException( String.format( diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/block/BlockManagerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/block/BlockManagerTest.java index 66bbbdbdb6b..d741b926f0e 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/block/BlockManagerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/block/BlockManagerTest.java @@ -941,8 +941,7 @@ void onDeneb_shouldStoreBlockWhenBlobSidecarsNotRequired() { verify(blobSidecarsAvailabilityChecker1).getAvailabilityCheckResult(); assertThat(localRecentChainData.retrieveBlockByRoot(block1.getRoot())) .isCompletedWithValue(Optional.of(block1.getMessage())); - assertThat(localRecentChainData.retrieveBlobSidecars(block1.getSlotAndBlockRoot())) - .isCompletedWithValue(Collections.emptyList()); + assertThat(localRecentChainData.getBlobSidecars(block1.getSlotAndBlockRoot())).isEmpty(); assertThat(localRecentChainData.retrieveEarliestBlobSidecarSlot()) .isCompletedWithValueMatching(Optional::isEmpty); } @@ -981,8 +980,7 @@ void onDeneb_shouldNotStoreBlockWhenBlobSidecarsIsInvalid() { verify(blobSidecarsAvailabilityChecker1).getAvailabilityCheckResult(); assertThat(localRecentChainData.retrieveBlockByRoot(block1.getRoot())) .isCompletedWithValue(Optional.empty()); - assertThat(localRecentChainData.retrieveBlobSidecars(block1.getSlotAndBlockRoot())) - .isCompletedWithValue(Collections.emptyList()); + assertThat(localRecentChainData.getBlobSidecars(block1.getSlotAndBlockRoot())).isEmpty(); assertThat(localRecentChainData.retrieveEarliestBlobSidecarSlot()) .isCompletedWithValueMatching(Optional::isEmpty); @@ -1022,8 +1020,7 @@ void onDeneb_shouldNotStoreBlockWhenBlobSidecarsIsNotAvailable() { verify(blobSidecarsAvailabilityChecker1).getAvailabilityCheckResult(); assertThat(localRecentChainData.retrieveBlockByRoot(block1.getRoot())) .isCompletedWithValue(Optional.empty()); - assertThat(localRecentChainData.retrieveBlobSidecars(block1.getSlotAndBlockRoot())) - .isCompletedWithValue(Collections.emptyList()); + assertThat(localRecentChainData.getBlobSidecars(block1.getSlotAndBlockRoot())).isEmpty(); assertThat(localRecentChainData.retrieveEarliestBlobSidecarSlot()) .isCompletedWithValueMatching(Optional::isEmpty); } @@ -1053,8 +1050,7 @@ void preDeneb_shouldNotWorryAboutBlobSidecars() { verify(blobSidecarsAvailabilityChecker1).getAvailabilityCheckResult(); assertThat(localRecentChainData.retrieveBlockByRoot(block1.getRoot())) .isCompletedWithValue(Optional.of(block1.getMessage())); - assertThat(localRecentChainData.retrieveBlobSidecars(block1.getSlotAndBlockRoot())) - .isCompletedWithValue(Collections.emptyList()); + assertThat(localRecentChainData.getBlobSidecars(block1.getSlotAndBlockRoot())).isEmpty(); assertThat(localRecentChainData.retrieveEarliestBlobSidecarSlot()) .isCompletedWithValueMatching(Optional::isEmpty); } @@ -1115,15 +1111,14 @@ private void assertThatStored( final BeaconBlock beaconBlock, final List<BlobSidecar> blobSidecars) { assertThat(localRecentChainData.retrieveBlockByRoot(beaconBlock.getRoot())) .isCompletedWithValue(Optional.of(beaconBlock)); - assertThat(localRecentChainData.retrieveBlobSidecars(beaconBlock.getSlotAndBlockRoot())) - .isCompletedWithValue(blobSidecars); + assertThat(localRecentChainData.getBlobSidecars(beaconBlock.getSlotAndBlockRoot())) + .contains(blobSidecars); } private void assertThatNothingStoredForSlotRoot(final SlotAndBlockRoot slotAndBlockRoot) { assertThat(localRecentChainData.retrieveBlockByRoot(slotAndBlockRoot.getBlockRoot())) .isCompletedWithValueMatching(Optional::isEmpty); - assertThat(localRecentChainData.retrieveBlobSidecars(slotAndBlockRoot)) - .isCompletedWithValueMatching(List::isEmpty); + assertThat(localRecentChainData.getBlobSidecars(slotAndBlockRoot)).isEmpty(); } private void assertImportBlockWithResult( diff --git a/networking/eth2/src/integration-test/java/tech/pegasys/teku/networking/eth2/AbstractRpcMethodIntegrationTest.java b/networking/eth2/src/integration-test/java/tech/pegasys/teku/networking/eth2/AbstractRpcMethodIntegrationTest.java index 2128d642343..276dfb26de7 100644 --- a/networking/eth2/src/integration-test/java/tech/pegasys/teku/networking/eth2/AbstractRpcMethodIntegrationTest.java +++ b/networking/eth2/src/integration-test/java/tech/pegasys/teku/networking/eth2/AbstractRpcMethodIntegrationTest.java @@ -19,7 +19,6 @@ import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.params.provider.Arguments; @@ -261,12 +260,13 @@ protected List<BlobSidecar> retrieveCanonicalBlobSidecarsFromPeerStorage( .flatMap(Optional::stream) .map(this::safeRetrieveBlobSidecars) .flatMap(Collection::stream) - .collect(Collectors.toUnmodifiableList()); + .toList(); } private List<BlobSidecar> safeRetrieveBlobSidecars(final SlotAndBlockRoot slotAndBlockRoot) { try { - return Waiter.waitFor(peerStorage.recentChainData().retrieveBlobSidecars(slotAndBlockRoot)); + return Waiter.waitFor( + peerStorage.chainStorage().getBlobSidecarsBySlotAndBlockRoot(slotAndBlockRoot)); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java b/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java index 2c5143f1280..be2aa7b9547 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java @@ -178,6 +178,11 @@ public SafeFuture<List<BlobSidecar>> getBlobSidecars( public SafeFuture<List<BlobSidecar>> getBlobSidecars( final SlotAndBlockRoot slotAndBlockRoot, final List<UInt64> indices) { + final Optional<List<BlobSidecar>> maybeBlobSidecars = + recentChainData.getBlobSidecars(slotAndBlockRoot); + if (maybeBlobSidecars.isPresent()) { + return SafeFuture.completedFuture(filterBlobSidecars(maybeBlobSidecars.get(), indices)); + } return historicalChainData .getBlobSidecarKeys(slotAndBlockRoot) .thenApply(keys -> filterBlobSidecarKeys(keys, indices)) @@ -192,6 +197,14 @@ public SafeFuture<List<BlobSidecar>> getAllBlobSidecars( .thenCompose(this::getAllBlobSidecars); } + private List<BlobSidecar> filterBlobSidecars( + final List<BlobSidecar> blobSidecars, final List<UInt64> indices) { + if (indices.isEmpty()) { + return blobSidecars; + } + return blobSidecars.stream().filter(key -> indices.contains(key.getIndex())).toList(); + } + private Stream<SlotAndBlockRootAndBlobIndex> filterBlobSidecarKeys( final List<SlotAndBlockRootAndBlobIndex> keys, final List<UInt64> indices) { if (indices.isEmpty()) { @@ -569,6 +582,14 @@ public SafeFuture<Optional<BlobSidecar>> getBlobSidecarByBlockRootAndIndex( public SafeFuture<Optional<BlobSidecar>> getBlobSidecarByKey( final SlotAndBlockRootAndBlobIndex key) { + final Optional<List<BlobSidecar>> maybeBlobSidecars = + recentChainData.getBlobSidecars(key.getSlotAndBlockRoot()); + if (maybeBlobSidecars.isPresent()) { + return key.getBlobIndex().isLessThan(maybeBlobSidecars.get().size()) + ? SafeFuture.completedFuture( + Optional.of(maybeBlobSidecars.get().get(key.getBlobIndex().intValue()))) + : SafeFuture.completedFuture(Optional.empty()); + } return historicalChainData.getBlobSidecar(key); } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java b/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java index 748a67e695d..7da78d1e3ce 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java @@ -28,7 +28,6 @@ import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; -import tech.pegasys.teku.dataproviders.lookup.BlobSidecarsProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.EarliestBlobSidecarSlotProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; @@ -77,7 +76,6 @@ public abstract class RecentChainData implements StoreUpdateHandler { private final BlockProvider blockProvider; private final StateAndBlockSummaryProvider stateProvider; - private final BlobSidecarsProvider blobSidecarsProvider; private final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider; protected final FinalizedCheckpointChannel finalizedCheckpointChannel; protected final StorageUpdateChannel storageUpdateChannel; @@ -106,7 +104,6 @@ public abstract class RecentChainData implements StoreUpdateHandler { final StoreConfig storeConfig, final BlockProvider blockProvider, final StateAndBlockSummaryProvider stateProvider, - final BlobSidecarsProvider blobSidecarsProvider, final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider, final StorageUpdateChannel storageUpdateChannel, final VoteUpdateChannel voteUpdateChannel, @@ -118,7 +115,6 @@ public abstract class RecentChainData implements StoreUpdateHandler { this.storeConfig = storeConfig; this.blockProvider = blockProvider; this.stateProvider = stateProvider; - this.blobSidecarsProvider = blobSidecarsProvider; this.earliestBlobSidecarSlotProvider = earliestBlobSidecarSlotProvider; this.voteUpdateChannel = voteUpdateChannel; this.chainHeadChannel = chainHeadChannel; @@ -154,7 +150,6 @@ public void initializeFromAnchorPoint(final AnchorPoint anchorPoint, final UInt6 .specProvider(spec) .blockProvider(blockProvider) .stateProvider(stateProvider) - .blobSidecarsProvider(blobSidecarsProvider) .earliestBlobSidecarSlotProvider(earliestBlobSidecarSlotProvider) .storeConfig(storeConfig) .build(); @@ -495,6 +490,11 @@ public Optional<Bytes32> getExecutionBlockHashForBlockRoot(final Bytes32 root) { return getForkChoiceStrategy().flatMap(forkChoice -> forkChoice.executionBlockHash(root)); } + public Optional<List<BlobSidecar>> getBlobSidecars(final SlotAndBlockRoot slotAndBlockRoot) { + return Optional.ofNullable(store) + .flatMap(s -> store.getBlobSidecarsIfAvailable(slotAndBlockRoot)); + } + public SafeFuture<Optional<BeaconBlock>> retrieveBlockByRoot(final Bytes32 root) { if (store == null) { return EmptyStoreResults.EMPTY_BLOCK_FUTURE; @@ -532,14 +532,6 @@ public SafeFuture<Optional<BeaconState>> retrieveStateInEffectAtSlot(final UInt6 return store.retrieveBlockState(rootAtSlot.get()); } - public SafeFuture<List<BlobSidecar>> retrieveBlobSidecars( - final SlotAndBlockRoot slotAndBlockRoot) { - if (store == null) { - return EmptyStoreResults.NO_BLOB_SIDECARS_FUTURE; - } - return store.retrieveBlobSidecars(slotAndBlockRoot); - } - public SafeFuture<Optional<UInt64>> retrieveEarliestBlobSidecarSlot() { if (store == null) { return EmptyStoreResults.NO_EARLIEST_BLOB_SIDECAR_SLOT_FUTURE; diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java b/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java index bd3fb0851a3..57aa724aa26 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java @@ -22,7 +22,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.hyperledger.besu.plugin.services.MetricsSystem; -import tech.pegasys.teku.dataproviders.lookup.BlobSidecarsProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; import tech.pegasys.teku.infrastructure.async.AsyncRunner; @@ -43,7 +42,6 @@ public class StorageBackedRecentChainData extends RecentChainData { private static final Logger LOG = LogManager.getLogger(); private final BlockProvider blockProvider; private final StateAndBlockSummaryProvider stateProvider; - private final BlobSidecarsProvider blobSidecarsProvider; private final StorageQueryChannel storageQueryChannel; private final StoreConfig storeConfig; @@ -63,7 +61,6 @@ public StorageBackedRecentChainData( storeConfig, storageQueryChannel::getHotBlocksByRoot, storageQueryChannel::getHotStateAndBlockSummaryByBlockRoot, - storageQueryChannel::getBlobSidecarsBySlotAndBlockRoot, storageQueryChannel::getEarliestAvailableBlobSidecarSlot, storageUpdateChannel, voteUpdateChannel, @@ -74,7 +71,6 @@ public StorageBackedRecentChainData( this.storageQueryChannel = storageQueryChannel; this.blockProvider = storageQueryChannel::getHotBlocksByRoot; this.stateProvider = storageQueryChannel::getHotStateAndBlockSummaryByBlockRoot; - this.blobSidecarsProvider = storageQueryChannel::getBlobSidecarsBySlotAndBlockRoot; } public static SafeFuture<RecentChainData> create( @@ -158,7 +154,6 @@ private SafeFuture<RecentChainData> processStoreFuture( .asyncRunner(asyncRunner) .blockProvider(blockProvider) .stateProvider(stateProvider) - .blobSidecarsProvider(blobSidecarsProvider) .storeConfig(storeConfig) .build(); setStore(store); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/store/CacheableStore.java b/storage/src/main/java/tech/pegasys/teku/storage/store/CacheableStore.java index b43a41c9451..5b01c536cf5 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/store/CacheableStore.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/store/CacheableStore.java @@ -14,11 +14,13 @@ package tech.pegasys.teku.storage.store; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Predicate; import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BlockAndCheckpoints; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.blocks.StateAndBlockSummary; @@ -38,6 +40,8 @@ public abstract class CacheableStore implements UpdatableStore { abstract void cacheStates(Map<Bytes32, StateAndBlockSummary> stateAndBlockSummaries); + abstract void cacheBlobSidecars(Map<SlotAndBlockRoot, List<BlobSidecar>> blobSidecarsMap); + abstract void cacheFinalizedOptimisticTransitionPayload( Optional<SlotAndExecutionPayloadSummary> finalizedOptimisticTransitionPayload); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/store/EmptyStoreResults.java b/storage/src/main/java/tech/pegasys/teku/storage/store/EmptyStoreResults.java index 595c08bd9cf..7a518982f6c 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/store/EmptyStoreResults.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/store/EmptyStoreResults.java @@ -13,13 +13,10 @@ package tech.pegasys.teku.storage.store; -import java.util.Collections; -import java.util.List; import java.util.Optional; import tech.pegasys.teku.dataproviders.generators.StateGenerationTask; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.StateAndBlockSummary; @@ -42,9 +39,6 @@ public abstract class EmptyStoreResults { public static final SafeFuture<Optional<StateGenerationTask>> EMPTY_STATE_GENERATION_TASK = SafeFuture.completedFuture(Optional.empty()); - public static final SafeFuture<List<BlobSidecar>> NO_BLOB_SIDECARS_FUTURE = - SafeFuture.completedFuture(Collections.emptyList()); - public static final SafeFuture<Optional<UInt64>> NO_EARLIEST_BLOB_SIDECAR_SLOT_FUTURE = SafeFuture.completedFuture(Optional.empty()); } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/store/Store.java b/storage/src/main/java/tech/pegasys/teku/storage/store/Store.java index 3e728f405d4..50bb9a743e8 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/store/Store.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/store/Store.java @@ -42,7 +42,6 @@ import tech.pegasys.teku.dataproviders.generators.StateAtSlotTask; import tech.pegasys.teku.dataproviders.generators.StateGenerationTask; import tech.pegasys.teku.dataproviders.generators.StateRegenerationBaseSelector; -import tech.pegasys.teku.dataproviders.lookup.BlobSidecarsProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.EarliestBlobSidecarSlotProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; @@ -87,15 +86,14 @@ class Store extends CacheableStore { private final MetricsSystem metricsSystem; private Optional<SettableGauge> blockCountGauge = Optional.empty(); - private Optional<SettableGauge> epochStatesCountGauge = Optional.empty(); + private Optional<SettableGauge> blobSidecarsBlocksCountGauge = Optional.empty(); private final Optional<Map<Bytes32, StateAndBlockSummary>> maybeEpochStates; private final Spec spec; private final StateAndBlockSummaryProvider stateProvider; private final BlockProvider blockProvider; - private final BlobSidecarsProvider blobSidecarsProvider; private final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider; private final ForkChoiceStrategy forkChoiceStrategy; @@ -103,6 +101,7 @@ class Store extends CacheableStore { private final CachingTaskQueue<Bytes32, StateAndBlockSummary> states; private final Map<Bytes32, SignedBeaconBlock> blocks; private final CachingTaskQueue<SlotAndBlockRoot, BeaconState> checkpointStates; + private final Map<SlotAndBlockRoot, List<BlobSidecar>> blobSidecars; private UInt64 timeMillis; private UInt64 genesisTime; private AnchorPoint finalizedAnchor; @@ -119,7 +118,6 @@ private Store( final int hotStatePersistenceFrequencyInEpochs, final BlockProvider blockProvider, final StateAndBlockSummaryProvider stateProvider, - final BlobSidecarsProvider blobSidecarsProvider, final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider, final CachingTaskQueue<Bytes32, StateAndBlockSummary> states, final Optional<Checkpoint> initialCheckpoint, @@ -133,7 +131,8 @@ private Store( final Map<UInt64, VoteTracker> votes, final Map<Bytes32, SignedBeaconBlock> blocks, final CachingTaskQueue<SlotAndBlockRoot, BeaconState> checkpointStates, - final Optional<Map<Bytes32, StateAndBlockSummary>> maybeEpochStates) { + final Optional<Map<Bytes32, StateAndBlockSummary>> maybeEpochStates, + final Map<SlotAndBlockRoot, List<BlobSidecar>> blobSidecars) { checkArgument( time.isGreaterThanOrEqualTo(genesisTime), "Time must be greater than or equal to genesisTime"); @@ -157,6 +156,7 @@ private Store( this.justifiedCheckpoint = justifiedCheckpoint; this.bestJustifiedCheckpoint = bestJustifiedCheckpoint; this.blocks = blocks; + this.blobSidecars = blobSidecars; this.highestVotedValidatorIndex = votes.keySet().stream().max(Comparator.naturalOrder()).orElse(UInt64.ZERO); this.votes = @@ -180,7 +180,6 @@ private Store( .orElseGet(Collections::emptyMap)), fromMap(this.blocks), blockProvider); - this.blobSidecarsProvider = blobSidecarsProvider; this.earliestBlobSidecarSlotProvider = earliestBlobSidecarSlotProvider; } @@ -190,7 +189,6 @@ public static UpdatableStore create( final Spec spec, final BlockProvider blockProvider, final StateAndBlockSummaryProvider stateAndBlockProvider, - final BlobSidecarsProvider blobSidecarsProvider, final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider, final Optional<Checkpoint> initialCheckpoint, final UInt64 time, @@ -215,11 +213,12 @@ public static UpdatableStore create( final CachingTaskQueue<Bytes32, StateAndBlockSummary> stateTaskQueue = CachingTaskQueue.create( asyncRunner, metricsSystem, "memory_states", config.getStateCacheSize()); - final Optional<Map<Bytes32, StateAndBlockSummary>> maybeEpochStates = config.getEpochStateCacheSize() > 0 ? Optional.of(LimitedMap.createSynchronized(config.getEpochStateCacheSize())) : Optional.empty(); + final Map<SlotAndBlockRoot, List<BlobSidecar>> blobSidecars = + LimitedMap.createSynchronized(config.getBlockCacheSize()); final UInt64 currentEpoch = spec.computeEpochAtSlot(spec.getCurrentSlot(time, genesisTime)); final ForkChoiceStrategy forkChoiceStrategy = @@ -239,7 +238,6 @@ public static UpdatableStore create( config.getHotStatePersistenceFrequencyInEpochs(), blockProvider, stateAndBlockProvider, - blobSidecarsProvider, earliestBlobSidecarSlotProvider, stateTaskQueue, initialCheckpoint, @@ -253,7 +251,8 @@ public static UpdatableStore create( votes, blocks, checkpointStateTaskQueue, - maybeEpochStates); + maybeEpochStates, + blobSidecars); } private static ProtoArray buildProtoArray( @@ -308,6 +307,13 @@ public void startMetrics() { TekuMetricCategory.STORAGE, "memory_block_count", "Number of beacon blocks held in the in-memory store")); + blobSidecarsBlocksCountGauge = + Optional.of( + SettableGauge.create( + metricsSystem, + TekuMetricCategory.STORAGE, + "memory_blob_sidecars_blocks_count", + "Number of complete blob sidecars blocks held in the in-memory store")); if (maybeEpochStates.isPresent()) { epochStatesCountGauge = @@ -497,7 +503,7 @@ public SafeFuture<Optional<SignedBeaconBlock>> retrieveSignedBlock(final Bytes32 return SafeFuture.completedFuture(inMemoryBlock); } - // Retrieve and cache block + // Retrieve block return blockProvider.getBlock(blockRoot); } @@ -568,9 +574,14 @@ public SafeFuture<Optional<BeaconState>> retrieveCheckpointState( } @Override - public SafeFuture<List<BlobSidecar>> retrieveBlobSidecars( + public Optional<List<BlobSidecar>> getBlobSidecarsIfAvailable( final SlotAndBlockRoot slotAndBlockRoot) { - return blobSidecarsProvider.getBlobSidecars(slotAndBlockRoot); + readLock.lock(); + try { + return Optional.ofNullable(blobSidecars.get(slotAndBlockRoot)); + } finally { + readLock.unlock(); + } } @Override @@ -614,6 +625,15 @@ void cacheStates(final Map<Bytes32, StateAndBlockSummary> stateAndBlockSummaries states.cacheAll(stateAndBlockSummaries); } + /** Non-synchronized, no lock, unsafe if Store is not locked externally */ + @Override + void cacheBlobSidecars(final Map<SlotAndBlockRoot, List<BlobSidecar>> blobSidecarsMap) { + blobSidecarsMap.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> blobSidecars.put(entry.getKey(), entry.getValue())); + blobSidecarsBlocksCountGauge.ifPresent(gauge -> gauge.set(blobSidecars.size())); + } + /** Non-synchronized, no lock, unsafe if Store is not locked externally */ @Override void cacheFinalizedOptimisticTransitionPayload( diff --git a/storage/src/main/java/tech/pegasys/teku/storage/store/StoreBuilder.java b/storage/src/main/java/tech/pegasys/teku/storage/store/StoreBuilder.java index ebabcef3997..f7a4c23a699 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/store/StoreBuilder.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/store/StoreBuilder.java @@ -21,7 +21,6 @@ import java.util.Optional; import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.plugin.services.MetricsSystem; -import tech.pegasys.teku.dataproviders.lookup.BlobSidecarsProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.EarliestBlobSidecarSlotProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; @@ -41,7 +40,6 @@ public class StoreBuilder { private Spec spec; private BlockProvider blockProvider; private StateAndBlockSummaryProvider stateAndBlockProvider; - private BlobSidecarsProvider blobSidecarsProvider; private EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider; private StoreConfig storeConfig = StoreConfig.createDefault(); @@ -112,7 +110,6 @@ public UpdatableStore build() { spec, blockProvider, stateAndBlockProvider, - blobSidecarsProvider, earliestBlobSidecarSlotProvider, anchor, time, @@ -176,12 +173,6 @@ public StoreBuilder stateProvider(final StateAndBlockSummaryProvider stateProvid return this; } - public StoreBuilder blobSidecarsProvider(final BlobSidecarsProvider blobSidecarsProvider) { - checkNotNull(blobSidecarsProvider); - this.blobSidecarsProvider = blobSidecarsProvider; - return this; - } - public StoreBuilder earliestBlobSidecarSlotProvider( final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider) { checkNotNull(earliestBlobSidecarSlotProvider); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransaction.java b/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransaction.java index 875f562568e..66e11cf0912 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransaction.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransaction.java @@ -93,11 +93,11 @@ public void putBlockAndState( final SignedBeaconBlock block, final BeaconState state, final BlockCheckpoints blockCheckpoints, - final List<BlobSidecar> blobSidecars, + final Optional<List<BlobSidecar>> blobSidecars, final Optional<UInt64> maybeEarliestBlobSidecarSlot) { blockData.put(block.getRoot(), new TransactionBlockData(block, state, blockCheckpoints)); if (!blobSidecars.isEmpty()) { - this.blobSidecars.put(block.getSlotAndBlockRoot(), blobSidecars); + this.blobSidecars.put(block.getSlotAndBlockRoot(), blobSidecars.get()); } if (needToUpdateEarliestBlobSidecarSlot(maybeEarliestBlobSidecarSlot)) { this.maybeEarliestBlobSidecarTransactionSlot = maybeEarliestBlobSidecarSlot; @@ -459,13 +459,10 @@ public Optional<SignedBeaconBlock> getBlockIfAvailable(final Bytes32 blockRoot) } @Override - public SafeFuture<List<BlobSidecar>> retrieveBlobSidecars( + public Optional<List<BlobSidecar>> getBlobSidecarsIfAvailable( final SlotAndBlockRoot slotAndBlockRoot) { - final Optional<List<BlobSidecar>> maybeBlobSidecars = - Optional.ofNullable(blobSidecars.get(slotAndBlockRoot)); - return maybeBlobSidecars - .map(SafeFuture::completedFuture) - .orElseGet(() -> store.retrieveBlobSidecars(slotAndBlockRoot)); + return Optional.ofNullable(blobSidecars.get(slotAndBlockRoot)) + .or(() -> store.getBlobSidecarsIfAvailable(slotAndBlockRoot)); } @Override diff --git a/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransactionUpdates.java b/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransactionUpdates.java index 71aa9fe8636..ba6c478eb45 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransactionUpdates.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/store/StoreTransactionUpdates.java @@ -109,6 +109,7 @@ public void applyToStore(final Store store, final UpdateResult updateResult) { tx.bestJustifiedCheckpoint.ifPresent(store::updateBestJustifiedCheckpoint); store.cacheBlocks(hotBlocks.values()); store.cacheStates(Maps.transformValues(hotBlockAndStates, this::blockAndStateAsSummary)); + store.cacheBlobSidecars(blobSidecars); if (optimisticTransitionBlockRootSet) { store.cacheFinalizedOptimisticTransitionPayload( updateResult.getFinalizedOptimisticTransitionPayload()); diff --git a/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java b/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java index 3a875f339b3..16f68fc8114 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java @@ -803,9 +803,9 @@ void retrieveBlobSidecars_shouldReturnBlobSidecarsWhenExists() { assertThat(blobSidecars).isNotEmpty(); storageSystem.chainUpdater().saveBlock(block, blobSidecars); - final SafeFuture<List<BlobSidecar>> blobSidecarsFuture = - recentChainData.retrieveBlobSidecars(block.getSlotAndBlockRoot()); - assertThat(blobSidecarsFuture).isCompletedWithValue(blobSidecars); + final Optional<List<BlobSidecar>> maybeBlobSidecars = + recentChainData.getBlobSidecars(block.getSlotAndBlockRoot()); + assertThat(maybeBlobSidecars).contains(blobSidecars); } @Test @@ -817,9 +817,9 @@ void retrieveBlobSidecars_shouldReturnEmptyBlobSidecarsWhenNotExists() { storageSystem.chainUpdater().saveBlock(block, blobSidecars); assertThat(blobSidecars).isEmpty(); - final SafeFuture<List<BlobSidecar>> blobSidecarsFuture = - recentChainData.retrieveBlobSidecars(block.getSlotAndBlockRoot()); - assertThat(blobSidecarsFuture).isCompletedWithValue(blobSidecars); + final Optional<List<BlobSidecar>> maybeBlobSidecars = + recentChainData.getBlobSidecars(block.getSlotAndBlockRoot()); + assertThat(maybeBlobSidecars).contains(blobSidecars); } @Test diff --git a/storage/src/test/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainDataTest.java b/storage/src/test/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainDataTest.java index d1cf4c567ca..8b602676966 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainDataTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainDataTest.java @@ -26,7 +26,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.Test; -import tech.pegasys.teku.dataproviders.lookup.BlobSidecarsProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -106,7 +105,6 @@ public void storageBackedClient_storeInitializeViaGetStoreRequest() .specProvider(spec) .blockProvider(BlockProvider.NOOP) .stateProvider(StateAndBlockSummaryProvider.NOOP) - .blobSidecarsProvider(BlobSidecarsProvider.NOOP) .storeConfig(storeConfig) .build(); StoreAssertions.assertStoresMatch(client.get().getStore(), expectedStore); @@ -154,7 +152,6 @@ public void storageBackedClient_storeInitializeViaNewGenesisState() .specProvider(spec) .blockProvider(BlockProvider.NOOP) .stateProvider(StateAndBlockSummaryProvider.NOOP) - .blobSidecarsProvider(BlobSidecarsProvider.NOOP) .storeConfig(storeConfig) .build(); client.get().initializeFromGenesis(initialState, UInt64.ZERO); @@ -202,8 +199,7 @@ public void storageBackedClient_storeInitializeViaGetStoreRequestAfterTimeout() .metricsSystem(new StubMetricsSystem()) .specProvider(spec) .blockProvider(BlockProvider.NOOP) - .stateProvider(StateAndBlockSummaryProvider.NOOP) - .blobSidecarsProvider(BlobSidecarsProvider.NOOP); + .stateProvider(StateAndBlockSummaryProvider.NOOP); storeRequestFuture.complete(Optional.of(storeData)); assertThat(client).isCompleted(); assertStoreInitialized(client.get()); diff --git a/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java b/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java index fc687d3750b..9d44544a06b 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java @@ -25,7 +25,6 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.stream.Collectors; -import tech.pegasys.teku.dataproviders.lookup.BlobSidecarsProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.EarliestBlobSidecarSlotProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; @@ -143,7 +142,6 @@ protected UpdatableStore createGenesisStore(final StoreConfig pruningOptions) { .metricsSystem(new StubMetricsSystem()) .specProvider(spec) .blockProvider(blockProviderFromChainBuilder()) - .blobSidecarsProvider(blobSidecarsProviderFromChainBuilder()) .earliestBlobSidecarSlotProvider(earliestBlobSidecarSlotProviderFromChainBuilder()) .stateProvider(StateAndBlockSummaryProvider.NOOP) .anchor(Optional.empty()) @@ -176,11 +174,6 @@ protected BlockProvider blockProviderFromChainBuilder() { .collect(Collectors.toMap(SignedBeaconBlock::getRoot, Function.identity()))); } - protected BlobSidecarsProvider blobSidecarsProviderFromChainBuilder() { - return (slotAndBlockRoot) -> - SafeFuture.completedFuture(chainBuilder.getBlobSidecars(slotAndBlockRoot.getBlockRoot())); - } - protected EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProviderFromChainBuilder() { return () -> SafeFuture.completedFuture(chainBuilder.getEarliestBlobSidecarSlot()); } diff --git a/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java b/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java index 25c0b247873..1c6eb07e0b5 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java @@ -26,7 +26,6 @@ import java.util.stream.Collectors; import org.apache.tuweni.bytes.Bytes32; import org.junit.jupiter.api.Test; -import tech.pegasys.teku.dataproviders.lookup.BlobSidecarsProvider; import tech.pegasys.teku.dataproviders.lookup.EarliestBlobSidecarSlotProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -62,7 +61,6 @@ public void create_timeLessThanGenesisTime() { spec, blockProviderFromChainBuilder(), StateAndBlockSummaryProvider.NOOP, - BlobSidecarsProvider.NOOP, EarliestBlobSidecarSlotProvider.NOOP, Optional.empty(), genesisTime.minus(1), diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java index 0e3c56e9d0f..7ed8b1c92fe 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java @@ -17,7 +17,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static tech.pegasys.teku.infrastructure.time.TimeUtilities.secondsToMillis; -import java.util.Collections; import java.util.List; import java.util.Optional; import tech.pegasys.teku.infrastructure.unsigned.UInt64; @@ -252,7 +251,7 @@ public void saveBlock(final SignedBlockAndState block) { block.getBlock(), block.getState(), spec.calculateBlockCheckpoints(block.getState()), - Collections.emptyList(), + Optional.empty(), Optional.empty()); assertThat(tx.commit()).isCompleted(); recentChainData @@ -271,7 +270,7 @@ public void saveOptimisticBlock(final SignedBlockAndState block) { block.getBlock(), block.getState(), spec.calculateBlockCheckpoints(block.getState()), - Collections.emptyList(), + Optional.empty(), Optional.empty()); assertThat(tx.commit()).isCompleted(); saveBlockTime(block); @@ -290,7 +289,7 @@ public void saveBlock( block.getBlock(), block.getState(), spec.calculateBlockCheckpoints(block.getState()), - blobSidecars, + Optional.of(blobSidecars), Optional.of(earliestBlobSidecarSlot)); assertThat(tx.commit()).isCompleted(); recentChainData diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/MemoryOnlyRecentChainData.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/MemoryOnlyRecentChainData.java index 584a5efe2ce..cfd32ec42a8 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/MemoryOnlyRecentChainData.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/MemoryOnlyRecentChainData.java @@ -18,7 +18,6 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.MetricsSystem; -import tech.pegasys.teku.dataproviders.lookup.BlobSidecarsProvider; import tech.pegasys.teku.dataproviders.lookup.BlockProvider; import tech.pegasys.teku.dataproviders.lookup.EarliestBlobSidecarSlotProvider; import tech.pegasys.teku.dataproviders.lookup.StateAndBlockSummaryProvider; @@ -51,7 +50,6 @@ private MemoryOnlyRecentChainData( storeConfig, BlockProvider.NOOP, StateAndBlockSummaryProvider.NOOP, - BlobSidecarsProvider.NOOP, EarliestBlobSidecarSlotProvider.NOOP, storageUpdateChannel, voteUpdateChannel, diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/store/StoreAssertions.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/store/StoreAssertions.java index bdb087873af..21c245a5ac1 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/store/StoreAssertions.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/store/StoreAssertions.java @@ -55,7 +55,9 @@ public static void assertStoresMatch( "checkpointStates", "forkChoiceStrategy", "maybeEpochStates", - "epochStatesCountGauge") + "epochStatesCountGauge", + "blobSidecars", + "blobSidecarsBlocksCountGauge") .isEqualTo(expectedState); assertThat(actualState.getOrderedBlockRoots()) .containsExactlyElementsOf(expectedState.getOrderedBlockRoots());