Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Save earliest slot block in a DB variable #8722

Merged
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
354c6bc
add time tracker to getEarliestAvailableBlockSlot cmd
gfukushima Oct 1, 2024
ba1278a
Store earliest block slot in db variable to reduce unnecessary iterat…
gfukushima Oct 14, 2024
fbf35fa
remove required from dbdebug get-earliest-available-block-slot timed …
gfukushima Oct 14, 2024
d454ed4
Merge branch 'master' into save-earliest-slot-block-in-dbvariable
gfukushima Oct 14, 2024
9d87500
fix broken tests
gfukushima Oct 15, 2024
8de8f4a
Merge branch 'master' into save-earliest-slot-block-in-dbvariable
gfukushima Oct 15, 2024
57336b0
get extra block when streaming so we can be certain of what is the ne…
gfukushima Oct 16, 2024
8ef6ec7
fetch earliest slot from the db
gfukushima Oct 16, 2024
d7901bf
add integration test to ensure we get the right earliest slot availab…
gfukushima Oct 16, 2024
fcc3444
fix codespell
gfukushima Oct 16, 2024
f4a6464
add time tracker to getEarliestAvailableBlockSlot cmd
gfukushima Oct 1, 2024
1f8c202
Store earliest block slot in db variable to reduce unnecessary iterat…
gfukushima Oct 14, 2024
5bb65b3
remove required from dbdebug get-earliest-available-block-slot timed …
gfukushima Oct 14, 2024
3b6e87d
fix broken tests
gfukushima Oct 15, 2024
94bfff7
get extra block when streaming so we can be certain of what is the ne…
gfukushima Oct 16, 2024
721b364
fetch earliest slot from the db
gfukushima Oct 16, 2024
7ff1fa9
add integration test to ensure we get the right earliest slot availab…
gfukushima Oct 16, 2024
d1e0230
fix codespell
gfukushima Oct 16, 2024
905c9e5
pass checkpoint earliest slot to use as stream limit to fetch earlies…
gfukushima Oct 17, 2024
527c270
Merge remote-tracking branch 'origin/save-earliest-slot-block-in-dbva…
gfukushima Oct 17, 2024
397d918
Merge branch 'master' into save-earliest-slot-block-in-dbvariable
gfukushima Oct 17, 2024
109d342
if for some reason there's no block after the last pruned reset varia…
gfukushima Oct 17, 2024
00db207
Merge remote-tracking branch 'origin/save-earliest-slot-block-in-dbva…
gfukushima Oct 17, 2024
ca03731
spotless
gfukushima Oct 17, 2024
a5a4afc
Merge branch 'master' into save-earliest-slot-block-in-dbvariable
gfukushima Oct 18, 2024
9df9590
Merge branch 'master' into save-earliest-slot-block-in-dbvariable
gfukushima Oct 21, 2024
5cae7c2
Merge branch 'master' into save-earliest-slot-block-in-dbvariable
gfukushima Oct 21, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2180,6 +2180,30 @@ public void pruneFinalizedBlocks_shouldRemoveFinalizedBlocks(final DatabaseConte
assertThat(lastPrunedSlot3).isEqualTo(UInt64.valueOf(4));
}

@TestTemplate
public void pruneFinalizedBlocks_UpdatesEarliestAvailableBlockSlot(final DatabaseContext context)
throws Exception {
initialize(context, StateStorageMode.ARCHIVE);
final List<SignedBlockAndState> blockAndStates = chainBuilder.generateBlocksUpToSlot(5);
addBlocks(blockAndStates);
// Block 7 skipped simulating it was an empty block
final SignedBlockAndState finalizedBlock = chainBuilder.generateBlockAtSlot(7);
addBlocks(finalizedBlock);
justifyAndFinalizeEpoch(
spec.computeEpochAtSlot(finalizedBlock.getSlot()).plus(1), finalizedBlock);
assertThat(database.getFinalizedBlockAtSlot(UInt64.valueOf(6))).isEmpty();

final UInt64 lastPrunedSlot1 = database.pruneFinalizedBlocks(UInt64.valueOf(3), 100);
assertThat(lastPrunedSlot1).isEqualTo(UInt64.valueOf(3));
assertThat(database.getEarliestAvailableBlockSlot()).isEqualTo(Optional.of(UInt64.valueOf(4)));

final UInt64 lastPrunedSlot2 = database.pruneFinalizedBlocks(UInt64.valueOf(5), 10);
assertThat(lastPrunedSlot2).isEqualTo(UInt64.valueOf(5));
// there's no slot 6 because that was purposely skipped so we expect the earliest available
// block slot to be 7
assertThat(database.getEarliestAvailableBlockSlot()).isEqualTo(Optional.of(UInt64.valueOf(7)));
}

private List<Map.Entry<Bytes32, UInt64>> getFinalizedStateRootsList() {
try (final Stream<Map.Entry<Bytes32, UInt64>> roots = database.getFinalizedStateRoots()) {
return roots.map(entry -> Map.entry(entry.getKey(), entry.getValue())).collect(toList());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -324,12 +324,34 @@ protected void storeFinalizedBlocksToDao(
.forEach(updater::addBlobSidecar);
});

needToUpdateEarliestBlockSlot(blocks.stream().findFirst())
.ifPresent(updater::setEarliestBlockSlot);
needToUpdateEarliestBlobSidecarSlot(maybeEarliestBlobSidecar)
.ifPresent(updater::setEarliestBlobSidecarSlot);
updater.commit();
}
}

private Optional<UInt64> needToUpdateEarliestBlockSlot(
final Optional<SignedBeaconBlock> maybeNewEarliestBlockSlot) {
// New value is absent - not updating
if (maybeNewEarliestBlockSlot.isEmpty()) {
return Optional.empty();
}
// New value is present, value from DB is absent - updating
final Optional<UInt64> maybeEarliestFinalizedBlockSlotDb = dao.getEarliestFinalizedBlockSlot();
if (maybeEarliestFinalizedBlockSlotDb.isEmpty()) {
return maybeNewEarliestBlockSlot.map(SignedBeaconBlock::getSlot);
}
// New value is smaller than value from DB - updating
final UInt64 newEarliestBlockSlot = maybeNewEarliestBlockSlot.get().getSlot();
if (newEarliestBlockSlot.isLessThan(maybeEarliestFinalizedBlockSlotDb.get())) {
return maybeNewEarliestBlockSlot.map(SignedBeaconBlock::getSlot);
} else {
return Optional.empty();
}
}

private Optional<UInt64> needToUpdateEarliestBlobSidecarSlot(
final Optional<UInt64> maybeNewEarliestBlobSidecarSlot) {
// New value is absent - not updating
Expand Down Expand Up @@ -395,6 +417,8 @@ private UInt64 pruneToBlock(final UInt64 lastSlotToPrune, final int pruneLimit)
LOG.debug("Pruning finalized blocks to slot {} (included)", lastSlotToPrune);
try (final Stream<SignedBeaconBlock> stream =
dao.streamFinalizedBlocks(UInt64.ZERO, lastSlotToPrune)) {
// get an extra block to set earliest finalized block slot available after pruning runs
// ensuring it is an existing block in the DB
blocksToPrune =
stream.limit(pruneLimit).map(block -> Pair.of(block.getSlot(), block.getRoot())).toList();
}
Expand Down Expand Up @@ -428,6 +452,16 @@ private void deleteFinalizedBlocks(final List<Pair<UInt64, Bytes32>> blocksToPru
pair -> updater.deleteFinalizedBlock(pair.getLeft(), pair.getRight()));
updater.commit();
}
// retrieve earliest finalized slot left after pruning and set it in the DB
final Optional<UInt64> earliestBlockSlot =
dao.getEarliestFinalizedBlock().map(SignedBeaconBlock::getSlot);

if (earliestBlockSlot.isPresent()) {
try (final FinalizedUpdater updater = finalizedUpdater()) {
updater.setEarliestBlockSlot(earliestBlockSlot.get());
updater.commit();
}
}
gfukushima marked this conversation as resolved.
Show resolved Hide resolved
}
}

Expand Down Expand Up @@ -579,6 +613,7 @@ public void storeInitialAnchor(final AnchorPoint initialAnchor) {
&& spec.atSlot(initialAnchor.getSlot())
.getMilestone()
.isGreaterThanOrEqualTo(SpecMilestone.DENEB)) {
updater.setEarliestBlockSlot(initialAnchor.getSlot());
updater.setEarliestBlobSidecarSlot(initialAnchor.getSlot());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,10 @@ public Optional<SignedBeaconBlock> getFinalizedBlockAtSlot(final UInt64 slot) {

@Override
public Optional<UInt64> getEarliestFinalizedBlockSlot() {
return db.getFirstEntry(schema.getColumnFinalizedBlocksBySlot()).map(ColumnEntry::getKey);
return db.get(schema.getVariableEarliestBlockSlot())
.or(
() ->
db.getFirstEntry(schema.getColumnFinalizedBlocksBySlot()).map(ColumnEntry::getKey));
}

@Override
Expand Down Expand Up @@ -649,6 +652,11 @@ public void setEarliestBlobSidecarSlot(final UInt64 slot) {
transaction.put(schema.getVariableEarliestBlobSidecarSlot(), slot);
}

@Override
public void setEarliestBlockSlot(final UInt64 slot) {
transaction.put(schema.getVariableEarliestBlockSlot(), slot);
}

@Override
public void commit() {
// Commit db updates
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,8 @@ interface FinalizedUpdater extends AutoCloseable {

void setEarliestBlobSidecarSlot(UInt64 slot);

void setEarliestBlockSlot(UInt64 slot);

void commit();

void cancel();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -588,6 +588,11 @@ public void setEarliestBlobSidecarSlot(final UInt64 slot) {
finalizedUpdater.setEarliestBlobSidecarSlot(slot);
}

@Override
public void setEarliestBlockSlot(final UInt64 slot) {
finalizedUpdater.setEarliestBlockSlot(slot);
}

@Override
public void addMinGenesisTimeBlock(final MinGenesisTimeBlockEvent event) {
hotUpdater.addMinGenesisTimeBlock(event);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,10 @@ public Optional<SignedBeaconBlock> getFinalizedBlockAtSlot(final UInt64 slot) {
}

public Optional<UInt64> getEarliestFinalizedBlockSlot() {
return db.getFirstEntry(schema.getColumnFinalizedBlocksBySlot()).map(ColumnEntry::getKey);
return db.get(schema.getVariableEarliestBlockSlot())
.or(
() ->
db.getFirstEntry(schema.getColumnFinalizedBlocksBySlot()).map(ColumnEntry::getKey));
}

public Optional<UInt64> getEarliestFinalizedStateSlot() {
Expand Down Expand Up @@ -412,6 +415,11 @@ public void setEarliestBlobSidecarSlot(final UInt64 slot) {
transaction.put(schema.getVariableEarliestBlobSidecarSlot(), slot);
}

@Override
public void setEarliestBlockSlot(final UInt64 slot) {
transaction.put(schema.getVariableEarliestBlockSlot(), slot);
}

@Override
public void commit() {
// Commit db updates
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ public interface SchemaCombined extends Schema {

KvStoreVariable<UInt64> getVariableEarliestBlobSidecarSlot();

KvStoreVariable<UInt64> getVariableEarliestBlockSlot();

KvStoreVariable<DepositTreeSnapshot> getVariableFinalizedDepositSnapshot();

Map<String, KvStoreColumn<?, ?>> getColumnMap();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,17 @@ public KvStoreVariable<UInt64> getVariableEarliestBlobSidecarSlot() {
return delegate.getVariableEarliestBlobSidecarSlot();
}

public KvStoreVariable<UInt64> getVariableEarliestBlockSlot() {
return delegate.getVariableEarliestBlockSlot();
}

public Map<String, KvStoreVariable<?>> getVariableMap() {
return Map.of(
"OPTIMISTIC_TRANSITION_BLOCK_SLOT",
getOptimisticTransitionBlockSlot(),
"EARLIEST_BLOB_SIDECAR_SLOT",
getVariableEarliestBlobSidecarSlot());
getVariableEarliestBlobSidecarSlot(),
"EARLIEST_BLOCK_SLOT_AVAILABLE",
getVariableEarliestBlockSlot());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ public abstract class V6SchemaCombined implements SchemaCombined {

private final KvStoreVariable<UInt64> optimisticTransitionBlockSlot;
private final KvStoreVariable<UInt64> earliestBlobSidecarSlot;
private final KvStoreVariable<UInt64> earliestBlockSlot;

protected V6SchemaCombined(final Spec spec, final int finalizedOffset) {
this.finalizedOffset = finalizedOffset;
Expand All @@ -100,6 +101,7 @@ protected V6SchemaCombined(final Spec spec, final int finalizedOffset) {

optimisticTransitionBlockSlot = KvStoreVariable.create(finalizedOffset + 1, UINT64_SERIALIZER);
earliestBlobSidecarSlot = KvStoreVariable.create(finalizedOffset + 2, UINT64_SERIALIZER);
earliestBlockSlot = KvStoreVariable.create(finalizedOffset + 3, UINT64_SERIALIZER);
}

@Override
Expand Down Expand Up @@ -192,6 +194,11 @@ public KvStoreVariable<UInt64> getVariableEarliestBlobSidecarSlot() {
return earliestBlobSidecarSlot;
}

@Override
public KvStoreVariable<UInt64> getVariableEarliestBlockSlot() {
return earliestBlockSlot;
}

@Override
public Map<String, KvStoreColumn<?, ?>> getColumnMap() {
return ImmutableMap.<String, KvStoreColumn<?, ?>>builder()
Expand Down Expand Up @@ -227,6 +234,7 @@ public Map<String, KvStoreVariable<?>> getVariableMap() {
.put("OPTIMISTIC_TRANSITION_BLOCK_SLOT", getOptimisticTransitionBlockSlot())
.put("FINALIZED_DEPOSIT_SNAPSHOT", getVariableFinalizedDepositSnapshot())
.put("EARLIEST_BLOB_SIDECAR_SLOT", getVariableEarliestBlobSidecarSlot())
.put("EARLIEST_BLOCK_SLOT_AVAILABLE", getVariableEarliestBlockSlot())
.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ public Map<String, KvStoreVariable<?>> getVariableMap() {
.put("OPTIMISTIC_TRANSITION_BLOCK_SLOT", getOptimisticTransitionBlockSlot())
.put("FINALIZED_DEPOSIT_SNAPSHOT", getVariableFinalizedDepositSnapshot())
.put("EARLIEST_BLOB_SIDECAR_SLOT", getVariableEarliestBlobSidecarSlot())
.put("EARLIEST_BLOCK_SLOT", getVariableEarliestBlockSlot())
.build();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -156,13 +156,33 @@ public int getFinalizedState(
footer = "Teku is licensed under the Apache License 2.0")
public int getEarliestAvailableBlockSlot(
@Mixin final BeaconNodeDataOptions beaconNodeDataOptions,
@Mixin final Eth2NetworkOptions eth2NetworkOptions)
@Mixin final Eth2NetworkOptions eth2NetworkOptions,
@Option(
names = {"--timed", "-t"},
description = "Prints the time taken to retrieve the earliest available block slot",
defaultValue = "true",
fallbackValue = "true",
showDefaultValue = Visibility.ALWAYS)
final boolean verbose)
throws Exception {
try (final Database database = createDatabase(beaconNodeDataOptions, eth2NetworkOptions)) {
Optional<UInt64> earliestAvailableBlockSlot = database.getEarliestAvailableBlockSlot();
earliestAvailableBlockSlot.ifPresent(System.out::println);
if (verbose) {
final long startTime = System.currentTimeMillis();
final Optional<UInt64> earliestAvailableBlockSlot =
database.getEarliestAvailableBlockSlot();
final long endTime = System.currentTimeMillis();
earliestAvailableBlockSlot.ifPresent(System.out::println);
System.out.println(
"Time taken to retrieve the earliest available block slot: "
+ (endTime - startTime)
+ "ms");
} else {
final Optional<UInt64> earliestAvailableBlockSlot =
database.getEarliestAvailableBlockSlot();
earliestAvailableBlockSlot.ifPresent(System.out::println);
}
return 0;
}
return 0;
}

@Command(
Expand Down