From 5e455b2f880e17d9e38f9639b171aee3a6137948 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 26 Nov 2024 09:22:37 +0700 Subject: [PATCH 1/3] fix: prune checkpoint states at syncing time --- packages/beacon-node/src/chain/chain.ts | 1 - .../stateCache/persistentCheckpointsCache.ts | 17 ++--------------- .../persistentCheckpointsCache.test.ts | 12 ++++++------ 3 files changed, 8 insertions(+), 22 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 4751770b3bfc..415158abb2d5 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -293,7 +293,6 @@ export class BeaconChain implements IBeaconChain { metrics, logger, clock, - shufflingCache: this.shufflingCache, blockStateCache, bufferPool: this.bufferPool, datastore: fileDataStore diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 4c9a8b0f1265..8b43bd53c456 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -9,7 +9,6 @@ import {AllocSource, BufferPool, BufferWithKey} from "../../util/bufferPool.js"; import {IClock} from "../../util/clock.js"; import {StateCloneOpts} from "../regen/interface.js"; import {serializeState} from "../serializeState.js"; -import {ShufflingCache} from "../shufflingCache.js"; import {CPStateDatastore, DatastoreKey, datastoreKeyToCheckpoint} from "./datastore/index.js"; import {MapTracker} from "./mapMetrics.js"; import {BlockStateCache, CacheItemType, CheckpointHex, CheckpointStateCache} from "./types.js"; @@ -17,8 +16,6 @@ import {BlockStateCache, CacheItemType, CheckpointHex, CheckpointStateCache} fro export type PersistentCheckpointStateCacheOpts = { /** Keep max n states in memory, persist the rest to disk */ maxCPStateEpochsInMemory?: number; - /** for testing only */ - processLateBlock?: boolean; }; type PersistentCheckpointStateCacheModules = { @@ -26,7 +23,6 @@ type PersistentCheckpointStateCacheModules = { logger: Logger; clock?: IClock | null; signal?: AbortSignal; - shufflingCache: ShufflingCache; datastore: CPStateDatastore; blockStateCache: BlockStateCache; bufferPool?: BufferPool | null; @@ -102,10 +98,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { private preComputedCheckpoint: string | null = null; private preComputedCheckpointHits: number | null = null; private readonly maxEpochsInMemory: number; - // only for testing, default false for production - private readonly processLateBlock: boolean; private readonly datastore: CPStateDatastore; - private readonly shufflingCache: ShufflingCache; private readonly blockStateCache: BlockStateCache; private readonly bufferPool?: BufferPool | null; @@ -115,7 +108,6 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { logger, clock, signal, - shufflingCache, datastore, blockStateCache, bufferPool, @@ -153,10 +145,8 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { throw new Error("maxEpochsInMemory must be >= 0"); } this.maxEpochsInMemory = opts.maxCPStateEpochsInMemory ?? DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY; - this.processLateBlock = opts.processLateBlock ?? false; // Specify different datastore for testing this.datastore = datastore; - this.shufflingCache = shufflingCache; this.blockStateCache = blockStateCache; this.bufferPool = bufferPool; } @@ -487,12 +477,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { // 2/3 of slot is the most free time of every slot, take that chance to persist checkpoint states // normally it should only persist checkpoint states at 2/3 of slot 0 of epoch await sleep(secToTwoThirdsSlot * 1000, this.signal); - } else if (!this.processLateBlock) { - // normally the block persist happens at 2/3 of slot 0 of epoch, if it's already late then just skip to allow other tasks to run - // there are plenty of chances in the same epoch to persist checkpoint states, also if block is late it could be reorged - this.logger.verbose("Skip persist checkpoint states", {blockSlot, root: blockRootHex}); - return 0; } + // at syncing time, it's critical to persist checkpoint states as soon as possible to avoid OOM during unfinality time + // if node is synced this is not a hot time because block comes late, we'll likely miss attestation already, or the block is orphaned const persistEpochs = sortedEpochs.slice(0, sortedEpochs.length - this.maxEpochsInMemory); for (const lowestEpoch of persistEpochs) { diff --git a/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts index 5ab3da532948..aecca3e45aaf 100644 --- a/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts +++ b/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts @@ -93,7 +93,7 @@ describe("PersistentCheckpointStateCache", () => { shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 2, processLateBlock: true} + {maxCPStateEpochsInMemory: 2} ); cache.add(cp0a, states["cp0a"]); cache.add(cp0b, states["cp0b"]); @@ -168,7 +168,7 @@ describe("PersistentCheckpointStateCache", () => { shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 2, processLateBlock: true} + {maxCPStateEpochsInMemory: 2} ); cache.add(cp0a, states["cp0a"]); cache.add(cp0b, states["cp0b"]); @@ -245,7 +245,7 @@ describe("PersistentCheckpointStateCache", () => { shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 2, processLateBlock: true} + {maxCPStateEpochsInMemory: 2} ); cache.add(cp0a, states["cp0a"]); cache.add(cp0b, states["cp0b"]); @@ -551,7 +551,7 @@ describe("PersistentCheckpointStateCache", () => { shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 1, processLateBlock: true} + {maxCPStateEpochsInMemory: 1} ); cache.add(cp0a, states["cp0a"]); cache.add(cp0b, states["cp0b"]); @@ -823,7 +823,7 @@ describe("PersistentCheckpointStateCache", () => { shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 0, processLateBlock: true} + {maxCPStateEpochsInMemory: 0} ); cache.add(cp0a, states["cp0a"]); cache.add(cp0b, states["cp0b"]); @@ -914,7 +914,7 @@ describe("PersistentCheckpointStateCache", () => { shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, - {maxCPStateEpochsInMemory: 0, processLateBlock: true} + {maxCPStateEpochsInMemory: 0} ); const root1a = Buffer.alloc(32, 100); From 8b7ba779ec130754f697ca89e55ec47876a22b29 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 26 Nov 2024 11:00:33 +0700 Subject: [PATCH 2/3] fix: lint --- .../src/chain/stateCache/persistentCheckpointsCache.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 8b43bd53c456..7fb67a758673 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -103,15 +103,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { private readonly bufferPool?: BufferPool | null; constructor( - { - metrics, - logger, - clock, - signal, - datastore, - blockStateCache, - bufferPool, - }: PersistentCheckpointStateCacheModules, + {metrics, logger, clock, signal, datastore, blockStateCache, bufferPool}: PersistentCheckpointStateCacheModules, opts: PersistentCheckpointStateCacheOpts ) { this.cache = new MapTracker(metrics?.cpStateCache); From 231a347e0b019b4f1848997818e51610487ec879 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 26 Nov 2024 11:08:40 +0700 Subject: [PATCH 3/3] fix: check-types in test --- .../chain/stateCache/persistentCheckpointsCache.test.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts index aecca3e45aaf..4e8b62013331 100644 --- a/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts +++ b/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts @@ -90,7 +90,6 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, {maxCPStateEpochsInMemory: 2} @@ -165,7 +164,6 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, {maxCPStateEpochsInMemory: 2} @@ -242,7 +240,6 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, {maxCPStateEpochsInMemory: 2} @@ -548,7 +545,6 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, {maxCPStateEpochsInMemory: 1} @@ -820,7 +816,6 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, {maxCPStateEpochsInMemory: 0} @@ -911,7 +906,6 @@ describe("PersistentCheckpointStateCache", () => { { datastore, logger: testLogger(), - shufflingCache: new ShufflingCache(), blockStateCache: new FIFOBlockStateCache({}, {}), }, {maxCPStateEpochsInMemory: 0}