From 1abc896e28acdea654f36d8d6c9368bcbac176f0 Mon Sep 17 00:00:00 2001 From: harkamal Date: Wed, 24 Jan 2024 17:38:11 +0530 Subject: [PATCH 1/7] feat: placeholder PR for electra add types stub and epoch config fix types --- .../test/spec/presets/fork.test.ts | 2 + .../test/spec/presets/transition.test.ts | 8 + .../upgradeLightClientHeader.test.ts | 33 +++- .../test/unit/network/fork.test.ts | 13 +- packages/beacon-node/test/utils/config.ts | 8 + .../config/src/chainConfig/configs/mainnet.ts | 4 + .../config/src/chainConfig/configs/minimal.ts | 4 + packages/config/src/chainConfig/types.ts | 6 + packages/config/src/forkConfig/index.ts | 10 +- packages/light-client/src/spec/utils.ts | 4 + packages/params/src/forkName.ts | 2 + .../test/unit/upgradeState.test.ts | 8 + packages/types/src/allForks/sszTypes.ts | 38 +++++ packages/types/src/electra/index.ts | 3 + packages/types/src/electra/sszTypes.ts | 148 ++++++++++++++++++ packages/types/src/electra/types.ts | 29 ++++ packages/types/src/sszTypes.ts | 1 + packages/types/src/types.ts | 1 + packages/validator/src/util/params.ts | 5 + 19 files changed, 323 insertions(+), 4 deletions(-) create mode 100644 packages/types/src/electra/index.ts create mode 100644 packages/types/src/electra/sszTypes.ts create mode 100644 packages/types/src/electra/types.ts diff --git a/packages/beacon-node/test/spec/presets/fork.test.ts b/packages/beacon-node/test/spec/presets/fork.test.ts index 228ab6a38935..c880d24bbbe3 100644 --- a/packages/beacon-node/test/spec/presets/fork.test.ts +++ b/packages/beacon-node/test/spec/presets/fork.test.ts @@ -35,6 +35,8 @@ const fork: TestRunnerFn = (forkNext) => { return slotFns.upgradeStateToCapella(preState as CachedBeaconStateBellatrix); case ForkName.deneb: return slotFns.upgradeStateToDeneb(preState as CachedBeaconStateCapella); + case ForkName.electra: + throw Error("not Implemented"); } }, options: { diff --git a/packages/beacon-node/test/spec/presets/transition.test.ts b/packages/beacon-node/test/spec/presets/transition.test.ts index 77919d76c3b1..1f98dbb41f58 100644 --- a/packages/beacon-node/test/spec/presets/transition.test.ts +++ b/packages/beacon-node/test/spec/presets/transition.test.ts @@ -102,6 +102,14 @@ function getTransitionConfig(fork: ForkName, forkEpoch: number): Partial${toFork}`, function () { + lcHeaderByFork[fromFork].beacon.slot = testSlots[fromFork]; + lcHeaderByFork[toFork].beacon.slot = testSlots[fromFork]; + + expect(() => { + upgradeLightClientHeader(config, toFork, lcHeaderByFork[fromFork]); + }).toThrow("Not Implemented"); + }); + } + + // Since electra is not implemented for loop is till deneb (Object.values(ForkName).length-1) + // Once electra is implemnted run for loop till Object.values(ForkName).length + + // for (let i = ForkSeq.altair; i < Object.values(ForkName).length; i++) { + + for (let i = ForkSeq.altair; i < Object.values(ForkName).length - 1; i++) { for (let j = i; j > 0; j--) { const fromFork = ForkName[ForkSeq[i] as ForkName]; const toFork = ForkName[ForkSeq[j] as ForkName]; diff --git a/packages/beacon-node/test/unit/network/fork.test.ts b/packages/beacon-node/test/unit/network/fork.test.ts index be748d2e8185..bbe1c0870d30 100644 --- a/packages/beacon-node/test/unit/network/fork.test.ts +++ b/packages/beacon-node/test/unit/network/fork.test.ts @@ -9,12 +9,14 @@ function getForkConfig({ bellatrix, capella, deneb, + electra, }: { phase0: number; altair: number; bellatrix: number; capella: number; deneb: number; + electra: number; }): BeaconConfig { const forks: Record = { phase0: { @@ -57,6 +59,14 @@ function getForkConfig({ prevVersion: Buffer.from([0, 0, 0, 3]), prevForkName: ForkName.capella, }, + electra: { + name: ForkName.electra, + seq: ForkSeq.electra, + epoch: electra, + version: Buffer.from([0, 0, 0, 5]), + prevVersion: Buffer.from([0, 0, 0, 4]), + prevForkName: ForkName.deneb, + }, }; const forksAscendingEpochOrder = Object.values(forks); const forksDescendingEpochOrder = Object.values(forks).reverse(); @@ -133,9 +143,10 @@ const testScenarios = [ for (const testScenario of testScenarios) { const {phase0, altair, bellatrix, capella, testCases} = testScenario; const deneb = Infinity; + const electra = Infinity; describe(`network / fork: phase0: ${phase0}, altair: ${altair}, bellatrix: ${bellatrix} capella: ${capella}`, () => { - const forkConfig = getForkConfig({phase0, altair, bellatrix, capella, deneb}); + const forkConfig = getForkConfig({phase0, altair, bellatrix, capella, deneb, electra}); const forks = forkConfig.forks; for (const testCase of testCases) { const {epoch, currentFork, nextFork, activeForks} = testCase; diff --git a/packages/beacon-node/test/utils/config.ts b/packages/beacon-node/test/utils/config.ts index 54c058d30722..2aad1c14c03e 100644 --- a/packages/beacon-node/test/utils/config.ts +++ b/packages/beacon-node/test/utils/config.ts @@ -31,5 +31,13 @@ export function getConfig(fork: ForkName, forkEpoch = 0): ChainForkConfig { CAPELLA_FORK_EPOCH: 0, DENEB_FORK_EPOCH: forkEpoch, }); + case ForkName.electra: + return createChainForkConfig({ + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: forkEpoch, + }); } } diff --git a/packages/config/src/chainConfig/configs/mainnet.ts b/packages/config/src/chainConfig/configs/mainnet.ts index 9d060330d201..8f5051394551 100644 --- a/packages/config/src/chainConfig/configs/mainnet.ts +++ b/packages/config/src/chainConfig/configs/mainnet.ts @@ -49,6 +49,10 @@ export const chainConfig: ChainConfig = { DENEB_FORK_VERSION: b("0x04000000"), DENEB_FORK_EPOCH: 269568, // March 13, 2024, 01:55:35pm UTC + // Electra + ELECTRA_FORK_VERSION: b("0x05000000"), + ELECTRA_FORK_EPOCH: Infinity, + // Time parameters // --------------------------------------------------------------- // 12 seconds diff --git a/packages/config/src/chainConfig/configs/minimal.ts b/packages/config/src/chainConfig/configs/minimal.ts index 6c0a13d8abb2..a8a9834c1592 100644 --- a/packages/config/src/chainConfig/configs/minimal.ts +++ b/packages/config/src/chainConfig/configs/minimal.ts @@ -46,6 +46,10 @@ export const chainConfig: ChainConfig = { DENEB_FORK_VERSION: b("0x04000001"), DENEB_FORK_EPOCH: Infinity, + // Electra + ELECTRA_FORK_VERSION: b("0x05000001"), + ELECTRA_FORK_EPOCH: Infinity, + // Time parameters // --------------------------------------------------------------- // [customized] Faster for testing purposes diff --git a/packages/config/src/chainConfig/types.ts b/packages/config/src/chainConfig/types.ts index 3e0844118290..384ef54ca994 100644 --- a/packages/config/src/chainConfig/types.ts +++ b/packages/config/src/chainConfig/types.ts @@ -40,6 +40,9 @@ export type ChainConfig = { // DENEB DENEB_FORK_VERSION: Uint8Array; DENEB_FORK_EPOCH: number; + // ELECTRA + ELECTRA_FORK_VERSION: Uint8Array; + ELECTRA_FORK_EPOCH: number; // Time parameters SECONDS_PER_SLOT: number; @@ -96,6 +99,9 @@ export const chainConfigTypes: SpecTypes = { // DENEB DENEB_FORK_VERSION: "bytes", DENEB_FORK_EPOCH: "number", + // ELECTRA + ELECTRA_FORK_VERSION: "bytes", + ELECTRA_FORK_EPOCH: "number", // Time parameters SECONDS_PER_SLOT: "number", diff --git a/packages/config/src/forkConfig/index.ts b/packages/config/src/forkConfig/index.ts index d630f1ddfc88..efb27ca1505f 100644 --- a/packages/config/src/forkConfig/index.ts +++ b/packages/config/src/forkConfig/index.ts @@ -55,10 +55,18 @@ export function createForkConfig(config: ChainConfig): ForkConfig { prevVersion: config.CAPELLA_FORK_VERSION, prevForkName: ForkName.capella, }; + const electra: ForkInfo = { + name: ForkName.electra, + seq: ForkSeq.electra, + epoch: config.ELECTRA_FORK_EPOCH, + version: config.ELECTRA_FORK_VERSION, + prevVersion: config.DENEB_FORK_VERSION, + prevForkName: ForkName.deneb, + }; /** Forks in order order of occurence, `phase0` first */ // Note: Downstream code relies on proper ordering. - const forks = {phase0, altair, bellatrix, capella, deneb}; + const forks = {phase0, altair, bellatrix, capella, deneb, electra}; // Prevents allocating an array on every getForkInfo() call const forksAscendingEpochOrder = Object.values(forks); diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index 2a5720a1f637..ed4b9e6e961d 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -103,6 +103,10 @@ export function upgradeLightClientHeader( // Break if no further upgradation is required else fall through if (ForkSeq[targetFork] <= ForkSeq.deneb) break; + + // eslint-disable-next-line no-fallthrough + case ForkName.electra: + throw Error("Not Implemented"); } return upgradedHeader; } diff --git a/packages/params/src/forkName.ts b/packages/params/src/forkName.ts index 142684c313f4..bbb72a7972fa 100644 --- a/packages/params/src/forkName.ts +++ b/packages/params/src/forkName.ts @@ -7,6 +7,7 @@ export enum ForkName { bellatrix = "bellatrix", capella = "capella", deneb = "deneb", + electra = "electra", } /** @@ -18,6 +19,7 @@ export enum ForkSeq { bellatrix = 2, capella = 3, deneb = 4, + electra = 5, } export type ForkPreLightClient = ForkName.phase0; diff --git a/packages/state-transition/test/unit/upgradeState.test.ts b/packages/state-transition/test/unit/upgradeState.test.ts index 2ea8eef182ac..75ba415c1bea 100644 --- a/packages/state-transition/test/unit/upgradeState.test.ts +++ b/packages/state-transition/test/unit/upgradeState.test.ts @@ -55,5 +55,13 @@ function getConfig(fork: ForkName, forkEpoch = 0): ChainForkConfig { CAPELLA_FORK_EPOCH: 0, DENEB_FORK_EPOCH: forkEpoch, }); + case ForkName.electra: + return createChainForkConfig({ + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: forkEpoch, + }); } } diff --git a/packages/types/src/allForks/sszTypes.ts b/packages/types/src/allForks/sszTypes.ts index 7174bc52e89c..6030215ac8ca 100644 --- a/packages/types/src/allForks/sszTypes.ts +++ b/packages/types/src/allForks/sszTypes.ts @@ -3,6 +3,7 @@ import {ssz as altair} from "../altair/index.js"; import {ssz as bellatrix} from "../bellatrix/index.js"; import {ssz as capella} from "../capella/index.js"; import {ssz as deneb} from "../deneb/index.js"; +import {ssz as electra} from "../electra/index.js"; /** * Index the ssz types that differ by fork @@ -44,6 +45,13 @@ export const allForks = { BeaconState: deneb.BeaconState, Metadata: altair.Metadata, }, + electra: { + BeaconBlockBody: electra.BeaconBlockBody, + BeaconBlock: electra.BeaconBlock, + SignedBeaconBlock: electra.SignedBeaconBlock, + BeaconState: electra.BeaconState, + Metadata: altair.Metadata, + }, }; /** @@ -85,6 +93,17 @@ export const allForksExecution = { SignedBuilderBid: deneb.SignedBuilderBid, SSEPayloadAttributes: deneb.SSEPayloadAttributes, }, + electra: { + BeaconBlockBody: electra.BeaconBlockBody, + BeaconBlock: electra.BeaconBlock, + SignedBeaconBlock: electra.SignedBeaconBlock, + BeaconState: electra.BeaconState, + ExecutionPayload: electra.ExecutionPayload, + ExecutionPayloadHeader: electra.ExecutionPayloadHeader, + BuilderBid: electra.BuilderBid, + SignedBuilderBid: electra.SignedBuilderBid, + SSEPayloadAttributes: electra.SSEPayloadAttributes, + }, }; /** @@ -107,6 +126,11 @@ export const allForksBlinded = { BeaconBlock: deneb.BlindedBeaconBlock, SignedBeaconBlock: deneb.SignedBlindedBeaconBlock, }, + electra: { + BeaconBlockBody: electra.BlindedBeaconBlockBody, + BeaconBlock: electra.BlindedBeaconBlock, + SignedBeaconBlock: electra.SignedBlindedBeaconBlock, + }, }; export const allForksLightClient = { @@ -150,6 +174,16 @@ export const allForksLightClient = { LightClientOptimisticUpdate: deneb.LightClientOptimisticUpdate, LightClientStore: deneb.LightClientStore, }, + electra: { + BeaconBlock: electra.BeaconBlock, + BeaconBlockBody: electra.BeaconBlockBody, + LightClientHeader: electra.LightClientHeader, + LightClientBootstrap: electra.LightClientBootstrap, + LightClientUpdate: electra.LightClientUpdate, + LightClientFinalityUpdate: electra.LightClientFinalityUpdate, + LightClientOptimisticUpdate: electra.LightClientOptimisticUpdate, + LightClientStore: electra.LightClientStore, + }, }; export const allForksBlobs = { @@ -157,4 +191,8 @@ export const allForksBlobs = { BlobSidecar: deneb.BlobSidecar, ExecutionPayloadAndBlobsBundle: deneb.ExecutionPayloadAndBlobsBundle, }, + electra: { + BlobSidecar: electra.BlobSidecar, + ExecutionPayloadAndBlobsBundle: electra.ExecutionPayloadAndBlobsBundle, + }, }; diff --git a/packages/types/src/electra/index.ts b/packages/types/src/electra/index.ts new file mode 100644 index 000000000000..7856cd729620 --- /dev/null +++ b/packages/types/src/electra/index.ts @@ -0,0 +1,3 @@ +export * from "./types.js"; +export * as ts from "./types.js"; +export * as ssz from "./sszTypes.js"; diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts new file mode 100644 index 000000000000..30690a499845 --- /dev/null +++ b/packages/types/src/electra/sszTypes.ts @@ -0,0 +1,148 @@ +import {ContainerType} from "@chainsafe/ssz"; +import {ssz as primitiveSsz} from "../primitive/index.js"; +import {ssz as denebSsz} from "../deneb/index.js"; + +const {BLSSignature} = primitiveSsz; + +export const ExecutionPayload = new ContainerType( + { + ...denebSsz.ExecutionPayload.fields, + }, + {typeName: "ExecutionPayload", jsonCase: "eth2"} +); + +export const ExecutionPayloadHeader = new ContainerType( + { + ...denebSsz.ExecutionPayloadHeader.fields, + }, + {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} +); + +export const BeaconBlockBody = new ContainerType( + { + ...denebSsz.BeaconBlockBody.fields, + }, + {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} +); + +export const BeaconBlock = new ContainerType( + { + ...denebSsz.BeaconBlock.fields, + }, + {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} +); + +export const SignedBeaconBlock = new ContainerType( + { + message: BeaconBlock, + signature: BLSSignature, + }, + {typeName: "SignedBeaconBlock", jsonCase: "eth2"} +); + +export const BlobSidecar = new ContainerType( + { + ...denebSsz.BlobSidecar.fields, + }, + {typeName: "BlobSidecar", jsonCase: "eth2"} +); + +export const BlindedBeaconBlockBody = new ContainerType( + { + ...denebSsz.BlindedBeaconBlockBody.fields, + }, + {typeName: "BlindedBeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} +); + +export const BlindedBeaconBlock = new ContainerType( + { + ...denebSsz.BlindedBeaconBlock.fields, + }, + {typeName: "BlindedBeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} +); + +export const SignedBlindedBeaconBlock = new ContainerType( + { + message: BlindedBeaconBlock, + signature: BLSSignature, + }, + {typeName: "SignedBlindedBeaconBlock", jsonCase: "eth2"} +); + +export const BuilderBid = new ContainerType( + { + ...denebSsz.BuilderBid.fields, + }, + {typeName: "BuilderBid", jsonCase: "eth2"} +); + +export const SignedBuilderBid = new ContainerType( + { + message: BuilderBid, + signature: BLSSignature, + }, + {typeName: "SignedBuilderBid", jsonCase: "eth2"} +); + +export const ExecutionPayloadAndBlobsBundle = new ContainerType( + { + ...denebSsz.ExecutionPayloadAndBlobsBundle.fields, + }, + {typeName: "ExecutionPayloadAndBlobsBundle", jsonCase: "eth2"} +); + +export const BeaconState = new ContainerType( + { + ...denebSsz.BeaconState.fields, + }, + {typeName: "BeaconState", jsonCase: "eth2"} +); + +export const LightClientHeader = new ContainerType( + { + ...denebSsz.LightClientHeader.fields, + }, + {typeName: "LightClientHeader", jsonCase: "eth2"} +); + +export const LightClientBootstrap = new ContainerType( + { + ...denebSsz.LightClientBootstrap.fields, + }, + {typeName: "LightClientBootstrap", jsonCase: "eth2"} +); + +export const LightClientUpdate = new ContainerType( + { + ...denebSsz.LightClientUpdate.fields, + }, + {typeName: "LightClientUpdate", jsonCase: "eth2"} +); + +export const LightClientFinalityUpdate = new ContainerType( + { + ...denebSsz.LightClientFinalityUpdate.fields, + }, + {typeName: "LightClientFinalityUpdate", jsonCase: "eth2"} +); + +export const LightClientOptimisticUpdate = new ContainerType( + { + ...denebSsz.LightClientOptimisticUpdate.fields, + }, + {typeName: "LightClientOptimisticUpdate", jsonCase: "eth2"} +); + +export const LightClientStore = new ContainerType( + { + ...denebSsz.LightClientStore.fields, + }, + {typeName: "LightClientStore", jsonCase: "eth2"} +); + +export const SSEPayloadAttributes = new ContainerType( + { + ...denebSsz.SSEPayloadAttributes.fields, + }, + {typeName: "SSEPayloadAttributes", jsonCase: "eth2"} +); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts new file mode 100644 index 000000000000..198259eed1dd --- /dev/null +++ b/packages/types/src/electra/types.ts @@ -0,0 +1,29 @@ +import {ValueOf} from "@chainsafe/ssz"; +import * as ssz from "./sszTypes.js"; + +export type BlobSidecar = ValueOf; +export type ExecutionPayloadAndBlobsBundle = ValueOf; + +export type ExecutionPayload = ValueOf; +export type ExecutionPayloadHeader = ValueOf; + +export type BeaconBlockBody = ValueOf; +export type BeaconBlock = ValueOf; +export type SignedBeaconBlock = ValueOf; + +export type BeaconState = ValueOf; + +export type BlindedBeaconBlockBody = ValueOf; +export type BlindedBeaconBlock = ValueOf; +export type SignedBlindedBeaconBlock = ValueOf; + +export type BuilderBid = ValueOf; +export type SignedBuilderBid = ValueOf; +export type SSEPayloadAttributes = ValueOf; + +export type LightClientHeader = ValueOf; +export type LightClientBootstrap = ValueOf; +export type LightClientUpdate = ValueOf; +export type LightClientFinalityUpdate = ValueOf; +export type LightClientOptimisticUpdate = ValueOf; +export type LightClientStore = ValueOf; diff --git a/packages/types/src/sszTypes.ts b/packages/types/src/sszTypes.ts index 2a7df948a447..ff3e77ab6947 100644 --- a/packages/types/src/sszTypes.ts +++ b/packages/types/src/sszTypes.ts @@ -4,6 +4,7 @@ export {ssz as altair} from "./altair/index.js"; export {ssz as bellatrix} from "./bellatrix/index.js"; export {ssz as capella} from "./capella/index.js"; export {ssz as deneb} from "./deneb/index.js"; +export {ssz as electra} from "./electra/index.js"; import {ssz as allForksSsz} from "./allForks/index.js"; export const allForks = allForksSsz.allForks; diff --git a/packages/types/src/types.ts b/packages/types/src/types.ts index e2e416fa3667..fbd7d5621da0 100644 --- a/packages/types/src/types.ts +++ b/packages/types/src/types.ts @@ -6,6 +6,7 @@ export {ts as altair} from "./altair/index.js"; export {ts as bellatrix} from "./bellatrix/index.js"; export {ts as capella} from "./capella/index.js"; export {ts as deneb} from "./deneb/index.js"; +export {ts as electra} from "./electra/index.js"; export {ts as allForks} from "./allForks/index.js"; diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index 006ae3fadbbb..f316ea1b270d 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -73,6 +73,7 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record Date: Fri, 15 Mar 2024 17:17:22 +0530 Subject: [PATCH 2/7] feat: add inclusion lists for censorship resistance --- .../src/api/impl/config/constants.ts | 3 + .../test/spec/presets/fork.test.ts | 3 +- packages/params/src/forkName.ts | 6 + packages/params/src/index.ts | 5 +- .../state-transition/src/cache/stateCache.ts | 2 + packages/state-transition/src/cache/types.ts | 6 +- packages/state-transition/src/slot/index.ts | 1 + .../src/slot/upgradeStateToElectra.ts | 36 ++++ .../state-transition/src/stateTransition.ts | 5 + packages/state-transition/src/types.ts | 2 + packages/state-transition/src/util/genesis.ts | 10 + .../test/unit/upgradeState.test.ts | 16 ++ packages/types/src/allForks/sszTypes.ts | 2 +- packages/types/src/allForks/types.ts | 178 ++++++++++++++---- packages/types/src/deneb/sszTypes.ts | 4 +- packages/types/src/electra/sszTypes.ts | 166 +++++++++++++--- packages/types/src/electra/types.ts | 14 +- 17 files changed, 378 insertions(+), 81 deletions(-) create mode 100644 packages/state-transition/src/slot/upgradeStateToElectra.ts diff --git a/packages/beacon-node/src/api/impl/config/constants.ts b/packages/beacon-node/src/api/impl/config/constants.ts index 87ffce91b4d9..755c42eee128 100644 --- a/packages/beacon-node/src/api/impl/config/constants.ts +++ b/packages/beacon-node/src/api/impl/config/constants.ts @@ -18,6 +18,7 @@ import { DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, DOMAIN_CONTRIBUTION_AND_PROOF, DOMAIN_BLS_TO_EXECUTION_CHANGE, + DOMAIN_INCLUSION_LIST_SUMMARY, DOMAIN_APPLICATION_BUILDER, TIMELY_SOURCE_FLAG_INDEX, TIMELY_TARGET_FLAG_INDEX, @@ -100,4 +101,6 @@ export const specConstants = { // Deneb types BLOB_TX_TYPE, VERSIONED_HASH_VERSION_KZG, + + DOMAIN_INCLUSION_LIST_SUMMARY, }; diff --git a/packages/beacon-node/test/spec/presets/fork.test.ts b/packages/beacon-node/test/spec/presets/fork.test.ts index c880d24bbbe3..c121e651fcea 100644 --- a/packages/beacon-node/test/spec/presets/fork.test.ts +++ b/packages/beacon-node/test/spec/presets/fork.test.ts @@ -5,6 +5,7 @@ import { CachedBeaconStateAltair, CachedBeaconStatePhase0, CachedBeaconStateCapella, + CachedBeaconStateDeneb, } from "@lodestar/state-transition"; import * as slotFns from "@lodestar/state-transition/slot"; import {phase0, ssz} from "@lodestar/types"; @@ -36,7 +37,7 @@ const fork: TestRunnerFn = (forkNext) => { case ForkName.deneb: return slotFns.upgradeStateToDeneb(preState as CachedBeaconStateCapella); case ForkName.electra: - throw Error("not Implemented"); + return slotFns.upgradeStateToElectra(preState as CachedBeaconStateDeneb); } }, options: { diff --git a/packages/params/src/forkName.ts b/packages/params/src/forkName.ts index bbb72a7972fa..31cb1229efb5 100644 --- a/packages/params/src/forkName.ts +++ b/packages/params/src/forkName.ts @@ -45,3 +45,9 @@ export type ForkBlobs = Exclude; export function isForkBlobs(fork: ForkName): fork is ForkBlobs { return isForkWithdrawals(fork) && fork !== ForkName.capella; } + +export type ForkPreILs = ForkPreBlobs | ForkName.deneb; +export type ForkILs = Exclude; +export function isForkILs(fork: ForkName): fork is ForkILs { + return isForkBlobs(fork) && fork !== ForkName.deneb; +} diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index 6a95e3ca632e..71c448dfcda1 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -128,7 +128,7 @@ export const DOMAIN_SYNC_COMMITTEE = Uint8Array.from([7, 0, 0, 0]); export const DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF = Uint8Array.from([8, 0, 0, 0]); export const DOMAIN_CONTRIBUTION_AND_PROOF = Uint8Array.from([9, 0, 0, 0]); export const DOMAIN_BLS_TO_EXECUTION_CHANGE = Uint8Array.from([10, 0, 0, 0]); -export const DOMAIN_BLOB_SIDECAR = Uint8Array.from([11, 0, 0, 0]); +export const DOMAIN_INCLUSION_LIST_SUMMARY = Uint8Array.from([11, 0, 0, 0]); // Application specific domains @@ -244,3 +244,6 @@ export const KZG_COMMITMENT_SUBTREE_INDEX0 = KZG_COMMITMENT_GINDEX0 - 2 ** KZG_C // ssz.deneb.BlobSidecars.elementType.fixedSize export const BLOBSIDECAR_FIXED_SIZE = ACTIVE_PRESET === PresetName.minimal ? 131672 : 131928; + +// might be rounded to nearest power of 2 +export const MAX_TRANSACTIONS_PER_INCLUSION_LIST = 143; diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index 8b45152a3646..3015011da554 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -11,6 +11,7 @@ import { BeaconStateBellatrix, BeaconStateCapella, BeaconStateDeneb, + BeaconStateElectra, } from "./types.js"; import {RewardCache, createEmptyRewardCache} from "./rewardCache.js"; @@ -131,6 +132,7 @@ export type CachedBeaconStateAltair = CachedBeaconState; export type CachedBeaconStateBellatrix = CachedBeaconState; export type CachedBeaconStateCapella = CachedBeaconState; export type CachedBeaconStateDeneb = CachedBeaconState; +export type CachedBeaconStateElectra = CachedBeaconState; export type CachedBeaconStateAllForks = CachedBeaconState; export type CachedBeaconStateExecutions = CachedBeaconState; diff --git a/packages/state-transition/src/cache/types.ts b/packages/state-transition/src/cache/types.ts index 39b1dbb4b45b..a865aa4f4183 100644 --- a/packages/state-transition/src/cache/types.ts +++ b/packages/state-transition/src/cache/types.ts @@ -7,6 +7,7 @@ export type BeaconStateAltair = CompositeViewDU; export type BeaconStateBellatrix = CompositeViewDU; export type BeaconStateCapella = CompositeViewDU; export type BeaconStateDeneb = CompositeViewDU; +export type BeaconStateElectra = CompositeViewDU; // Union at the TreeViewDU level // - Works well as function argument and as generic type for allForks functions @@ -18,8 +19,9 @@ export type BeaconStateAllForks = | BeaconStateAltair | BeaconStateBellatrix | BeaconStateCapella - | BeaconStateDeneb; + | BeaconStateDeneb + | BeaconStateElectra; -export type BeaconStateExecutions = BeaconStateBellatrix | BeaconStateCapella | BeaconStateDeneb; +export type BeaconStateExecutions = BeaconStateBellatrix | BeaconStateCapella | BeaconStateDeneb | BeaconStateElectra; export type ShufflingGetter = (shufflingEpoch: Epoch, dependentRoot: RootHex) => EpochShuffling | null; diff --git a/packages/state-transition/src/slot/index.ts b/packages/state-transition/src/slot/index.ts index 6c4add1d1230..b05bd7ac93f2 100644 --- a/packages/state-transition/src/slot/index.ts +++ b/packages/state-transition/src/slot/index.ts @@ -7,6 +7,7 @@ export {upgradeStateToAltair} from "./upgradeStateToAltair.js"; export {upgradeStateToBellatrix} from "./upgradeStateToBellatrix.js"; export {upgradeStateToCapella} from "./upgradeStateToCapella.js"; export {upgradeStateToDeneb} from "./upgradeStateToDeneb.js"; +export {upgradeStateToElectra} from "./upgradeStateToElectra.js"; /** * Dial state to next slot. Common for all forks diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts new file mode 100644 index 000000000000..1dde905bb7fe --- /dev/null +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -0,0 +1,36 @@ +import {ssz} from "@lodestar/types"; +import {getCachedBeaconState} from "../cache/stateCache.js"; +import {CachedBeaconStateDeneb} from "../types.js"; +import {CachedBeaconStateElectra} from "../types.js"; + +/** + * Upgrade a state from Capella to Deneb. + */ +export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): CachedBeaconStateElectra { + const {config} = stateDeneb; + + const stateDenebNode = ssz.deneb.BeaconState.commitViewDU(stateDeneb); + const stateElectraView = ssz.electra.BeaconState.getViewDU(stateDenebNode); + + const stateElectra = getCachedBeaconState(stateElectraView, stateDeneb); + + stateElectra.fork = ssz.phase0.Fork.toViewDU({ + previousVersion: stateDeneb.fork.currentVersion, + currentVersion: config.ELECTRA_FORK_VERSION, + epoch: stateDeneb.epochCtx.epoch, + }); + + // TODO ELECTRA: check if this is following is required, since it seemed to be an issue in deneb state u[grade + // (see upgradeStateToDeneb) + // + // stateElectra.latestExecutionPayloadHeader = ssz.electra.BeaconState.fields.latestExecutionPayloadHeader.toViewDU({ + // ...stateDeneb.latestExecutionPayloadHeader.toValue(), + // previousInclusionListSummaryRoot: ssz.Root.defaultValue(), + // }); + + stateElectra.commit(); + // Clear cache to ensure the cache of capella fields is not used by new deneb fields + stateElectra["clearCache"](); + + return stateElectra; +} diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index b3f3b41eb865..049f04dce287 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -9,6 +9,7 @@ import { CachedBeaconStateAltair, CachedBeaconStateBellatrix, CachedBeaconStateCapella, + CachedBeaconStateDeneb, } from "./types.js"; import {computeEpochAtSlot} from "./util/index.js"; import {verifyProposerSignature} from "./signatureSets/index.js"; @@ -18,6 +19,7 @@ import { upgradeStateToBellatrix, upgradeStateToCapella, upgradeStateToDeneb, + upgradeStateToElectra, } from "./slot/index.js"; import {processBlock} from "./block/index.js"; import {EpochTransitionStep, processEpoch} from "./epoch/index.js"; @@ -230,6 +232,9 @@ function processSlotsWithTransientCache( if (stateSlot === config.DENEB_FORK_EPOCH) { postState = upgradeStateToDeneb(postState as CachedBeaconStateCapella) as CachedBeaconStateAllForks; } + if (stateSlot === config.ELECTRA_FORK_EPOCH) { + postState = upgradeStateToElectra(postState as CachedBeaconStateDeneb) as CachedBeaconStateAllForks; + } } else { postState.slot++; } diff --git a/packages/state-transition/src/types.ts b/packages/state-transition/src/types.ts index 6b6b1f6260b2..d3a1ed69a7a9 100644 --- a/packages/state-transition/src/types.ts +++ b/packages/state-transition/src/types.ts @@ -9,6 +9,7 @@ export type { CachedBeaconStateBellatrix, CachedBeaconStateCapella, CachedBeaconStateDeneb, + CachedBeaconStateElectra, } from "./cache/stateCache.js"; export type { @@ -19,4 +20,5 @@ export type { BeaconStateBellatrix, BeaconStateCapella, BeaconStateDeneb, + BeaconStateElectra, } from "./cache/types.js"; diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index 1041c33d0eb3..1edbab34ad57 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -214,6 +214,7 @@ export function initializeBeaconStateFromEth1( | typeof ssz.bellatrix.ExecutionPayloadHeader | typeof ssz.capella.ExecutionPayloadHeader | typeof ssz.deneb.ExecutionPayloadHeader + | typeof ssz.electra.ExecutionPayloadHeader > ): CachedBeaconStateAllForks { const stateView = getGenesisBeaconState( @@ -284,6 +285,15 @@ export function initializeBeaconStateFromEth1( ssz.deneb.ExecutionPayloadHeader.defaultViewDU(); } + if (GENESIS_SLOT >= config.ELECTRA_FORK_EPOCH) { + const stateElectra = state as CompositeViewDU; + stateElectra.fork.previousVersion = config.ELECTRA_FORK_VERSION; + stateElectra.fork.currentVersion = config.ELECTRA_FORK_VERSION; + stateElectra.latestExecutionPayloadHeader = + (executionPayloadHeader as CompositeViewDU) ?? + ssz.electra.ExecutionPayloadHeader.defaultViewDU(); + } + state.commit(); return state; diff --git a/packages/state-transition/test/unit/upgradeState.test.ts b/packages/state-transition/test/unit/upgradeState.test.ts index 75ba415c1bea..df9b052542f9 100644 --- a/packages/state-transition/test/unit/upgradeState.test.ts +++ b/packages/state-transition/test/unit/upgradeState.test.ts @@ -5,6 +5,7 @@ import {createBeaconConfig, ChainForkConfig, createChainForkConfig} from "@lodes import {config as chainConfig} from "@lodestar/config/default"; import {upgradeStateToDeneb} from "../../src/slot/upgradeStateToDeneb.js"; +import {upgradeStateToElectra} from "../../src/slot/upgradeStateToElectra.js"; import {createCachedBeaconState} from "../../src/cache/stateCache.js"; import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; @@ -24,6 +25,21 @@ describe("upgradeState", () => { const newState = upgradeStateToDeneb(stateView); expect(() => newState.toValue()).not.toThrow(); }); + it("upgradeStateToElectra", () => { + const denebState = ssz.deneb.BeaconState.defaultViewDU(); + const config = getConfig(ForkName.deneb); + const stateView = createCachedBeaconState( + denebState, + { + config: createBeaconConfig(config, denebState.genesisValidatorsRoot), + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }, + {skipSyncCommitteeCache: true} + ); + const newState = upgradeStateToElectra(stateView); + expect(() => newState.toValue()).not.toThrow(); + }); }); const ZERO_HASH = Buffer.alloc(32, 0); diff --git a/packages/types/src/allForks/sszTypes.ts b/packages/types/src/allForks/sszTypes.ts index 6030215ac8ca..84c6bb86ce5f 100644 --- a/packages/types/src/allForks/sszTypes.ts +++ b/packages/types/src/allForks/sszTypes.ts @@ -192,7 +192,7 @@ export const allForksBlobs = { ExecutionPayloadAndBlobsBundle: deneb.ExecutionPayloadAndBlobsBundle, }, electra: { - BlobSidecar: electra.BlobSidecar, + BlobSidecar: deneb.BlobSidecar, ExecutionPayloadAndBlobsBundle: electra.ExecutionPayloadAndBlobsBundle, }, }; diff --git a/packages/types/src/allForks/types.ts b/packages/types/src/allForks/types.ts index 59768a5a3308..d097f6db2936 100644 --- a/packages/types/src/allForks/types.ts +++ b/packages/types/src/allForks/types.ts @@ -4,12 +4,14 @@ import {ts as altair} from "../altair/index.js"; import {ts as bellatrix} from "../bellatrix/index.js"; import {ts as capella} from "../capella/index.js"; import {ts as deneb} from "../deneb/index.js"; +import {ts as electra} from "../electra/index.js"; import {ssz as phase0Ssz} from "../phase0/index.js"; import {ssz as altairSsz} from "../altair/index.js"; import {ssz as bellatrixSsz} from "../bellatrix/index.js"; import {ssz as capellaSsz} from "../capella/index.js"; import {ssz as denebSsz} from "../deneb/index.js"; +import {ssz as electraSsz} from "../electra/index.js"; // Re-export union types for types that are _known_ to differ @@ -18,52 +20,69 @@ export type BeaconBlockBody = | altair.BeaconBlockBody | bellatrix.BeaconBlockBody | capella.BeaconBlockBody - | deneb.BeaconBlockBody; + | deneb.BeaconBlockBody + | electra.BeaconBlockBody; export type BeaconBlock = | phase0.BeaconBlock | altair.BeaconBlock | bellatrix.BeaconBlock | capella.BeaconBlock - | deneb.BeaconBlock; + | deneb.BeaconBlock + | electra.BeaconBlock; export type SignedBeaconBlock = | phase0.SignedBeaconBlock | altair.SignedBeaconBlock | bellatrix.SignedBeaconBlock | capella.SignedBeaconBlock - | deneb.SignedBeaconBlock; + | deneb.SignedBeaconBlock + | electra.SignedBeaconBlock; export type BeaconState = | phase0.BeaconState | altair.BeaconState | bellatrix.BeaconState | capella.BeaconState - | deneb.BeaconState; + | deneb.BeaconState + | electra.BeaconState; export type Metadata = phase0.Metadata | altair.Metadata; // For easy reference in the assemble block for building payloads -export type ExecutionBlockBody = bellatrix.BeaconBlockBody | capella.BeaconBlockBody | deneb.BeaconBlockBody; +export type ExecutionBlockBody = + | bellatrix.BeaconBlockBody + | capella.BeaconBlockBody + | deneb.BeaconBlockBody + | electra.BeaconBlockBody; // These two additional types will also change bellatrix forward -export type ExecutionPayload = bellatrix.ExecutionPayload | capella.ExecutionPayload | deneb.ExecutionPayload; +export type ExecutionPayload = + | bellatrix.ExecutionPayload + | capella.ExecutionPayload + | deneb.ExecutionPayload + | electra.ExecutionPayload; export type ExecutionPayloadHeader = | bellatrix.ExecutionPayloadHeader | capella.ExecutionPayloadHeader - | deneb.ExecutionPayloadHeader; + | deneb.ExecutionPayloadHeader + | electra.ExecutionPayloadHeader; // Blinded types that will change across forks export type BlindedBeaconBlockBody = | bellatrix.BlindedBeaconBlockBody | capella.BlindedBeaconBlockBody - | deneb.BlindedBeaconBlockBody; -export type BlindedBeaconBlock = bellatrix.BlindedBeaconBlock | capella.BlindedBeaconBlock | deneb.BlindedBeaconBlock; + | deneb.BlindedBeaconBlockBody + | electra.BlindedBeaconBlockBody; +export type BlindedBeaconBlock = + | bellatrix.BlindedBeaconBlock + | capella.BlindedBeaconBlock + | deneb.BlindedBeaconBlock + | electra.BlindedBeaconBlock; export type SignedBlindedBeaconBlock = | bellatrix.SignedBlindedBeaconBlock | capella.SignedBlindedBeaconBlock - | deneb.SignedBlindedBeaconBlock; + | deneb.SignedBlindedBeaconBlock + | electra.SignedBlindedBeaconBlock; // Full or blinded types -export type FullOrBlindedExecutionPayload = - | bellatrix.FullOrBlindedExecutionPayload - | capella.FullOrBlindedExecutionPayload; +export type FullOrBlindedExecutionPayload = ExecutionPayload | ExecutionPayloadHeader; export type FullOrBlindedBeaconBlockBody = BeaconBlockBody | BlindedBeaconBlockBody; export type FullOrBlindedBeaconBlock = BeaconBlock | BlindedBeaconBlock; export type FullOrBlindedSignedBeaconBlock = SignedBeaconBlock | SignedBlindedBeaconBlock; @@ -80,30 +99,52 @@ export type SignedBeaconBlockOrContents = SignedBeaconBlock | SignedBlockContent export type FullOrBlindedBeaconBlockOrContents = BeaconBlockOrContents | BlindedBeaconBlock; -export type BuilderBid = bellatrix.BuilderBid | capella.BuilderBid | deneb.BuilderBid; -export type SignedBuilderBid = bellatrix.SignedBuilderBid | capella.SignedBuilderBid | deneb.SignedBuilderBid; -export type ExecutionPayloadAndBlobsBundle = deneb.ExecutionPayloadAndBlobsBundle; +export type BuilderBid = bellatrix.BuilderBid | capella.BuilderBid | deneb.BuilderBid | electra.BuilderBid; +export type SignedBuilderBid = + | bellatrix.SignedBuilderBid + | capella.SignedBuilderBid + | deneb.SignedBuilderBid + | electra.SignedBuilderBid; +export type ExecutionPayloadAndBlobsBundle = + | deneb.ExecutionPayloadAndBlobsBundle + | electra.ExecutionPayloadAndBlobsBundle; -export type LightClientHeader = altair.LightClientHeader | capella.LightClientHeader | deneb.LightClientHeader; +export type LightClientHeader = + | altair.LightClientHeader + | capella.LightClientHeader + | deneb.LightClientHeader + | electra.LightClientHeader; export type LightClientBootstrap = | altair.LightClientBootstrap | capella.LightClientBootstrap - | deneb.LightClientBootstrap; -export type LightClientUpdate = altair.LightClientUpdate | capella.LightClientUpdate | deneb.LightClientUpdate; + | deneb.LightClientBootstrap + | electra.LightClientBootstrap; +export type LightClientUpdate = + | altair.LightClientUpdate + | capella.LightClientUpdate + | deneb.LightClientUpdate + | electra.LightClientUpdate; export type LightClientFinalityUpdate = | altair.LightClientFinalityUpdate | capella.LightClientFinalityUpdate - | deneb.LightClientFinalityUpdate; + | deneb.LightClientFinalityUpdate + | electra.LightClientFinalityUpdate; export type LightClientOptimisticUpdate = | altair.LightClientOptimisticUpdate | capella.LightClientOptimisticUpdate - | deneb.LightClientOptimisticUpdate; -export type LightClientStore = altair.LightClientStore | capella.LightClientStore | deneb.LightClientStore; + | deneb.LightClientOptimisticUpdate + | electra.LightClientOptimisticUpdate; +export type LightClientStore = + | altair.LightClientStore + | capella.LightClientStore + | deneb.LightClientStore + | electra.LightClientStore; export type SSEPayloadAttributes = | bellatrix.SSEPayloadAttributes | capella.SSEPayloadAttributes - | deneb.SSEPayloadAttributes; + | deneb.SSEPayloadAttributes + | electra.SSEPayloadAttributes; /** * Types known to change between forks @@ -128,7 +169,12 @@ export type AllForksBlindedTypes = { }; export type AllForksLightClient = { - BeaconBlock: altair.BeaconBlock | bellatrix.BeaconBlock | capella.BeaconBlock | deneb.BeaconBlock; + BeaconBlock: + | altair.BeaconBlock + | bellatrix.BeaconBlock + | capella.BeaconBlock + | deneb.BeaconBlock + | electra.BeaconBlock; LightClientHeader: LightClientHeader; LightClientBootstrap: LightClientBootstrap; LightClientUpdate: LightClientUpdate; @@ -138,8 +184,12 @@ export type AllForksLightClient = { }; export type AllForksExecution = { - BeaconBlock: bellatrix.BeaconBlock | capella.BeaconBlock | deneb.BeaconBlock; - BeaconBlockBody: bellatrix.BeaconBlockBody | capella.BeaconBlockBody | deneb.BeaconBlockBody; + BeaconBlock: bellatrix.BeaconBlock | capella.BeaconBlock | deneb.BeaconBlock | electra.BeaconBlock; + BeaconBlockBody: + | bellatrix.BeaconBlockBody + | capella.BeaconBlockBody + | deneb.BeaconBlockBody + | electra.BeaconBlockBody; }; /** @@ -178,6 +228,7 @@ export type AllForksSSZTypes = { | typeof bellatrixSsz.BeaconBlockBody | typeof capellaSsz.BeaconBlockBody | typeof denebSsz.BeaconBlockBody + | typeof electraSsz.BeaconBlockBody >; BeaconBlock: AllForksTypeOf< | typeof phase0Ssz.BeaconBlock @@ -185,6 +236,7 @@ export type AllForksSSZTypes = { | typeof bellatrixSsz.BeaconBlock | typeof capellaSsz.BeaconBlock | typeof denebSsz.BeaconBlock + | typeof electraSsz.BeaconBlock >; SignedBeaconBlock: AllForksTypeOf< | typeof phase0Ssz.SignedBeaconBlock @@ -192,6 +244,7 @@ export type AllForksSSZTypes = { | typeof bellatrixSsz.SignedBeaconBlock | typeof capellaSsz.SignedBeaconBlock | typeof denebSsz.SignedBeaconBlock + | typeof electraSsz.SignedBeaconBlock >; BeaconState: AllForksTypeOf< | typeof phase0Ssz.BeaconState @@ -199,57 +252,86 @@ export type AllForksSSZTypes = { | typeof bellatrixSsz.BeaconState | typeof capellaSsz.BeaconState | typeof denebSsz.BeaconState + | typeof electraSsz.BeaconState >; Metadata: AllForksTypeOf; }; export type AllForksExecutionSSZTypes = { BeaconBlockBody: AllForksTypeOf< - typeof bellatrixSsz.BeaconBlockBody | typeof capellaSsz.BeaconBlockBody | typeof denebSsz.BeaconBlockBody + | typeof bellatrixSsz.BeaconBlockBody + | typeof capellaSsz.BeaconBlockBody + | typeof denebSsz.BeaconBlockBody + | typeof electraSsz.BeaconBlockBody >; BeaconBlock: AllForksTypeOf< - typeof bellatrixSsz.BeaconBlock | typeof capellaSsz.BeaconBlock | typeof denebSsz.BeaconBlock + | typeof bellatrixSsz.BeaconBlock + | typeof capellaSsz.BeaconBlock + | typeof denebSsz.BeaconBlock + | typeof electraSsz.BeaconBlock >; SignedBeaconBlock: AllForksTypeOf< - typeof bellatrixSsz.SignedBeaconBlock | typeof capellaSsz.SignedBeaconBlock | typeof denebSsz.SignedBeaconBlock + | typeof bellatrixSsz.SignedBeaconBlock + | typeof capellaSsz.SignedBeaconBlock + | typeof denebSsz.SignedBeaconBlock + | typeof electraSsz.SignedBeaconBlock >; BeaconState: AllForksTypeOf< - typeof bellatrixSsz.BeaconState | typeof capellaSsz.BeaconState | typeof denebSsz.BeaconState + | typeof bellatrixSsz.BeaconState + | typeof capellaSsz.BeaconState + | typeof denebSsz.BeaconState + | typeof electraSsz.BeaconState >; ExecutionPayload: AllForksTypeOf< - typeof bellatrixSsz.ExecutionPayload | typeof capellaSsz.ExecutionPayload | typeof denebSsz.ExecutionPayload + | typeof bellatrixSsz.ExecutionPayload + | typeof capellaSsz.ExecutionPayload + | typeof denebSsz.ExecutionPayload + | typeof electraSsz.ExecutionPayload >; ExecutionPayloadHeader: AllForksTypeOf< | typeof bellatrixSsz.ExecutionPayloadHeader | typeof capellaSsz.ExecutionPayloadHeader | typeof denebSsz.ExecutionPayloadHeader + | typeof electraSsz.ExecutionPayloadHeader >; BuilderBid: AllForksTypeOf< - typeof bellatrixSsz.BuilderBid | typeof capellaSsz.BuilderBid | typeof denebSsz.BuilderBid + | typeof bellatrixSsz.BuilderBid + | typeof capellaSsz.BuilderBid + | typeof denebSsz.BuilderBid + | typeof electraSsz.BuilderBid >; SignedBuilderBid: AllForksTypeOf< - typeof bellatrixSsz.SignedBuilderBid | typeof capellaSsz.SignedBuilderBid | typeof denebSsz.SignedBuilderBid + | typeof bellatrixSsz.SignedBuilderBid + | typeof capellaSsz.SignedBuilderBid + | typeof denebSsz.SignedBuilderBid + | typeof electraSsz.SignedBuilderBid >; SSEPayloadAttributes: AllForksTypeOf< | typeof bellatrixSsz.SSEPayloadAttributes | typeof capellaSsz.SSEPayloadAttributes | typeof denebSsz.SSEPayloadAttributes + | typeof electraSsz.SSEPayloadAttributes >; }; export type AllForksBlindedSSZTypes = { BeaconBlockBody: AllForksTypeOf< | typeof bellatrixSsz.BlindedBeaconBlockBody - | typeof capellaSsz.BlindedBeaconBlock - | typeof denebSsz.BlindedBeaconBlock + | typeof capellaSsz.BlindedBeaconBlockBody + | typeof denebSsz.BlindedBeaconBlockBody + | typeof electraSsz.BlindedBeaconBlockBody >; BeaconBlock: AllForksTypeOf< - typeof bellatrixSsz.BlindedBeaconBlock | typeof capellaSsz.BlindedBeaconBlock | typeof denebSsz.BlindedBeaconBlock + | typeof bellatrixSsz.BlindedBeaconBlock + | typeof capellaSsz.BlindedBeaconBlock + | typeof denebSsz.BlindedBeaconBlock + | typeof electraSsz.BlindedBeaconBlock >; SignedBeaconBlock: AllForksTypeOf< | typeof bellatrixSsz.SignedBlindedBeaconBlock | typeof capellaSsz.SignedBlindedBeaconBlock | typeof denebSsz.SignedBlindedBeaconBlock + | typeof electraSsz.SignedBlindedBeaconBlock >; }; @@ -259,40 +341,56 @@ export type AllForksLightClientSSZTypes = { | typeof bellatrixSsz.BeaconBlock | typeof capellaSsz.BeaconBlock | typeof denebSsz.BeaconBlock + | typeof electraSsz.BeaconBlock >; BeaconBlockBody: AllForksTypeOf< | typeof altairSsz.BeaconBlockBody | typeof bellatrixSsz.BeaconBlockBody | typeof capellaSsz.BeaconBlockBody | typeof denebSsz.BeaconBlockBody + | typeof electraSsz.BeaconBlockBody >; LightClientHeader: AllForksTypeOf< - typeof altairSsz.LightClientHeader | typeof capellaSsz.LightClientHeader | typeof denebSsz.LightClientHeader + | typeof altairSsz.LightClientHeader + | typeof capellaSsz.LightClientHeader + | typeof denebSsz.LightClientHeader + | typeof electraSsz.LightClientHeader >; LightClientBootstrap: AllForksTypeOf< | typeof altairSsz.LightClientBootstrap | typeof capellaSsz.LightClientBootstrap | typeof denebSsz.LightClientBootstrap + | typeof electraSsz.LightClientBootstrap >; LightClientUpdate: AllForksTypeOf< - typeof altairSsz.LightClientUpdate | typeof capellaSsz.LightClientUpdate | typeof denebSsz.LightClientUpdate + | typeof altairSsz.LightClientUpdate + | typeof capellaSsz.LightClientUpdate + | typeof denebSsz.LightClientUpdate + | typeof electraSsz.LightClientUpdate >; LightClientFinalityUpdate: AllForksTypeOf< | typeof altairSsz.LightClientFinalityUpdate | typeof capellaSsz.LightClientFinalityUpdate | typeof denebSsz.LightClientFinalityUpdate + | typeof electraSsz.LightClientFinalityUpdate >; LightClientOptimisticUpdate: AllForksTypeOf< | typeof altairSsz.LightClientOptimisticUpdate | typeof capellaSsz.LightClientOptimisticUpdate | typeof denebSsz.LightClientOptimisticUpdate + | typeof electraSsz.LightClientOptimisticUpdate >; LightClientStore: AllForksTypeOf< - typeof altairSsz.LightClientStore | typeof capellaSsz.LightClientStore | typeof denebSsz.LightClientStore + | typeof altairSsz.LightClientStore + | typeof capellaSsz.LightClientStore + | typeof denebSsz.LightClientStore + | typeof electraSsz.LightClientStore >; }; export type AllForksBlobsSSZTypes = { BlobSidecar: AllForksTypeOf; - ExecutionPayloadAndBlobsBundle: AllForksTypeOf; + ExecutionPayloadAndBlobsBundle: AllForksTypeOf< + typeof denebSsz.ExecutionPayloadAndBlobsBundle | typeof electraSsz.ExecutionPayloadAndBlobsBundle + >; }; diff --git a/packages/types/src/deneb/sszTypes.ts b/packages/types/src/deneb/sszTypes.ts index c93c3f145beb..052db6a3f2d9 100644 --- a/packages/types/src/deneb/sszTypes.ts +++ b/packages/types/src/deneb/sszTypes.ts @@ -98,7 +98,7 @@ export const BeaconBlockBody = new ContainerType( export const BeaconBlock = new ContainerType( { - ...capellaSsz.BeaconBlock.fields, + ...phase0Ssz.BeaconBlock.fields, body: BeaconBlockBody, // Modified in DENEB }, {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} @@ -149,7 +149,7 @@ export const BlindedBeaconBlockBody = new ContainerType( export const BlindedBeaconBlock = new ContainerType( { - ...capellaSsz.BlindedBeaconBlock.fields, + ...bellatrixSsz.BlindedBeaconBlock.fields, body: BlindedBeaconBlockBody, // Modified in DENEB }, {typeName: "BlindedBeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 30690a499845..bdbade4a94a5 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -1,12 +1,43 @@ -import {ContainerType} from "@chainsafe/ssz"; +import {ContainerType, ListCompositeType} from "@chainsafe/ssz"; +import { + EPOCHS_PER_SYNC_COMMITTEE_PERIOD, + SLOTS_PER_EPOCH, + HISTORICAL_ROOTS_LIMIT, + MAX_TRANSACTIONS_PER_INCLUSION_LIST, +} from "@lodestar/params"; import {ssz as primitiveSsz} from "../primitive/index.js"; +import {ssz as phase0Ssz} from "../phase0/index.js"; +import {ssz as altairSsz} from "../altair/index.js"; +import {ssz as bellatrixSsz} from "../bellatrix/index.js"; +import {ssz as capellaSsz} from "../capella/index.js"; import {ssz as denebSsz} from "../deneb/index.js"; -const {BLSSignature} = primitiveSsz; +const {BLSSignature, ExecutionAddress, Root, UintBn256, BLSPubkey, ValidatorIndex, Slot, UintNum64} = primitiveSsz; + +export const ILSummary = new ListCompositeType(ExecutionAddress, MAX_TRANSACTIONS_PER_INCLUSION_LIST); +export const ILTransactions = new ListCompositeType(bellatrixSsz.Transaction, MAX_TRANSACTIONS_PER_INCLUSION_LIST); + +export const SignedInclusionListSummary = new ContainerType( + { + summary: ILSummary, + signature: BLSSignature, + }, + {typeName: "SignedInclusionListSummary", jsonCase: "eth2"} +); + +export const NewInclusionListRequest = new ContainerType( + { + inclusionList: ILTransactions, + summary: ILSummary, + parentBlockHash: Root, + }, + {typeName: "NewInclusionListRequest", jsonCase: "eth2"} +); export const ExecutionPayload = new ContainerType( { ...denebSsz.ExecutionPayload.fields, + previousInclusionListSummary: SignedInclusionListSummary, }, {typeName: "ExecutionPayload", jsonCase: "eth2"} ); @@ -14,20 +45,25 @@ export const ExecutionPayload = new ContainerType( export const ExecutionPayloadHeader = new ContainerType( { ...denebSsz.ExecutionPayloadHeader.fields, + previousInclusionListSummaryRoot: Root, }, {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} ); export const BeaconBlockBody = new ContainerType( { - ...denebSsz.BeaconBlockBody.fields, + ...altairSsz.BeaconBlockBody.fields, + executionPayload: ExecutionPayload, // Modified in ELECTRA + blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, + blobKzgCommitments: denebSsz.BlobKzgCommitments, }, {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} ); export const BeaconBlock = new ContainerType( { - ...denebSsz.BeaconBlock.fields, + ...phase0Ssz.BeaconBlock.fields, + body: BeaconBlockBody, // Modified in ELECTRA }, {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} ); @@ -40,23 +76,20 @@ export const SignedBeaconBlock = new ContainerType( {typeName: "SignedBeaconBlock", jsonCase: "eth2"} ); -export const BlobSidecar = new ContainerType( - { - ...denebSsz.BlobSidecar.fields, - }, - {typeName: "BlobSidecar", jsonCase: "eth2"} -); - export const BlindedBeaconBlockBody = new ContainerType( { - ...denebSsz.BlindedBeaconBlockBody.fields, + ...altairSsz.BeaconBlockBody.fields, + executionPayloadHeader: ExecutionPayloadHeader, // Modified in ELECTRA + blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, + blobKzgCommitments: denebSsz.BlobKzgCommitments, }, {typeName: "BlindedBeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} ); export const BlindedBeaconBlock = new ContainerType( { - ...denebSsz.BlindedBeaconBlock.fields, + ...bellatrixSsz.BlindedBeaconBlock.fields, + body: BlindedBeaconBlockBody, // Modified in ELECTRA }, {typeName: "BlindedBeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} ); @@ -71,78 +104,153 @@ export const SignedBlindedBeaconBlock = new ContainerType( export const BuilderBid = new ContainerType( { - ...denebSsz.BuilderBid.fields, + header: ExecutionPayloadHeader, + blobKzgCommitments: denebSsz.BlobKzgCommitments, + value: UintBn256, + pubkey: BLSPubkey, }, {typeName: "BuilderBid", jsonCase: "eth2"} ); -export const SignedBuilderBid = new ContainerType( +export const ExecutionPayloadAndBlobsBundle = new ContainerType( { - message: BuilderBid, - signature: BLSSignature, + executionPayload: ExecutionPayload, + blobsBundle: denebSsz.BlobsBundle, }, - {typeName: "SignedBuilderBid", jsonCase: "eth2"} + {typeName: "ExecutionPayloadAndBlobsBundle", jsonCase: "eth2"} ); -export const ExecutionPayloadAndBlobsBundle = new ContainerType( +export const SignedBuilderBid = new ContainerType( { - ...denebSsz.ExecutionPayloadAndBlobsBundle.fields, + message: BuilderBid, + signature: BLSSignature, }, - {typeName: "ExecutionPayloadAndBlobsBundle", jsonCase: "eth2"} + {typeName: "SignedBuilderBid", jsonCase: "eth2"} ); +// We don't spread capella.BeaconState fields since we need to replace +// latestExecutionPayloadHeader and we cannot keep order doing that export const BeaconState = new ContainerType( { - ...denebSsz.BeaconState.fields, + genesisTime: UintNum64, + genesisValidatorsRoot: Root, + slot: primitiveSsz.Slot, + fork: phase0Ssz.Fork, + // History + latestBlockHeader: phase0Ssz.BeaconBlockHeader, + blockRoots: phase0Ssz.HistoricalBlockRoots, + stateRoots: phase0Ssz.HistoricalStateRoots, + // historical_roots Frozen in Capella, replaced by historical_summaries + historicalRoots: new ListCompositeType(Root, HISTORICAL_ROOTS_LIMIT), + // Eth1 + eth1Data: phase0Ssz.Eth1Data, + eth1DataVotes: phase0Ssz.Eth1DataVotes, + eth1DepositIndex: UintNum64, + // Registry + validators: phase0Ssz.Validators, + balances: phase0Ssz.Balances, + randaoMixes: phase0Ssz.RandaoMixes, + // Slashings + slashings: phase0Ssz.Slashings, + // Participation + previousEpochParticipation: altairSsz.EpochParticipation, + currentEpochParticipation: altairSsz.EpochParticipation, + // Finality + justificationBits: phase0Ssz.JustificationBits, + previousJustifiedCheckpoint: phase0Ssz.Checkpoint, + currentJustifiedCheckpoint: phase0Ssz.Checkpoint, + finalizedCheckpoint: phase0Ssz.Checkpoint, + // Inactivity + inactivityScores: altairSsz.InactivityScores, + // Sync + currentSyncCommittee: altairSsz.SyncCommittee, + nextSyncCommittee: altairSsz.SyncCommittee, + // Execution + latestExecutionPayloadHeader: ExecutionPayloadHeader, // Modified in ELECTRA + // Withdrawals + nextWithdrawalIndex: capellaSsz.BeaconState.fields.nextWithdrawalIndex, + nextWithdrawalValidatorIndex: capellaSsz.BeaconState.fields.nextWithdrawalValidatorIndex, + // Deep history valid from Capella onwards + historicalSummaries: capellaSsz.BeaconState.fields.historicalSummaries, + // Inclusion list validation + previousProposerIndex: ValidatorIndex, // New in Electra }, {typeName: "BeaconState", jsonCase: "eth2"} ); export const LightClientHeader = new ContainerType( { - ...denebSsz.LightClientHeader.fields, + beacon: phase0Ssz.BeaconBlockHeader, + execution: ExecutionPayloadHeader, + executionBranch: capellaSsz.LightClientHeader.fields.executionBranch, }, {typeName: "LightClientHeader", jsonCase: "eth2"} ); export const LightClientBootstrap = new ContainerType( { - ...denebSsz.LightClientBootstrap.fields, + header: LightClientHeader, + currentSyncCommittee: altairSsz.SyncCommittee, + currentSyncCommitteeBranch: altairSsz.LightClientBootstrap.fields.currentSyncCommitteeBranch, }, {typeName: "LightClientBootstrap", jsonCase: "eth2"} ); export const LightClientUpdate = new ContainerType( { - ...denebSsz.LightClientUpdate.fields, + attestedHeader: LightClientHeader, + nextSyncCommittee: altairSsz.SyncCommittee, + nextSyncCommitteeBranch: altairSsz.LightClientUpdate.fields.nextSyncCommitteeBranch, + finalizedHeader: LightClientHeader, + finalityBranch: altairSsz.LightClientUpdate.fields.finalityBranch, + syncAggregate: altairSsz.SyncAggregate, + signatureSlot: Slot, }, {typeName: "LightClientUpdate", jsonCase: "eth2"} ); export const LightClientFinalityUpdate = new ContainerType( { - ...denebSsz.LightClientFinalityUpdate.fields, + attestedHeader: LightClientHeader, + finalizedHeader: LightClientHeader, + finalityBranch: altairSsz.LightClientFinalityUpdate.fields.finalityBranch, + syncAggregate: altairSsz.SyncAggregate, + signatureSlot: Slot, }, {typeName: "LightClientFinalityUpdate", jsonCase: "eth2"} ); export const LightClientOptimisticUpdate = new ContainerType( { - ...denebSsz.LightClientOptimisticUpdate.fields, + attestedHeader: LightClientHeader, + syncAggregate: altairSsz.SyncAggregate, + signatureSlot: Slot, }, {typeName: "LightClientOptimisticUpdate", jsonCase: "eth2"} ); export const LightClientStore = new ContainerType( { - ...denebSsz.LightClientStore.fields, + snapshot: LightClientBootstrap, + validUpdates: new ListCompositeType(LightClientUpdate, EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH), }, {typeName: "LightClientStore", jsonCase: "eth2"} ); +// PayloadAttributes primarily for SSE event +export const PayloadAttributes = new ContainerType( + { + ...capellaSsz.PayloadAttributes.fields, + parentBeaconBlockRoot: Root, + previousInclusionListSummary: SignedInclusionListSummary, + }, + {typeName: "PayloadAttributes", jsonCase: "eth2"} +); + export const SSEPayloadAttributes = new ContainerType( { - ...denebSsz.SSEPayloadAttributes.fields, + ...bellatrixSsz.SSEPayloadAttributesCommon.fields, + payloadAttributes: PayloadAttributes, }, {typeName: "SSEPayloadAttributes", jsonCase: "eth2"} ); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index 198259eed1dd..e5ecf598db49 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -1,8 +1,10 @@ import {ValueOf} from "@chainsafe/ssz"; import * as ssz from "./sszTypes.js"; -export type BlobSidecar = ValueOf; -export type ExecutionPayloadAndBlobsBundle = ValueOf; +export type ILSummary = ValueOf; +export type ILTransactions = ValueOf; +export type SignedInclusionListSummary = ValueOf; +export type NewInclusionListRequest = ValueOf; export type ExecutionPayload = ValueOf; export type ExecutionPayloadHeader = ValueOf; @@ -11,15 +13,15 @@ export type BeaconBlockBody = ValueOf; export type BeaconBlock = ValueOf; export type SignedBeaconBlock = ValueOf; -export type BeaconState = ValueOf; - export type BlindedBeaconBlockBody = ValueOf; export type BlindedBeaconBlock = ValueOf; export type SignedBlindedBeaconBlock = ValueOf; export type BuilderBid = ValueOf; +export type ExecutionPayloadAndBlobsBundle = ValueOf; export type SignedBuilderBid = ValueOf; -export type SSEPayloadAttributes = ValueOf; + +export type BeaconState = ValueOf; export type LightClientHeader = ValueOf; export type LightClientBootstrap = ValueOf; @@ -27,3 +29,5 @@ export type LightClientUpdate = ValueOf; export type LightClientFinalityUpdate = ValueOf; export type LightClientOptimisticUpdate = ValueOf; export type LightClientStore = ValueOf; + +export type SSEPayloadAttributes = ValueOf; From c7f3ec6a827ea4ecdabf37ff5d50a4bb7a7adce7 Mon Sep 17 00:00:00 2001 From: harkamal Date: Sat, 16 Mar 2024 21:44:01 +0530 Subject: [PATCH 3/7] extend blockinput for blobs and il data --- .../beacon-node/src/chain/blocks/types.ts | 64 +++++++----- .../chain/seenCache/seenGossipBlockInput.ts | 99 +++++++++++-------- 2 files changed, 96 insertions(+), 67 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index e2c7b5a32e0a..7416331e10c1 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -1,7 +1,7 @@ import {CachedBeaconStateAllForks, computeEpochAtSlot, DataAvailableStatus} from "@lodestar/state-transition"; import {MaybeValidExecutionStatus} from "@lodestar/fork-choice"; -import {allForks, deneb, Slot, RootHex} from "@lodestar/types"; -import {ForkSeq} from "@lodestar/params"; +import {allForks, deneb, electra, Slot, RootHex} from "@lodestar/types"; +import {ForkSeq, ForkName, ForkPreBlobs, ForkILs} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; export enum BlockInputType { @@ -23,20 +23,38 @@ export enum GossipedInputType { blob = "blob", } +type ForkBlobsInfo = {fork: ForkName.deneb}; +type ForkILsInfo = {fork: ForkILs}; + export type BlobsCache = Map; -export type BlockInputBlobs = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]}; -type CachedBlobs = { - blobsCache: BlobsCache; - availabilityPromise: Promise; - resolveAvailability: (blobs: BlockInputBlobs) => void; -}; + +type BlobsData = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]}; +type ILsData = BlobsData & {ilSummary: electra.ILSummary; inclusionLists: electra.ILTransactions[]}; + +export type BlockInputDataBlobs = ForkBlobsInfo & BlobsData; +export type BlockInputDataIls = ForkILsInfo & ILsData; +export type BlockInputData = BlockInputDataBlobs | BlockInputDataIls; + +type BlobsInputCache = {blobsCache: BlobsCache}; +type ForkILsCache = BlobsInputCache & {ilSummary?: electra.ILSummary; inclusionLists: electra.ILTransactions[]}; + +export type BlockInputCacheBlobs = ForkBlobsInfo & BlobsInputCache; +export type BlockInputCacheILs = ForkILsInfo & ForkILsCache; +export type BlockInputCache = (ForkBlobsInfo & BlobsInputCache) | (ForkILsInfo & ForkILsCache); + +type Availability = {availabilityPromise: Promise; resolveAvailability: (data: T) => void}; +export type CachedData = + | (ForkBlobsInfo & BlobsInputCache & Availability) + | (ForkILsInfo & ForkILsCache & Availability); export type BlockInput = {block: allForks.SignedBeaconBlock; source: BlockSource; blockBytes: Uint8Array | null} & ( | {type: BlockInputType.preDeneb} - | ({type: BlockInputType.postDeneb} & BlockInputBlobs) - | ({type: BlockInputType.blobsPromise} & CachedBlobs) + | ({type: BlockInputType.postDeneb} & {blockData: BlockInputData}) + | ({type: BlockInputType.blobsPromise} & {cachedData: CachedData}) ); -export type NullBlockInput = {block: null; blockRootHex: RootHex; blockInputPromise: Promise} & CachedBlobs; +export type NullBlockInput = {block: null; blockRootHex: RootHex; blockInputPromise: Promise} & { + cachedData: CachedData; +}; export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clockSlot: Slot): boolean { return ( @@ -67,10 +85,9 @@ export const getBlockInput = { postDeneb( config: ChainForkConfig, block: allForks.SignedBeaconBlock, - source: BlockSource, - blobs: deneb.BlobSidecars, blockBytes: Uint8Array | null, - blobsBytes: (Uint8Array | null)[] + blockData: BlockInputData, + source: BlockSource ): BlockInput { if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) { throw Error(`Pre Deneb block slot ${block.message.slot}`); @@ -78,21 +95,18 @@ export const getBlockInput = { return { type: BlockInputType.postDeneb, block, - source, - blobs, blockBytes, - blobsBytes, + blockData, + source, }; }, blobsPromise( config: ChainForkConfig, block: allForks.SignedBeaconBlock, - source: BlockSource, - blobsCache: BlobsCache, blockBytes: Uint8Array | null, - availabilityPromise: Promise, - resolveAvailability: (blobs: BlockInputBlobs) => void + cachedData: CachedData, + source: BlockSource ): BlockInput { if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) { throw Error(`Pre Deneb block slot ${block.message.slot}`); @@ -100,16 +114,14 @@ export const getBlockInput = { return { type: BlockInputType.blobsPromise, block, - source, - blobsCache, blockBytes, - availabilityPromise, - resolveAvailability, + source, + cachedData, }; }, }; -export function getBlockInputBlobs(blobsCache: BlobsCache): BlockInputBlobs { +export function getBlockInputBlobs(blobsCache: BlobsCache): BlobsData { const blobs = []; const blobsBytes = []; diff --git a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts index c652eaad9a9b..9694aace5032 100644 --- a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts +++ b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts @@ -2,15 +2,15 @@ import {toHexString} from "@chainsafe/ssz"; import {deneb, RootHex, ssz, allForks} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {pruneSetToMax} from "@lodestar/utils"; -import {BLOBSIDECAR_FIXED_SIZE, ForkSeq} from "@lodestar/params"; +import {BLOBSIDECAR_FIXED_SIZE, ForkSeq, ForkName} from "@lodestar/params"; import { BlockInput, NullBlockInput, getBlockInput, BlockSource, - BlockInputBlobs, - BlobsCache, + BlockInputDataBlobs, + CachedData, GossipedInputType, getBlockInputBlobs, } from "../blocks/types.js"; @@ -28,10 +28,7 @@ type GossipedBlockInput = type BlockInputCacheType = { block?: allForks.SignedBeaconBlock; blockBytes?: Uint8Array | null; - blobsCache: BlobsCache; - // blobs promise and its callback cached for delayed resolution - availabilityPromise: Promise; - resolveAvailability: (blobs: BlockInputBlobs) => void; + cachedData?: CachedData; // block promise and its callback cached for delayed resolution blockInputPromise: Promise; resolveBlockInput: (blockInput: BlockInput) => void; @@ -76,25 +73,29 @@ export class SeenGossipBlockInput { } { let blockHex; let blockCache; + let fork; if (gossipedInput.type === GossipedInputType.block) { const {signedBlock, blockBytes} = gossipedInput; + fork = config.getForkName(signedBlock.message.slot); blockHex = toHexString( config.getForkTypes(signedBlock.message.slot).BeaconBlock.hashTreeRoot(signedBlock.message) ); - blockCache = this.blockInputCache.get(blockHex) ?? getEmptyBlockInputCacheEntry(); + blockCache = this.blockInputCache.get(blockHex) ?? getEmptyBlockInputCacheEntry(fork); blockCache.block = signedBlock; blockCache.blockBytes = blockBytes; } else { const {blobSidecar, blobBytes} = gossipedInput; const blockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(blobSidecar.signedBlockHeader.message); + fork = config.getForkName(blobSidecar.signedBlockHeader.message.slot); + blockHex = toHexString(blockRoot); - blockCache = this.blockInputCache.get(blockHex) ?? getEmptyBlockInputCacheEntry(); + blockCache = this.blockInputCache.get(blockHex) ?? getEmptyBlockInputCacheEntry(fork); // TODO: freetheblobs check if its the same blob or a duplicate and throw/take actions - blockCache.blobsCache.set(blobSidecar.index, { + blockCache.cachedData?.blobsCache.set(blobSidecar.index, { blobSidecar, // easily splice out the unsigned message as blob is a fixed length type blobBytes: blobBytes?.slice(0, BLOBSIDECAR_FIXED_SIZE) ?? null, @@ -105,23 +106,21 @@ export class SeenGossipBlockInput { this.blockInputCache.set(blockHex, blockCache); } - const { - block: signedBlock, - blockBytes, - blobsCache, - availabilityPromise, - resolveAvailability, - blockInputPromise, - resolveBlockInput, - } = blockCache; + const {block: signedBlock, blockBytes, blockInputPromise, resolveBlockInput, cachedData} = blockCache; if (signedBlock !== undefined) { - if (config.getForkSeq(signedBlock.message.slot) < ForkSeq.deneb) { + if (ForkSeq[fork] < ForkSeq.deneb) { return { blockInput: getBlockInput.preDeneb(config, signedBlock, BlockSource.gossip, blockBytes ?? null), blockInputMeta: {pending: null, haveBlobs: 0, expectedBlobs: 0}, }; } + + if (cachedData === undefined) { + throw Error("Missing cached Data for deneb+ block"); + } + const {blobsCache} = cachedData; + // block is available, check if all blobs have shown up const {slot, body} = signedBlock.message; const {blobKzgCommitments} = body as deneb.BeaconBlockBody; @@ -135,16 +134,20 @@ export class SeenGossipBlockInput { if (blobKzgCommitments.length === blobsCache.size) { const allBlobs = getBlockInputBlobs(blobsCache); - resolveAvailability(allBlobs); + const blockData = {fork: ForkName.deneb, ...allBlobs} as BlockInputDataBlobs; + if (cachedData.fork === ForkName.deneb) { + cachedData.resolveAvailability(blockData); + } else { + throw Error("il availability not implemented"); + } metrics?.syncUnknownBlock.resolveAvailabilitySource.inc({source: BlockInputAvailabilitySource.GOSSIP}); - const {blobs, blobsBytes} = allBlobs; + const {blobs} = allBlobs; const blockInput = getBlockInput.postDeneb( config, signedBlock, - BlockSource.gossip, - blobs, blockBytes ?? null, - blobsBytes + blockData, + BlockSource.gossip ); resolveBlockInput(blockInput); @@ -156,11 +159,9 @@ export class SeenGossipBlockInput { const blockInput = getBlockInput.blobsPromise( config, signedBlock, - BlockSource.gossip, - blobsCache, blockBytes ?? null, - availabilityPromise, - resolveAvailability + cachedData, + BlockSource.gossip ); resolveBlockInput(blockInput); @@ -174,14 +175,17 @@ export class SeenGossipBlockInput { }; } } else { + if (cachedData === undefined) { + throw Error("Missing cachedData for deneb+ blobs"); + } + const {blobsCache} = cachedData; + // will need to wait for the block to showup return { blockInput: { block: null, blockRootHex: blockHex, - blobsCache, - availabilityPromise, - resolveAvailability, + cachedData, blockInputPromise, }, blockInputMeta: {pending: GossipedInputType.block, haveBlobs: blobsCache.size, expectedBlobs: null}, @@ -190,22 +194,35 @@ export class SeenGossipBlockInput { } } -function getEmptyBlockInputCacheEntry(): BlockInputCacheType { +function getEmptyBlockInputCacheEntry(fork: ForkName): BlockInputCacheType { // Capture both the promise and its callbacks for blockInput and final availability // It is not spec'ed but in tests in Firefox and NodeJS the promise constructor is run immediately let resolveBlockInput: ((block: BlockInput) => void) | null = null; const blockInputPromise = new Promise((resolveCB) => { resolveBlockInput = resolveCB; }); - - let resolveAvailability: ((blobs: BlockInputBlobs) => void) | null = null; - const availabilityPromise = new Promise((resolveCB) => { - resolveAvailability = resolveCB; - }); - - if (resolveAvailability === null || resolveBlockInput === null) { + if (resolveBlockInput === null) { throw Error("Promise Constructor was not executed immediately"); } + + if (ForkSeq[fork] < ForkSeq.deneb) { + return {blockInputPromise, resolveBlockInput}; + } + const blobsCache = new Map(); - return {blockInputPromise, resolveBlockInput, availabilityPromise, resolveAvailability, blobsCache}; + + if (fork === ForkName.deneb) { + let resolveAvailability: ((blobs: BlockInputDataBlobs) => void) | null = null; + const availabilityPromise = new Promise((resolveCB) => { + resolveAvailability = resolveCB; + }); + + if (resolveAvailability === null) { + throw Error("Promise Constructor was not executed immediately"); + } + const cachedData: CachedData = {fork, blobsCache, availabilityPromise, resolveAvailability}; + return {blockInputPromise, resolveBlockInput, cachedData}; + } else { + throw Error("il cache not implemented"); + } } From 40bd276965fdf2ad984587154989db5c3023519e Mon Sep 17 00:00:00 2001 From: harkamal Date: Sun, 17 Mar 2024 03:33:49 +0530 Subject: [PATCH 4/7] add bit of gossip, data availability handling, fix build and run --- .../src/api/impl/beacon/blocks/index.ts | 8 +- .../src/chain/blocks/importBlock.ts | 2 +- .../beacon-node/src/chain/blocks/types.ts | 3 +- .../blocks/verifyBlocksDataAvailability.ts | 20 +++- .../src/chain/blocks/writeBlockInputToDb.ts | 6 +- .../chain/seenCache/seenGossipBlockInput.ts | 34 ++++-- .../src/chain/validation/inclusionList.ts | 7 ++ .../src/network/gossip/interface.ts | 6 +- .../beacon-node/src/network/gossip/topic.ts | 10 ++ .../src/network/processor/gossipHandlers.ts | 110 +++++++++++++++++- .../network/processor/gossipQueues/index.ts | 1 + .../src/network/processor/index.ts | 1 + .../reqresp/beaconBlocksMaybeBlobsByRange.ts | 11 +- .../reqresp/beaconBlocksMaybeBlobsByRoot.ts | 28 +++-- packages/beacon-node/src/sync/range/chain.ts | 2 +- packages/beacon-node/src/sync/unknownBlock.ts | 2 +- packages/types/src/electra/sszTypes.ts | 1 + 17 files changed, 212 insertions(+), 40 deletions(-) create mode 100644 packages/beacon-node/src/chain/validation/inclusionList.ts diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 79e258960b2c..a0d236e2a015 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -1,7 +1,7 @@ import {fromHexString, toHexString} from "@chainsafe/ssz"; import {routes, ServerApi, ResponseFormat} from "@lodestar/api"; import {computeTimeAtSlot, reconstructFullBlockOrContents} from "@lodestar/state-transition"; -import {SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; +import {SLOTS_PER_HISTORICAL_ROOT, ForkName} from "@lodestar/params"; import {sleep, toHex} from "@lodestar/utils"; import {allForks, deneb, isSignedBlockContents, ProducedBlockSource} from "@lodestar/types"; import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput} from "../../../../chain/blocks/types.js"; @@ -50,11 +50,9 @@ export function getBeaconBlockApi({ blockForImport = getBlockInput.postDeneb( config, signedBlock, - BlockSource.api, - blobSidecars, - // don't bundle any bytes for block and blobs null, - blobSidecars.map(() => null) + {fork: ForkName.deneb, blobs: blobSidecars, blobsBytes: [null]}, + BlockSource.api ); } else { signedBlock = signedBlockOrContents; diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index d82448a4e932..4c6509038de9 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -103,7 +103,7 @@ export async function importBlock( }); if (blockInput.type === BlockInputType.postDeneb) { - for (const blobSidecar of blockInput.blobs) { + for (const blobSidecar of blockInput.blockData.blobs) { const {index, kzgCommitment} = blobSidecar; this.emitter.emit(routes.events.EventType.blobSidecar, { blockRoot: blockRootHex, diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index 7416331e10c1..75a83f170d67 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -1,7 +1,7 @@ import {CachedBeaconStateAllForks, computeEpochAtSlot, DataAvailableStatus} from "@lodestar/state-transition"; import {MaybeValidExecutionStatus} from "@lodestar/fork-choice"; import {allForks, deneb, electra, Slot, RootHex} from "@lodestar/types"; -import {ForkSeq, ForkName, ForkPreBlobs, ForkILs} from "@lodestar/params"; +import {ForkSeq, ForkName, ForkILs} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; export enum BlockInputType { @@ -21,6 +21,7 @@ export enum BlockSource { export enum GossipedInputType { block = "block", blob = "blob", + ilist = "ilist", } type ForkBlobsInfo = {fork: ForkName.deneb}; diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts index de7a9575ce06..f005f18b2f6c 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts @@ -1,6 +1,7 @@ import {computeTimeAtSlot, DataAvailableStatus} from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; import {deneb, UintNum64} from "@lodestar/types"; +import {ForkName} from "@lodestar/params"; import {Logger} from "@lodestar/utils"; import {BlockError, BlockErrorCode} from "../errors/index.js"; import {validateBlobSidecars} from "../validation/blobSidecar.js"; @@ -78,12 +79,21 @@ async function maybeValidateBlobs( const {block} = blockInput; const blockSlot = block.message.slot; - const blobsData = - blockInput.type === BlockInputType.postDeneb - ? blockInput - : await raceWithCutoff(chain, blockInput, blockInput.availabilityPromise); - const {blobs} = blobsData; + let blockData; + if (blockInput.type === BlockInputType.postDeneb) { + blockData = blockInput.blockData; + } else { + const {cachedData} = blockInput; + // weird that typescript is getting confused doing the same thing but with + // differing promise types, need to separate the case out + if (cachedData.fork === ForkName.deneb) { + blockData = await raceWithCutoff(chain, blockInput, cachedData.availabilityPromise); + } else { + blockData = await raceWithCutoff(chain, blockInput, cachedData.availabilityPromise); + } + } + const {blobs} = blockData; const {blobKzgCommitments} = (block as deneb.SignedBeaconBlock).message.body; const beaconBlockRoot = chain.config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message); diff --git a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts index 0b94d32b84ec..069518abe198 100644 --- a/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts +++ b/packages/beacon-node/src/chain/blocks/writeBlockInputToDb.ts @@ -32,9 +32,9 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInput: BlockI if (blockInput.type === BlockInputType.postDeneb || blockInput.type === BlockInputType.blobsPromise) { const blobSidecars = blockInput.type == BlockInputType.postDeneb - ? blockInput.blobs + ? blockInput.blockData.blobs : // At this point of import blobs are available and can be safely awaited - (await blockInput.availabilityPromise).blobs; + (await blockInput.cachedData.availabilityPromise).blobs; // NOTE: Old blobs are pruned on archive fnPromises.push(this.db.blobSidecars.add({blockRoot, slot: block.message.slot, blobSidecars})); @@ -64,7 +64,7 @@ export async function removeEagerlyPersistedBlockInputs(this: BeaconChain, block blockToRemove.push(block); if (type === BlockInputType.postDeneb) { - const blobSidecars = blockInput.blobs; + const blobSidecars = blockInput.blockData.blobs; blobsToRemove.push({blockRoot, slot: block.message.slot, blobSidecars}); } } diff --git a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts index 9694aace5032..38259bd4e75b 100644 --- a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts +++ b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts @@ -1,8 +1,8 @@ import {toHexString} from "@chainsafe/ssz"; -import {deneb, RootHex, ssz, allForks} from "@lodestar/types"; +import {deneb, electra, RootHex, ssz, allForks} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {pruneSetToMax} from "@lodestar/utils"; -import {BLOBSIDECAR_FIXED_SIZE, ForkSeq, ForkName} from "@lodestar/params"; +import {BLOBSIDECAR_FIXED_SIZE, ForkSeq, ForkName, isForkBlobs, isForkILs} from "@lodestar/params"; import { BlockInput, @@ -13,6 +13,7 @@ import { CachedData, GossipedInputType, getBlockInputBlobs, + BlockInputDataIls, } from "../blocks/types.js"; import {Metrics} from "../../metrics/index.js"; @@ -23,7 +24,12 @@ export enum BlockInputAvailabilitySource { type GossipedBlockInput = | {type: GossipedInputType.block; signedBlock: allForks.SignedBeaconBlock; blockBytes: Uint8Array | null} - | {type: GossipedInputType.blob; blobSidecar: deneb.BlobSidecar; blobBytes: Uint8Array | null}; + | {type: GossipedInputType.blob; blobSidecar: deneb.BlobSidecar; blobBytes: Uint8Array | null} + | { + type: GossipedInputType.ilist; + inclusionList: electra.NewInclusionListRequest; + inclusionListBytes: Uint8Array | null; + }; type BlockInputCacheType = { block?: allForks.SignedBeaconBlock; @@ -75,7 +81,9 @@ export class SeenGossipBlockInput { let blockCache; let fork; - if (gossipedInput.type === GossipedInputType.block) { + if (gossipedInput.type === GossipedInputType.ilist) { + throw Error("Inclusion list gossip handling not implemented"); + } else if (gossipedInput.type === GossipedInputType.block) { const {signedBlock, blockBytes} = gossipedInput; fork = config.getForkName(signedBlock.message.slot); @@ -205,13 +213,13 @@ function getEmptyBlockInputCacheEntry(fork: ForkName): BlockInputCacheType { throw Error("Promise Constructor was not executed immediately"); } - if (ForkSeq[fork] < ForkSeq.deneb) { + if (!isForkBlobs(fork)) { return {blockInputPromise, resolveBlockInput}; } - const blobsCache = new Map(); - if (fork === ForkName.deneb) { + if (!isForkILs(fork)) { + // blobs availability let resolveAvailability: ((blobs: BlockInputDataBlobs) => void) | null = null; const availabilityPromise = new Promise((resolveCB) => { resolveAvailability = resolveCB; @@ -223,6 +231,16 @@ function getEmptyBlockInputCacheEntry(fork: ForkName): BlockInputCacheType { const cachedData: CachedData = {fork, blobsCache, availabilityPromise, resolveAvailability}; return {blockInputPromise, resolveBlockInput, cachedData}; } else { - throw Error("il cache not implemented"); + // il availability (with blobs) + let resolveAvailability: ((blobs: BlockInputDataIls) => void) | null = null; + const availabilityPromise = new Promise((resolveCB) => { + resolveAvailability = resolveCB; + }); + + if (resolveAvailability === null) { + throw Error("Promise Constructor was not executed immediately"); + } + const cachedData: CachedData = {fork, blobsCache, availabilityPromise, resolveAvailability, inclusionLists: []}; + return {blockInputPromise, resolveBlockInput, cachedData}; } } diff --git a/packages/beacon-node/src/chain/validation/inclusionList.ts b/packages/beacon-node/src/chain/validation/inclusionList.ts new file mode 100644 index 000000000000..83b8f3ba7916 --- /dev/null +++ b/packages/beacon-node/src/chain/validation/inclusionList.ts @@ -0,0 +1,7 @@ +import {electra} from "@lodestar/types"; +import {IBeaconChain} from "../interface.js"; + +export async function validateGossipInclusionList( + _chain: IBeaconChain, + _inclusionList: electra.NewInclusionListRequest +): Promise {} diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts index df26c2328c70..ced5ea6ff59c 100644 --- a/packages/beacon-node/src/network/gossip/interface.ts +++ b/packages/beacon-node/src/network/gossip/interface.ts @@ -2,7 +2,7 @@ import {Libp2p} from "libp2p"; import {Message, TopicValidatorResult} from "@libp2p/interface"; import {PeerIdStr} from "@chainsafe/libp2p-gossipsub/types"; import {ForkName} from "@lodestar/params"; -import {allForks, altair, capella, deneb, phase0, Slot} from "@lodestar/types"; +import {allForks, altair, capella, deneb, electra, phase0, Slot} from "@lodestar/types"; import {BeaconConfig} from "@lodestar/config"; import {Logger} from "@lodestar/utils"; import {IBeaconChain} from "../../chain/index.js"; @@ -13,6 +13,7 @@ import {GossipActionError} from "../../chain/errors/gossipValidation.js"; export enum GossipType { beacon_block = "beacon_block", blob_sidecar = "blob_sidecar", + inclusion_list = "inclusion_list", beacon_aggregate_and_proof = "beacon_aggregate_and_proof", beacon_attestation = "beacon_attestation", voluntary_exit = "voluntary_exit", @@ -41,6 +42,7 @@ export interface IGossipTopic { export type GossipTopicTypeMap = { [GossipType.beacon_block]: {type: GossipType.beacon_block}; [GossipType.blob_sidecar]: {type: GossipType.blob_sidecar; index: number}; + [GossipType.inclusion_list]: {type: GossipType.inclusion_list}; [GossipType.beacon_aggregate_and_proof]: {type: GossipType.beacon_aggregate_and_proof}; [GossipType.beacon_attestation]: {type: GossipType.beacon_attestation; subnet: number}; [GossipType.voluntary_exit]: {type: GossipType.voluntary_exit}; @@ -71,6 +73,7 @@ export type SSZTypeOfGossipTopic = T extends {type: infer export type GossipTypeMap = { [GossipType.beacon_block]: allForks.SignedBeaconBlock; [GossipType.blob_sidecar]: deneb.BlobSidecar; + [GossipType.inclusion_list]: electra.NewInclusionListRequest; [GossipType.beacon_aggregate_and_proof]: phase0.SignedAggregateAndProof; [GossipType.beacon_attestation]: phase0.Attestation; [GossipType.voluntary_exit]: phase0.SignedVoluntaryExit; @@ -86,6 +89,7 @@ export type GossipTypeMap = { export type GossipFnByType = { [GossipType.beacon_block]: (signedBlock: allForks.SignedBeaconBlock) => Promise | void; [GossipType.blob_sidecar]: (blobSidecar: deneb.BlobSidecar) => Promise | void; + [GossipType.inclusion_list]: (inclusionList: electra.NewInclusionListRequest) => Promise | void; [GossipType.beacon_aggregate_and_proof]: (aggregateAndProof: phase0.SignedAggregateAndProof) => Promise | void; [GossipType.beacon_attestation]: (attestation: phase0.Attestation) => Promise | void; [GossipType.voluntary_exit]: (voluntaryExit: phase0.SignedVoluntaryExit) => Promise | void; diff --git a/packages/beacon-node/src/network/gossip/topic.ts b/packages/beacon-node/src/network/gossip/topic.ts index c5cd68ffa1de..0c52e142b432 100644 --- a/packages/beacon-node/src/network/gossip/topic.ts +++ b/packages/beacon-node/src/network/gossip/topic.ts @@ -61,6 +61,7 @@ export function stringifyGossipTopic(forkDigestContext: ForkDigestContext, topic function stringifyGossipTopicType(topic: GossipTopic): string { switch (topic.type) { case GossipType.beacon_block: + case GossipType.inclusion_list: case GossipType.beacon_aggregate_and_proof: case GossipType.voluntary_exit: case GossipType.proposer_slashing: @@ -86,6 +87,8 @@ export function getGossipSSZType(topic: GossipTopic) { return ssz[topic.fork].SignedBeaconBlock; case GossipType.blob_sidecar: return ssz.deneb.BlobSidecar; + case GossipType.inclusion_list: + return ssz.electra.NewInclusionListRequest; case GossipType.beacon_aggregate_and_proof: return ssz.phase0.SignedAggregateAndProof; case GossipType.beacon_attestation: @@ -162,6 +165,7 @@ export function parseGossipTopic(forkDigestContext: ForkDigestContext, topicStr: // Inline-d the parseGossipTopicType() function since spreading the resulting object x4 the time to parse a topicStr switch (gossipTypeStr) { case GossipType.beacon_block: + case GossipType.inclusion_list: case GossipType.beacon_aggregate_and_proof: case GossipType.voluntary_exit: case GossipType.proposer_slashing: @@ -212,6 +216,11 @@ export function getCoreTopicsAtFork( {type: GossipType.attester_slashing}, ]; + // electra + if (ForkSeq[fork] >= ForkSeq.electra) { + topics.push({type: GossipType.inclusion_list}); + } + // After Deneb also track blob_sidecar_{index} if (ForkSeq[fork] >= ForkSeq.deneb) { for (let index = 0; index < MAX_BLOBS_PER_BLOCK; index++) { @@ -261,6 +270,7 @@ function parseEncodingStr(encodingStr: string): GossipEncoding { // TODO: Review which yes, and which not export const gossipTopicIgnoreDuplicatePublishError: Record = { [GossipType.beacon_block]: true, + [GossipType.inclusion_list]: true, [GossipType.blob_sidecar]: true, [GossipType.beacon_aggregate_and_proof]: true, [GossipType.beacon_attestation]: true, diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 519e314df055..b772aef01d15 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -1,7 +1,7 @@ import {toHexString} from "@chainsafe/ssz"; import {BeaconConfig, ChainForkConfig} from "@lodestar/config"; import {LogLevel, Logger, prettyBytes} from "@lodestar/utils"; -import {Root, Slot, ssz, allForks, deneb, UintNum64} from "@lodestar/types"; +import {Root, Slot, ssz, allForks, deneb, electra, UintNum64} from "@lodestar/types"; import {ForkName, ForkSeq} from "@lodestar/params"; import {routes} from "@lodestar/api"; import {computeTimeAtSlot} from "@lodestar/state-transition"; @@ -46,6 +46,7 @@ import {PeerAction} from "../peers/index.js"; import {validateLightClientFinalityUpdate} from "../../chain/validation/lightClientFinalityUpdate.js"; import {validateLightClientOptimisticUpdate} from "../../chain/validation/lightClientOptimisticUpdate.js"; import {validateGossipBlobSidecar} from "../../chain/validation/blobSidecar.js"; +import {validateGossipInclusionList} from "../../chain/validation/inclusionList.js"; import { BlockInput, GossipedInputType, @@ -252,6 +253,62 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } } + async function validateInclusionList( + inclusionList: electra.NewInclusionListRequest, + inclusionListBytes: Uint8Array, + peerIdStr: string, + seenTimestampSec: number + ): Promise { + const slot = inclusionList.slot; + const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec); + const recvToValLatency = Date.now() / 1000 - seenTimestampSec; + + const {blockInput, blockInputMeta} = chain.seenGossipBlockInput.getGossipBlockInput( + config, + { + type: GossipedInputType.ilist, + inclusionList, + inclusionListBytes, + }, + metrics + ); + + try { + await validateGossipInclusionList(chain, inclusionList); + const recvToValidation = Date.now() / 1000 - seenTimestampSec; + const validationTime = recvToValidation - recvToValLatency; + + metrics?.gossipBlob.recvToValidation.observe(recvToValidation); + metrics?.gossipBlob.validationTime.observe(validationTime); + + logger.debug("Received inclusion list", { + curentSlot: chain.clock.currentSlot, + peerId: peerIdStr, + delaySec, + ...blockInputMeta, + recvToValLatency, + recvToValidation, + validationTime, + }); + + return blockInput; + } catch (e) { + if (e instanceof BlobSidecarGossipError) { + // Don't trigger this yet if full block and blobs haven't arrived yet + if (e.type.code === BlobSidecarErrorCode.PARENT_UNKNOWN && blockInput.block !== null) { + logger.debug("Gossip inclusion list has error", {code: e.type.code}); + events.emit(NetworkEvent.unknownBlockParent, {blockInput, peer: peerIdStr}); + } + + if (e.action === GossipAction.REJECT) { + chain.persistInvalidSszValue(ssz.electra.NewInclusionListRequest, inclusionList, "gossip_reject_slot"); + } + } + + throw e; + } + } + function handleValidBeaconBlock(blockInput: BlockInput, peerIdStr: string, seenTimestampSec: number): void { const signedBlock = blockInput.block; @@ -408,6 +465,57 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } }, + [GossipType.inclusion_list]: async ({ + gossipData, + topic, + peerIdStr, + seenTimestampSec, + }: GossipHandlerParamGeneric) => { + const {serializedData} = gossipData; + const inclusionList = sszDeserialize(topic, serializedData); + const ilSlot = inclusionList.slot; + + if (config.getForkSeq(ilSlot) < ForkSeq.electra) { + throw new GossipActionError(GossipAction.REJECT, {code: "PRE_ELECTRA_IL"}); + } + + const blockInput = await validateInclusionList(inclusionList, serializedData, peerIdStr, seenTimestampSec); + if (blockInput.block !== null) { + // we can just queue up the blockInput in the processor, but block gossip handler would have already + // queued it up. + // + // handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec); + } else { + // wait for the block to arrive till some cutoff else emit unknownBlockInput event + chain.logger.debug("Block not yet available, racing with cutoff", {ilSlot}); + const normalBlockInput = await raceWithCutoff( + chain, + ilSlot, + blockInput.blockInputPromise, + BLOCK_AVAILABILITY_CUTOFF_MS + ).catch((_e) => { + return null; + }); + + if (normalBlockInput !== null) { + chain.logger.debug("Block corresponding to inclusion list is now available for processing", {ilSlot}); + // we can directly send it for processing but block gossip handler will queue it up anyway + // if we see any issues later, we can send it to handleValidBeaconBlock + // + // handleValidBeaconBlock(normalBlockInput, peerIdStr, seenTimestampSec); + // + // however we can emit the event which will atleast add the peer to the list of peers to pull + // data from + if (normalBlockInput.type === BlockInputType.blobsPromise) { + events.emit(NetworkEvent.unknownBlockInput, {blockInput: normalBlockInput, peer: peerIdStr}); + } + } else { + chain.logger.debug("Block not available till BLOCK_AVAILABILITY_CUTOFF_MS", {ilSlot}); + events.emit(NetworkEvent.unknownBlockInput, {blockInput, peer: peerIdStr}); + } + } + }, + [GossipType.beacon_aggregate_and_proof]: async ({ gossipData, topic, diff --git a/packages/beacon-node/src/network/processor/gossipQueues/index.ts b/packages/beacon-node/src/network/processor/gossipQueues/index.ts index 366b23b30679..130968340b1e 100644 --- a/packages/beacon-node/src/network/processor/gossipQueues/index.ts +++ b/packages/beacon-node/src/network/processor/gossipQueues/index.ts @@ -39,6 +39,7 @@ const defaultGossipQueueOpts: { type: QueueType.FIFO, dropOpts: {type: DropType.count, count: 1}, }, + [GossipType.inclusion_list]: {maxLength: 1024, type: QueueType.FIFO, dropOpts: {type: DropType.count, count: 1}}, // lighthoue has aggregate_queue 4096 and unknown_block_aggregate_queue 1024, we use single queue [GossipType.beacon_aggregate_and_proof]: { maxLength: 5120, diff --git a/packages/beacon-node/src/network/processor/index.ts b/packages/beacon-node/src/network/processor/index.ts index 420360920518..6002c6ceb775 100644 --- a/packages/beacon-node/src/network/processor/index.ts +++ b/packages/beacon-node/src/network/processor/index.ts @@ -65,6 +65,7 @@ type WorkOpts = { const executeGossipWorkOrderObj: Record = { [GossipType.beacon_block]: {bypassQueue: true}, [GossipType.blob_sidecar]: {bypassQueue: true}, + [GossipType.inclusion_list]: {bypassQueue: true}, [GossipType.beacon_aggregate_and_proof]: {}, [GossipType.voluntary_exit]: {}, [GossipType.bls_to_execution_change]: {}, diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts index e57c3e5b7c8e..717b689d2b0d 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts @@ -1,6 +1,6 @@ import {ChainForkConfig} from "@lodestar/config"; import {deneb, Epoch, phase0, allForks, Slot} from "@lodestar/types"; -import {ForkSeq} from "@lodestar/params"; +import {ForkSeq, ForkName} from "@lodestar/params"; import {computeEpochAtSlot} from "@lodestar/state-transition"; import {BlockInput, BlockSource, getBlockInput} from "../../chain/blocks/types.js"; @@ -99,10 +99,13 @@ export function matchBlockWithBlobs( getBlockInput.postDeneb( config, block.data, - blockSource, - blobSidecars, null, - Array.from({length: blobKzgCommitmentsLen}, () => null) + { + fork: "deneb" as ForkName.deneb, + blobs: blobSidecars, + blobsBytes: Array.from({length: blobKzgCommitmentsLen}, () => null), + }, + blockSource ) ); } diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts index 9aa262732042..220a2cc00083 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts @@ -1,7 +1,7 @@ import {fromHexString} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {phase0, deneb} from "@lodestar/types"; -import {ForkSeq} from "@lodestar/params"; +import {ForkSeq, ForkName} from "@lodestar/params"; import { BlockInput, BlockInputType, @@ -62,17 +62,21 @@ export async function unavailableBeaconBlobsByRoot( } // resolve the block if thats unavailable - let block, blobsCache, blockBytes, resolveAvailability; + let block, blockBytes, cachedData; if (unavailableBlockInput.block === null) { const allBlocks = await network.sendBeaconBlocksByRoot(peerId, [fromHexString(unavailableBlockInput.blockRootHex)]); block = allBlocks[0].data; blockBytes = allBlocks[0].bytes; - ({blobsCache, resolveAvailability} = unavailableBlockInput); + cachedData = unavailableBlockInput.cachedData; } else { - ({block, blobsCache, resolveAvailability, blockBytes} = unavailableBlockInput); + ({block, cachedData, blockBytes} = unavailableBlockInput); + if (cachedData === undefined) { + throw Error("Missing cachedData for deneb+ unavailable BlockInput"); + } } // resolve missing blobs + const {blobsCache} = cachedData; const blobIdentifiers: deneb.BlobIdentifier[] = []; const slot = block.message.slot; const blockRoot = config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message); @@ -98,13 +102,19 @@ export async function unavailableBeaconBlobsByRoot( // check and see if all blobs are now available and in that case resolve availability // if not this will error and the leftover blobs will be tried from another peer - const allBlobs = getBlockInputBlobs(blobsCache); - const {blobs, blobsBytes} = allBlobs; + const blobsData = getBlockInputBlobs(blobsCache); + const {blobs, blobsBytes} = blobsData; if (blobs.length !== blobKzgCommitmentsLen) { throw Error(`Not all blobs fetched missingBlobs=${blobKzgCommitmentsLen - blobs.length}`); } - resolveAvailability(allBlobs); - metrics?.syncUnknownBlock.resolveAvailabilitySource.inc({source: BlockInputAvailabilitySource.UNKNOWN_SYNC}); - return getBlockInput.postDeneb(config, block, BlockSource.byRoot, blobs, blockBytes, blobsBytes); + if (cachedData.fork === ForkName.deneb) { + const blockData = {fork: cachedData.fork, blobs, blobsBytes}; + + cachedData.resolveAvailability(blockData); + metrics?.syncUnknownBlock.resolveAvailabilitySource.inc({source: BlockInputAvailabilitySource.UNKNOWN_SYNC}); + return getBlockInput.postDeneb(config, block, blockBytes, blockData, BlockSource.byRoot); + } else { + throw Error("electra resolution not implemented"); + } } diff --git a/packages/beacon-node/src/sync/range/chain.ts b/packages/beacon-node/src/sync/range/chain.ts index ed67004bd128..b8757cb141e5 100644 --- a/packages/beacon-node/src/sync/range/chain.ts +++ b/packages/beacon-node/src/sync/range/chain.ts @@ -404,7 +404,7 @@ export class SyncChain { const blobs = res.result.reduce((acc, blockInput) => { hasPostDenebBlocks ||= blockInput.type === BlockInputType.postDeneb; return hasPostDenebBlocks - ? acc + (blockInput.type === BlockInputType.postDeneb ? blockInput.blobs.length : 0) + ? acc + (blockInput.type === BlockInputType.postDeneb ? blockInput.blockData.blobs.length : 0) : 0; }, 0); const downloadInfo = {blocks: res.result.length}; diff --git a/packages/beacon-node/src/sync/unknownBlock.ts b/packages/beacon-node/src/sync/unknownBlock.ts index c252f3087c82..68e7c63f5a96 100644 --- a/packages/beacon-node/src/sync/unknownBlock.ts +++ b/packages/beacon-node/src/sync/unknownBlock.ts @@ -529,7 +529,7 @@ export class UnknownBlockSync { .BeaconBlock.hashTreeRoot(unavailableBlock.message); blockRootHex = toHexString(blockRoot); blobKzgCommitmentsLen = (unavailableBlock.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length; - pendingBlobs = blobKzgCommitmentsLen - unavailableBlockInput.blobsCache.size; + pendingBlobs = blobKzgCommitmentsLen - unavailableBlockInput.cachedData.blobsCache.size; } let lastError: Error | null = null; diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index bdbade4a94a5..58f07ebd484f 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -27,6 +27,7 @@ export const SignedInclusionListSummary = new ContainerType( export const NewInclusionListRequest = new ContainerType( { + slot: Slot, inclusionList: ILTransactions, summary: ILSummary, parentBlockHash: Root, From 13f6ccc893c265a1d56746acd33dfe7984f7a3a3 Mon Sep 17 00:00:00 2001 From: harkamal Date: Sun, 17 Mar 2024 16:33:57 +0530 Subject: [PATCH 5/7] add the IL publishing --- .../src/api/impl/beacon/blocks/index.ts | 44 ++++++++++++------- packages/beacon-node/src/network/interface.ts | 3 +- packages/beacon-node/src/network/network.ts | 9 +++- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index a0d236e2a015..70d404f30453 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -1,9 +1,9 @@ import {fromHexString, toHexString} from "@chainsafe/ssz"; import {routes, ServerApi, ResponseFormat} from "@lodestar/api"; import {computeTimeAtSlot, reconstructFullBlockOrContents} from "@lodestar/state-transition"; -import {SLOTS_PER_HISTORICAL_ROOT, ForkName} from "@lodestar/params"; +import {SLOTS_PER_HISTORICAL_ROOT, isForkBlobs, isForkILs} from "@lodestar/params"; import {sleep, toHex} from "@lodestar/utils"; -import {allForks, deneb, isSignedBlockContents, ProducedBlockSource} from "@lodestar/types"; +import {allForks, deneb, electra, isSignedBlockContents, ProducedBlockSource, ssz} from "@lodestar/types"; import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput} from "../../../../chain/blocks/types.js"; import {promiseAllMaybeAsync} from "../../../../util/promises.js"; import {isOptimisticBlock} from "../../../../util/forkChoice.js"; @@ -42,21 +42,35 @@ export function getBeaconBlockApi({ opts: PublishBlockOpts = {} ) => { const seenTimestampSec = Date.now() / 1000; - let blockForImport: BlockInput, signedBlock: allForks.SignedBeaconBlock, blobSidecars: deneb.BlobSidecars; + const signedBlock = isSignedBlockContents(signedBlockOrContents) + ? signedBlockOrContents.signedBlock + : signedBlockOrContents; + // if block is locally produced, full or blinded, it already is 'consensus' validated as it went through + // state transition to produce the stateRoot + const slot = signedBlock.message.slot; + const fork = config.getForkName(slot); + let blockForImport: BlockInput, blobSidecars: deneb.BlobSidecars, inclusionLists: electra.NewInclusionListRequest[]; if (isSignedBlockContents(signedBlockOrContents)) { - ({signedBlock} = signedBlockOrContents); blobSidecars = computeBlobSidecars(config, signedBlock, signedBlockOrContents); - blockForImport = getBlockInput.postDeneb( - config, - signedBlock, - null, - {fork: ForkName.deneb, blobs: blobSidecars, blobsBytes: [null]}, - BlockSource.api - ); + let blockData; + if (!isForkBlobs(fork)) { + throw Error(`Invalid fork=${fork} for publishing signedBlockOrContents`); + } else if (!isForkILs(fork)) { + inclusionLists = []; + blockData = {fork, blobs: blobSidecars, blobsBytes: [null]}; + } else { + // stub the values + const ilSummary = ssz.electra.ILSummary.defaultValue(); + const ilTransactions = [ssz.electra.ILTransactions.defaultValue()]; + inclusionLists = [ssz.electra.NewInclusionListRequest.defaultValue()]; + blockData = {fork, blobs: blobSidecars, blobsBytes: [null], ilSummary, inclusionLists: ilTransactions}; + } + + blockForImport = getBlockInput.postDeneb(config, signedBlock, null, blockData, BlockSource.api); } else { - signedBlock = signedBlockOrContents; blobSidecars = []; + inclusionLists = []; // TODO: Once API supports submitting data as SSZ, replace null with blockBytes blockForImport = getBlockInput.preDeneb(config, signedBlock, BlockSource.api, null); } @@ -64,10 +78,6 @@ export function getBeaconBlockApi({ // check what validations have been requested before broadcasting and publishing the block // TODO: add validation time to metrics const broadcastValidation = opts.broadcastValidation ?? routes.beacon.BroadcastValidation.gossip; - // if block is locally produced, full or blinded, it already is 'consensus' validated as it went through - // state transition to produce the stateRoot - const slot = signedBlock.message.slot; - const fork = config.getForkName(slot); const blockRoot = toHex(chain.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message)); // bodyRoot should be the same to produced block const bodyRoot = toHex(chain.config.getForkTypes(slot).BeaconBlockBody.hashTreeRoot(signedBlock.message.body)); @@ -191,6 +201,8 @@ export function getBeaconBlockApi({ // b) they might require more hops to reach recipients in peerDAS kind of setup where // blobs might need to hop between nodes because of partial subnet subscription ...blobSidecars.map((blobSidecar) => () => network.publishBlobSidecar(blobSidecar)), + // republish the inclusion list even though it might have been already published + ...inclusionLists.map((inclusionList) => () => network.publishInclusionList(inclusionList)), () => network.publishBeaconBlock(signedBlock) as Promise, () => // there is no rush to persist block since we published it to gossip anyway diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index aeeb61f1feb2..8242d045f3c7 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -16,7 +16,7 @@ import { import type {AddressManager, ConnectionManager, Registrar, TransportManager} from "@libp2p/interface-internal"; import type {Datastore} from "interface-datastore"; import {Identify} from "@chainsafe/libp2p-identify"; -import {Slot, SlotRootHex, allForks, altair, capella, deneb, phase0} from "@lodestar/types"; +import {Slot, SlotRootHex, allForks, altair, capella, deneb, electra, phase0} from "@lodestar/types"; import {PeerIdStr} from "../util/peerId.js"; import {INetworkEventBus} from "./events.js"; import {INetworkCorePublic} from "./core/types.js"; @@ -61,6 +61,7 @@ export interface INetwork extends INetworkCorePublic { // Gossip publishBeaconBlock(signedBlock: allForks.SignedBeaconBlock): Promise; publishBlobSidecar(blobSidecar: deneb.BlobSidecar): Promise; + publishInclusionList(inclusionList: electra.NewInclusionListRequest): Promise; publishBeaconAggregateAndProof(aggregateAndProof: phase0.SignedAggregateAndProof): Promise; publishBeaconAttestation(attestation: phase0.Attestation, subnet: number): Promise; publishVoluntaryExit(voluntaryExit: phase0.SignedVoluntaryExit): Promise; diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index f4b57fe0c658..889d3c102b40 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -5,7 +5,7 @@ import {BeaconConfig} from "@lodestar/config"; import {sleep} from "@lodestar/utils"; import {LoggerNode} from "@lodestar/logger/node"; import {computeStartSlotAtEpoch, computeTimeAtSlot} from "@lodestar/state-transition"; -import {phase0, allForks, deneb, altair, Root, capella, SlotRootHex} from "@lodestar/types"; +import {phase0, allForks, deneb, electra, altair, Root, capella, SlotRootHex} from "@lodestar/types"; import {routes} from "@lodestar/api"; import {ResponseIncoming} from "@lodestar/reqresp"; import {ForkSeq, MAX_BLOBS_PER_BLOCK} from "@lodestar/params"; @@ -302,6 +302,13 @@ export class Network implements INetwork { }); } + async publishInclusionList(inclusionList: electra.NewInclusionListRequest): Promise { + const fork = this.config.getForkName(inclusionList.slot); + return this.publishGossip({type: GossipType.inclusion_list, fork}, inclusionList, { + ignoreDuplicatePublishError: true, + }); + } + async publishBeaconAggregateAndProof(aggregateAndProof: phase0.SignedAggregateAndProof): Promise { const fork = this.config.getForkName(aggregateAndProof.message.aggregate.data.slot); return this.publishGossip( From c515cb72504ca50fe7c9bbbc197ffddd8e5ab71c Mon Sep 17 00:00:00 2001 From: harkamal Date: Sun, 17 Mar 2024 21:49:02 +0530 Subject: [PATCH 6/7] add ability and stubs to validate and process IL --- .../src/api/impl/beacon/blocks/index.ts | 4 + packages/beacon-node/src/chain/chain.ts | 5 ++ .../src/chain/errors/inclusionListError.ts | 21 +++++ .../beacon-node/src/chain/errors/index.ts | 1 + packages/beacon-node/src/chain/interface.ts | 2 + .../chain/seenCache/seenGossipBlockInput.ts | 25 ++++-- .../src/execution/engine/disabled.ts | 4 + .../beacon-node/src/execution/engine/http.ts | 9 +- .../src/execution/engine/interface.ts | 3 +- .../src/metrics/metrics/lodestar.ts | 19 ++++- .../src/network/processor/gossipHandlers.ts | 83 +++++++++++++++++-- 11 files changed, 163 insertions(+), 13 deletions(-) create mode 100644 packages/beacon-node/src/chain/errors/inclusionListError.ts diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 70d404f30453..d9ff900ae194 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -64,6 +64,10 @@ export function getBeaconBlockApi({ const ilSummary = ssz.electra.ILSummary.defaultValue(); const ilTransactions = [ssz.electra.ILTransactions.defaultValue()]; inclusionLists = [ssz.electra.NewInclusionListRequest.defaultValue()]; + inclusionLists[0].slot = slot; + inclusionLists[0].parentBlockHash = ( + signedBlock as electra.SignedBeaconBlock + ).message.body.executionPayload.parentHash; blockData = {fork, blobs: blobSidecars, blobsBytes: [null], ilSummary, inclusionLists: ilTransactions}; } diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 08743165cd05..41f3829041d8 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -24,6 +24,7 @@ import { Epoch, ValidatorIndex, deneb, + electra, Wei, bellatrix, isBlindedBeaconBlock, @@ -630,6 +631,10 @@ export class BeaconChain implements IBeaconChain { return this.blockProcessor.processBlocksJob(blocks, opts); } + async processInclusionList(inclusionList: electra.NewInclusionListRequest, _opts?: ImportBlockOpts): Promise { + await this.executionEngine.notifyNewInclusionList(inclusionList); + } + getStatus(): phase0.Status { const head = this.forkChoice.getHead(); const finalizedCheckpoint = this.forkChoice.getFinalizedCheckpoint(); diff --git a/packages/beacon-node/src/chain/errors/inclusionListError.ts b/packages/beacon-node/src/chain/errors/inclusionListError.ts new file mode 100644 index 000000000000..43d094cc9a19 --- /dev/null +++ b/packages/beacon-node/src/chain/errors/inclusionListError.ts @@ -0,0 +1,21 @@ +import {Slot, RootHex, ValidatorIndex} from "@lodestar/types"; +import {GossipActionError} from "./gossipValidation.js"; + +export enum InclusionListErrorCode { + ALREADY_KNOWN = "INCLUSION_LIST_ERROR_ALREADY_KNOWN", + PARENT_UNKNOWN = "INCLUSION_LIST_ERROR_PARENT_UNKNOWN", + NOT_LATER_THAN_PARENT = "INCLUSION_LIST_ERROR_NOT_LATER_THAN_PARENT", + PROPOSAL_SIGNATURE_INVALID = "INCLUSION_LIST_ERROR_PROPOSAL_SIGNATURE_INVALID", + INCORRECT_PROPOSER = "INCLUSION_LIST_INCORRECT_PROPOSER", + EXECUTION_ENGINE_ERROR = "INCLUSION_LIST_EXECUTION_ENGINE_ERROR", +} + +export type InclusionListErrorType = + | {code: InclusionListErrorCode.ALREADY_KNOWN; root: RootHex} + | {code: InclusionListErrorCode.PARENT_UNKNOWN; parentRoot: RootHex} + | {code: InclusionListErrorCode.NOT_LATER_THAN_PARENT; parentSlot: Slot; slot: Slot} + | {code: InclusionListErrorCode.PROPOSAL_SIGNATURE_INVALID} + | {code: InclusionListErrorCode.INCORRECT_PROPOSER; proposerIndex: ValidatorIndex} + | {code: InclusionListErrorCode.EXECUTION_ENGINE_ERROR}; + +export class InclusionListGossipError extends GossipActionError {} diff --git a/packages/beacon-node/src/chain/errors/index.ts b/packages/beacon-node/src/chain/errors/index.ts index 1bd8f8577305..cd46ad5296d1 100644 --- a/packages/beacon-node/src/chain/errors/index.ts +++ b/packages/beacon-node/src/chain/errors/index.ts @@ -1,6 +1,7 @@ export * from "./attestationError.js"; export * from "./attesterSlashingError.js"; export * from "./blobSidecarError.js"; +export * from "./inclusionListError.js"; export * from "./blockError.js"; export * from "./gossipValidation.js"; export * from "./proposerSlashingError.js"; diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 55f5ebf485a2..7df7158bc022 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -12,6 +12,7 @@ import { Wei, capella, altair, + electra, } from "@lodestar/types"; import { BeaconStateAllForks, @@ -176,6 +177,7 @@ export interface IBeaconChain { processBlock(block: BlockInput, opts?: ImportBlockOpts): Promise; /** Process a chain of blocks until complete */ processChainSegment(blocks: BlockInput[], opts?: ImportBlockOpts): Promise; + processInclusionList(inclusionList: electra.NewInclusionListRequest, opts?: ImportBlockOpts): Promise; getStatus(): phase0.Status; diff --git a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts index 38259bd4e75b..ef8d063f6bef 100644 --- a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts +++ b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts @@ -32,6 +32,7 @@ type GossipedBlockInput = }; type BlockInputCacheType = { + fork: ForkName; block?: allForks.SignedBeaconBlock; blockBytes?: Uint8Array | null; cachedData?: CachedData; @@ -55,9 +56,13 @@ const MAX_GOSSIPINPUT_CACHE = 5; */ export class SeenGossipBlockInput { private blockInputCache = new Map(); + private seenILsByParentHash = new Map(); + private blockInputRootByParentHash = new Map(); prune(): void { pruneSetToMax(this.blockInputCache, MAX_GOSSIPINPUT_CACHE); + pruneSetToMax(this.seenILsByParentHash, MAX_GOSSIPINPUT_CACHE); + pruneSetToMax(this.blockInputRootByParentHash, MAX_GOSSIPINPUT_CACHE); } hasBlock(blockRoot: RootHex): boolean { @@ -76,13 +81,23 @@ export class SeenGossipBlockInput { | { blockInput: NullBlockInput; blockInputMeta: {pending: GossipedInputType.block; haveBlobs: number; expectedBlobs: null}; - } { + } + | null { let blockHex; let blockCache; let fork; if (gossipedInput.type === GossipedInputType.ilist) { - throw Error("Inclusion list gossip handling not implemented"); + const {inclusionList} = gossipedInput; + const parentBlockHashHex = toHexString(inclusionList.parentBlockHash); + this.seenILsByParentHash.set(parentBlockHashHex, inclusionList); + + blockHex = this.blockInputRootByParentHash.get(parentBlockHashHex); + blockCache = blockHex ? this.blockInputCache.get(blockHex) : undefined; + if (blockHex === undefined || blockCache === undefined) { + return null; + } + fork = blockCache.fork; } else if (gossipedInput.type === GossipedInputType.block) { const {signedBlock, blockBytes} = gossipedInput; fork = config.getForkName(signedBlock.message.slot); @@ -214,7 +229,7 @@ function getEmptyBlockInputCacheEntry(fork: ForkName): BlockInputCacheType { } if (!isForkBlobs(fork)) { - return {blockInputPromise, resolveBlockInput}; + return {fork, blockInputPromise, resolveBlockInput}; } const blobsCache = new Map(); @@ -229,7 +244,7 @@ function getEmptyBlockInputCacheEntry(fork: ForkName): BlockInputCacheType { throw Error("Promise Constructor was not executed immediately"); } const cachedData: CachedData = {fork, blobsCache, availabilityPromise, resolveAvailability}; - return {blockInputPromise, resolveBlockInput, cachedData}; + return {fork, blockInputPromise, resolveBlockInput, cachedData}; } else { // il availability (with blobs) let resolveAvailability: ((blobs: BlockInputDataIls) => void) | null = null; @@ -241,6 +256,6 @@ function getEmptyBlockInputCacheEntry(fork: ForkName): BlockInputCacheType { throw Error("Promise Constructor was not executed immediately"); } const cachedData: CachedData = {fork, blobsCache, availabilityPromise, resolveAvailability, inclusionLists: []}; - return {blockInputPromise, resolveBlockInput, cachedData}; + return {fork, blockInputPromise, resolveBlockInput, cachedData}; } } diff --git a/packages/beacon-node/src/execution/engine/disabled.ts b/packages/beacon-node/src/execution/engine/disabled.ts index 82bf84c37d33..eb81cf5cf1dc 100644 --- a/packages/beacon-node/src/execution/engine/disabled.ts +++ b/packages/beacon-node/src/execution/engine/disabled.ts @@ -7,6 +7,10 @@ export class ExecutionEngineDisabled implements IExecutionEngine { throw Error("Execution engine disabled"); } + async notifyNewInclusionList(): Promise { + throw Error("Execution engine disabled"); + } + async notifyForkchoiceUpdate(): Promise { throw Error("Execution engine disabled"); } diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index f5ec03f41626..829693901bfc 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -1,4 +1,5 @@ -import {Root, RootHex, allForks, Wei} from "@lodestar/types"; +import {toHexString} from "@chainsafe/ssz"; +import {Root, RootHex, allForks, Wei, electra} from "@lodestar/types"; import {SLOTS_PER_EPOCH, ForkName, ForkSeq} from "@lodestar/params"; import {Logger} from "@lodestar/logger"; import { @@ -144,6 +145,12 @@ export class ExecutionEngineHttp implements IExecutionEngine { }); } + async notifyNewInclusionList(inclusionList: electra.NewInclusionListRequest): Promise { + // return dummy status, don't use latestValidHash anyway + const latestValidHash = toHexString(inclusionList.parentBlockHash); + return {status: ExecutionPayloadStatus.VALID, latestValidHash, validationError: null}; + } + /** * `engine_newPayloadV1` * From: https://github.com/ethereum/execution-apis/blob/v1.0.0-alpha.6/src/engine/specification.md#engine_newpayloadv1 diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index e5f612fc0965..d0fd737f8fa7 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -1,6 +1,6 @@ import {ForkName} from "@lodestar/params"; import {KZGCommitment, Blob, KZGProof} from "@lodestar/types/deneb"; -import {Root, RootHex, allForks, capella, Wei} from "@lodestar/types"; +import {Root, RootHex, allForks, capella, electra, Wei} from "@lodestar/types"; import {DATA} from "../../eth1/provider/utils.js"; import {PayloadIdCache, PayloadId, WithdrawalV1} from "./payloadIdCache.js"; @@ -90,6 +90,7 @@ export type VersionedHashes = Uint8Array[]; */ export interface IExecutionEngine { payloadIdCache: PayloadIdCache; + notifyNewInclusionList(InclusionList: electra.NewInclusionListRequest): Promise; /** * A state transition function which applies changes to the self.execution_state. * Returns ``True`` iff ``execution_payload`` is valid with respect to ``self.execution_state``. diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 4f82969cff81..1c39db5035b2 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -2,7 +2,7 @@ import {EpochTransitionStep, StateCloneSource, StateHashTreeRootSource} from "@l import {allForks} from "@lodestar/types"; import {BlockSource} from "../../chain/blocks/types.js"; import {JobQueueItemType} from "../../chain/bls/index.js"; -import {BlockErrorCode} from "../../chain/errors/index.js"; +import {BlockErrorCode, InclusionListErrorCode} from "../../chain/errors/index.js"; import {InsertOutcome} from "../../chain/opPools/types.js"; import {RegenCaller, RegenFnName} from "../../chain/regen/interface.js"; import {ReprocessStatus} from "../../chain/reprocess.js"; @@ -777,6 +777,23 @@ export function createLodestarMetrics( buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5, 2, 4], }), }, + gossipInclusionList: { + recvToValidation: register.histogram({ + name: "lodestar_gossip_inclusion_list_received_to_gossip_validate", + help: "Time elapsed between inclusion list received and validation", + buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5, 2, 4], + }), + validationTime: register.histogram({ + name: "lodestar_gossip_inclusion_list_gossip_validate_time", + help: "Time elapsed for inclusion list validation", + buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5, 2, 4], + }), + processInclusionListErrors: register.gauge<{error: InclusionListErrorCode | "NOT_INCLUSION_LIST_ERROR"}>({ + name: "lodestar_gossip_inclusion_list_process_errors", + help: "Count of errors, by error type, while processing inclusionlist", + labelNames: ["error"], + }), + }, importBlock: { persistBlockNoSerializedDataCount: register.gauge({ name: "lodestar_import_block_persist_block_no_serialized_data_count", diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index b772aef01d15..b95a745be1be 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -15,6 +15,8 @@ import { BlockGossipError, BlobSidecarErrorCode, BlobSidecarGossipError, + InclusionListGossipError, + InclusionListErrorCode, GossipAction, GossipActionError, SyncCommitteeError, @@ -136,6 +138,10 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler }, metrics ); + if (blockInputRes === null) { + throw Error("Invalid blockInputRes=null returned for gossip block in cache processing"); + } + const blockInput = blockInputRes.blockInput; // blockInput can't be returned null, improve by enforcing via return types if (blockInput.block === null) { @@ -200,7 +206,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec); const recvToValLatency = Date.now() / 1000 - seenTimestampSec; - const {blockInput, blockInputMeta} = chain.seenGossipBlockInput.getGossipBlockInput( + const blockInputRes = chain.seenGossipBlockInput.getGossipBlockInput( config, { type: GossipedInputType.blob, @@ -209,7 +215,11 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler }, metrics ); + if (blockInputRes === null) { + throw Error("Invalid blockInputRes=null returned for gossip blob in cache processing"); + } + const {blockInput, blockInputMeta} = blockInputRes; try { await validateGossipBlobSidecar(chain, blobSidecar, gossipIndex); const recvToValidation = Date.now() / 1000 - seenTimestampSec; @@ -258,12 +268,12 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler inclusionListBytes: Uint8Array, peerIdStr: string, seenTimestampSec: number - ): Promise { + ): Promise { const slot = inclusionList.slot; const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec); const recvToValLatency = Date.now() / 1000 - seenTimestampSec; - const {blockInput, blockInputMeta} = chain.seenGossipBlockInput.getGossipBlockInput( + const blockInputRes = chain.seenGossipBlockInput.getGossipBlockInput( config, { type: GossipedInputType.ilist, @@ -273,7 +283,9 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler metrics ); + const blockInput = blockInputRes?.blockInput ?? null; try { + const blockInputMeta = blockInputRes?.blockInputMeta ?? {}; await validateGossipInclusionList(chain, inclusionList); const recvToValidation = Date.now() / 1000 - seenTimestampSec; const validationTime = recvToValidation - recvToValLatency; @@ -295,7 +307,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } catch (e) { if (e instanceof BlobSidecarGossipError) { // Don't trigger this yet if full block and blobs haven't arrived yet - if (e.type.code === BlobSidecarErrorCode.PARENT_UNKNOWN && blockInput.block !== null) { + if (e.type.code === BlobSidecarErrorCode.PARENT_UNKNOWN && blockInput !== null && blockInput.block !== null) { logger.debug("Gossip inclusion list has error", {code: e.type.code}); events.emit(NetworkEvent.unknownBlockParent, {blockInput, peer: peerIdStr}); } @@ -388,6 +400,64 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler }); } + function handleValidInclusionList( + inclusionList: electra.NewInclusionListRequest, + peerIdStr: string, + seenTimestampSec: number + ): void { + // Handler - MUST NOT `await`, to allow validation result to be propagated + + // metrics?.registerInclusionList(OpSource.gossip, seenTimestampSec, inclusionList); + // if blobs are not yet fully available start an aggressive blob pull + + chain + .processInclusionList(inclusionList, { + // block may be downloaded and processed by UnknownBlockSync + ignoreIfKnown: true, + // proposer signature already checked in validateBeaconBlock() + validProposerSignature: true, + blsVerifyOnMainThread: true, + // to track block process steps + seenTimestampSec, + }) + .then(() => { + // Returns the delay between the start of `block.slot` and `current time` + const delaySec = chain.clock.secFromSlot(inclusionList.slot); + metrics?.gossipBlock.elapsedTimeTillProcessed.observe(delaySec); + chain.seenGossipBlockInput.prune(); + }) + .catch((e) => { + // Adjust verbosity based on error type + let logLevel: LogLevel; + + if (e instanceof InclusionListGossipError) { + switch (e.type.code) { + // ALREADY_KNOWN should not happen with ignoreIfKnown=true above + // PARENT_UNKNOWN should not happen, we handled this in validateBeaconBlock() function above + case InclusionListErrorCode.ALREADY_KNOWN: + case InclusionListErrorCode.PARENT_UNKNOWN: + case InclusionListErrorCode.EXECUTION_ENGINE_ERROR: + // Errors might indicate an issue with our node or the connected EL client + logLevel = LogLevel.error; + break; + default: + // TODO: Should it use PeerId or string? + core.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadGossipInclusionList"); + // Misbehaving peer, but could highlight an issue in another client + logLevel = LogLevel.warn; + } + } else { + // Any unexpected error + logLevel = LogLevel.error; + } + metrics?.gossipInclusionList.processInclusionListErrors.inc({ + error: e instanceof InclusionListGossipError ? e.type.code : "NOT_INCLUSION_LIST_ERROR", + }); + logger[logLevel]("Error receiving inclusion list", {slot: inclusionList.slot, peer: peerIdStr}, e as Error); + chain.seenGossipBlockInput.prune(); + }); + } + return { [GossipType.beacon_block]: async ({ gossipData, @@ -480,7 +550,10 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } const blockInput = await validateInclusionList(inclusionList, serializedData, peerIdStr, seenTimestampSec); - if (blockInput.block !== null) { + handleValidInclusionList(inclusionList, peerIdStr, seenTimestampSec); + if (blockInput === null) { + return; + } else if (blockInput.block !== null) { // we can just queue up the blockInput in the processor, but block gossip handler would have already // queued it up. // From d57939f4cf24e4dc9844fd667d3a0a3091c4aa08 Mon Sep 17 00:00:00 2001 From: harkamal Date: Mon, 25 Mar 2024 14:39:49 +0530 Subject: [PATCH 7/7] hookup the entire il moving parts except signing --- .../src/api/impl/beacon/blocks/index.ts | 51 ++- .../src/chain/blocks/importBlock.ts | 55 ++- .../beacon-node/src/chain/blocks/types.ts | 16 +- .../src/chain/blocks/verifyBlock.ts | 2 +- .../blocks/verifyBlocksExecutionPayloads.ts | 40 +- packages/beacon-node/src/chain/chain.ts | 13 +- .../beacon-node/src/chain/forkChoice/index.ts | 8 +- packages/beacon-node/src/chain/interface.ts | 6 +- .../chain/produceBlock/produceBlockBody.ts | 13 + .../chain/seenCache/seenGossipBlockInput.ts | 65 ++- .../src/chain/validation/inclusionList.ts | 2 +- .../src/execution/engine/disabled.ts | 4 + .../beacon-node/src/execution/engine/http.ts | 79 +++- .../src/execution/engine/interface.ts | 26 +- .../beacon-node/src/execution/engine/mock.ts | 14 + .../beacon-node/src/execution/engine/types.ts | 59 ++- .../src/network/gossip/interface.ts | 4 +- .../beacon-node/src/network/gossip/topic.ts | 2 +- packages/beacon-node/src/network/interface.ts | 2 +- packages/beacon-node/src/network/network.ts | 7 +- .../src/network/processor/gossipHandlers.ts | 41 +- .../reqresp/beaconBlocksMaybeBlobsByRange.ts | 39 +- packages/beacon-node/src/node/notifier.ts | 2 +- .../el-interop/ethereumjsdocker/post-merge.sh | 2 +- .../scripts/el-interop/gethdocker/il.tmpl | 35 ++ .../beacon-node/test/sim/4844-interop.test.ts | 3 +- .../beacon-node/test/sim/ils-interop.test.ts | 397 ++++++++++++++++++ packages/beacon-node/test/utils/logger.ts | 2 +- .../fork-choice/src/forkChoice/forkChoice.ts | 43 +- .../fork-choice/src/forkChoice/interface.ts | 16 +- packages/fork-choice/src/index.ts | 2 +- .../fork-choice/src/protoArray/interface.ts | 41 +- .../fork-choice/src/protoArray/protoArray.ts | 18 +- .../state-transition/src/util/execution.ts | 11 +- packages/types/src/electra/sszTypes.ts | 45 +- packages/types/src/electra/types.ts | 7 +- 36 files changed, 1018 insertions(+), 154 deletions(-) create mode 100644 packages/beacon-node/test/scripts/el-interop/gethdocker/il.tmpl create mode 100644 packages/beacon-node/test/sim/ils-interop.test.ts diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index d9ff900ae194..c123a4298aa3 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -4,7 +4,15 @@ import {computeTimeAtSlot, reconstructFullBlockOrContents} from "@lodestar/state import {SLOTS_PER_HISTORICAL_ROOT, isForkBlobs, isForkILs} from "@lodestar/params"; import {sleep, toHex} from "@lodestar/utils"; import {allForks, deneb, electra, isSignedBlockContents, ProducedBlockSource, ssz} from "@lodestar/types"; -import {BlockSource, getBlockInput, ImportBlockOpts, BlockInput} from "../../../../chain/blocks/types.js"; +import { + BlockSource, + getBlockInput, + ImportBlockOpts, + BlockInput, + BlockInputILType, + BlockInputDataBlobs, + BlockInputDataIls, +} from "../../../../chain/blocks/types.js"; import {promiseAllMaybeAsync} from "../../../../util/promises.js"; import {isOptimisticBlock} from "../../../../util/forkChoice.js"; import {computeBlobSidecars} from "../../../../util/blobs.js"; @@ -50,31 +58,44 @@ export function getBeaconBlockApi({ const slot = signedBlock.message.slot; const fork = config.getForkName(slot); - let blockForImport: BlockInput, blobSidecars: deneb.BlobSidecars, inclusionLists: electra.NewInclusionListRequest[]; + let blockForImport: BlockInput, + blobSidecars: deneb.BlobSidecars, + signedInclusionList: electra.SignedInclusionList | null; if (isSignedBlockContents(signedBlockOrContents)) { blobSidecars = computeBlobSidecars(config, signedBlock, signedBlockOrContents); let blockData; if (!isForkBlobs(fork)) { throw Error(`Invalid fork=${fork} for publishing signedBlockOrContents`); } else if (!isForkILs(fork)) { - inclusionLists = []; - blockData = {fork, blobs: blobSidecars, blobsBytes: [null]}; + signedInclusionList = null; + blockData = {fork, blobs: blobSidecars, blobsBytes: [null]} as BlockInputDataBlobs; } else { - // stub the values - const ilSummary = ssz.electra.ILSummary.defaultValue(); - const ilTransactions = [ssz.electra.ILTransactions.defaultValue()]; - inclusionLists = [ssz.electra.NewInclusionListRequest.defaultValue()]; - inclusionLists[0].slot = slot; - inclusionLists[0].parentBlockHash = ( - signedBlock as electra.SignedBeaconBlock - ).message.body.executionPayload.parentHash; - blockData = {fork, blobs: blobSidecars, blobsBytes: [null], ilSummary, inclusionLists: ilTransactions}; + // pick the IL withsout signing anything for now + signedInclusionList = ssz.electra.SignedInclusionList.defaultValue(); + const blockHash = toHexString( + (signedBlock as electra.SignedBeaconBlock).message.body.executionPayload.blockHash + ); + const producedList = chain.producedInclusionList.get(blockHash); + if (producedList === undefined) { + throw Error("No Inclusion List produced for the block"); + } + + signedInclusionList.message.signedSummary.message = producedList.ilSummary; + signedInclusionList.message.transactions = producedList.ilTransactions; + + blockData = { + fork, + blobs: blobSidecars, + blobsBytes: [null], + ilType: BlockInputILType.actualIL, + inclusionList: signedInclusionList.message, + } as BlockInputDataIls; } blockForImport = getBlockInput.postDeneb(config, signedBlock, null, blockData, BlockSource.api); } else { blobSidecars = []; - inclusionLists = []; + signedInclusionList = null; // TODO: Once API supports submitting data as SSZ, replace null with blockBytes blockForImport = getBlockInput.preDeneb(config, signedBlock, BlockSource.api, null); } @@ -206,7 +227,7 @@ export function getBeaconBlockApi({ // blobs might need to hop between nodes because of partial subnet subscription ...blobSidecars.map((blobSidecar) => () => network.publishBlobSidecar(blobSidecar)), // republish the inclusion list even though it might have been already published - ...inclusionLists.map((inclusionList) => () => network.publishInclusionList(inclusionList)), + () => (signedInclusionList !== null ? network.publishInclusionList(signedInclusionList) : Promise.resolve(0)), () => network.publishBeaconBlock(signedBlock) as Promise, () => // there is no rush to persist block since we published it to gossip anyway diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 4c6509038de9..3ca7c9e00a91 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -1,6 +1,6 @@ import {toHexString} from "@chainsafe/ssz"; -import {capella, ssz, allForks, altair} from "@lodestar/types"; -import {ForkSeq, INTERVALS_PER_SLOT, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {capella, ssz, allForks, altair, electra} from "@lodestar/types"; +import {ForkSeq, INTERVALS_PER_SLOT, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH, isForkILs, ForkName} from "@lodestar/params"; import { CachedBeaconStateAltair, computeEpochAtSlot, @@ -9,7 +9,14 @@ import { RootCache, } from "@lodestar/state-transition"; import {routes} from "@lodestar/api"; -import {ForkChoiceError, ForkChoiceErrorCode, EpochDifference, AncestorStatus} from "@lodestar/fork-choice"; +import { + ForkChoiceError, + ForkChoiceErrorCode, + EpochDifference, + AncestorStatus, + InclusionListStatus, + ExecutionStatus, +} from "@lodestar/fork-choice"; import {isErrorAborted} from "@lodestar/utils"; import {ZERO_HASH_HEX} from "../../constants/index.js"; import {toCheckpointHex} from "../stateCache/index.js"; @@ -19,7 +26,7 @@ import {kzgCommitmentToVersionedHash} from "../../util/blobs.js"; import {ChainEvent, ReorgEventData} from "../emitter.js"; import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js"; import type {BeaconChain} from "../chain.js"; -import {FullyVerifiedBlock, ImportBlockOpts, AttestationImportOpt, BlockInputType} from "./types.js"; +import {FullyVerifiedBlock, ImportBlockOpts, AttestationImportOpt, BlockInputType, BlockInputILType} from "./types.js"; import {getCheckpointFromState} from "./utils/checkpoint.js"; import {writeBlockInputToDb} from "./writeBlockInputToDb.js"; @@ -75,17 +82,43 @@ export async function importBlock( await writeBlockInputToDb.call(this, [blockInput]); } + let ilStatus; + let inclusionList: electra.InclusionList | undefined = undefined; + if (blockInput.type === BlockInputType.preDeneb) { + ilStatus = InclusionListStatus.PreIL; + } else { + const blockData = + blockInput.type === BlockInputType.postDeneb + ? blockInput.blockData + : await blockInput.cachedData.availabilityPromise; + if (blockData.fork === ForkName.deneb) { + ilStatus = InclusionListStatus.PreIL; + } else { + if (blockData.ilType === BlockInputILType.childBlock || blockData.ilType === BlockInputILType.syncing) { + // if child is valid, the onBlock on protoArray will auto propagate the valid child upwards + ilStatus = InclusionListStatus.Syncing; + } else if (blockData.ilType === BlockInputILType.actualIL) { + // if IL was available and the block is valid, IL can be marked valid as IL would have been validated + // in the verification + // + // TODO : build and bundle te ilStatus in verify execution payload section itself + ilStatus = executionStatus === ExecutionStatus.Valid ? InclusionListStatus.Valid : InclusionListStatus.Syncing; + inclusionList = blockData.inclusionList; + } else { + throw Error("Parsing error, il"); + } + } + } + // 2. Import block to fork choice // Should compute checkpoint balances before forkchoice.onBlock this.checkpointBalancesCache.processState(blockRootHex, postState); - const blockSummary = this.forkChoice.onBlock( - block.message, - postState, - blockDelaySec, - this.clock.currentSlot, - executionStatus - ); + const blockSummary = this.forkChoice.onBlock(block.message, postState, blockDelaySec, this.clock.currentSlot, { + executionStatus, + ilStatus, + inclusionList, + }); // This adds the state necessary to process the next block // Some block event handlers require state being in state cache so need to do this before emitting EventType.block diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index 75a83f170d67..ffc5f8907372 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -24,20 +24,32 @@ export enum GossipedInputType { ilist = "ilist", } +export enum BlockInputILType { + childBlock = "childBlock", + actualIL = "actualIL", + syncing = "syncing", +} + type ForkBlobsInfo = {fork: ForkName.deneb}; type ForkILsInfo = {fork: ForkILs}; export type BlobsCache = Map; type BlobsData = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Array | null)[]}; -type ILsData = BlobsData & {ilSummary: electra.ILSummary; inclusionLists: electra.ILTransactions[]}; +type ILsData = BlobsData & + ( + | {ilType: BlockInputILType.childBlock | BlockInputILType.syncing} + | {ilType: BlockInputILType.actualIL; inclusionList: electra.InclusionList} + ); export type BlockInputDataBlobs = ForkBlobsInfo & BlobsData; export type BlockInputDataIls = ForkILsInfo & ILsData; export type BlockInputData = BlockInputDataBlobs | BlockInputDataIls; type BlobsInputCache = {blobsCache: BlobsCache}; -type ForkILsCache = BlobsInputCache & {ilSummary?: electra.ILSummary; inclusionLists: electra.ILTransactions[]}; +type ForkILsCache = BlobsInputCache & { + inclusionList?: electra.InclusionList; +}; export type BlockInputCacheBlobs = ForkBlobsInfo & BlobsInputCache; export type BlockInputCacheILs = ForkILsInfo & ForkILsCache; diff --git a/packages/beacon-node/src/chain/blocks/verifyBlock.ts b/packages/beacon-node/src/chain/blocks/verifyBlock.ts index 658ac05d3908..961765ce3ebd 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlock.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlock.ts @@ -98,7 +98,7 @@ export async function verifyBlocksInEpoch( ] = await Promise.all([ // Execution payloads opts.skipVerifyExecutionPayload !== true - ? verifyBlocksExecutionPayload(this, parentBlock, blocks, preState0, abortController.signal, opts) + ? verifyBlocksExecutionPayload(this, parentBlock, blocksInput, preState0, abortController.signal, opts) : Promise.resolve({ execAborted: null, executionStatuses: blocks.map((_blk) => ExecutionStatus.Syncing), diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts index 8f386ef191d2..4bcbcc51de00 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts @@ -6,7 +6,7 @@ import { isMergeTransitionBlock as isMergeTransitionBlockFn, isExecutionEnabled, } from "@lodestar/state-transition"; -import {bellatrix, allForks, Slot, deneb} from "@lodestar/types"; +import {bellatrix, allForks, Slot, deneb, electra} from "@lodestar/types"; import { IForkChoice, assertValidTerminalPowBlock, @@ -15,10 +15,11 @@ import { MaybeValidExecutionStatus, LVHValidResponse, LVHInvalidResponse, + InclusionListStatus, } from "@lodestar/fork-choice"; import {ChainForkConfig} from "@lodestar/config"; import {ErrorAborted, Logger} from "@lodestar/utils"; -import {ForkSeq, SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY} from "@lodestar/params"; +import {ForkSeq, SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY, isForkILs, ForkName} from "@lodestar/params"; import {IExecutionEngine} from "../../execution/engine/interface.js"; import {BlockError, BlockErrorCode} from "../errors/index.js"; @@ -28,7 +29,7 @@ import {BlockProcessOpts} from "../options.js"; import {ExecutionPayloadStatus} from "../../execution/engine/interface.js"; import {IEth1ForBlockProduction} from "../../eth1/index.js"; import {Metrics} from "../../metrics/metrics.js"; -import {ImportBlockOpts} from "./types.js"; +import {BlockInput, BlockInputType, ImportBlockOpts, BlockInputILType} from "./types.js"; export type VerifyBlockExecutionPayloadModules = { eth1: IEth1ForBlockProduction; @@ -68,17 +69,18 @@ type VerifyBlockExecutionResponse = export async function verifyBlocksExecutionPayload( chain: VerifyBlockExecutionPayloadModules, parentBlock: ProtoBlock, - blocks: allForks.SignedBeaconBlock[], + blocksInput: BlockInput[], preState0: CachedBeaconStateAllForks, signal: AbortSignal, opts: BlockProcessOpts & ImportBlockOpts ): Promise { const executionStatuses: MaybeValidExecutionStatus[] = []; + const blocks = blocksInput.map((blockInput) => blockInput.block); let mergeBlockFound: bellatrix.BeaconBlock | null = null; const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000); // Error in the same way as verifyBlocksSanityChecks if empty blocks - if (blocks.length === 0) { + if (blocksInput.length === 0) { throw Error("Empty partiallyVerifiedBlocks"); } @@ -150,8 +152,9 @@ export async function verifyBlocksExecutionPayload( parentBlock.executionStatus !== ExecutionStatus.PreMerge || lastBlock.message.slot + safeSlotsToImportOptimistically < currentSlot; - for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) { - const block = blocks[blockIndex]; + for (let blockIndex = 0; blockIndex < blocksInput.length; blockIndex++) { + const blockInput = blocksInput[blockIndex]; + const block = blockInput.block; // If blocks are invalid in consensus the main promise could resolve before this loop ends. // In that case stop sending blocks to execution engine if (signal.aborted) { @@ -159,7 +162,7 @@ export async function verifyBlocksExecutionPayload( } const verifyResponse = await verifyBlockExecutionPayload( chain, - block, + blockInput, preState0, opts, isOptimisticallySafe, @@ -274,12 +277,13 @@ export async function verifyBlocksExecutionPayload( */ export async function verifyBlockExecutionPayload( chain: VerifyBlockExecutionPayloadModules, - block: allForks.SignedBeaconBlock, + blockInput: BlockInput, preState0: CachedBeaconStateAllForks, opts: BlockProcessOpts, isOptimisticallySafe: boolean, currentSlot: Slot ): Promise { + const block = blockInput.block; /** Not null if execution is enabled */ const executionPayloadEnabled = isExecutionStateType(preState0) && @@ -312,6 +316,24 @@ export async function verifyBlockExecutionPayload( parentBlockRoot ); + let inclusionList: electra.InclusionList | null = null; + if ( + blockInput.type === BlockInputType.postDeneb && + blockInput.blockData.fork !== ForkName.deneb && + blockInput.blockData.ilType === BlockInputILType.actualIL + ) { + inclusionList = blockInput.blockData.inclusionList; + } else if (blockInput.type === BlockInputType.blobsPromise && blockInput.cachedData.fork !== ForkName.deneb) { + inclusionList = blockInput.cachedData.inclusionList ?? null; + } + + if (inclusionList !== null) { + const ilResult = await chain.executionEngine.notifyNewInclusionList( + inclusionList.signedSummary.message, + inclusionList.transactions + ); + } + chain.metrics?.engineNotifyNewPayloadResult.inc({result: execResult.status}); switch (execResult.status) { diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 41f3829041d8..444843de1471 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -146,6 +146,10 @@ export class BeaconChain implements IBeaconChain { // actual publish readonly producedBlockRoot = new Map(); readonly producedBlindedBlockRoot = new Set(); + readonly producedInclusionList = new Map< + RootHex, + {ilSummary: electra.InclusionListSummary; ilTransactions: electra.ILTransactions} + >(); readonly opts: IChainOptions; @@ -631,8 +635,9 @@ export class BeaconChain implements IBeaconChain { return this.blockProcessor.processBlocksJob(blocks, opts); } - async processInclusionList(inclusionList: electra.NewInclusionListRequest, _opts?: ImportBlockOpts): Promise { - await this.executionEngine.notifyNewInclusionList(inclusionList); + async processInclusionList(signedInclusionList: electra.SignedInclusionList, _opts?: ImportBlockOpts): Promise { + const inclusionList = signedInclusionList.message; + await this.executionEngine.notifyNewInclusionList(inclusionList.signedSummary.message, inclusionList.transactions); } getStatus(): phase0.Status { @@ -910,6 +915,10 @@ export class BeaconChain implements IBeaconChain { this.metrics?.blockProductionCaches.producedContentsCache.set(this.producedContentsCache.size); } + if (this.config.getForkSeq(slot) >= ForkSeq.deneb) { + pruneSetToMax(this.producedInclusionList, this.opts.maxCachedProducedRoots ?? DEFAULT_MAX_CACHED_PRODUCED_ROOTS); + } + const metrics = this.metrics; if (metrics && (slot + 1) % SLOTS_PER_EPOCH === 0) { // On the last slot of the epoch diff --git a/packages/beacon-node/src/chain/forkChoice/index.ts b/packages/beacon-node/src/chain/forkChoice/index.ts index 7e195a84922d..17699e6aa59c 100644 --- a/packages/beacon-node/src/chain/forkChoice/index.ts +++ b/packages/beacon-node/src/chain/forkChoice/index.ts @@ -8,6 +8,7 @@ import { ExecutionStatus, JustifiedBalancesGetter, ForkChoiceOpts, + InclusionListStatus, } from "@lodestar/fork-choice"; import { CachedBeaconStateAllForks, @@ -83,8 +84,13 @@ export function initializeForkChoice( executionPayloadBlockHash: toHexString(state.latestExecutionPayloadHeader.blockHash), executionPayloadNumber: state.latestExecutionPayloadHeader.blockNumber, executionStatus: blockHeader.slot === GENESIS_SLOT ? ExecutionStatus.Valid : ExecutionStatus.Syncing, + ilStatus: InclusionListStatus.PreIL, } - : {executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}), + : { + executionPayloadBlockHash: null, + executionStatus: ExecutionStatus.PreMerge, + ilStatus: InclusionListStatus.PreIL, + }), }, currentSlot ), diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 7df7158bc022..418d5c4dec6d 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -117,6 +117,10 @@ export interface IBeaconChain { readonly producedBlockRoot: Map; readonly shufflingCache: ShufflingCache; readonly producedBlindedBlockRoot: Set; + readonly producedInclusionList: Map< + BlockHash, + {ilSummary: electra.InclusionListSummary; ilTransactions: electra.ILTransactions} + >; readonly opts: IChainOptions; /** Stop beacon chain processing */ @@ -177,7 +181,7 @@ export interface IBeaconChain { processBlock(block: BlockInput, opts?: ImportBlockOpts): Promise; /** Process a chain of blocks until complete */ processChainSegment(blocks: BlockInput[], opts?: ImportBlockOpts): Promise; - processInclusionList(inclusionList: electra.NewInclusionListRequest, opts?: ImportBlockOpts): Promise; + processInclusionList(inclusionList: electra.SignedInclusionList, opts?: ImportBlockOpts): Promise; getStatus(): phase0.Status; diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 7697e89807ea..3c40d0e7fc71 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -1,3 +1,4 @@ +import {toHexString} from "@chainsafe/ssz"; import { Bytes32, allForks, @@ -254,6 +255,18 @@ export async function produceBlockBody( await sleep(PAYLOAD_GENERATION_TIME_MS); } + // fetch the IL and cache it, it its not ready by the time of block signing + // then once can just proceed with empty IL + const parentHashRes = await getExecutionPayloadParentHash(this, currentState as CachedBeaconStateExecutions); + if (parentHashRes.isPremerge) { + throw Error("Execution builder disabled pre-merge"); + } + + const {parentHash} = parentHashRes; + this.executionEngine.getInclusionList(parentHash).then((ilRes) => { + this.producedInclusionList.set(toHexString(parentHash), ilRes); + }); + const engineRes = await this.executionEngine.getPayload(fork, payloadId); const {executionPayload, blobsBundle} = engineRes; shouldOverrideBuilder = engineRes.shouldOverrideBuilder; diff --git a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts index ef8d063f6bef..7068e142f718 100644 --- a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts +++ b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts @@ -2,7 +2,7 @@ import {toHexString} from "@chainsafe/ssz"; import {deneb, electra, RootHex, ssz, allForks} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {pruneSetToMax} from "@lodestar/utils"; -import {BLOBSIDECAR_FIXED_SIZE, ForkSeq, ForkName, isForkBlobs, isForkILs} from "@lodestar/params"; +import {BLOBSIDECAR_FIXED_SIZE, ForkName, isForkBlobs, isForkILs} from "@lodestar/params"; import { BlockInput, @@ -14,6 +14,7 @@ import { GossipedInputType, getBlockInputBlobs, BlockInputDataIls, + BlockInputILType, } from "../blocks/types.js"; import {Metrics} from "../../metrics/index.js"; @@ -27,7 +28,7 @@ type GossipedBlockInput = | {type: GossipedInputType.blob; blobSidecar: deneb.BlobSidecar; blobBytes: Uint8Array | null} | { type: GossipedInputType.ilist; - inclusionList: electra.NewInclusionListRequest; + signedInclusionList: electra.SignedInclusionList; inclusionListBytes: Uint8Array | null; }; @@ -56,7 +57,7 @@ const MAX_GOSSIPINPUT_CACHE = 5; */ export class SeenGossipBlockInput { private blockInputCache = new Map(); - private seenILsByParentHash = new Map(); + private seenILsByParentHash = new Map(); private blockInputRootByParentHash = new Map(); prune(): void { @@ -76,7 +77,11 @@ export class SeenGossipBlockInput { ): | { blockInput: BlockInput; - blockInputMeta: {pending: GossipedInputType.blob | null; haveBlobs: number; expectedBlobs: number}; + blockInputMeta: { + pending: GossipedInputType.blob | GossipedInputType.ilist | null; + haveBlobs: number; + expectedBlobs: number; + }; } | { blockInput: NullBlockInput; @@ -88,9 +93,9 @@ export class SeenGossipBlockInput { let fork; if (gossipedInput.type === GossipedInputType.ilist) { - const {inclusionList} = gossipedInput; - const parentBlockHashHex = toHexString(inclusionList.parentBlockHash); - this.seenILsByParentHash.set(parentBlockHashHex, inclusionList); + const {signedInclusionList} = gossipedInput; + const parentBlockHashHex = toHexString(signedInclusionList.message.signedSummary.message.parentHash); + this.seenILsByParentHash.set(parentBlockHashHex, signedInclusionList); blockHex = this.blockInputRootByParentHash.get(parentBlockHashHex); blockCache = blockHex ? this.blockInputCache.get(blockHex) : undefined; @@ -132,15 +137,15 @@ export class SeenGossipBlockInput { const {block: signedBlock, blockBytes, blockInputPromise, resolveBlockInput, cachedData} = blockCache; if (signedBlock !== undefined) { - if (ForkSeq[fork] < ForkSeq.deneb) { + if (!isForkBlobs(fork)) { return { blockInput: getBlockInput.preDeneb(config, signedBlock, BlockSource.gossip, blockBytes ?? null), blockInputMeta: {pending: null, haveBlobs: 0, expectedBlobs: 0}, }; } - if (cachedData === undefined) { - throw Error("Missing cached Data for deneb+ block"); + if (cachedData === undefined || !isForkBlobs(cachedData.fork)) { + throw Error("Missing or Invalid fork cached Data for deneb+ block"); } const {blobsCache} = cachedData; @@ -157,26 +162,42 @@ export class SeenGossipBlockInput { if (blobKzgCommitments.length === blobsCache.size) { const allBlobs = getBlockInputBlobs(blobsCache); - const blockData = {fork: ForkName.deneb, ...allBlobs} as BlockInputDataBlobs; + const {blobs} = allBlobs; + let blockInput; + + let blockData; if (cachedData.fork === ForkName.deneb) { + blockData = {fork: cachedData.fork, ...allBlobs} as BlockInputDataBlobs; cachedData.resolveAvailability(blockData); } else { - throw Error("il availability not implemented"); + const parentBlockHash = toHexString( + (signedBlock as electra.SignedBeaconBlock).message.body.executionPayload.parentHash + ); + const signedIL = this.seenILsByParentHash.get(parentBlockHash); + if (signedIL === undefined) { + blockData = {fork: cachedData.fork, ...allBlobs, ilType: BlockInputILType.syncing} as BlockInputDataIls; + } else { + blockData = { + fork: cachedData.fork, + ...allBlobs, + ilType: BlockInputILType.actualIL, + inclusionList: signedIL.message, + } as BlockInputDataIls; + cachedData.resolveAvailability(blockData); + } } + metrics?.syncUnknownBlock.resolveAvailabilitySource.inc({source: BlockInputAvailabilitySource.GOSSIP}); - const {blobs} = allBlobs; - const blockInput = getBlockInput.postDeneb( - config, - signedBlock, - blockBytes ?? null, - blockData, - BlockSource.gossip - ); + blockInput = getBlockInput.postDeneb(config, signedBlock, blockBytes ?? null, blockData, BlockSource.gossip); resolveBlockInput(blockInput); return { blockInput, - blockInputMeta: {pending: null, haveBlobs: blobs.length, expectedBlobs: blobKzgCommitments.length}, + blockInputMeta: { + pending: GossipedInputType.ilist, + haveBlobs: blobsCache.size, + expectedBlobs: blobKzgCommitments.length, + }, }; } else { const blockInput = getBlockInput.blobsPromise( @@ -255,7 +276,7 @@ function getEmptyBlockInputCacheEntry(fork: ForkName): BlockInputCacheType { if (resolveAvailability === null) { throw Error("Promise Constructor was not executed immediately"); } - const cachedData: CachedData = {fork, blobsCache, availabilityPromise, resolveAvailability, inclusionLists: []}; + const cachedData: CachedData = {fork, blobsCache, availabilityPromise, resolveAvailability}; return {fork, blockInputPromise, resolveBlockInput, cachedData}; } } diff --git a/packages/beacon-node/src/chain/validation/inclusionList.ts b/packages/beacon-node/src/chain/validation/inclusionList.ts index 83b8f3ba7916..2ecef79b9a87 100644 --- a/packages/beacon-node/src/chain/validation/inclusionList.ts +++ b/packages/beacon-node/src/chain/validation/inclusionList.ts @@ -3,5 +3,5 @@ import {IBeaconChain} from "../interface.js"; export async function validateGossipInclusionList( _chain: IBeaconChain, - _inclusionList: electra.NewInclusionListRequest + _inclusionList: electra.SignedInclusionList ): Promise {} diff --git a/packages/beacon-node/src/execution/engine/disabled.ts b/packages/beacon-node/src/execution/engine/disabled.ts index eb81cf5cf1dc..70c58a28887e 100644 --- a/packages/beacon-node/src/execution/engine/disabled.ts +++ b/packages/beacon-node/src/execution/engine/disabled.ts @@ -19,6 +19,10 @@ export class ExecutionEngineDisabled implements IExecutionEngine { throw Error("Execution engine disabled"); } + async getInclusionList(): Promise { + throw Error("Execution engine disabled"); + } + async getBlobsBundle(): Promise { throw Error("Execution engine disabled"); } diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index 829693901bfc..a516fca573d7 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -12,7 +12,7 @@ import { import {Metrics} from "../../metrics/index.js"; import {JobItemQueue} from "../../util/queue/index.js"; import {EPOCHS_PER_BATCH} from "../../sync/constants.js"; -import {numToQuantity} from "../../eth1/provider/utils.js"; +import {bytesToData, dataToBytes, numToQuantity} from "../../eth1/provider/utils.js"; import { ExecutionPayloadStatus, ExecutePayloadResponse, @@ -22,6 +22,7 @@ import { BlobsBundle, VersionedHashes, ExecutionEngineState, + InclusionListResponse, } from "./interface.js"; import {PayloadIdCache} from "./payloadIdCache.js"; import { @@ -35,6 +36,8 @@ import { ExecutionPayloadBody, assertReqSizeLimit, deserializeExecutionPayloadBody, + serializeInclusionListSummary, + parseInclusionListSummary, } from "./types.js"; import {getExecutionEngineState} from "./utils.js"; @@ -86,8 +89,10 @@ const QUEUE_MAX_LENGTH = EPOCHS_PER_BATCH * SLOTS_PER_EPOCH * 2; // Define static options once to prevent extra allocations const notifyNewPayloadOpts: ReqOpts = {routeId: "notifyNewPayload"}; +const notifyInclusionListOpts: ReqOpts = {routeId: "notifyInclusionList"}; const forkchoiceUpdatedV1Opts: ReqOpts = {routeId: "forkchoiceUpdated"}; const getPayloadOpts: ReqOpts = {routeId: "getPayload"}; +const getInclusionListOpts: ReqOpts = {routeId: "getInclusionList"}; /** * based on Ethereum JSON-RPC API and inherits the following properties of this standard: @@ -145,10 +150,51 @@ export class ExecutionEngineHttp implements IExecutionEngine { }); } - async notifyNewInclusionList(inclusionList: electra.NewInclusionListRequest): Promise { - // return dummy status, don't use latestValidHash anyway - const latestValidHash = toHexString(inclusionList.parentBlockHash); - return {status: ExecutionPayloadStatus.VALID, latestValidHash, validationError: null}; + async notifyNewInclusionList( + inclusionListSummary: electra.InclusionListSummary, + transactions: electra.ILTransactions + ): Promise { + const method = "engine_newInclusionListV1"; + const serializedSummary = serializeInclusionListSummary(inclusionListSummary); + const serializedTransactions = transactions.map((trans) => bytesToData(trans)); + const engineRequest = { + method, + params: [serializedSummary, serializedTransactions] as EngineApiRpcParamTypes[typeof method], + methodOpts: notifyInclusionListOpts, + } as EngineRequest; + + const {status, validationError} = await ( + this.rpcFetchQueue.push(engineRequest) as Promise + ).catch((e: Error) => { + if (e instanceof HttpRpcError || e instanceof ErrorJsonRpcResponse) { + return {status: ExecutionPayloadStatus.ELERROR, validationError: e.message}; + } else { + return {status: ExecutionPayloadStatus.UNAVAILABLE, validationError: e.message}; + } + }); + + switch (status) { + case ExecutionPayloadStatus.SYNCING: + case ExecutionPayloadStatus.ACCEPTED: + case ExecutionPayloadStatus.VALID: + return {status, validationError: null}; + + case ExecutionPayloadStatus.INVALID: + return {status, validationError}; + + case ExecutionPayloadStatus.UNAVAILABLE: + case ExecutionPayloadStatus.ELERROR: + return { + status, + validationError: validationError ?? "Unknown ELERROR", + }; + + default: + return { + status: ExecutionPayloadStatus.ELERROR, + validationError: `Invalid EL status on executePayload: ${status}`, + }; + } } /** @@ -395,6 +441,29 @@ export class ExecutionEngineHttp implements IExecutionEngine { return parseExecutionPayload(fork, payloadResponse); } + async getInclusionList( + parentHash: Root + ): Promise<{ilSummary: electra.InclusionListSummary; ilTransactions: electra.ILTransactions}> { + const method = "engine_getInclusionListV1"; + const parentHashData = bytesToData(parentHash); + + const {inclusionListSummary, inclusionListTransactions} = await this.rpc.fetchWithRetries< + EngineApiRpcReturnTypes[typeof method], + EngineApiRpcParamTypes[typeof method] + >( + { + method, + params: [parentHashData], + }, + getInclusionListOpts + ); + + const ilSummary = parseInclusionListSummary(inclusionListSummary); + const ilTransactions = inclusionListTransactions.map((trans) => dataToBytes(trans, null)); + + return {ilSummary, ilTransactions}; + } + async prunePayloadIdCache(): Promise { this.payloadIdCache.prune(); } diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index d0fd737f8fa7..cfcab4ac8760 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -55,6 +55,24 @@ export type ExecutePayloadResponse = validationError: string; }; +export type InclusionListResponse = + | { + status: + | ExecutionPayloadStatus.ACCEPTED + | ExecutionPayloadStatus.INVALID + | ExecutionPayloadStatus.SYNCING + | ExecutionPayloadStatus.VALID; + validationError: null; + } + | { + status: ExecutionPayloadStatus.INVALID; + validationError: string; + } + | { + status: ExecutionPayloadStatus.ELERROR | ExecutionPayloadStatus.UNAVAILABLE; + validationError: string; + }; + export type ForkChoiceUpdateStatus = | ExecutionPayloadStatus.VALID | ExecutionPayloadStatus.INVALID @@ -90,7 +108,13 @@ export type VersionedHashes = Uint8Array[]; */ export interface IExecutionEngine { payloadIdCache: PayloadIdCache; - notifyNewInclusionList(InclusionList: electra.NewInclusionListRequest): Promise; + notifyNewInclusionList( + InclusionListSummary: electra.InclusionListSummary, + transactions: electra.ILTransactions + ): Promise; + getInclusionList( + parentHash: Root + ): Promise<{ilSummary: electra.InclusionListSummary; ilTransactions: electra.ILTransactions}>; /** * A state transition function which applies changes to the self.execution_state. * Returns ``True`` iff ``execution_payload`` is valid with respect to ``self.execution_state``. diff --git a/packages/beacon-node/src/execution/engine/mock.ts b/packages/beacon-node/src/execution/engine/mock.ts index 5779713435a5..75e762447c01 100644 --- a/packages/beacon-node/src/execution/engine/mock.ts +++ b/packages/beacon-node/src/execution/engine/mock.ts @@ -96,9 +96,23 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend { engine_getPayloadV3: this.getPayload.bind(this), engine_getPayloadBodiesByHashV1: this.getPayloadBodiesByHash.bind(this), engine_getPayloadBodiesByRangeV1: this.getPayloadBodiesByRange.bind(this), + engine_newInclusionListV1: this.newInclusionListV1.bind(this), + engine_getInclusionListV1: this.getInclusionListV1.bind(this), }; } + private newInclusionListV1( + ..._args: EngineApiRpcParamTypes["engine_newInclusionListV1"] + ): EngineApiRpcReturnTypes["engine_newInclusionListV1"] { + throw Error("not implemented"); + } + + private getInclusionListV1( + ..._args: EngineApiRpcParamTypes["engine_getInclusionListV1"] + ): EngineApiRpcReturnTypes["engine_getInclusionListV1"] { + throw Error("not implemented"); + } + private getPayloadBodiesByHash( _blockHex: EngineApiRpcParamTypes["engine_getPayloadBodiesByHashV1"][0] ): EngineApiRpcReturnTypes["engine_getPayloadBodiesByHashV1"] { diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 72a0100f7a51..9da16d08660a 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -1,4 +1,4 @@ -import {allForks, capella, deneb, Wei, bellatrix, Root} from "@lodestar/types"; +import {allForks, capella, deneb, electra, Wei, bellatrix, Root, ssz} from "@lodestar/types"; import { BYTES_PER_LOGS_BLOOM, FIELD_ELEMENTS_PER_BLOB, @@ -19,6 +19,13 @@ import { import {ExecutionPayloadStatus, BlobsBundle, PayloadAttributes, VersionedHashes} from "./interface.js"; import {WithdrawalV1} from "./payloadIdCache.js"; +type InclusionListSummaryV1 = { + slot: QUANTITY; + proposerIndex: QUANTITY; + parentHash: DATA; + summary: {address: DATA; nonce: QUANTITY}[]; +}; + /* eslint-disable @typescript-eslint/naming-convention */ export type EngineApiRpcParamTypes = { @@ -62,6 +69,12 @@ export type EngineApiRpcParamTypes = { * 2. count: QUANTITY, 64 bits - Number of blocks to return */ engine_getPayloadBodiesByRangeV1: [start: QUANTITY, count: QUANTITY]; + + engine_newInclusionListV1: [ + inclusionListSummary: InclusionListSummaryV1, + inclusionListTransactions: ExecutionPayloadRpc["transactions"], + ]; + engine_getInclusionListV1: [DATA]; }; export type PayloadStatus = { @@ -70,6 +83,11 @@ export type PayloadStatus = { validationError: string | null; }; +export type InclusionListStatus = { + status: ExecutionPayloadStatus; + validationError: string | null; +}; + export type EngineApiRpcReturnTypes = { /** * Object - Response object: @@ -100,6 +118,12 @@ export type EngineApiRpcReturnTypes = { engine_getPayloadBodiesByHashV1: (ExecutionPayloadBodyRpc | null)[]; engine_getPayloadBodiesByRangeV1: (ExecutionPayloadBodyRpc | null)[]; + + engine_newInclusionListV1: InclusionListStatus; + engine_getInclusionListV1: { + inclusionListSummary: InclusionListSummaryV1; + inclusionListTransactions: ExecutionPayloadRpc["transactions"]; + }; }; type ExecutionPayloadRpcWithValue = { @@ -134,6 +158,7 @@ export type ExecutionPayloadRpc = { blobGasUsed?: QUANTITY; // DENEB excessBlobGas?: QUANTITY; // DENEB parentBeaconBlockRoot?: QUANTITY; // DENEB + previousInclusionListSummary?: InclusionListSummaryV1; }; export type WithdrawalRpc = { @@ -201,6 +226,30 @@ export function serializeVersionedHashes(vHashes: VersionedHashes): VersionedHas return vHashes.map(bytesToData); } +export function serializeInclusionListSummary(ilSummary: electra.InclusionListSummary): InclusionListSummaryV1 { + return { + slot: numToQuantity(ilSummary.slot), + proposerIndex: numToQuantity(ilSummary.proposerIndex), + parentHash: bytesToData(ilSummary.parentHash), + summary: ilSummary.summary.map(({address, nonce}) => ({ + address: bytesToData(address), + nonce: numToQuantity(nonce), + })), + }; +} + +export function parseInclusionListSummary(rpcSummary: InclusionListSummaryV1): electra.InclusionListSummary { + return { + slot: quantityToNum(rpcSummary.slot), + proposerIndex: quantityToNum(rpcSummary.proposerIndex), + parentHash: dataToBytes(rpcSummary.parentHash, 32), + summary: rpcSummary.summary.map(({address, nonce}) => ({ + address: dataToBytes(address, 20), + nonce: quantityToNum(nonce), + })), + }; +} + export function hasPayloadValue(response: ExecutionPayloadResponse): response is ExecutionPayloadRpcWithValue { return (response as ExecutionPayloadRpcWithValue).blockValue !== undefined; } @@ -279,6 +328,14 @@ export function parseExecutionPayload( (executionPayload as deneb.ExecutionPayload).excessBlobGas = quantityToBigint(excessBlobGas); } + // inject parent summary + const {previousInclusionListSummary} = data; + if (ForkSeq[fork] >= ForkSeq.electra) { + (executionPayload as electra.ExecutionPayload).previousInclusionListSummary = previousInclusionListSummary + ? parseInclusionListSummary(previousInclusionListSummary) + : ssz.electra.InclusionListSummary.defaultValue(); + } + return {executionPayload, executionPayloadValue, blobsBundle, shouldOverrideBuilder}; } diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts index ced5ea6ff59c..66f7f452dd5e 100644 --- a/packages/beacon-node/src/network/gossip/interface.ts +++ b/packages/beacon-node/src/network/gossip/interface.ts @@ -73,7 +73,7 @@ export type SSZTypeOfGossipTopic = T extends {type: infer export type GossipTypeMap = { [GossipType.beacon_block]: allForks.SignedBeaconBlock; [GossipType.blob_sidecar]: deneb.BlobSidecar; - [GossipType.inclusion_list]: electra.NewInclusionListRequest; + [GossipType.inclusion_list]: electra.SignedInclusionList; [GossipType.beacon_aggregate_and_proof]: phase0.SignedAggregateAndProof; [GossipType.beacon_attestation]: phase0.Attestation; [GossipType.voluntary_exit]: phase0.SignedVoluntaryExit; @@ -89,7 +89,7 @@ export type GossipTypeMap = { export type GossipFnByType = { [GossipType.beacon_block]: (signedBlock: allForks.SignedBeaconBlock) => Promise | void; [GossipType.blob_sidecar]: (blobSidecar: deneb.BlobSidecar) => Promise | void; - [GossipType.inclusion_list]: (inclusionList: electra.NewInclusionListRequest) => Promise | void; + [GossipType.inclusion_list]: (inclusionList: electra.SignedInclusionList) => Promise | void; [GossipType.beacon_aggregate_and_proof]: (aggregateAndProof: phase0.SignedAggregateAndProof) => Promise | void; [GossipType.beacon_attestation]: (attestation: phase0.Attestation) => Promise | void; [GossipType.voluntary_exit]: (voluntaryExit: phase0.SignedVoluntaryExit) => Promise | void; diff --git a/packages/beacon-node/src/network/gossip/topic.ts b/packages/beacon-node/src/network/gossip/topic.ts index 0c52e142b432..20cf1d44eae3 100644 --- a/packages/beacon-node/src/network/gossip/topic.ts +++ b/packages/beacon-node/src/network/gossip/topic.ts @@ -88,7 +88,7 @@ export function getGossipSSZType(topic: GossipTopic) { case GossipType.blob_sidecar: return ssz.deneb.BlobSidecar; case GossipType.inclusion_list: - return ssz.electra.NewInclusionListRequest; + return ssz.electra.SignedInclusionList; case GossipType.beacon_aggregate_and_proof: return ssz.phase0.SignedAggregateAndProof; case GossipType.beacon_attestation: diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index 8242d045f3c7..8b34f79517e4 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -61,7 +61,7 @@ export interface INetwork extends INetworkCorePublic { // Gossip publishBeaconBlock(signedBlock: allForks.SignedBeaconBlock): Promise; publishBlobSidecar(blobSidecar: deneb.BlobSidecar): Promise; - publishInclusionList(inclusionList: electra.NewInclusionListRequest): Promise; + publishInclusionList(inclusionList: electra.SignedInclusionList): Promise; publishBeaconAggregateAndProof(aggregateAndProof: phase0.SignedAggregateAndProof): Promise; publishBeaconAttestation(attestation: phase0.Attestation, subnet: number): Promise; publishVoluntaryExit(voluntaryExit: phase0.SignedVoluntaryExit): Promise; diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index 889d3c102b40..212be11e7c6c 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -302,9 +302,10 @@ export class Network implements INetwork { }); } - async publishInclusionList(inclusionList: electra.NewInclusionListRequest): Promise { - const fork = this.config.getForkName(inclusionList.slot); - return this.publishGossip({type: GossipType.inclusion_list, fork}, inclusionList, { + async publishInclusionList(signedInclusionList: electra.SignedInclusionList): Promise { + const slot = signedInclusionList.message.signedSummary.message.slot; + const fork = this.config.getForkName(slot); + return this.publishGossip({type: GossipType.inclusion_list, fork}, signedInclusionList, { ignoreDuplicatePublishError: true, }); } diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index b95a745be1be..345b5cd32372 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -264,12 +264,12 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } async function validateInclusionList( - inclusionList: electra.NewInclusionListRequest, + signedInclusionList: electra.SignedInclusionList, inclusionListBytes: Uint8Array, peerIdStr: string, seenTimestampSec: number ): Promise { - const slot = inclusionList.slot; + const slot = signedInclusionList.message.signedSummary.message.slot; const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec); const recvToValLatency = Date.now() / 1000 - seenTimestampSec; @@ -277,7 +277,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler config, { type: GossipedInputType.ilist, - inclusionList, + signedInclusionList, inclusionListBytes, }, metrics @@ -286,7 +286,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler const blockInput = blockInputRes?.blockInput ?? null; try { const blockInputMeta = blockInputRes?.blockInputMeta ?? {}; - await validateGossipInclusionList(chain, inclusionList); + await validateGossipInclusionList(chain, signedInclusionList); const recvToValidation = Date.now() / 1000 - seenTimestampSec; const validationTime = recvToValidation - recvToValLatency; @@ -313,7 +313,11 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } if (e.action === GossipAction.REJECT) { - chain.persistInvalidSszValue(ssz.electra.NewInclusionListRequest, inclusionList, "gossip_reject_slot"); + chain.persistInvalidSszValue( + ssz.electra.SignedInclusionList, + signedInclusionList, + `gossip_reject_slot_${slot}` + ); } } @@ -401,7 +405,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler } function handleValidInclusionList( - inclusionList: electra.NewInclusionListRequest, + signedInclusionList: electra.SignedInclusionList, peerIdStr: string, seenTimestampSec: number ): void { @@ -409,9 +413,10 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler // metrics?.registerInclusionList(OpSource.gossip, seenTimestampSec, inclusionList); // if blobs are not yet fully available start an aggressive blob pull + const slot = signedInclusionList.message.signedSummary.message.slot; chain - .processInclusionList(inclusionList, { + .processInclusionList(signedInclusionList, { // block may be downloaded and processed by UnknownBlockSync ignoreIfKnown: true, // proposer signature already checked in validateBeaconBlock() @@ -422,7 +427,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler }) .then(() => { // Returns the delay between the start of `block.slot` and `current time` - const delaySec = chain.clock.secFromSlot(inclusionList.slot); + const delaySec = chain.clock.secFromSlot(slot); metrics?.gossipBlock.elapsedTimeTillProcessed.observe(delaySec); chain.seenGossipBlockInput.prune(); }) @@ -453,7 +458,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler metrics?.gossipInclusionList.processInclusionListErrors.inc({ error: e instanceof InclusionListGossipError ? e.type.code : "NOT_INCLUSION_LIST_ERROR", }); - logger[logLevel]("Error receiving inclusion list", {slot: inclusionList.slot, peer: peerIdStr}, e as Error); + logger[logLevel]("Error receiving inclusion list", {slot, peer: peerIdStr}, e as Error); chain.seenGossipBlockInput.prune(); }); } @@ -542,15 +547,15 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler seenTimestampSec, }: GossipHandlerParamGeneric) => { const {serializedData} = gossipData; - const inclusionList = sszDeserialize(topic, serializedData); - const ilSlot = inclusionList.slot; + const signedInclusionList = sszDeserialize(topic, serializedData); + const slot = signedInclusionList.message.signedSummary.message.slot; - if (config.getForkSeq(ilSlot) < ForkSeq.electra) { + if (config.getForkSeq(slot) < ForkSeq.electra) { throw new GossipActionError(GossipAction.REJECT, {code: "PRE_ELECTRA_IL"}); } - const blockInput = await validateInclusionList(inclusionList, serializedData, peerIdStr, seenTimestampSec); - handleValidInclusionList(inclusionList, peerIdStr, seenTimestampSec); + const blockInput = await validateInclusionList(signedInclusionList, serializedData, peerIdStr, seenTimestampSec); + handleValidInclusionList(signedInclusionList, peerIdStr, seenTimestampSec); if (blockInput === null) { return; } else if (blockInput.block !== null) { @@ -560,10 +565,10 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler // handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec); } else { // wait for the block to arrive till some cutoff else emit unknownBlockInput event - chain.logger.debug("Block not yet available, racing with cutoff", {ilSlot}); + chain.logger.debug("Block not yet available for seen inclusion list, racing with cutoff", {slot}); const normalBlockInput = await raceWithCutoff( chain, - ilSlot, + slot, blockInput.blockInputPromise, BLOCK_AVAILABILITY_CUTOFF_MS ).catch((_e) => { @@ -571,7 +576,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler }); if (normalBlockInput !== null) { - chain.logger.debug("Block corresponding to inclusion list is now available for processing", {ilSlot}); + chain.logger.debug("Block corresponding to inclusion list is now available for processing", {slot}); // we can directly send it for processing but block gossip handler will queue it up anyway // if we see any issues later, we can send it to handleValidBeaconBlock // @@ -583,7 +588,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler events.emit(NetworkEvent.unknownBlockInput, {blockInput: normalBlockInput, peer: peerIdStr}); } } else { - chain.logger.debug("Block not available till BLOCK_AVAILABILITY_CUTOFF_MS", {ilSlot}); + chain.logger.debug("Inclusion list seen but block not available till BLOCK_AVAILABILITY_CUTOFF_MS", {slot}); events.emit(NetworkEvent.unknownBlockInput, {blockInput, peer: peerIdStr}); } } diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts index 717b689d2b0d..6242a67bd571 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts @@ -1,9 +1,16 @@ import {ChainForkConfig} from "@lodestar/config"; import {deneb, Epoch, phase0, allForks, Slot} from "@lodestar/types"; -import {ForkSeq, ForkName} from "@lodestar/params"; +import {ForkSeq, isForkBlobs, isForkILs} from "@lodestar/params"; import {computeEpochAtSlot} from "@lodestar/state-transition"; -import {BlockInput, BlockSource, getBlockInput} from "../../chain/blocks/types.js"; +import { + BlockInput, + BlockSource, + getBlockInput, + BlockInputILType, + BlockInputDataBlobs, + BlockInputDataIls, +} from "../../chain/blocks/types.js"; import {PeerIdStr} from "../../util/peerId.js"; import {INetwork, WithBytes} from "../interface.js"; @@ -72,7 +79,8 @@ export function matchBlockWithBlobs( // Assuming that the blocks and blobs will come in same sorted order for (let i = 0; i < allBlocks.length; i++) { const block = allBlocks[i]; - if (config.getForkSeq(block.data.message.slot) < ForkSeq.deneb) { + const fork = config.getForkName(block.data.message.slot); + if (!isForkBlobs(fork)) { blockInputs.push(getBlockInput.preDeneb(config, block.data, blockSource, block.bytes)); } else { const blobSidecars: deneb.BlobSidecar[] = []; @@ -94,20 +102,21 @@ export function matchBlockWithBlobs( ); } - // TODO DENEB: instead of null, pass payload in bytes - blockInputs.push( - getBlockInput.postDeneb( - config, - block.data, - null, - { - fork: "deneb" as ForkName.deneb, + const blockData = isForkILs(fork) + ? ({ + fork, + blobs: blobSidecars, + blobsBytes: Array.from({length: blobKzgCommitmentsLen}, () => null), + ilType: BlockInputILType.childBlock, + } as BlockInputDataIls) + : ({ + fork, blobs: blobSidecars, blobsBytes: Array.from({length: blobKzgCommitmentsLen}, () => null), - }, - blockSource - ) - ); + } as BlockInputDataBlobs); + + // TODO DENEB: instead of null, pass payload in bytes + blockInputs.push(getBlockInput.postDeneb(config, block.data, null, blockData, blockSource)); } } diff --git a/packages/beacon-node/src/node/notifier.ts b/packages/beacon-node/src/node/notifier.ts index 8c393a4fcb05..a150e6534cf8 100644 --- a/packages/beacon-node/src/node/notifier.ts +++ b/packages/beacon-node/src/node/notifier.ts @@ -173,7 +173,7 @@ function getHeadExecutionInfo( if (clockEpoch < config.BELLATRIX_FORK_EPOCH) { return []; } else { - const executionStatusStr = headInfo.executionStatus.toLowerCase(); + const executionStatusStr = `${headInfo.executionStatus.toLowerCase()}-il-${headInfo.ilStatus.toLowerCase()}`; // Add execution status to notifier only if head is on/post bellatrix if (isExecutionCachedStateType(headState)) { diff --git a/packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/post-merge.sh b/packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/post-merge.sh index fbf9dcaaf929..3495a4dd0755 100755 --- a/packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/post-merge.sh +++ b/packages/beacon-node/test/scripts/el-interop/ethereumjsdocker/post-merge.sh @@ -5,4 +5,4 @@ currentDir=$(pwd) . $scriptDir/common-setup.sh -docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) --name custom-execution --network host -v $currentDir/$DATA_DIR:/data $EL_BINARY_DIR --dataDir /data/ethereumjs --gethGenesis /data/genesis.json --rpc --rpcEngineAddr 0.0.0.0 --rpcAddr 0.0.0.0 --rpcEngine --jwt-secret /data/jwtsecret --logLevel debug --isSingleNode +docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) --name custom-execution --network host -v $currentDir/$DATA_DIR:/data $EL_BINARY_DIR --dataDir /data/ethereumjs --gethGenesis /data/genesis.json --rpc --rpcEngineAddr 0.0.0.0 --rpcAddr 0.0.0.0 --rpcEngine --jwt-secret /data/jwtsecret --logLevel info --isSingleNode diff --git a/packages/beacon-node/test/scripts/el-interop/gethdocker/il.tmpl b/packages/beacon-node/test/scripts/el-interop/gethdocker/il.tmpl new file mode 100644 index 000000000000..966505524d81 --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/gethdocker/il.tmpl @@ -0,0 +1,35 @@ +{ +"config": { +"chainId":1, +"homesteadBlock":0, +"eip150Block":0, +"eip155Block":0, +"eip158Block":0, +"byzantiumBlock":0, +"constantinopleBlock":0, +"petersburgBlock":0, +"istanbulBlock":0, +"muirGlacierBlock":0, +"berlinBlock":0, +"londonBlock":0, +"shanghaiTime":0, +"cancunTime": 0, +"pragueTime": 0, +"terminalTotalDifficulty":${TTD}, +"terminalTotalDifficultyPassed": true +}, +"nonce":"0x42", +"timestamp":"0x0", +"extraData":"0x0000000000000000000000000000000000000000000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", +"gasLimit":"0x1C9C380", +"difficulty":"0x400000000", +"mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000", +"coinbase":"0x0000000000000000000000000000000000000000", +"alloc":{ +"0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b":{"balance":"0x6d6172697573766477000000"} +}, +"number":"0x0", +"gasUsed":"0x0", +"parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", +"baseFeePerGas":"0x7" +} \ No newline at end of file diff --git a/packages/beacon-node/test/sim/4844-interop.test.ts b/packages/beacon-node/test/sim/4844-interop.test.ts index 23a01bf1bb33..ae4ad5f7494c 100644 --- a/packages/beacon-node/test/sim/4844-interop.test.ts +++ b/packages/beacon-node/test/sim/4844-interop.test.ts @@ -38,8 +38,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { ); } vi.setConfig({testTimeout: 1000 * 60 * 10, hookTimeout: 1000 * 60 * 10}); - - const dataPath = fs.mkdtempSync("lodestar-test-4844"); + const dataPath = fs.mkdtempSync(`${process.env.DATA_DIR_PREFIX}lodestar-test-4844`); const elSetupConfig = { elScriptDir: process.env.EL_SCRIPT_DIR, elBinaryDir: process.env.EL_BINARY_DIR, diff --git a/packages/beacon-node/test/sim/ils-interop.test.ts b/packages/beacon-node/test/sim/ils-interop.test.ts new file mode 100644 index 000000000000..d024fd30d87a --- /dev/null +++ b/packages/beacon-node/test/sim/ils-interop.test.ts @@ -0,0 +1,397 @@ +import fs from "node:fs"; +import {describe, it, vi, afterAll, afterEach} from "vitest"; +import {fromHexString} from "@chainsafe/ssz"; +import {LogLevel, sleep} from "@lodestar/utils"; +import {TimestampFormatCode} from "@lodestar/logger"; +import {SLOTS_PER_EPOCH} from "@lodestar/params"; +import {ChainConfig} from "@lodestar/config"; +import {computeStartSlotAtEpoch} from "@lodestar/state-transition"; +import {Epoch, Slot} from "@lodestar/types"; +import {ValidatorProposerConfig} from "@lodestar/validator"; + +import {ChainEvent} from "../../src/chain/index.js"; +import {ClockEvent} from "../../src/util/clock.js"; + +import {testLogger, TestLoggerOpts} from "../utils/logger.js"; +import {getDevBeaconNode} from "../utils/node/beacon.js"; +import {BeaconRestApiServerOpts} from "../../src/api/index.js"; +import {simTestInfoTracker} from "../utils/node/simTest.js"; +import {getAndInitDevValidators} from "../utils/node/validator.js"; +import {BeaconNode, Eth1Provider} from "../../src/index.js"; +import {ZERO_HASH} from "../../src/constants/index.js"; +import {runEL, ELStartMode, ELClient, sendRawTransactionBig} from "../utils/runEl.js"; +import {logFilesDir} from "./params.js"; +import {shell} from "./shell.js"; + +// NOTE: How to run +// DATA_DIR_PREFIX=mergetests/ DEV_RUN=true EL_BINARY_DIR=ethpandaops/geth:7547-exp EL_SCRIPT_DIR=gethdocker yarn vitest --run test/sim/ils-interop.test.ts +// ``` + +/* eslint-disable no-console, @typescript-eslint/naming-convention */ + +const jwtSecretHex = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"; +const blobTxsPath = "./test/sim/data/blobs.txt"; +describe("executionEngine / ExecutionEngineHttp", function () { + if (!process.env.EL_BINARY_DIR || !process.env.EL_SCRIPT_DIR) { + throw Error( + `EL ENV must be provided, EL_BINARY_DIR: ${process.env.EL_BINARY_DIR}, EL_SCRIPT_DIR: ${process.env.EL_SCRIPT_DIR}` + ); + } + vi.setConfig({testTimeout: 1000 * 60 * 10, hookTimeout: 1000 * 60 * 10}); + const dataPath = fs.mkdtempSync(`${process.env.DATA_DIR_PREFIX}lodestar-test-il`); + const elSetupConfig = { + elScriptDir: process.env.EL_SCRIPT_DIR, + elBinaryDir: process.env.EL_BINARY_DIR, + }; + const elRunOptions = { + dataPath, + jwtSecretHex, + enginePort: parseInt(process.env.ENGINE_PORT ?? "8551"), + ethPort: parseInt(process.env.ETH_PORT ?? "8545"), + }; + + const controller = new AbortController(); + afterAll(async () => { + controller?.abort(); + await shell(`sudo rm -rf ${dataPath}`); + }); + + const afterEachCallbacks: (() => Promise | void)[] = []; + afterEach(async () => { + while (afterEachCallbacks.length > 0) { + const callback = afterEachCallbacks.pop(); + if (callback) await callback(); + } + }); + + it("Post-merge, run for a few blocks", async function () { + console.log("\n\nPost-merge, run for a few blocks\n\n"); + const {elClient, tearDownCallBack} = await runEL( + {...elSetupConfig, mode: ELStartMode.PostMerge, genesisTemplate: "il.tmpl"}, + {...elRunOptions, ttd: BigInt(0)}, + controller.signal + ); + afterEachCallbacks.push(() => tearDownCallBack()); + + await runNodeWithEL({ + elClient, + denebEpoch: 0, + testName: "post-merge", + }); + }); + + async function runNodeWithEL({ + elClient, + denebEpoch, + testName, + }: { + elClient: ELClient; + denebEpoch: Epoch; + testName: string; + }): Promise { + const {genesisBlockHash, ttd, engineRpcUrl, ethRpcUrl} = elClient; + const validatorClientCount = 1; + const validatorsPerClient = 8; + + const testParams: Pick = { + SECONDS_PER_SLOT: 2, + }; + + // Just finish the run within first epoch as we only need to test if withdrawals started + const expectedEpochsToFinish = 1; + // 1 epoch of margin of error + const epochsOfMargin = 1; + const timeoutSetupMargin = 30 * 1000; // Give extra 30 seconds of margin + + // delay a bit so that test is over the startup cpu surge that can cause timeouts + // somehow this seems to be dependent on the number of the bns we start which calls + // for some debugging + const genesisSlotsDelay = 4; + + // On the emprical runs 11 blobs are processed, leaving 3 blobs marging + const expectedBlobs = 8; + + // Keep timeout high for variour sync modes to be tested + const timeout = + ((epochsOfMargin + 10 * expectedEpochsToFinish) * SLOTS_PER_EPOCH + genesisSlotsDelay) * + testParams.SECONDS_PER_SLOT * + 1000; + vi.setConfig({testTimeout: timeout + 2 * timeoutSetupMargin}); + + const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT; + + const testLoggerOpts: TestLoggerOpts = { + level: LogLevel.info, + file: { + filepath: `${logFilesDir}/4844-${testName}.log`, + level: LogLevel.debug, + }, + timestampFormat: { + format: TimestampFormatCode.EpochSlot, + genesisTime, + slotsPerEpoch: SLOTS_PER_EPOCH, + secondsPerSlot: testParams.SECONDS_PER_SLOT, + }, + }; + const loggerNodeA = testLogger("Node-A", testLoggerOpts); + + const bn = await getDevBeaconNode({ + params: { + ...testParams, + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: 0, + TERMINAL_TOTAL_DIFFICULTY: ttd, + }, + options: { + api: {rest: {enabled: true} as BeaconRestApiServerOpts}, + sync: {isSingleNode: true}, + network: {allowPublishToZeroPeers: true, rateLimitMultiplier: 0}, + // Now eth deposit/merge tracker methods directly available on engine endpoints + eth1: {enabled: false, providerUrls: [engineRpcUrl], jwtSecretHex}, + executionEngine: {urls: [engineRpcUrl], jwtSecretHex}, + chain: {suggestedFeeRecipient: "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, + }, + validatorCount: validatorClientCount * validatorsPerClient, + logger: loggerNodeA, + genesisTime, + eth1BlockHash: fromHexString(genesisBlockHash), + withEth1Credentials: true, + }); + + afterEachCallbacks.push(async function () { + await bn.close(); + await sleep(1000); + }); + + const stopInfoTracker = simTestInfoTracker(bn, loggerNodeA); + const valProposerConfig = { + defaultConfig: { + feeRecipient: "0xcccccccccccccccccccccccccccccccccccccccc", + }, + } as ValidatorProposerConfig; + const {data: bnIdentity} = await bn.api.node.getNetworkIdentity(); + + const {validators} = await getAndInitDevValidators({ + logPrefix: "Node-A", + node: bn, + validatorsPerClient, + validatorClientCount, + startIndex: 0, + testLoggerOpts, + valProposerConfig, + }); + + afterEachCallbacks.push(async function () { + await Promise.all(validators.map((v) => v.close())); + }); + + // Start range sync from the bn but using the same execution node + const loggerNodeB = testLogger("Node-B", { + ...testLoggerOpts, + file: { + filepath: `${logFilesDir}/4844-${testName}-B.log`, + level: LogLevel.debug, + }, + }); + const unknownSyncBN = await getDevBeaconNode({ + params: { + ...testParams, + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: 0, + TERMINAL_TOTAL_DIFFICULTY: ttd, + }, + options: { + api: {rest: {enabled: false} as BeaconRestApiServerOpts}, + network: {allowPublishToZeroPeers: true, discv5: null}, + // Now eth deposit/merge tracker methods directly available on engine endpoints + eth1: {enabled: false, providerUrls: [engineRpcUrl], jwtSecretHex}, + executionEngine: {urls: [engineRpcUrl], jwtSecretHex}, + chain: { + disableImportExecutionFcU: true, + }, + }, + validatorCount: validatorClientCount * validatorsPerClient, + logger: loggerNodeB, + genesisTime, + eth1BlockHash: fromHexString(genesisBlockHash), + withEth1Credentials: true, + }); + + // Start range sync from the bn but using the same execution node + const loggerNodeC = testLogger("Node-C", { + ...testLoggerOpts, + level: LogLevel.debug, + file: { + filepath: `${logFilesDir}/4844-${testName}-C.log`, + level: LogLevel.debug, + }, + }); + const rangeSyncBN = await getDevBeaconNode({ + params: { + ...testParams, + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: 0, + TERMINAL_TOTAL_DIFFICULTY: ttd, + }, + options: { + api: {rest: {enabled: false} as BeaconRestApiServerOpts}, + network: {allowPublishToZeroPeers: true, discv5: null}, + // Now eth deposit/merge tracker methods directly available on engine endpoints + eth1: {enabled: false, providerUrls: [engineRpcUrl], jwtSecretHex}, + executionEngine: {urls: [engineRpcUrl], jwtSecretHex}, + chain: { + disableImportExecutionFcU: true, + }, + }, + validatorCount: validatorClientCount * validatorsPerClient, + logger: loggerNodeC, + genesisTime, + eth1BlockHash: fromHexString(genesisBlockHash), + withEth1Credentials: true, + }); + + const blobTxs = getBlobTxsFromFile(blobTxsPath); + const blobTxsIdx = 0; + + // bn.chain.clock.on(ClockEvent.slot, (slot) => { + // // send raw tx every other slot + // if (slot > 0 && slot % 2 === 1 && blobTxs[blobTxsIdx] !== undefined) { + // sendRawTransactionBig(ethRpcUrl, blobTxs[blobTxsIdx], `${dataPath}/blobTx-${blobTxsIdx}.json`) + // .then(() => { + // // increment if blobTx has been transmitted successfully + // blobTxsIdx++; + // }) + // .catch((e) => { + // loggerNodeA.error("failed to send raw blob tx", {slot, blobTxsIdx}, e); + // }); + // }4844-i + // }); + + // let bn run for some time and then connect rangeSyncBN + await new Promise((resolve, _reject) => { + bn.chain.clock.on(ClockEvent.epoch, (epoch) => { + // Resolve only if the finalized checkpoint includes execution payload + if (epoch >= expectedEpochsToFinish) { + console.log(`\nGot event ${ClockEvent.epoch}, stopping validators and nodes\n`); + resolve(); + } + }); + }); + + // unknownSyncBN should startup in Synced mode and the gossip should cause unknown parent error + // resulting into sync by root of all the parents + await unknownSyncBN.api.lodestar.connectPeer(bnIdentity.peerId, bnIdentity.p2pAddresses); + await new Promise((resolve, _reject) => { + unknownSyncBN.chain.emitter.on(ChainEvent.forkChoiceFinalized, (finalizedCheckpoint) => { + // Resolve only if the finalized checkpoint includes execution payload + if (finalizedCheckpoint.epoch >= expectedEpochsToFinish) { + console.log(`\nGot event ${ChainEvent.forkChoiceFinalized}, stopping validators and nodes\n`); + resolve(); + } + }); + }); + + // rangeSyncBN should start in syncing mode and range sync through req/resp + await rangeSyncBN.api.lodestar.connectPeer(bnIdentity.peerId, bnIdentity.p2pAddresses); + await new Promise((resolve, _reject) => { + rangeSyncBN.chain.emitter.on(ChainEvent.forkChoiceFinalized, (finalizedCheckpoint) => { + // Resolve only if the finalized checkpoint includes execution payload + if (finalizedCheckpoint.epoch >= expectedEpochsToFinish) { + console.log(`\nGot event ${ChainEvent.forkChoiceFinalized}, stopping validators and nodes\n`); + resolve(); + } + }); + }); + + const bnHeadSlot = Math.min( + bn.chain.forkChoice.getHead().slot, + unknownSyncBN.chain.forkChoice.getHead().slot, + rangeSyncBN.chain.forkChoice.getHead().slot + ); + const foundBlobs = await retrieveCanonicalBlobs(bn, computeStartSlotAtEpoch(denebEpoch), bnHeadSlot); + + const foundBlobsUnknownSync = await retrieveCanonicalBlobs( + unknownSyncBN, + computeStartSlotAtEpoch(denebEpoch), + bnHeadSlot + ); + + const foundBlobsRangeSyncBN = await retrieveCanonicalBlobs( + rangeSyncBN, + computeStartSlotAtEpoch(denebEpoch), + bnHeadSlot + ); + + if (foundBlobs !== foundBlobsUnknownSync || foundBlobs !== foundBlobsRangeSyncBN) { + throw Error( + `Blobs not synced foundBlobs=${foundBlobs} foundBlobsUnknownSync=${foundBlobsUnknownSync} foundBlobsRangeSyncBN=${foundBlobsRangeSyncBN}` + ); + } + + // Stop chain and un-subscribe events so the execution engine won't update it's head + // Allow some time to broadcast finalized events and complete the importBlock routine + await Promise.all(validators.map((v) => v.close())); + await bn.close(); + await sleep(500); + + if (bn.chain.beaconProposerCache.get(1) !== "0xcccccccccccccccccccccccccccccccccccccccc") { + throw Error("Invalid feeRecipient set at BN"); + } + + // Assertions to make sure the end state is good + // 1. The proper head is set + const rpc = new Eth1Provider({DEPOSIT_CONTRACT_ADDRESS: ZERO_HASH}, {providerUrls: [engineRpcUrl], jwtSecretHex}); + const consensusHead = bn.chain.forkChoice.getHead(); + const executionHeadBlock = await rpc.getBlockByNumber("latest"); + + if (!executionHeadBlock) throw Error("Execution has not head block"); + if (consensusHead.executionPayloadBlockHash !== executionHeadBlock.hash) { + throw Error( + "Consensus head not equal to execution head: " + + JSON.stringify({ + executionHeadBlockHash: executionHeadBlock.hash, + consensusHeadExecutionPayloadBlockHash: consensusHead.executionPayloadBlockHash, + consensusHeadSlot: consensusHead.slot, + }) + ); + } + + // Simple check to confirm that withdrawals were mostly processed + if (foundBlobs < expectedBlobs) { + throw Error(`4844 blobs ${foundBlobs} < ${expectedBlobs}`); + } + + // wait for 1 slot to print current epoch stats + await sleep(1 * bn.config.SECONDS_PER_SLOT * 1000); + stopInfoTracker(); + console.log("\n\nDone\n\n"); + } +}); + +async function retrieveCanonicalBlobs(bn: BeaconNode, fromSlot: Slot, toSlot: Slot): Promise { + let eip4844Blobs = 0; + for (let slot = fromSlot; slot <= toSlot; slot++) { + const blobSideCars = await bn.api.beacon.getBlobSidecars(slot).catch((_e: Error) => { + return null; + }); + if (blobSideCars) { + eip4844Blobs += blobSideCars.data.length; + } + } + + return eip4844Blobs; +} + +function getBlobTxsFromFile(blobsPath: string): string[] { + const file = fs.readFileSync(blobsPath, "utf-8"); + return file.split("\n").filter((txn) => txn.length > 0); +} diff --git a/packages/beacon-node/test/utils/logger.ts b/packages/beacon-node/test/utils/logger.ts index 1c1526514565..fcfa34b80765 100644 --- a/packages/beacon-node/test/utils/logger.ts +++ b/packages/beacon-node/test/utils/logger.ts @@ -21,6 +21,6 @@ export const testLogger = (module?: string, opts?: TestLoggerOpts): LoggerNode = opts.module = module; } const level = getEnvLogLevel(); - opts.level = level ?? LogLevel.info; + opts.level = opts?.level ?? level ?? LogLevel.info; return getNodeLogger(opts); }; diff --git a/packages/fork-choice/src/forkChoice/forkChoice.ts b/packages/fork-choice/src/forkChoice/forkChoice.ts index 374bc65542ee..8fbaa8119492 100644 --- a/packages/fork-choice/src/forkChoice/forkChoice.ts +++ b/packages/fork-choice/src/forkChoice/forkChoice.ts @@ -1,7 +1,7 @@ import {toHexString} from "@chainsafe/ssz"; import {fromHex} from "@lodestar/utils"; import {SLOTS_PER_HISTORICAL_ROOT, SLOTS_PER_EPOCH, INTERVALS_PER_SLOT} from "@lodestar/params"; -import {bellatrix, Slot, ValidatorIndex, phase0, allForks, ssz, RootHex, Epoch, Root} from "@lodestar/types"; +import {bellatrix, Slot, ValidatorIndex, phase0, allForks, ssz, RootHex, Epoch, Root, electra} from "@lodestar/types"; import { computeSlotsSinceEpochStart, computeStartSlotAtEpoch, @@ -26,6 +26,8 @@ import { MaybeValidExecutionStatus, LVHExecResponse, ProtoNode, + InclusionListStatus, + MayBeValidExecutionPayloadStatuses, } from "../protoArray/interface.js"; import {ProtoArray} from "../protoArray/protoArray.js"; import {ProtoArrayError, ProtoArrayErrorCode} from "../protoArray/errors.js"; @@ -290,7 +292,11 @@ export class ForkChoice implements IForkChoice { state: CachedBeaconStateAllForks, blockDelaySec: number, currentSlot: Slot, - executionStatus: MaybeValidExecutionStatus + payloadStatusesInfo: { + executionStatus: MaybeValidExecutionStatus; + ilStatus: InclusionListStatus; + inclusionList?: electra.InclusionList; + } ): ProtoBlock { const {parentRoot, slot} = block; const parentRootHex = toHexString(parentRoot); @@ -465,9 +471,13 @@ export class ForkChoice implements IForkChoice { ? { executionPayloadBlockHash: toHexString(block.body.executionPayload.blockHash), executionPayloadNumber: block.body.executionPayload.blockNumber, - executionStatus: this.getPostMergeExecStatus(executionStatus), + ...this.getPostMergeExecStatuses(payloadStatusesInfo), } - : {executionPayloadBlockHash: null, executionStatus: this.getPreMergeExecStatus(executionStatus)}), + : { + executionPayloadBlockHash: null, + executionStatus: this.getPreMergeExecStatus(payloadStatusesInfo.executionStatus), + ilStatus: InclusionListStatus.PreIL, + }), }; this.protoArray.onBlock(protoBlock, currentSlot); @@ -905,14 +915,31 @@ export class ForkChoice implements IForkChoice { return executionStatus; } - private getPostMergeExecStatus( - executionStatus: MaybeValidExecutionStatus - ): ExecutionStatus.Valid | ExecutionStatus.Syncing { + private getPostMergeExecStatuses({ + executionStatus, + ilStatus, + inclusionList, + }: { + executionStatus: MaybeValidExecutionStatus; + ilStatus: InclusionListStatus; + inclusionList?: electra.InclusionList; + }): MayBeValidExecutionPayloadStatuses { if (executionStatus === ExecutionStatus.PreMerge) throw Error( `Invalid post-merge execution status: expected: ${ExecutionStatus.Syncing} or ${ExecutionStatus.Valid} , got ${executionStatus}` ); - return executionStatus; + + if (ilStatus === InclusionListStatus.ValidChild) { + throw Error("ValidChild status can only be marked as exec status propagation"); + } + if ([InclusionListStatus.PreIL, InclusionListStatus.Syncing].includes(ilStatus)) { + return {executionStatus, ilStatus} as MayBeValidExecutionPayloadStatuses; + } else { + if (inclusionList === undefined) { + throw Error("Missing inclusionList for valid ilStatus"); + } + return {executionStatus, ilStatus, inclusionList} as MayBeValidExecutionPayloadStatuses; + } } /** diff --git a/packages/fork-choice/src/forkChoice/interface.ts b/packages/fork-choice/src/forkChoice/interface.ts index fffbc3e4007f..6e9a127c9029 100644 --- a/packages/fork-choice/src/forkChoice/interface.ts +++ b/packages/fork-choice/src/forkChoice/interface.ts @@ -1,7 +1,13 @@ import {EffectiveBalanceIncrements} from "@lodestar/state-transition"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; -import {Epoch, Slot, ValidatorIndex, phase0, allForks, Root, RootHex} from "@lodestar/types"; -import {ProtoBlock, MaybeValidExecutionStatus, LVHExecResponse, ProtoNode} from "../protoArray/interface.js"; +import {Epoch, Slot, ValidatorIndex, phase0, allForks, Root, RootHex, electra} from "@lodestar/types"; +import { + ProtoBlock, + MaybeValidExecutionStatus, + LVHExecResponse, + ProtoNode, + InclusionListStatus, +} from "../protoArray/interface.js"; import {CheckpointWithHex} from "./store.js"; export type CheckpointHex = { @@ -108,7 +114,11 @@ export interface IForkChoice { state: CachedBeaconStateAllForks, blockDelaySec: number, currentSlot: Slot, - executionStatus: MaybeValidExecutionStatus + payloadStatusesInfo: { + executionStatus: MaybeValidExecutionStatus; + ilStatus: InclusionListStatus; + inclusionList?: electra.InclusionList; + } ): ProtoBlock; /** * Register `attestation` with the fork choice DAG so that it may influence future calls to `getHead`. diff --git a/packages/fork-choice/src/index.ts b/packages/fork-choice/src/index.ts index ff0711599a54..4a130ac2c6d8 100644 --- a/packages/fork-choice/src/index.ts +++ b/packages/fork-choice/src/index.ts @@ -7,7 +7,7 @@ export type { LVHValidResponse, LVHInvalidResponse, } from "./protoArray/interface.js"; -export {ExecutionStatus} from "./protoArray/interface.js"; +export {ExecutionStatus, InclusionListStatus} from "./protoArray/interface.js"; export {ForkChoice, type ForkChoiceOpts, assertValidTerminalPowBlock} from "./forkChoice/forkChoice.js"; export { diff --git a/packages/fork-choice/src/protoArray/interface.ts b/packages/fork-choice/src/protoArray/interface.ts index 003a3c8f9f1e..649099326642 100644 --- a/packages/fork-choice/src/protoArray/interface.ts +++ b/packages/fork-choice/src/protoArray/interface.ts @@ -1,4 +1,4 @@ -import {Epoch, Slot, RootHex, UintNum64} from "@lodestar/types"; +import {Epoch, Slot, RootHex, UintNum64, electra} from "@lodestar/types"; // RootHex is a root as a hex string // Used for lightweight and easy comparison @@ -35,13 +35,44 @@ export type LVHExecResponse = LVHValidResponse | LVHInvalidResponse; export type MaybeValidExecutionStatus = Exclude; -export type BlockExecution = +/** + * We only track if a valid inclusion list is available or unavail which is + * basically syncing. when an EL shows valid status for a block, the IL + * is valid for all the ancestors and can be constructed if need be if one + * has to propose from the Valid + */ +export enum InclusionListStatus { + PreIL = "PreIL", + Syncing = "Syncing", + Valid = "Valid", + // valid because there is a valid child/decensant + ValidChild = "ValidChild", +} + +export type MayBeValidExecutionPayloadStatuses = + | { + executionStatus: ExecutionStatus.Valid | ExecutionStatus.Syncing; + ilStatus: InclusionListStatus.Syncing | InclusionListStatus.PreIL; + } + | {executionStatus: ExecutionStatus.Valid; ilStatus: InclusionListStatus.ValidChild; validILChild: number} | { + executionStatus: ExecutionStatus.Valid; + ilStatus: InclusionListStatus.Valid; + inclusionList: electra.ILTransactions; + }; + +export type BlockExecution = + | ({ executionPayloadBlockHash: RootHex; executionPayloadNumber: UintNum64; - executionStatus: Exclude; - } - | {executionPayloadBlockHash: null; executionStatus: ExecutionStatus.PreMerge}; + } & ( + | { + executionStatus: ExecutionStatus.Invalid; + ilStatus: InclusionListStatus.Syncing | InclusionListStatus.PreIL; + } + | MayBeValidExecutionPayloadStatuses + )) + | {executionPayloadBlockHash: null; executionStatus: ExecutionStatus.PreMerge; ilStatus: InclusionListStatus.PreIL}; /** * A block that is to be applied to the fork choice * diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index eaa86b2f0ee1..bb275c790592 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -4,7 +4,14 @@ import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-trans import {GENESIS_EPOCH} from "@lodestar/params"; import {ForkChoiceError, ForkChoiceErrorCode} from "../forkChoice/errors.js"; -import {ProtoBlock, ProtoNode, HEX_ZERO_HASH, ExecutionStatus, LVHExecResponse} from "./interface.js"; +import { + ProtoBlock, + ProtoNode, + HEX_ZERO_HASH, + ExecutionStatus, + LVHExecResponse, + InclusionListStatus, +} from "./interface.js"; import {ProtoArrayError, ProtoArrayErrorCode, LVHExecError, LVHExecErrorCode} from "./errors.js"; export const DEFAULT_PRUNE_THRESHOLD = 0; @@ -312,13 +319,15 @@ export class ProtoArray { private propagateValidExecutionStatusByIndex(validNodeIndex: number): void { let nodeIndex: number | undefined = validNodeIndex; + let childNodeIndex: number | undefined = undefined; // propagate till we keep encountering syncing status while (nodeIndex !== undefined) { const node = this.getNodeFromIndex(nodeIndex); if (node.executionStatus === ExecutionStatus.PreMerge || node.executionStatus === ExecutionStatus.Valid) { break; } - this.validateNodeByIndex(nodeIndex); + this.validateNodeByIndex(nodeIndex, childNodeIndex); + childNodeIndex = nodeIndex; nodeIndex = node.parent; } } @@ -419,7 +428,7 @@ export class ProtoArray { return invalidNode; } - private validateNodeByIndex(nodeIndex: number): ProtoNode { + private validateNodeByIndex(nodeIndex: number, validChildIndex?: number): ProtoNode { const validNode = this.getNodeFromIndex(nodeIndex); if (validNode.executionStatus === ExecutionStatus.Invalid) { this.lvhError = { @@ -433,6 +442,9 @@ export class ProtoArray { }); } else if (validNode.executionStatus === ExecutionStatus.Syncing) { validNode.executionStatus = ExecutionStatus.Valid; + if (validChildIndex !== undefined && validNode.ilStatus === InclusionListStatus.Syncing) { + Object.assign(validNode, {ilStatus: InclusionListStatus.ValidChild, validILChild: validChildIndex}); + } } return validNode; } diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index 7ac4da4aeecb..b110024b61ed 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -1,4 +1,4 @@ -import {allForks, bellatrix, capella, deneb, isBlindedBeaconBlockBody, ssz} from "@lodestar/types"; +import {allForks, bellatrix, capella, deneb, electra, isBlindedBeaconBlockBody, ssz} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; import { @@ -163,12 +163,17 @@ export function executionPayloadToPayloadHeader( if (fork >= ForkSeq.deneb) { // https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#process_execution_payload (bellatrixPayloadFields as deneb.ExecutionPayloadHeader).blobGasUsed = ( - payload as deneb.ExecutionPayloadHeader | deneb.ExecutionPayload + payload as deneb.ExecutionPayload ).blobGasUsed; (bellatrixPayloadFields as deneb.ExecutionPayloadHeader).excessBlobGas = ( - payload as deneb.ExecutionPayloadHeader | deneb.ExecutionPayload + payload as deneb.ExecutionPayload ).excessBlobGas; } + if (fork >= ForkSeq.electra) { + (bellatrixPayloadFields as electra.ExecutionPayloadHeader).previousInclusionListSummaryRoot = + ssz.electra.InclusionListSummary.hashTreeRoot((payload as electra.ExecutionPayload).previousInclusionListSummary); + } + return bellatrixPayloadFields; } diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 58f07ebd484f..3d5e2529ad1f 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -12,33 +12,56 @@ import {ssz as bellatrixSsz} from "../bellatrix/index.js"; import {ssz as capellaSsz} from "../capella/index.js"; import {ssz as denebSsz} from "../deneb/index.js"; -const {BLSSignature, ExecutionAddress, Root, UintBn256, BLSPubkey, ValidatorIndex, Slot, UintNum64} = primitiveSsz; +const {BLSSignature, ExecutionAddress, Root, UintBn256, BLSPubkey, Slot, UintNum64, ValidatorIndex} = primitiveSsz; -export const ILSummary = new ListCompositeType(ExecutionAddress, MAX_TRANSACTIONS_PER_INCLUSION_LIST); +export const InclusionListSummaryEntry = new ContainerType( + { + address: ExecutionAddress, + nonce: UintNum64, + }, + {typeName: "InclusionListSummaryEntry", jsonCase: "eth2"} +); +export const ILSummaryEntryList = new ListCompositeType(InclusionListSummaryEntry, MAX_TRANSACTIONS_PER_INCLUSION_LIST); export const ILTransactions = new ListCompositeType(bellatrixSsz.Transaction, MAX_TRANSACTIONS_PER_INCLUSION_LIST); +export const InclusionListSummary = new ContainerType( + { + slot: Slot, + proposerIndex: ValidatorIndex, + parentHash: Root, + summary: ILSummaryEntryList, + }, + {typeName: "InclusionListSummary", jsonCase: "eth2"} +); + export const SignedInclusionListSummary = new ContainerType( { - summary: ILSummary, + message: InclusionListSummary, signature: BLSSignature, }, {typeName: "SignedInclusionListSummary", jsonCase: "eth2"} ); -export const NewInclusionListRequest = new ContainerType( +export const InclusionList = new ContainerType( { - slot: Slot, - inclusionList: ILTransactions, - summary: ILSummary, - parentBlockHash: Root, + signedSummary: SignedInclusionListSummary, + transactions: ILTransactions, }, - {typeName: "NewInclusionListRequest", jsonCase: "eth2"} + {typeName: "InclusionList", jsonCase: "eth2"} +); + +export const SignedInclusionList = new ContainerType( + { + message: InclusionList, + signature: BLSSignature, + }, + {typeName: "SignedInclusionList", jsonCase: "eth2"} ); export const ExecutionPayload = new ContainerType( { ...denebSsz.ExecutionPayload.fields, - previousInclusionListSummary: SignedInclusionListSummary, + previousInclusionListSummary: InclusionListSummary, }, {typeName: "ExecutionPayload", jsonCase: "eth2"} ); @@ -173,8 +196,6 @@ export const BeaconState = new ContainerType( nextWithdrawalValidatorIndex: capellaSsz.BeaconState.fields.nextWithdrawalValidatorIndex, // Deep history valid from Capella onwards historicalSummaries: capellaSsz.BeaconState.fields.historicalSummaries, - // Inclusion list validation - previousProposerIndex: ValidatorIndex, // New in Electra }, {typeName: "BeaconState", jsonCase: "eth2"} ); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index e5ecf598db49..573aba02548e 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -1,10 +1,13 @@ import {ValueOf} from "@chainsafe/ssz"; import * as ssz from "./sszTypes.js"; -export type ILSummary = ValueOf; +export type InclusionListSummaryEntry = ValueOf; +export type ILSummaryEntryList = ValueOf; export type ILTransactions = ValueOf; +export type InclusionListSummary = ValueOf; export type SignedInclusionListSummary = ValueOf; -export type NewInclusionListRequest = ValueOf; +export type InclusionList = ValueOf; +export type SignedInclusionList = ValueOf; export type ExecutionPayload = ValueOf; export type ExecutionPayloadHeader = ValueOf;