From a0ce52155c5f53b582409b608bed5e43e70079e8 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 19 Dec 2024 22:58:58 +0100 Subject: [PATCH 1/8] Hardcode Bellatrix TTD blocks for all known networks --- .../networks/Eth2NetworkConfiguration.java | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java b/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java index 1842ee7dff5..a1df6344002 100644 --- a/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java +++ b/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java @@ -802,7 +802,11 @@ public Builder applyMainnetNetworkDefaults() { // Nimbus "enr:-LK4QA8FfhaAjlb_BXsXxSfiysR7R52Nhi9JBt4F8SPssu8hdE1BXQQEtVDC3qStCW60LSO7hEsVHv5zm8_6Vnjhcn0Bh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhAN4aBKJc2VjcDI1NmsxoQJerDhsJ-KxZ8sHySMOCmTO6sHM3iCFQ6VMvLTe948MyYN0Y3CCI4yDdWRwgiOM", - "enr:-LK4QKWrXTpV9T78hNG6s8AM6IO4XH9kFT91uZtFg1GcsJ6dKovDOr1jtAAFPnS2lvNltkOGA9k29BUN7lFh_sjuc9QBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhANAdd-Jc2VjcDI1NmsxoQLQa6ai7y9PMN5hpLe5HmiJSlYzMuzP7ZhwRiwHvqNXdoN0Y3CCI4yDdWRwgiOM"); + "enr:-LK4QKWrXTpV9T78hNG6s8AM6IO4XH9kFT91uZtFg1GcsJ6dKovDOr1jtAAFPnS2lvNltkOGA9k29BUN7lFh_sjuc9QBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpC1MD8qAAAAAP__________gmlkgnY0gmlwhANAdd-Jc2VjcDI1NmsxoQLQa6ai7y9PMN5hpLe5HmiJSlYzMuzP7ZhwRiwHvqNXdoN0Y3CCI4yDdWRwgiOM") + .terminalBlockHashEpochOverride(UInt64.valueOf(146875)) + .terminalBlockHashOverride( + Bytes32.fromHexString( + "0x55b11b918355b1ef9c5db810302ebad0bf2544255b530cdce90674d5887bb286")); } private Builder applySepoliaNetworkDefaults() { @@ -827,7 +831,11 @@ private Builder applySepoliaNetworkDefaults() { // Another bootnode "enr:-L64QC9Hhov4DhQ7mRukTOz4_jHm4DHlGL726NWH4ojH1wFgEwSin_6H95Gs6nW2fktTWbPachHJ6rUFu0iJNgA0SB2CARqHYXR0bmV0c4j__________4RldGgykDb6UBOQAABx__________-CaWSCdjSCaXCEA-2vzolzZWNwMjU2azGhA17lsUg60R776rauYMdrAz383UUgESoaHEzMkvm4K6k6iHN5bmNuZXRzD4N0Y3CCIyiDdWRwgiMo", // Lodestart bootnode - "enr:-KG4QJejf8KVtMeAPWFhN_P0c4efuwu1pZHELTveiXUeim6nKYcYcMIQpGxxdgT2Xp9h-M5pr9gn2NbbwEAtxzu50Y8BgmlkgnY0gmlwhEEVkQCDaXA2kCoBBPnAEJg4AAAAAAAAAAGJc2VjcDI1NmsxoQLEh_eVvk07AQABvLkTGBQTrrIOQkzouMgSBtNHIRUxOIN1ZHCCIyiEdWRwNoIjKA"); + "enr:-KG4QJejf8KVtMeAPWFhN_P0c4efuwu1pZHELTveiXUeim6nKYcYcMIQpGxxdgT2Xp9h-M5pr9gn2NbbwEAtxzu50Y8BgmlkgnY0gmlwhEEVkQCDaXA2kCoBBPnAEJg4AAAAAAAAAAGJc2VjcDI1NmsxoQLEh_eVvk07AQABvLkTGBQTrrIOQkzouMgSBtNHIRUxOIN1ZHCCIyiEdWRwNoIjKA") + .terminalBlockHashEpochOverride(UInt64.valueOf(3599)) + .terminalBlockHashOverride( + Bytes32.fromHexString( + "0xd07cce9785d39c0dd2409b7d8e69d6bff26a69a0fa5308ac781c63ffe2a37bc1")); } private Builder applyLuksoNetworkDefaults() { @@ -860,7 +868,11 @@ public Builder applyGnosisNetworkDefaults() { "enr:-Ly4QCD5D99p36WafgTSxB6kY7D2V1ca71C49J4VWI2c8UZCCPYBvNRWiv0-HxOcbpuUdwPVhyWQCYm1yq2ZH0ukCbQBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCCS-QxAgAAZP__________gmlkgnY0gmlwhI1eYVSJc2VjcDI1NmsxoQJJMSV8iSZ8zvkgbi8cjIGEUVJeekLqT0LQha_co-siT4hzeW5jbmV0cwCDdGNwgiMog3VkcIIjKA", "enr:-KK4QKXJq1QOVWuJAGige4uaT8LRPQGCVRf3lH3pxjaVScMRUfFW1eiiaz8RwOAYvw33D4EX-uASGJ5QVqVCqwccxa-Bi4RldGgykCGm-DYDAABk__________-CaWSCdjSCaXCEM0QnzolzZWNwMjU2azGhAhNvrRkpuK4MWTf3WqiOXSOePL8Zc-wKVpZ9FQx_BDadg3RjcIIjKIN1ZHCCIyg", "enr:-LO4QO87Rn2ejN3SZdXkx7kv8m11EZ3KWWqoIN5oXwQ7iXR9CVGd1dmSyWxOL1PGsdIqeMf66OZj4QGEJckSi6okCdWBpIdhdHRuZXRziAAAAABgAAAAhGV0aDKQPr_UhAQAAGT__________4JpZIJ2NIJpcIQj0iX1iXNlY3AyNTZrMaEDd-_eqFlWWJrUfEp8RhKT9NxdYaZoLHvsp3bbejPyOoeDdGNwgiMog3VkcIIjKA", - "enr:-LK4QIJUAxX9uNgW4ACkq8AixjnSTcs9sClbEtWRq9F8Uy9OEExsr4ecpBTYpxX66cMk6pUHejCSX3wZkK2pOCCHWHEBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpA-v9SEBAAAZP__________gmlkgnY0gmlwhCPSnDuJc2VjcDI1NmsxoQNuaAjFE-ANkH3pbeBdPiEIwjR5kxFuKaBWxHkqFuPz5IN0Y3CCIyiDdWRwgiMo"); + "enr:-LK4QIJUAxX9uNgW4ACkq8AixjnSTcs9sClbEtWRq9F8Uy9OEExsr4ecpBTYpxX66cMk6pUHejCSX3wZkK2pOCCHWHEBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpA-v9SEBAAAZP__________gmlkgnY0gmlwhCPSnDuJc2VjcDI1NmsxoQNuaAjFE-ANkH3pbeBdPiEIwjR5kxFuKaBWxHkqFuPz5IN0Y3CCIyiDdWRwgiMo") + .terminalBlockHashEpochOverride(UInt64.valueOf(394147)) + .terminalBlockHashOverride( + Bytes32.fromHexString( + "0xf5cff68065ac6014bb7c9aa731d4d4084de0994f807ac1df3856308b3c9b2b48")); } public Builder applyChiadoNetworkDefaults() { @@ -891,7 +903,11 @@ public Builder applyChiadoNetworkDefaults() { // GnosisDAO Bootnode: 35.206.174.92 "enr:-KG4QF7z4LUdMfgwvh-fS-MDv_1hPSUCqGfyOWGLNJuoBHKFAMSHz8geQn8v3qDDbuSQKud3WIAjKqR4gqJoLBUEJ08ZhGV0aDKQDc1ElgAAAG___________4JpZIJ2NIJpcIQjzq5ciXNlY3AyNTZrMaECt7YO363pV54d3QdgnluL5kxzhCR_k0yM9C-G6bqMGoKDdGNwgiMog3VkcIIjKA", // GnosisDAO Bootnode: 35.210.126.23 - "enr:-LK4QCUTEmZrT1AgCKdyVgwnHL5J0VSoxsyjruAtwo-owBTBVEOyAnQRVNXlcW5aL-ycntk5oHDrKCR-DXZAlUAKpjEBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCdM7Z1BAAAb___________gmlkgnY0gmlwhCPSfheJc2VjcDI1NmsxoQNpdf8U9pzsU9m6Hzgd1rmTI-On-QImJnkZBGqDp4org4N0Y3CCIyiDdWRwgiMo"); + "enr:-LK4QCUTEmZrT1AgCKdyVgwnHL5J0VSoxsyjruAtwo-owBTBVEOyAnQRVNXlcW5aL-ycntk5oHDrKCR-DXZAlUAKpjEBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCdM7Z1BAAAb___________gmlkgnY0gmlwhCPSfheJc2VjcDI1NmsxoQNpdf8U9pzsU9m6Hzgd1rmTI-On-QImJnkZBGqDp4org4N0Y3CCIyiDdWRwgiMo") + .terminalBlockHashEpochOverride(UInt64.valueOf(27264)) + .terminalBlockHashOverride( + Bytes32.fromHexString( + "0x39f44fc16dc964e8d2d1637b99e12992be4a4f766a66658da730e20e511ced64")); } private Builder applyHoleskyNetworkDefaults() { From b085281b19eae881a32dc1004c90e2901d215546 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 20 Dec 2024 18:35:10 +0100 Subject: [PATCH 2/8] reset should affect total difficult overrides --- .../tech/pegasys/teku/networks/Eth2NetworkConfiguration.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java b/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java index a1df6344002..2457f35dc9c 100644 --- a/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java +++ b/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java @@ -751,6 +751,9 @@ private Builder reset() { eth1DepositContractAddress = null; eth1DepositContractDeployBlock = Optional.empty(); trustedSetup = Optional.empty(); + terminalBlockHashOverride = Optional.empty(); + terminalBlockHashEpochOverride = Optional.empty(); + totalTerminalDifficultyOverride = Optional.empty(); return this; } From 8cccf019ffd7a13cc0e86dec511fa48011b3d7dc Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 20 Dec 2024 19:02:51 +0100 Subject: [PATCH 3/8] fix test for default (mainnet) --- .../pegasys/teku/cli/options/Eth2NetworkOptionsTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/teku/src/test/java/tech/pegasys/teku/cli/options/Eth2NetworkOptionsTest.java b/teku/src/test/java/tech/pegasys/teku/cli/options/Eth2NetworkOptionsTest.java index d90f2eb03e0..495133ccc55 100644 --- a/teku/src/test/java/tech/pegasys/teku/cli/options/Eth2NetworkOptionsTest.java +++ b/teku/src/test/java/tech/pegasys/teku/cli/options/Eth2NetworkOptionsTest.java @@ -127,14 +127,16 @@ void shouldUseAlwaysSendPayloadAttributesIfSpecified() { } @Test - void shouldMergeTransitionsOverrideBeEmptyByDefault() { + void shouldMergeTransitionsOverrideContainsMainnetTransitionByDefault() { final TekuConfiguration config = getTekuConfigurationFromArguments(); assertThat(config.eth2NetworkConfiguration().getTotalTerminalDifficultyOverride()) .isEqualTo(Optional.empty()); assertThat(config.eth2NetworkConfiguration().getTerminalBlockHashOverride()) - .isEqualTo(Optional.empty()); + .contains( + Bytes32.fromHexString( + "0x55b11b918355b1ef9c5db810302ebad0bf2544255b530cdce90674d5887bb286")); assertThat(config.eth2NetworkConfiguration().getTerminalBlockHashEpochOverride()) - .isEqualTo(Optional.empty()); + .contains(UInt64.valueOf(146875)); } @Test From c118e705a939ef0d1e50595b3d8eaf9995bd365e Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 20 Dec 2024 21:07:09 +0100 Subject: [PATCH 4/8] Add verification EL is never called when transition block hash set --- .../bellatrix/helpers/BellatrixTransitionHelpersTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/bellatrix/helpers/BellatrixTransitionHelpersTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/bellatrix/helpers/BellatrixTransitionHelpersTest.java index a95bc2e08ae..f4139234907 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/bellatrix/helpers/BellatrixTransitionHelpersTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/bellatrix/helpers/BellatrixTransitionHelpersTest.java @@ -16,6 +16,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import static tech.pegasys.teku.infrastructure.async.SafeFuture.completedFuture; @@ -97,6 +98,8 @@ void shouldBeValidWhenTerminalBlockHashMatchesInActivationEpoch() { .getBellatrixTransitionHelpers() .orElseThrow() .validateMergeBlock(executionLayer, payload, blockSlot); + // Verify EL is never called if hash is configured and matches + verifyNoMoreInteractions(executionLayer); assertPayloadResultStatus(result, ExecutionPayloadStatus.VALID); } From e0a1e8d543798d8e5789c9a1fcbff8d23f2f0547 Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Sat, 28 Dec 2024 12:41:06 +0200 Subject: [PATCH 5/8] No delay when fetching blobs with known block and no attempt to recover blobs for unknown block (#8927) --- CHANGELOG.md | 1 + .../deneb/helpers/MiscHelpersDeneb.java | 30 +++ .../blobs/BlobSidecarManagerImpl.java | 4 +- .../blobs/BlockBlobSidecarsTracker.java | 101 ++++---- .../BlockBlobSidecarsTrackersPoolImpl.java | 224 ++++++++---------- .../blobs/BlockBlobSidecarsTrackerTest.java | 130 ++-------- ...BlockBlobSidecarsTrackersPoolImplTest.java | 151 +++--------- 7 files changed, 240 insertions(+), 401 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d7628c3eec..a6cc88b16f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,5 +10,6 @@ ### Additions and Improvements - Optimized blobs validation pipeline +- Remove delay when fetching blobs from the local EL on block arrival ### Bug Fixes \ No newline at end of file diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java index ebf9e188b45..87953843804 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java @@ -37,8 +37,12 @@ import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarSchema; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; +import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlockHeader; import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBody; +import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodyDeneb; import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodySchemaDeneb; +import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; +import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.BlobIdentifier; import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment; import tech.pegasys.teku.spec.datastructures.type.SszKZGProof; import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; @@ -266,6 +270,32 @@ public BlobSidecar constructBlobSidecar( index, blob, commitment, proof, signedBeaconBlock.asHeader(), kzgCommitmentInclusionProof); } + public BlobSidecar constructBlobSidecarFromBlobAndProof( + final BlobIdentifier blobIdentifier, + final BlobAndProof blobAndProof, + final BeaconBlockBodyDeneb beaconBlockBodyDeneb, + final SignedBeaconBlockHeader signedBeaconBlockHeader) { + + final SszKZGCommitment sszKZGCommitment = + beaconBlockBodyDeneb.getBlobKzgCommitments().get(blobIdentifier.getIndex().intValue()); + + final BlobSidecar blobSidecar = + blobSidecarSchema.create( + blobIdentifier.getIndex(), + blobAndProof.blob(), + sszKZGCommitment, + new SszKZGProof(blobAndProof.proof()), + signedBeaconBlockHeader, + computeKzgCommitmentInclusionProof(blobIdentifier.getIndex(), beaconBlockBodyDeneb)); + + blobSidecar.markSignatureAsValidated(); + blobSidecar.markKzgCommitmentInclusionProofAsValidated(); + // assume kzg validation done by local EL + blobSidecar.markKzgAsValidated(); + + return blobSidecar; + } + public boolean verifyBlobKzgCommitmentInclusionProof(final BlobSidecar blobSidecar) { if (blobSidecar.isKzgCommitmentInclusionProofValidated()) { return true; diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java index fc20d0c60e8..22609e00b69 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarManagerImpl.java @@ -66,9 +66,7 @@ public BlobSidecarManagerImpl( invalidBlobSidecarRoots, (tracker) -> new ForkChoiceBlobSidecarsAvailabilityChecker(spec, recentChainData, tracker, kzg), - // we don't care to set maxBlobsPerBlock since it isn't used with this immediate validation - // flow - (block) -> new BlockBlobSidecarsTracker(block.getSlotAndBlockRoot(), UInt64.ZERO)); + (block) -> new BlockBlobSidecarsTracker(block.getSlotAndBlockRoot())); } @VisibleForTesting diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java index 64c89d91ba1..83faba90209 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTracker.java @@ -40,13 +40,15 @@ public class BlockBlobSidecarsTracker { private static final Logger LOG = LogManager.getLogger(); + private static final UInt64 CREATION_TIMING_IDX = UInt64.MAX_VALUE; private static final UInt64 BLOCK_ARRIVAL_TIMING_IDX = CREATION_TIMING_IDX.decrement(); - private static final UInt64 RPC_FETCH_TIMING_IDX = BLOCK_ARRIVAL_TIMING_IDX.decrement(); - private static final UInt64 LOCAL_EL_FETCH_TIMING_IDX = RPC_FETCH_TIMING_IDX.decrement(); + private static final UInt64 RPC_BLOCK_FETCH_TIMING_IDX = BLOCK_ARRIVAL_TIMING_IDX.decrement(); + private static final UInt64 RPC_BLOBS_FETCH_TIMING_IDX = RPC_BLOCK_FETCH_TIMING_IDX.decrement(); + private static final UInt64 LOCAL_EL_BLOBS_FETCH_TIMING_IDX = + RPC_BLOBS_FETCH_TIMING_IDX.decrement(); private final SlotAndBlockRoot slotAndBlockRoot; - private final UInt64 maxBlobsPerBlock; private final AtomicReference> block = new AtomicReference<>(Optional.empty()); @@ -56,8 +58,9 @@ public class BlockBlobSidecarsTracker { private final NavigableMap blobSidecars = new ConcurrentSkipListMap<>(); private final SafeFuture blobSidecarsComplete = new SafeFuture<>(); - private volatile boolean rpcFetchTriggered = false; - private volatile boolean localElFetchTriggered = false; + private volatile boolean localElBlobsFetchTriggered = false; + private volatile boolean rpcBlockFetchTriggered = false; + private volatile boolean rpcBlobsFetchTriggered = false; private final Optional> maybeDebugTimings; @@ -69,12 +72,9 @@ public class BlockBlobSidecarsTracker { * tracker instance will be used, so no synchronization is required * * @param slotAndBlockRoot slot and block root to create tracker for - * @param maxBlobsPerBlock max number of blobs per block for the slot */ - public BlockBlobSidecarsTracker( - final SlotAndBlockRoot slotAndBlockRoot, final UInt64 maxBlobsPerBlock) { + public BlockBlobSidecarsTracker(final SlotAndBlockRoot slotAndBlockRoot) { this.slotAndBlockRoot = slotAndBlockRoot; - this.maxBlobsPerBlock = maxBlobsPerBlock; if (LOG.isDebugEnabled()) { // don't need a concurrent hashmap since we'll interact with it from synchronized BlobSidecar // pool methods @@ -112,31 +112,13 @@ public Optional getBlobSidecar(final UInt64 index) { public Stream getMissingBlobSidecars() { final Optional blockCommitmentsCount = getBlockKzgCommitmentsCount(); - if (blockCommitmentsCount.isPresent()) { - return UInt64.range(UInt64.ZERO, UInt64.valueOf(blockCommitmentsCount.get())) - .filter(blobIndex -> !blobSidecars.containsKey(blobIndex)) - .map(blobIndex -> new BlobIdentifier(slotAndBlockRoot.getBlockRoot(), blobIndex)); - } + checkState(blockCommitmentsCount.isPresent(), "Block must be known to call this method"); - if (blobSidecars.isEmpty()) { - return Stream.of(); - } - - // We may return maxBlobsPerBlock because we don't know the block - return UInt64.range(UInt64.ZERO, maxBlobsPerBlock) + return UInt64.range(UInt64.ZERO, UInt64.valueOf(blockCommitmentsCount.get())) .filter(blobIndex -> !blobSidecars.containsKey(blobIndex)) .map(blobIndex -> new BlobIdentifier(slotAndBlockRoot.getBlockRoot(), blobIndex)); } - public Stream getUnusedBlobSidecarsForBlock() { - final Optional blockCommitmentsCount = getBlockKzgCommitmentsCount(); - checkState(blockCommitmentsCount.isPresent(), "Block must me known to call this method"); - - final UInt64 firstUnusedIndex = UInt64.valueOf(blockCommitmentsCount.get()); - return UInt64.range(firstUnusedIndex, maxBlobsPerBlock) - .map(blobIndex -> new BlobIdentifier(slotAndBlockRoot.getBlockRoot(), blobIndex)); - } - public boolean add(final BlobSidecar blobSidecar) { checkArgument( blobSidecar.getBlockRoot().equals(slotAndBlockRoot.getBlockRoot()), @@ -247,24 +229,35 @@ public boolean isComplete() { return blobSidecarsComplete.isDone(); } - public boolean isRpcFetchTriggered() { - return rpcFetchTriggered; + public boolean isLocalElBlobsFetchTriggered() { + return localElBlobsFetchTriggered; } - public void setRpcFetchTriggered() { - this.rpcFetchTriggered = true; + public void setLocalElBlobsFetchTriggered() { + this.localElBlobsFetchTriggered = true; maybeDebugTimings.ifPresent( - debugTimings -> debugTimings.put(RPC_FETCH_TIMING_IDX, System.currentTimeMillis())); + debugTimings -> + debugTimings.put(LOCAL_EL_BLOBS_FETCH_TIMING_IDX, System.currentTimeMillis())); } - public boolean isLocalElFetchTriggered() { - return localElFetchTriggered; + public boolean isRpcBlockFetchTriggered() { + return rpcBlockFetchTriggered; } - public void setLocalElFetchTriggered() { - this.localElFetchTriggered = true; + public void setRpcBlockFetchTriggered() { + this.rpcBlockFetchTriggered = true; maybeDebugTimings.ifPresent( - debugTimings -> debugTimings.put(LOCAL_EL_FETCH_TIMING_IDX, System.currentTimeMillis())); + debugTimings -> debugTimings.put(RPC_BLOCK_FETCH_TIMING_IDX, System.currentTimeMillis())); + } + + public boolean isRpcBlobsFetchTriggered() { + return rpcBlobsFetchTriggered; + } + + public void setRpcBlobsFetchTriggered() { + this.rpcBlobsFetchTriggered = true; + maybeDebugTimings.ifPresent( + debugTimings -> debugTimings.put(RPC_BLOBS_FETCH_TIMING_IDX, System.currentTimeMillis())); } private boolean areBlobsComplete() { @@ -315,22 +308,31 @@ private void printDebugTimings(final Map debugTimings) { .append(debugTimings.getOrDefault(BLOCK_ARRIVAL_TIMING_IDX, 0L) - creationTime) .append("ms - "); - if (debugTimings.containsKey(LOCAL_EL_FETCH_TIMING_IDX)) { + if (debugTimings.containsKey(LOCAL_EL_BLOBS_FETCH_TIMING_IDX)) { + timingsReport + .append("Local EL blobs fetch delay ") + .append(debugTimings.get(LOCAL_EL_BLOBS_FETCH_TIMING_IDX) - creationTime) + .append("ms - "); + } else { + timingsReport.append("Local EL blobs fetch wasn't required - "); + } + + if (debugTimings.containsKey(RPC_BLOCK_FETCH_TIMING_IDX)) { timingsReport - .append("Local EL fetch delay ") - .append(debugTimings.get(LOCAL_EL_FETCH_TIMING_IDX) - creationTime) + .append("RPC block fetch delay ") + .append(debugTimings.get(RPC_BLOCK_FETCH_TIMING_IDX) - creationTime) .append("ms - "); } else { - timingsReport.append("Local EL fetch wasn't required - "); + timingsReport.append("RPC block fetch wasn't required - "); } - if (debugTimings.containsKey(RPC_FETCH_TIMING_IDX)) { + if (debugTimings.containsKey(RPC_BLOBS_FETCH_TIMING_IDX)) { timingsReport - .append("RPC fetch delay ") - .append(debugTimings.get(RPC_FETCH_TIMING_IDX) - creationTime) + .append("RPC blobs fetch delay ") + .append(debugTimings.get(RPC_BLOBS_FETCH_TIMING_IDX) - creationTime) .append("ms"); } else { - timingsReport.append("RPC fetch wasn't required"); + timingsReport.append("RPC blobs fetch wasn't required"); } LOG.debug(timingsReport.toString()); @@ -342,8 +344,9 @@ public String toString() { .add("slotAndBlockRoot", slotAndBlockRoot) .add("isBlockPresent", block.get().isPresent()) .add("isComplete", isComplete()) - .add("rpcFetchTriggered", rpcFetchTriggered) - .add("localElFetchTriggered", localElFetchTriggered) + .add("localElBlobsFetchTriggered", localElBlobsFetchTriggered) + .add("rpcBlockFetchTriggered", rpcBlockFetchTriggered) + .add("rpcBlobsFetchTriggered", rpcBlobsFetchTriggered) .add("blockImportOnCompletionEnabled", blockImportOnCompletionEnabled.get()) .add( "blobSidecars", diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java index 0bfb291d0e2..652e5fc9607 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java @@ -48,9 +48,7 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.SpecVersion; -import tech.pegasys.teku.spec.config.SpecConfigDeneb; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarSchema; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlockHeader; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; @@ -58,7 +56,6 @@ import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof; import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.BlobIdentifier; import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment; -import tech.pegasys.teku.spec.datastructures.type.SszKZGProof; import tech.pegasys.teku.spec.executionlayer.ExecutionLayerChannel; import tech.pegasys.teku.spec.logic.versions.deneb.helpers.MiscHelpersDeneb; import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; @@ -92,6 +89,7 @@ public class BlockBlobSidecarsTrackersPoolImpl extends AbstractIgnoringFutureHis static final String GAUGE_BLOB_SIDECARS_LABEL = "blob_sidecars"; static final String GAUGE_BLOB_SIDECARS_TRACKERS_LABEL = "blob_sidecars_trackers"; + // RPC fetching delay timings static final UInt64 MAX_WAIT_RELATIVE_TO_ATT_DUE_MILLIS = UInt64.valueOf(1500); static final UInt64 MIN_WAIT_MILLIS = UInt64.valueOf(500); static final UInt64 TARGET_WAIT_MILLIS = UInt64.valueOf(1000); @@ -154,7 +152,7 @@ public class BlockBlobSidecarsTrackersPoolImpl extends AbstractIgnoringFutureHis this.maxTrackers = maxTrackers; this.sizeGauge = sizeGauge; this.poolStatsCounters = poolStatsCounters; - this.trackerFactory = (slotAndBlockRoot) -> createTracker(spec, slotAndBlockRoot); + this.trackerFactory = BlockBlobSidecarsTracker::new; initMetrics(sizeGauge, poolStatsCounters); } @@ -209,15 +207,6 @@ private static void initMetrics( }); } - private static BlockBlobSidecarsTracker createTracker( - final Spec spec, final SlotAndBlockRoot slotAndBlockRoot) { - return new BlockBlobSidecarsTracker( - slotAndBlockRoot, - UInt64.valueOf( - SpecConfigDeneb.required(spec.atSlot(slotAndBlockRoot.getSlot()).getConfig()) - .getMaxBlobsPerBlock())); - } - @Override public synchronized void onNewBlobSidecar( final BlobSidecar blobSidecar, final RemoteOrigin remoteOrigin) { @@ -230,12 +219,25 @@ public synchronized void onNewBlobSidecar( final SlotAndBlockRoot slotAndBlockRoot = blobSidecar.getSlotAndBlockRoot(); - final BlockBlobSidecarsTracker blobSidecarsTracker = - getOrCreateBlobSidecarsTracker( - slotAndBlockRoot, - newTracker -> onFirstSeen(slotAndBlockRoot, Optional.of(remoteOrigin)), - existingTracker -> {}); + getOrCreateBlobSidecarsTracker( + slotAndBlockRoot, + newTracker -> { + addBlobSidecarToTracker(newTracker, slotAndBlockRoot, blobSidecar, remoteOrigin); + onFirstSeen(slotAndBlockRoot, Optional.of(remoteOrigin)); + }, + existingTracker -> + addBlobSidecarToTracker(existingTracker, slotAndBlockRoot, blobSidecar, remoteOrigin)); + + if (orderedBlobSidecarsTrackers.add(slotAndBlockRoot)) { + sizeGauge.set(orderedBlobSidecarsTrackers.size(), GAUGE_BLOB_SIDECARS_TRACKERS_LABEL); + } + } + private void addBlobSidecarToTracker( + final BlockBlobSidecarsTracker blobSidecarsTracker, + final SlotAndBlockRoot slotAndBlockRoot, + final BlobSidecar blobSidecar, + final RemoteOrigin remoteOrigin) { if (blobSidecarsTracker.add(blobSidecar)) { sizeGauge.set(++totalBlobSidecars, GAUGE_BLOB_SIDECARS_LABEL); countBlobSidecar(remoteOrigin); @@ -246,10 +248,6 @@ public synchronized void onNewBlobSidecar( } else { countDuplicateBlobSidecar(remoteOrigin); } - - if (orderedBlobSidecarsTrackers.add(slotAndBlockRoot)) { - sizeGauge.set(orderedBlobSidecarsTrackers.size(), GAUGE_BLOB_SIDECARS_TRACKERS_LABEL); - } } private void publishRecoveredBlobSidecar(final BlobSidecar blobSidecar) { @@ -326,7 +324,7 @@ public synchronized void onCompletedBlockAndBlobSidecars( long addedBlobs = blobSidecars.stream() - .map( + .filter( blobSidecar -> { final boolean isNew = blobSidecarsTracker.add(blobSidecar); if (isNew) { @@ -335,7 +333,6 @@ public synchronized void onCompletedBlockAndBlobSidecars( } return isNew; }) - .filter(Boolean::booleanValue) .count(); totalBlobSidecars += (int) addedBlobs; sizeGauge.set(totalBlobSidecars, GAUGE_BLOB_SIDECARS_LABEL); @@ -427,7 +424,13 @@ public synchronized Optional getBlock(final Bytes32 blockRoot @Override public synchronized Set getAllRequiredBlobSidecars() { return blockBlobSidecarsTrackers.values().stream() - .flatMap(BlockBlobSidecarsTracker::getMissingBlobSidecars) + .flatMap( + tracker -> { + if (tracker.getBlock().isEmpty()) { + return Stream.empty(); + } + return tracker.getMissingBlobSidecars(); + }) .collect(Collectors.toSet()); } @@ -474,6 +477,7 @@ public synchronized int getTotalBlobSidecarsTrackers() { return blockBlobSidecarsTrackers.size(); } + @SuppressWarnings("FutureReturnValueIgnored") private BlockBlobSidecarsTracker internalOnNewBlock( final SignedBeaconBlock block, final Optional remoteOrigin) { final SlotAndBlockRoot slotAndBlockRoot = block.getSlotAndBlockRoot(); @@ -495,25 +499,22 @@ private BlockBlobSidecarsTracker internalOnNewBlock( countBlock(remoteOrigin); - if (existingTracker.isRpcFetchTriggered()) { - // block has been set for the first time and we previously triggered fetching of - // missing blobSidecars. So we may have requested to fetch more sidecars - // than the block actually requires. Let's drop them. - existingTracker - .getUnusedBlobSidecarsForBlock() - .forEach( - blobIdentifier -> - requiredBlobSidecarDroppedSubscribers.deliver( - RequiredBlobSidecarDroppedSubscriber::onRequiredBlobSidecarDropped, - blobIdentifier)); - - // if we attempted to fetch via RPC, we missed the opportunity to complete the - // tracker via local EL (local El fetch requires the block to be known) - // Let's try now - if (!existingTracker.isLocalElFetchTriggered() && !existingTracker.isComplete()) { - fetchMissingContentFromLocalEL(slotAndBlockRoot) - .finish(this::logLocalElBlobsLookupFailure); - } + if (!existingTracker.isComplete()) { + // we missed the opportunity to complete the blob sidecars via local EL and RPC + // (since the block is required to be known) Let's try now + asyncRunner.runAsync( + () -> + fetchMissingBlobsFromLocalEL(slotAndBlockRoot) + .handleException(this::logLocalElBlobsLookupFailure) + .thenRun( + () -> { + // only run if RPC block fetch has happened ( no blobs RPC fetch + // has occurred) + if (existingTracker.isRpcBlockFetchTriggered()) { + fetchMissingBlockOrBlobsFromRPC(slotAndBlockRoot); + } + }) + .finish(this::logBlockOrBlobsRPCFailure)); } }); @@ -575,7 +576,7 @@ private BlockBlobSidecarsTracker getOrCreateBlobSidecarsTracker( } private void makeRoomForNewTracker() { - while (blockBlobSidecarsTrackers.size() > (maxTrackers - 1)) { + while (blockBlobSidecarsTrackers.size() > maxTrackers - 1) { final SlotAndBlockRoot toRemove = orderedBlobSidecarsTrackers.pollFirst(); if (toRemove == null) { break; @@ -584,6 +585,7 @@ private void makeRoomForNewTracker() { } } + @SuppressWarnings("FutureReturnValueIgnored") private void onFirstSeen( final SlotAndBlockRoot slotAndBlockRoot, final Optional remoteOrigin) { final boolean isLocalBlockProduction = @@ -591,27 +593,22 @@ private void onFirstSeen( if (isLocalBlockProduction) { return; } + // delay RPC fetching + final SafeFuture rpcFetchDelay = + asyncRunner.getDelayedFuture(calculateRpcFetchDelay(slotAndBlockRoot)); - final Duration fetchDelay = calculateFetchDelay(slotAndBlockRoot); - - asyncRunner - .runAfterDelay( - () -> - this.fetchMissingContentFromLocalEL(slotAndBlockRoot) - .handleException(this::logLocalElBlobsLookupFailure) - .thenRun(() -> this.fetchMissingContentFromRemotePeers(slotAndBlockRoot)), - fetchDelay) - .finish( - error -> - LOG.error("An error occurred while attempting to fetch missing blobs.", error)); - } - - private void logLocalElBlobsLookupFailure(final Throwable error) { - LOG.warn("Local EL blobs lookup failed: {}", getRootCauseMessage(error)); + asyncRunner.runAsync( + () -> + // fetch blobs from EL with no delay + fetchMissingBlobsFromLocalEL(slotAndBlockRoot) + .handleException(this::logLocalElBlobsLookupFailure) + .thenCompose(__ -> rpcFetchDelay) + .thenRun(() -> fetchMissingBlockOrBlobsFromRPC(slotAndBlockRoot)) + .finish(this::logBlockOrBlobsRPCFailure)); } @VisibleForTesting - Duration calculateFetchDelay(final SlotAndBlockRoot slotAndBlockRoot) { + Duration calculateRpcFetchDelay(final SlotAndBlockRoot slotAndBlockRoot) { final UInt64 slot = slotAndBlockRoot.getSlot(); if (slot.isLessThan(getCurrentSlot())) { @@ -641,28 +638,21 @@ Duration calculateFetchDelay(final SlotAndBlockRoot slotAndBlockRoot) { return Duration.ofMillis(finalTime.minus(nowMillis).intValue()); } - private synchronized SafeFuture fetchMissingContentFromLocalEL( + private synchronized SafeFuture fetchMissingBlobsFromLocalEL( final SlotAndBlockRoot slotAndBlockRoot) { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = blockBlobSidecarsTrackers.get(slotAndBlockRoot.getBlockRoot()); - if (blockBlobSidecarsTracker == null) { - return SafeFuture.COMPLETE; - } - - if (blockBlobSidecarsTracker.isComplete()) { - return SafeFuture.COMPLETE; - } - - if (blockBlobSidecarsTracker.getBlock().isEmpty()) { + if (blockBlobSidecarsTracker == null + || blockBlobSidecarsTracker.isComplete() + || blockBlobSidecarsTracker.getBlock().isEmpty()) { return SafeFuture.COMPLETE; } - blockBlobSidecarsTracker.setLocalElFetchTriggered(); + final List missingBlobsIdentifiers = + blockBlobSidecarsTracker.getMissingBlobSidecars().toList(); final SpecVersion specVersion = spec.atSlot(slotAndBlockRoot.getSlot()); - final BlobSidecarSchema blobSidecarSchema = - specVersion.getSchemaDefinitions().toVersionDeneb().orElseThrow().getBlobSidecarSchema(); final MiscHelpersDeneb miscHelpersDeneb = specVersion.miscHelpers().toVersionDeneb().orElseThrow(); final SignedBeaconBlockHeader signedBeaconBlockHeader = @@ -679,9 +669,6 @@ private synchronized SafeFuture fetchMissingContentFromLocalEL( final SszList sszKZGCommitments = beaconBlockBodyDeneb.getBlobKzgCommitments(); - final List missingBlobsIdentifiers = - blockBlobSidecarsTracker.getMissingBlobSidecars().toList(); - final List versionedHashes = missingBlobsIdentifiers.stream() .map( @@ -692,6 +679,8 @@ private synchronized SafeFuture fetchMissingContentFromLocalEL( .getKZGCommitment())) .toList(); + blockBlobSidecarsTracker.setLocalElBlobsFetchTriggered(); + poolStatsCounters .labels(COUNTER_SIDECAR_TYPE, COUNTER_LOCAL_EL_FETCH_SUBTYPE) .inc(versionedHashes.size()); @@ -708,16 +697,15 @@ private synchronized SafeFuture fetchMissingContentFromLocalEL( for (int index = 0; index < blobAndProofs.size(); index++) { final Optional blobAndProof = blobAndProofs.get(index); + final BlobIdentifier blobIdentifier = missingBlobsIdentifiers.get(index); if (blobAndProof.isEmpty()) { - LOG.trace("Blob not found on local EL: {}", missingBlobsIdentifiers.get(index)); + LOG.trace("Blob not found on local EL: {}", blobIdentifier); continue; } final BlobSidecar blobSidecar = - createBlobSidecarFromBlobAndProof( - blobSidecarSchema, - miscHelpersDeneb, - missingBlobsIdentifiers.get(index), + miscHelpersDeneb.constructBlobSidecarFromBlobAndProof( + blobIdentifier, blobAndProof.get(), beaconBlockBodyDeneb, signedBeaconBlockHeader); @@ -726,57 +714,32 @@ private synchronized SafeFuture fetchMissingContentFromLocalEL( }); } - private BlobSidecar createBlobSidecarFromBlobAndProof( - final BlobSidecarSchema blobSidecarSchema, - final MiscHelpersDeneb miscHelpersDeneb, - final BlobIdentifier blobIdentifier, - final BlobAndProof blobAndProof, - final BeaconBlockBodyDeneb beaconBlockBodyDeneb, - final SignedBeaconBlockHeader signedBeaconBlockHeader) { - - final SszKZGCommitment sszKZGCommitment = - beaconBlockBodyDeneb.getBlobKzgCommitments().get(blobIdentifier.getIndex().intValue()); - - final BlobSidecar blobSidecar = - blobSidecarSchema.create( - blobIdentifier.getIndex(), - blobAndProof.blob(), - sszKZGCommitment, - new SszKZGProof(blobAndProof.proof()), - signedBeaconBlockHeader, - miscHelpersDeneb.computeKzgCommitmentInclusionProof( - blobIdentifier.getIndex(), beaconBlockBodyDeneb)); - - blobSidecar.markSignatureAsValidated(); - blobSidecar.markKzgCommitmentInclusionProofAsValidated(); - // assume kzg validation done by local EL - blobSidecar.markKzgAsValidated(); - - return blobSidecar; + private void logLocalElBlobsLookupFailure(final Throwable error) { + LOG.warn("Local EL blobs lookup failed: {}", getRootCauseMessage(error)); } - private synchronized void fetchMissingContentFromRemotePeers( + private synchronized void fetchMissingBlockOrBlobsFromRPC( final SlotAndBlockRoot slotAndBlockRoot) { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = blockBlobSidecarsTrackers.get(slotAndBlockRoot.getBlockRoot()); - if (blockBlobSidecarsTracker == null) { + if (blockBlobSidecarsTracker == null || blockBlobSidecarsTracker.isComplete()) { return; } - if (blockBlobSidecarsTracker.isComplete()) { - return; - } + if (blockBlobSidecarsTracker.getBlock().isEmpty()) { - blockBlobSidecarsTracker.setRpcFetchTriggered(); + blockBlobSidecarsTracker.setRpcBlockFetchTriggered(); - if (blockBlobSidecarsTracker.getBlock().isEmpty()) { poolStatsCounters.labels(COUNTER_BLOCK_TYPE, COUNTER_RPC_FETCH_SUBTYPE).inc(); requiredBlockRootSubscribers.deliver( RequiredBlockRootSubscriber::onRequiredBlockRoot, blockBlobSidecarsTracker.getSlotAndBlockRoot().getBlockRoot()); + return; } + blockBlobSidecarsTracker.setRpcBlobsFetchTriggered(); + blockBlobSidecarsTracker .getMissingBlobSidecars() .forEach( @@ -787,24 +750,27 @@ private synchronized void fetchMissingContentFromRemotePeers( }); } - private void dropMissingContent(final BlockBlobSidecarsTracker blockBlobSidecarsTracker) { + private void logBlockOrBlobsRPCFailure(final Throwable error) { + LOG.error("An error occurred while attempting to fetch block or blobs via RPC.", error); + } - if (!blockBlobSidecarsTracker.isRpcFetchTriggered()) { - return; - } + private void dropMissingContent(final BlockBlobSidecarsTracker blockBlobSidecarsTracker) { - if (blockBlobSidecarsTracker.getBlock().isEmpty()) { + if (blockBlobSidecarsTracker.isRpcBlockFetchTriggered() + && blockBlobSidecarsTracker.getBlock().isEmpty()) { requiredBlockRootDroppedSubscribers.deliver( RequiredBlockRootDroppedSubscriber::onRequiredBlockRootDropped, blockBlobSidecarsTracker.getSlotAndBlockRoot().getBlockRoot()); } - blockBlobSidecarsTracker - .getMissingBlobSidecars() - .forEach( - blobIdentifier -> - requiredBlobSidecarDroppedSubscribers.deliver( - RequiredBlobSidecarDroppedSubscriber::onRequiredBlobSidecarDropped, - blobIdentifier)); + if (blockBlobSidecarsTracker.isRpcBlobsFetchTriggered()) { + blockBlobSidecarsTracker + .getMissingBlobSidecars() + .forEach( + blobIdentifier -> + requiredBlobSidecarDroppedSubscribers.deliver( + RequiredBlobSidecarDroppedSubscriber::onRequiredBlobSidecarDropped, + blobIdentifier)); + } } } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java index 275fb2fcada..662c97d818d 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/blobs/BlockBlobSidecarsTrackerTest.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; @@ -65,11 +64,10 @@ public class BlockBlobSidecarsTrackerTest { @Test void isNotCompletedJustAfterCreation() { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + new BlockBlobSidecarsTracker(slotAndBlockRoot); SafeFutureAssert.assertThatSafeFuture(blockBlobSidecarsTracker.getCompletionFuture()) .isNotCompleted(); - assertThat(blockBlobSidecarsTracker.getMissingBlobSidecars()).isEmpty(); assertThat(blockBlobSidecarsTracker.getBlock()).isEmpty(); assertThat(blockBlobSidecarsTracker.getBlobSidecars()).isEmpty(); assertThat(blockBlobSidecarsTracker.getSlotAndBlockRoot()).isEqualTo(slotAndBlockRoot); @@ -81,7 +79,7 @@ void isNotCompletedJustAfterCreation() { @Test void setBlock_shouldAcceptCorrectBlock() { BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + new BlockBlobSidecarsTracker(slotAndBlockRoot); blockBlobSidecarsTracker.setBlock(block); SafeFutureAssert.assertThatSafeFuture(blockBlobSidecarsTracker.getCompletionFuture()) @@ -95,7 +93,7 @@ void setBlock_shouldAcceptCorrectBlock() { @Test void setBlock_shouldThrowWithWrongBlock() { BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(dataStructureUtil.randomSlotAndBlockRoot(), maxBlobsPerBlock); + new BlockBlobSidecarsTracker(dataStructureUtil.randomSlotAndBlockRoot()); assertThatThrownBy(() -> blockBlobSidecarsTracker.setBlock(block)) .isInstanceOf(IllegalArgumentException.class); } @@ -103,7 +101,7 @@ void setBlock_shouldThrowWithWrongBlock() { @Test void setBlock_shouldAcceptBlockTwice() { BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + new BlockBlobSidecarsTracker(slotAndBlockRoot); blockBlobSidecarsTracker.setBlock(block); blockBlobSidecarsTracker.setBlock(block); assertThat(blockBlobSidecarsTracker.getBlock()).isEqualTo(Optional.of(block)); @@ -115,7 +113,7 @@ void setBlock_immediatelyCompletesWithBlockWithoutBlobs() { final SlotAndBlockRoot slotAndBlockRoot = block.getSlotAndBlockRoot(); final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + new BlockBlobSidecarsTracker(slotAndBlockRoot); final SafeFuture completionFuture = blockBlobSidecarsTracker.getCompletionFuture(); SafeFutureAssert.assertThatSafeFuture(completionFuture).isNotCompleted(); @@ -135,7 +133,7 @@ void getCompletionFuture_returnsIndependentFutures() { final SlotAndBlockRoot slotAndBlockRoot = block.getSlotAndBlockRoot(); final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + new BlockBlobSidecarsTracker(slotAndBlockRoot); final SafeFuture completionFuture1 = blockBlobSidecarsTracker.getCompletionFuture(); final SafeFuture completionFuture2 = blockBlobSidecarsTracker.getCompletionFuture(); final SafeFuture completionFuture3 = blockBlobSidecarsTracker.getCompletionFuture(); @@ -168,23 +166,18 @@ void getCompletionFuture_returnsIndependentFutures() { @Test void add_shouldWorkTillCompletionWhenAddingBlobsBeforeBlockIsSet() { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock.plus(1)); - final BlobSidecar toAdd = blobSidecarsForBlock.get(0); + new BlockBlobSidecarsTracker(slotAndBlockRoot); + final BlobSidecar toAdd = blobSidecarsForBlock.getFirst(); final Map added = new HashMap<>(); final SafeFuture completionFuture = blockBlobSidecarsTracker.getCompletionFuture(); added.put(toAdd.getIndex(), toAdd); blockBlobSidecarsTracker.add(toAdd); - // we don't know the block, missing blobs are max blobs minus the blob we already have - final Set potentialMissingBlobs = - UInt64.range(UInt64.valueOf(1), maxBlobsPerBlock.plus(1)) - .map(index -> new BlobIdentifier(slotAndBlockRoot.getBlockRoot(), index)) - .collect(Collectors.toSet()); - SafeFutureAssert.assertThatSafeFuture(completionFuture).isNotCompleted(); - assertThat(blockBlobSidecarsTracker.getMissingBlobSidecars()) - .containsExactlyInAnyOrderElementsOf(potentialMissingBlobs); + assertThatThrownBy(blockBlobSidecarsTracker::getMissingBlobSidecars) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Block must be known to call this method"); assertThat(blockBlobSidecarsTracker.getBlobSidecars()) .containsExactlyInAnyOrderEntriesOf(added); @@ -224,7 +217,7 @@ void add_shouldWorkTillCompletionWhenAddingBlobsBeforeBlockIsSet() { @Test void add_shouldWorkWhenBlockIsSetFirst() { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + new BlockBlobSidecarsTracker(slotAndBlockRoot); final SafeFuture completionFuture = blockBlobSidecarsTracker.getCompletionFuture(); blockBlobSidecarsTracker.setBlock(block); @@ -249,7 +242,7 @@ void add_shouldWorkWhenBlockIsSetFirst() { @Test void add_shouldThrowWhenAddingInconsistentBlobSidecar() { BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(dataStructureUtil.randomSlotAndBlockRoot(), maxBlobsPerBlock); + new BlockBlobSidecarsTracker(dataStructureUtil.randomSlotAndBlockRoot()); assertThatThrownBy(() -> blockBlobSidecarsTracker.add(dataStructureUtil.randomBlobSidecar())) .isInstanceOf(IllegalArgumentException.class); } @@ -257,108 +250,29 @@ void add_shouldThrowWhenAddingInconsistentBlobSidecar() { @Test void add_shouldAcceptAcceptSameBlobSidecarTwice() { BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + new BlockBlobSidecarsTracker(slotAndBlockRoot); blockBlobSidecarsTracker.setBlock(block); blockBlobSidecarsTracker.setBlock(block); assertThat(blockBlobSidecarsTracker.getBlock()).isEqualTo(Optional.of(block)); } @Test - void getMissingBlobSidecars_shouldReturnPartialBlobsIdentifierWhenBlockIsUnknown() { + void getMissingBlobSidecars_shouldThrowWhenBlockIsUnknown() { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + new BlockBlobSidecarsTracker(slotAndBlockRoot); final BlobSidecar toAdd = blobSidecarsForBlock.get(2); blockBlobSidecarsTracker.add(toAdd); - final List knownMissing = - blobIdentifiersForBlock.stream() - .filter(blobIdentifier -> !blobIdentifier.getIndex().equals(UInt64.valueOf(2))) - .collect(Collectors.toList()); - - assertThat(blockBlobSidecarsTracker.getMissingBlobSidecars()) - .containsExactlyInAnyOrderElementsOf(knownMissing); - } - - @Test - void getUnusedBlobSidecarsForBlock_shouldReturnShouldFailIfBlockIsUnknown() { - final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); - - assertThatThrownBy(blockBlobSidecarsTracker::getUnusedBlobSidecarsForBlock) - .isInstanceOf(IllegalStateException.class); - } - - @Test - void getUnusedBlobSidecarsForBlock_shouldReturnEmptySetIfBlockIsFull() { - final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); - - blockBlobSidecarsTracker.setBlock(block); - - assertThat(blockBlobSidecarsTracker.getUnusedBlobSidecarsForBlock()).isEmpty(); - } - - @Test - void getUnusedBlobSidecarsForBlock_shouldReturnUnusedBlobSpace() { - final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock.plus(2)); - - blockBlobSidecarsTracker.setBlock(block); - - final Set expectedUnusedBlobs = - UInt64.range(maxBlobsPerBlock, maxBlobsPerBlock.plus(2)) - .map(index -> new BlobIdentifier(slotAndBlockRoot.getBlockRoot(), index)) - .collect(Collectors.toSet()); - - assertThat(blockBlobSidecarsTracker.getUnusedBlobSidecarsForBlock()) - .containsExactlyInAnyOrderElementsOf(expectedUnusedBlobs); - } - - @Test - void getUnusedBlobSidecarsForBlock_shouldReturnAllMaxBlobsPerBlockIfBlockIsEmpty() { - - final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlockWithEmptyCommitments(); - final SlotAndBlockRoot slotAndBlockRoot = block.getSlotAndBlockRoot(); - - final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock.plus(2)); - - blockBlobSidecarsTracker.setBlock(block); - - final Set expectedUnusedBlobs = - UInt64.range(UInt64.ZERO, maxBlobsPerBlock.plus(2)) - .map(index -> new BlobIdentifier(slotAndBlockRoot.getBlockRoot(), index)) - .collect(Collectors.toSet()); - - assertThat(blockBlobSidecarsTracker.getUnusedBlobSidecarsForBlock()) - .containsExactlyInAnyOrderElementsOf(expectedUnusedBlobs); - } - - @Test - void getMissingBlobSidecars_shouldRespectMaxBlobsPerBlock() { - final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); - final BlobSidecar toAdd = - dataStructureUtil - .createRandomBlobSidecarBuilder() - .signedBeaconBlockHeader(block.asHeader()) - .index(UInt64.valueOf(100)) - .build(); - - blockBlobSidecarsTracker.add(toAdd); - - final List knownMissing = - blobIdentifiersForBlock.subList(0, maxBlobsPerBlock.intValue()); - - assertThat(blockBlobSidecarsTracker.getMissingBlobSidecars()) - .containsExactlyInAnyOrderElementsOf(knownMissing); + assertThatThrownBy(blockBlobSidecarsTracker::getMissingBlobSidecars) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Block must be known to call this method"); } @Test void shouldNotIgnoreExcessiveBlobSidecarWhenBlockIsUnknownAndWePruneItLater() { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + new BlockBlobSidecarsTracker(slotAndBlockRoot); final BlobSidecar legitBlobSidecar = createBlobSidecar(UInt64.valueOf(2)); final BlobSidecar excessiveBlobSidecar1 = createBlobSidecar(maxBlobsPerBlock); @@ -382,7 +296,7 @@ void shouldNotIgnoreExcessiveBlobSidecarWhenBlockIsUnknownAndWePruneItLater() { @Test void add_shouldIgnoreExcessiveBlobSidecarWhenBlockIsKnown() { final BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + new BlockBlobSidecarsTracker(slotAndBlockRoot); blockBlobSidecarsTracker.setBlock(block); @@ -400,7 +314,7 @@ void add_shouldIgnoreExcessiveBlobSidecarWhenBlockIsKnown() { @Test void enableBlockImportOnCompletion_shouldImportOnlyOnceWhenCalled() { BlockBlobSidecarsTracker blockBlobSidecarsTracker = - new BlockBlobSidecarsTracker(slotAndBlockRoot, maxBlobsPerBlock); + new BlockBlobSidecarsTracker(slotAndBlockRoot); blockBlobSidecarsTracker.setBlock(block); diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java index 2dad121a9eb..65b0c20c6d6 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImplTest.java @@ -15,6 +15,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -123,6 +124,8 @@ public void setup() { blockBlobSidecarsTrackersPool.subscribeRequiredBlobSidecarDropped( requiredBlobSidecarDroppedEvents::add); blockBlobSidecarsTrackersPool.subscribeNewBlobSidecar(newBlobSidecarEvents::add); + when(executionLayer.engineGetBlobs(any(), eq(currentSlot))) + .thenReturn(SafeFuture.completedFuture(List.of())); when(blobSidecarPublisher.apply(any())).thenReturn(SafeFuture.COMPLETE); setSlot(currentSlot); } @@ -463,7 +466,7 @@ public void onCompletedBlockAndBlobSidecars_shouldCreateTrackerIgnoringHistorica assertThat(blockBlobSidecarsTracker.getBlobSidecars().values()) .containsExactlyInAnyOrderElementsOf(blobSidecars); assertThat(blockBlobSidecarsTracker.getBlock()).isPresent(); - assertThat(blockBlobSidecarsTracker.isRpcFetchTriggered()).isFalse(); + assertThat(blockBlobSidecarsTracker.isRpcBlockFetchTriggered()).isFalse(); assertThatSafeFuture(blockBlobSidecarsTracker.getCompletionFuture()).isCompleted(); assertBlobSidecarsCount(blobSidecars.size()); @@ -493,7 +496,7 @@ public void onCompletedBlockAndBlobSidecars_shouldNotTriggerFetch() { assertThat(blockBlobSidecarsTracker.getBlobSidecars().values()) .containsExactlyInAnyOrderElementsOf(blobSidecars); assertThat(blockBlobSidecarsTracker.getBlock()).isPresent(); - assertThat(blockBlobSidecarsTracker.isRpcFetchTriggered()).isFalse(); + assertThat(blockBlobSidecarsTracker.isRpcBlockFetchTriggered()).isFalse(); assertThatSafeFuture(blockBlobSidecarsTracker.getCompletionFuture()).isNotCompleted(); assertBlobSidecarsCount(0); @@ -511,7 +514,7 @@ public void onCompletedBlockAndBlobSidecars_shouldNotTriggerFetch() { blockBlobSidecarsTrackersPool.getOrCreateBlockBlobSidecarsTracker(block); assertThat(blockBlobSidecarsTracker.getBlock()).isPresent(); - assertThat(blockBlobSidecarsTracker.isRpcFetchTriggered()).isFalse(); + assertThat(blockBlobSidecarsTracker.isRpcBlockFetchTriggered()).isFalse(); assertThatSafeFuture(blockBlobSidecarsTracker.getCompletionFuture()).isNotCompleted(); assertBlobSidecarsCount(0); @@ -671,7 +674,8 @@ void shouldFetchMissingBlobSidecarsFromLocalELFirst() { (slotAndRoot) -> { when(tracker.add(any())).thenReturn(true); when(tracker.getMissingBlobSidecars()) - .thenAnswer(__ -> missingBlobIdentifiers.stream()); + .thenAnswer(__ -> missingBlobIdentifiers.stream()) + .thenAnswer(__ -> Stream.empty()); when(tracker.getBlock()).thenReturn(Optional.of(block)); return tracker; }); @@ -694,7 +698,7 @@ void shouldFetchMissingBlobSidecarsFromLocalELFirst() { assertThat(requiredBlobSidecarDroppedEvents).isEmpty(); // local el fetch triggered - verify(tracker).setLocalElFetchTriggered(); + verify(tracker).setLocalElBlobsFetchTriggered(); // prepare partial response of 3 blobAndProofs final List> blobAndProofsFromEL = @@ -794,7 +798,7 @@ void shouldFetchMissingBlobSidecarsViaRPCWhenELLookupFails() { } @Test - void shouldFetchMissingBlockAndBlobSidecars() { + void shouldFetchMissingBlock() { final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(currentSlot); final BlobSidecar blobSidecar = dataStructureUtil @@ -803,14 +807,8 @@ void shouldFetchMissingBlockAndBlobSidecars() { .index(UInt64.valueOf(2)) .build(); - final Set missingBlobs = - Set.of( - new BlobIdentifier(block.getRoot(), UInt64.ONE), - new BlobIdentifier(block.getRoot(), UInt64.ZERO)); - final BlockBlobSidecarsTracker mockedTracker = mock(BlockBlobSidecarsTracker.class); when(mockedTracker.getBlock()).thenReturn(Optional.empty()); - when(mockedTracker.getMissingBlobSidecars()).thenReturn(missingBlobs.stream()); when(mockedTracker.getSlotAndBlockRoot()).thenReturn(block.getSlotAndBlockRoot()); mockedTrackersFactory = Optional.of((__) -> mockedTracker); @@ -821,91 +819,14 @@ void shouldFetchMissingBlockAndBlobSidecars() { asyncRunner.executeQueuedActions(); - verify(mockedTracker).setRpcFetchTriggered(); - verify(mockedTracker, never()).setLocalElFetchTriggered(); + verify(mockedTracker).setRpcBlockFetchTriggered(); + verify(mockedTracker, never()).setLocalElBlobsFetchTriggered(); assertThat(requiredBlockRootEvents).containsExactly(block.getRoot()); - assertThat(requiredBlobSidecarEvents).containsExactlyElementsOf(missingBlobs); assertStats("block", "rpc_fetch", 1); - assertStats("blob_sidecar", "rpc_fetch", missingBlobs.size()); assertThat(requiredBlockRootDroppedEvents).isEmpty(); - assertThat(requiredBlobSidecarDroppedEvents).isEmpty(); - } - - @Test - void shouldDropBlobSidecarsThatHasBeenFetchedButNotPresentInBlock() { - final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(currentSlot); - - final SlotAndBlockRoot slotAndBlockRoot = new SlotAndBlockRoot(currentSlot, block.getRoot()); - final BlobSidecar blobSidecar = - dataStructureUtil - .createRandomBlobSidecarBuilder() - .signedBeaconBlockHeader(block.asHeader()) - .index(UInt64.valueOf(2)) - .build(); - - final Set blobsNotPresentInBlock = - Set.of( - new BlobIdentifier(slotAndBlockRoot.getBlockRoot(), UInt64.valueOf(2)), - new BlobIdentifier(slotAndBlockRoot.getBlockRoot(), UInt64.valueOf(3))); - - mockedTrackersFactory = - Optional.of( - (slotAndRoot) -> { - BlockBlobSidecarsTracker tracker = mock(BlockBlobSidecarsTracker.class); - when(tracker.getBlock()).thenReturn(Optional.empty()); - when(tracker.getSlotAndBlockRoot()).thenReturn(slotAndBlockRoot); - when(tracker.setBlock(block)).thenReturn(true); - when(tracker.isRpcFetchTriggered()).thenReturn(true); - when(tracker.getUnusedBlobSidecarsForBlock()) - .thenReturn(blobsNotPresentInBlock.stream()); - return tracker; - }); - - blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); - - blockBlobSidecarsTrackersPool.onNewBlock(block, Optional.empty()); - - assertThat(requiredBlobSidecarDroppedEvents).containsExactlyElementsOf(blobsNotPresentInBlock); - } - - @Test - void shouldNotDropUnusedBlobSidecarsIfFetchingHasNotOccurred() { - final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(currentSlot); - - final SlotAndBlockRoot slotAndBlockRoot = new SlotAndBlockRoot(currentSlot, block.getRoot()); - final BlobSidecar blobSidecar = - dataStructureUtil - .createRandomBlobSidecarBuilder() - .signedBeaconBlockHeader(block.asHeader()) - .index(UInt64.valueOf(2)) - .build(); - - final Set blobsNotUserInBlock = - Set.of( - new BlobIdentifier(slotAndBlockRoot.getBlockRoot(), UInt64.valueOf(2)), - new BlobIdentifier(slotAndBlockRoot.getBlockRoot(), UInt64.valueOf(3))); - - mockedTrackersFactory = - Optional.of( - (slotAndRoot) -> { - BlockBlobSidecarsTracker tracker = mock(BlockBlobSidecarsTracker.class); - when(tracker.getBlock()).thenReturn(Optional.empty()); - when(tracker.getSlotAndBlockRoot()).thenReturn(slotAndBlockRoot); - when(tracker.setBlock(block)).thenReturn(true); - when(tracker.isRpcFetchTriggered()).thenReturn(false); - when(tracker.getUnusedBlobSidecarsForBlock()) - .thenReturn(blobsNotUserInBlock.stream()); - return tracker; - }); - - blockBlobSidecarsTrackersPool.onNewBlobSidecar(blobSidecar, RemoteOrigin.GOSSIP); - - blockBlobSidecarsTrackersPool.onNewBlock(block, Optional.empty()); - - assertThat(requiredBlobSidecarDroppedEvents).isEmpty(); } @Test @@ -945,10 +866,10 @@ void shouldDropPossiblyFetchedBlobSidecars() { Optional.of( (slotAndRoot) -> { BlockBlobSidecarsTracker tracker = mock(BlockBlobSidecarsTracker.class); - when(tracker.getMissingBlobSidecars()).thenReturn(missingBlobs.stream()); + when(tracker.getMissingBlobSidecars()).thenAnswer(__ -> missingBlobs.stream()); when(tracker.getBlock()).thenReturn(Optional.of(block)); when(tracker.getSlotAndBlockRoot()).thenReturn(block.getSlotAndBlockRoot()); - when(tracker.isRpcFetchTriggered()).thenReturn(true); + when(tracker.isRpcBlobsFetchTriggered()).thenReturn(true); return tracker; }); @@ -967,7 +888,7 @@ void shouldDropPossiblyFetchedBlobSidecars() { } @Test - void shouldTryToFetchFromLocalELWhenBlockArrivesAfterRPCFetch() { + void shouldTryToFetchBlobSidecarsWhenBlockArrivesAfterRPCFetch() { final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(currentSlot); final Set missingBlobs = @@ -991,8 +912,8 @@ void shouldTryToFetchFromLocalELWhenBlockArrivesAfterRPCFetch() { when(tracker.getBlock()).thenReturn(Optional.empty()); when(tracker.setBlock(any())).thenReturn(true); when(tracker.getSlotAndBlockRoot()).thenReturn(block.getSlotAndBlockRoot()); - when(tracker.isRpcFetchTriggered()).thenReturn(true); - when(tracker.isLocalElFetchTriggered()).thenReturn(false); + when(tracker.isRpcBlockFetchTriggered()).thenReturn(true); + when(tracker.isLocalElBlobsFetchTriggered()).thenReturn(true); return tracker; }); @@ -1001,7 +922,7 @@ void shouldTryToFetchFromLocalELWhenBlockArrivesAfterRPCFetch() { assertThat(asyncRunner.hasDelayedActions()).isTrue(); asyncRunner.executeQueuedActions(); - verify(tracker, never()).setLocalElFetchTriggered(); + verify(tracker, never()).setLocalElBlobsFetchTriggered(); when(tracker.getBlock()).thenReturn(Optional.of(block)); @@ -1011,7 +932,10 @@ void shouldTryToFetchFromLocalELWhenBlockArrivesAfterRPCFetch() { blockBlobSidecarsTrackersPool.onNewBlock(block, Optional.empty()); - verify(tracker).setLocalElFetchTriggered(); + assertThat(asyncRunner.hasDelayedActions()).isTrue(); + asyncRunner.executeQueuedActions(); + + verify(tracker).setLocalElBlobsFetchTriggered(); verify(executionLayer).engineGetBlobs(any(), any()); } @@ -1032,7 +956,7 @@ void shouldDropPossiblyFetchedBlock() { when(tracker.getBlock()).thenReturn(Optional.empty()); when(tracker.getSlotAndBlockRoot()) .thenReturn(signedBeaconBlock.getSlotAndBlockRoot()); - when(tracker.isRpcFetchTriggered()).thenReturn(true); + when(tracker.isRpcBlockFetchTriggered()).thenReturn(true); return tracker; }); @@ -1067,7 +991,7 @@ void shouldNotDropPossiblyFetchedBlockIfFetchHasNotOccurred() { when(tracker.getBlock()).thenReturn(Optional.empty()); when(tracker.getSlotAndBlockRoot()) .thenReturn(signedBeaconBlock.getSlotAndBlockRoot()); - when(tracker.isRpcFetchTriggered()).thenReturn(false); + when(tracker.isRpcBlockFetchTriggered()).thenReturn(false); return tracker; }); @@ -1093,14 +1017,15 @@ void shouldRespectTargetWhenBlockIsEarly() { // blocks arrives at slot start timeProvider.advanceTimeBySeconds(startSlotInSeconds.longValue()); - final Duration fetchDelay = blockBlobSidecarsTrackersPool.calculateFetchDelay(slotAndBlockRoot); + final Duration fetchDelay = + blockBlobSidecarsTrackersPool.calculateRpcFetchDelay(slotAndBlockRoot); // we can wait the full target assertThat(fetchDelay).isEqualTo(Duration.ofMillis(TARGET_WAIT_MILLIS.longValue())); } @Test - void calculateFetchDelay_shouldRespectMinimumWhenBlockIsLate() { + void calculateBlockFetchDelay_shouldRespectMinimumWhenRpcIsLate() { final SlotAndBlockRoot slotAndBlockRoot = new SlotAndBlockRoot(currentSlot, dataStructureUtil.randomBytes32()); @@ -1112,14 +1037,15 @@ void calculateFetchDelay_shouldRespectMinimumWhenBlockIsLate() { // blocks arrives 200ms before attestation due timeProvider.advanceTimeByMillis(startSlotInMillis.plus(3_800).longValue()); - final Duration fetchDelay = blockBlobSidecarsTrackersPool.calculateFetchDelay(slotAndBlockRoot); + final Duration fetchDelay = + blockBlobSidecarsTrackersPool.calculateRpcFetchDelay(slotAndBlockRoot); // we can wait the full target assertThat(fetchDelay).isEqualTo(Duration.ofMillis(MIN_WAIT_MILLIS.longValue())); } @Test - void calculateFetchDelay_shouldRespectTargetWhenBlockIsVeryLate() { + void calculateBlockFetchDelay_shouldRespectTargetWhenRpcIsVeryLate() { final SlotAndBlockRoot slotAndBlockRoot = new SlotAndBlockRoot(currentSlot, dataStructureUtil.randomBytes32()); @@ -1130,14 +1056,15 @@ void calculateFetchDelay_shouldRespectTargetWhenBlockIsVeryLate() { // blocks arrives 1s after attestation due timeProvider.advanceTimeBySeconds(startSlotInSeconds.plus(5).longValue()); - final Duration fetchDelay = blockBlobSidecarsTrackersPool.calculateFetchDelay(slotAndBlockRoot); + final Duration fetchDelay = + blockBlobSidecarsTrackersPool.calculateRpcFetchDelay(slotAndBlockRoot); // we can wait the full target assertThat(fetchDelay).isEqualTo(Duration.ofMillis(TARGET_WAIT_MILLIS.longValue())); } @Test - void calculateFetchDelay_shouldRespectAttestationDueLimit() { + void calculateRpcFetchDelay_shouldRespectAttestationDueLimit() { final SlotAndBlockRoot slotAndBlockRoot = new SlotAndBlockRoot(currentSlot, dataStructureUtil.randomBytes32()); @@ -1157,7 +1084,8 @@ void calculateFetchDelay_shouldRespectAttestationDueLimit() { timeProvider.advanceTimeByMillis(blockArrivalTimeMillis.longValue()); - final Duration fetchDelay = blockBlobSidecarsTrackersPool.calculateFetchDelay(slotAndBlockRoot); + final Duration fetchDelay = + blockBlobSidecarsTrackersPool.calculateRpcFetchDelay(slotAndBlockRoot); // we can only wait 200ms less than target assertThat(fetchDelay) @@ -1166,11 +1094,12 @@ void calculateFetchDelay_shouldRespectAttestationDueLimit() { } @Test - void calculateFetchDelay_shouldReturnZeroIfSlotIsOld() { + void calculateRpcFetchDelay_shouldReturnZeroIfSlotIsOld() { final SlotAndBlockRoot slotAndBlockRoot = new SlotAndBlockRoot(currentSlot.minus(1), dataStructureUtil.randomBytes32()); - final Duration fetchDelay = blockBlobSidecarsTrackersPool.calculateFetchDelay(slotAndBlockRoot); + final Duration fetchDelay = + blockBlobSidecarsTrackersPool.calculateRpcFetchDelay(slotAndBlockRoot); assertThat(fetchDelay).isEqualTo(Duration.ZERO); } @@ -1400,8 +1329,6 @@ private BlockBlobSidecarsTracker trackerFactory(final SlotAndBlockRoot slotAndBl if (mockedTrackersFactory.isPresent()) { return mockedTrackersFactory.get().apply(slotAndBlockRoot); } - return new BlockBlobSidecarsTracker( - slotAndBlockRoot, - UInt64.valueOf(spec.getMaxBlobsPerBlockForHighestMilestone().orElseThrow())); + return new BlockBlobSidecarsTracker(slotAndBlockRoot); } } From b4f7c2dbf108ac34dfd9841b2a9bf7fa9d47cebe Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Mon, 30 Dec 2024 12:54:23 +0200 Subject: [PATCH 6/8] Remove unused classes/modules (#8959) --- .../api/MinGenesisTimeBlockEventChannel.java | 20 --------- data/serializer/build.gradle | 2 - .../schema/interfaces/SignedBlindedBlock.java | 33 -------------- ethereum/execution-types/build.gradle | 2 +- ethereum/jackson-deserializers/build.gradle | 5 --- .../jackson/Eth1AddressDeserializer.java | 35 --------------- .../suppliers/type/Bytes32Supplier.java | 26 ----------- .../suppliers/type/UInt64Supplier.java | 26 ----------- infrastructure/jackson/build.gradle | 8 ---- .../bytes/ByteArrayDeserializer.java | 44 ------------------- .../bytes/ByteArraySerializer.java | 33 -------------- .../bytes/Bytes20Deserializer.java | 29 ------------ .../bytes/Bytes20Serializer.java | 30 ------------- .../bytes/Bytes4Deserializer.java | 29 ------------ .../deserializers/bytes/Bytes4Serializer.java | 30 ------------- .../bytes/BytesDeserializer.java | 29 ------------ .../bytes/DoubleDeserializer.java | 28 ------------ .../deserializers/bytes/DoubleSerializer.java | 40 ----------------- .../uints/UInt256Deserializer.java | 35 --------------- .../uints/UInt256Serializer.java | 29 ------------ .../service/serviceutils/NoopService.java | 29 ------------ infrastructure/yaml/build.gradle | 1 - .../teku/data/yaml}/Bytes32Deserializer.java | 4 +- .../teku/data/yaml}/BytesSerializer.java | 5 ++- .../teku/data/yaml}/UInt64Deserializer.java | 4 +- .../teku/data/yaml}/UInt64Serializer.java | 7 +-- .../pegasys/teku/data/yaml/YamlProvider.java | 36 ++------------- settings.gradle | 2 - validator/client/build.gradle | 1 - .../teku/validator/SetGasLimitException.java | 21 --------- .../loader/BLSPublicKeyDeserializer.java | 1 + .../loader/BLSPublicKeySerializer.java | 1 + .../loader}/Bytes48KeyDeserializer.java | 2 +- .../loader/ProposerConfigLoader.java | 7 ++- ...dedBlockEndpointNotAvailableException.java | 16 ------- .../BlockProductionV3FailedException.java | 16 ------- 36 files changed, 21 insertions(+), 645 deletions(-) delete mode 100644 beacon/pow/src/main/java/tech/pegasys/teku/beacon/pow/api/MinGenesisTimeBlockEventChannel.java delete mode 100644 data/serializer/src/main/java/tech/pegasys/teku/api/schema/interfaces/SignedBlindedBlock.java delete mode 100644 ethereum/jackson-deserializers/build.gradle delete mode 100644 ethereum/jackson-deserializers/src/main/java/tech/pegasys/teku/ethereum/jackson/Eth1AddressDeserializer.java delete mode 100644 ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/type/Bytes32Supplier.java delete mode 100644 ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/type/UInt64Supplier.java delete mode 100644 infrastructure/jackson/build.gradle delete mode 100644 infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/ByteArrayDeserializer.java delete mode 100644 infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/ByteArraySerializer.java delete mode 100644 infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes20Deserializer.java delete mode 100644 infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes20Serializer.java delete mode 100644 infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes4Deserializer.java delete mode 100644 infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes4Serializer.java delete mode 100644 infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/BytesDeserializer.java delete mode 100644 infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/DoubleDeserializer.java delete mode 100644 infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/DoubleSerializer.java delete mode 100644 infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt256Deserializer.java delete mode 100644 infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt256Serializer.java delete mode 100644 infrastructure/serviceutils/src/main/java/tech/pegasys/teku/service/serviceutils/NoopService.java rename infrastructure/{jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes => yaml/src/main/java/tech/pegasys/teku/data/yaml}/Bytes32Deserializer.java (90%) rename infrastructure/{jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes => yaml/src/main/java/tech/pegasys/teku/data/yaml}/BytesSerializer.java (90%) rename infrastructure/{jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints => yaml/src/main/java/tech/pegasys/teku/data/yaml}/UInt64Deserializer.java (90%) rename infrastructure/{jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints => yaml/src/main/java/tech/pegasys/teku/data/yaml}/UInt64Serializer.java (87%) delete mode 100644 validator/client/src/main/java/tech/pegasys/teku/validator/SetGasLimitException.java rename {infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes => validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader}/Bytes48KeyDeserializer.java (94%) delete mode 100644 validator/remote/src/main/java/tech/pegasys/teku/validator/remote/typedef/BlindedBlockEndpointNotAvailableException.java delete mode 100644 validator/remote/src/main/java/tech/pegasys/teku/validator/remote/typedef/BlockProductionV3FailedException.java diff --git a/beacon/pow/src/main/java/tech/pegasys/teku/beacon/pow/api/MinGenesisTimeBlockEventChannel.java b/beacon/pow/src/main/java/tech/pegasys/teku/beacon/pow/api/MinGenesisTimeBlockEventChannel.java deleted file mode 100644 index 7ba101fc9f1..00000000000 --- a/beacon/pow/src/main/java/tech/pegasys/teku/beacon/pow/api/MinGenesisTimeBlockEventChannel.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.beacon.pow.api; - -import tech.pegasys.teku.ethereum.pow.api.MinGenesisTimeBlockEvent; - -public interface MinGenesisTimeBlockEventChannel { - void onMinGenesisTimeBlock(MinGenesisTimeBlockEvent event); -} diff --git a/data/serializer/build.gradle b/data/serializer/build.gradle index 4302d4d8e80..f24eb8983ed 100644 --- a/data/serializer/build.gradle +++ b/data/serializer/build.gradle @@ -3,11 +3,9 @@ dependencies { api 'io.swagger.core.v3:swagger-annotations' implementation project(':ethereum:execution-types') - implementation project(':ethereum:jackson-deserializers') implementation project(':ethereum:spec') implementation project(':infrastructure:bls') implementation project(':infrastructure:bytes') - implementation project(':infrastructure:jackson') implementation project(':infrastructure:async') implementation 'io.tmio:tuweni-units' diff --git a/data/serializer/src/main/java/tech/pegasys/teku/api/schema/interfaces/SignedBlindedBlock.java b/data/serializer/src/main/java/tech/pegasys/teku/api/schema/interfaces/SignedBlindedBlock.java deleted file mode 100644 index 3429bdb17a7..00000000000 --- a/data/serializer/src/main/java/tech/pegasys/teku/api/schema/interfaces/SignedBlindedBlock.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.api.schema.interfaces; - -import io.swagger.v3.oas.annotations.media.Schema; -import tech.pegasys.teku.api.schema.altair.SignedBeaconBlockAltair; -import tech.pegasys.teku.api.schema.bellatrix.SignedBlindedBeaconBlockBellatrix; -import tech.pegasys.teku.api.schema.capella.SignedBlindedBeaconBlockCapella; -import tech.pegasys.teku.api.schema.deneb.SignedBlindedBeaconBlockDeneb; -import tech.pegasys.teku.api.schema.electra.SignedBlindedBeaconBlockElectra; -import tech.pegasys.teku.api.schema.phase0.SignedBeaconBlockPhase0; - -@Schema( - oneOf = { - SignedBeaconBlockPhase0.class, - SignedBeaconBlockAltair.class, - SignedBlindedBeaconBlockBellatrix.class, - SignedBlindedBeaconBlockCapella.class, - SignedBlindedBeaconBlockDeneb.class, - SignedBlindedBeaconBlockElectra.class - }) -public interface SignedBlindedBlock {} diff --git a/ethereum/execution-types/build.gradle b/ethereum/execution-types/build.gradle index c8ce4745c85..b9503a7dba8 100644 --- a/ethereum/execution-types/build.gradle +++ b/ethereum/execution-types/build.gradle @@ -3,7 +3,7 @@ dependencies { implementation project(':infrastructure:crypto') implementation project(':infrastructure:json') - testFixturesApi project(':infrastructure:jackson') + testFixturesApi 'com.fasterxml.jackson.core:jackson-databind' testImplementation project(':ethereum:json-types') testImplementation testFixtures(project(':infrastructure:json')) diff --git a/ethereum/jackson-deserializers/build.gradle b/ethereum/jackson-deserializers/build.gradle deleted file mode 100644 index 1b32471b6bc..00000000000 --- a/ethereum/jackson-deserializers/build.gradle +++ /dev/null @@ -1,5 +0,0 @@ -dependencies { - implementation project(':infrastructure:bytes') - implementation project(':infrastructure:jackson') - implementation project(':ethereum:execution-types') -} \ No newline at end of file diff --git a/ethereum/jackson-deserializers/src/main/java/tech/pegasys/teku/ethereum/jackson/Eth1AddressDeserializer.java b/ethereum/jackson-deserializers/src/main/java/tech/pegasys/teku/ethereum/jackson/Eth1AddressDeserializer.java deleted file mode 100644 index 5f789385890..00000000000 --- a/ethereum/jackson-deserializers/src/main/java/tech/pegasys/teku/ethereum/jackson/Eth1AddressDeserializer.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.ethereum.jackson; - -import com.fasterxml.jackson.core.JacksonException; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonMappingException; -import java.io.IOException; -import tech.pegasys.teku.ethereum.execution.types.Eth1Address; - -public class Eth1AddressDeserializer extends JsonDeserializer { - - @Override - public Eth1Address deserialize(final JsonParser p, final DeserializationContext ctxt) - throws JacksonException { - try { - return Eth1Address.fromHexString(p.getValueAsString()); - } catch (RuntimeException | IOException ex) { - throw new JsonMappingException(p, ex.getMessage(), ex); - } - } -} diff --git a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/type/Bytes32Supplier.java b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/type/Bytes32Supplier.java deleted file mode 100644 index 1ea025dc6f9..00000000000 --- a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/type/Bytes32Supplier.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.spec.propertytest.suppliers.type; - -import net.jqwik.api.Arbitraries; -import net.jqwik.api.Arbitrary; -import net.jqwik.api.ArbitrarySupplier; -import org.apache.tuweni.bytes.Bytes32; - -public class Bytes32Supplier implements ArbitrarySupplier { - @Override - public Arbitrary get() { - return Arbitraries.bytes().array(byte[].class).ofSize(32).map(Bytes32::wrap); - } -} diff --git a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/type/UInt64Supplier.java b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/type/UInt64Supplier.java deleted file mode 100644 index d4d3f11851e..00000000000 --- a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/type/UInt64Supplier.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.spec.propertytest.suppliers.type; - -import net.jqwik.api.Arbitraries; -import net.jqwik.api.Arbitrary; -import net.jqwik.api.ArbitrarySupplier; -import tech.pegasys.teku.infrastructure.unsigned.UInt64; - -public class UInt64Supplier implements ArbitrarySupplier { - @Override - public Arbitrary get() { - return Arbitraries.longs().map(UInt64::fromLongBits); - } -} diff --git a/infrastructure/jackson/build.gradle b/infrastructure/jackson/build.gradle deleted file mode 100644 index 3f1451d6923..00000000000 --- a/infrastructure/jackson/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -dependencies { - api 'com.fasterxml.jackson.core:jackson-databind' - api 'io.swagger.core.v3:swagger-annotations' - - implementation project(':infrastructure:bytes') - - implementation 'io.tmio:tuweni-units' -} \ No newline at end of file diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/ByteArrayDeserializer.java b/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/ByteArrayDeserializer.java deleted file mode 100644 index 5b64a8a0003..00000000000 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/ByteArrayDeserializer.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; - -import static com.google.common.base.Preconditions.checkArgument; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.ObjectCodec; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonNode; -import java.io.IOException; - -public class ByteArrayDeserializer extends JsonDeserializer { - - @Override - public byte[] deserialize(final JsonParser p, final DeserializationContext ctxt) - throws IOException { - ObjectCodec oc = p.getCodec(); - JsonNode node = oc.readTree(p); - checkArgument(node.isArray(), "Expected array but did not appear to be one!"); - byte[] data = new byte[node.size()]; - for (int i = 0; i < node.size(); i++) { - final int current = node.get(i).asInt(-1); - checkArgument( - current >= 0 && current <= 255, - "Expected %s to be a byte value between 0 and 255 inclusive", - node.get(i)); - data[i] = (byte) current; - } - return data; - } -} diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/ByteArraySerializer.java b/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/ByteArraySerializer.java deleted file mode 100644 index 1c97c40d7fc..00000000000 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/ByteArraySerializer.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import java.io.IOException; - -public class ByteArraySerializer extends JsonSerializer { - @Override - public void serialize( - final byte[] value, final JsonGenerator gen, final SerializerProvider serializers) - throws IOException { - gen.writeStartArray(); - for (int i = 0; i < value.length; i++) { - final int s = value[i] & 0xff; - gen.writeString(String.format("%d", s)); - } - gen.writeEndArray(); - } -} diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes20Deserializer.java b/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes20Deserializer.java deleted file mode 100644 index e9d53b33024..00000000000 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes20Deserializer.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import java.io.IOException; -import tech.pegasys.teku.infrastructure.bytes.Bytes20; - -public class Bytes20Deserializer extends JsonDeserializer { - - @Override - public Bytes20 deserialize(final JsonParser p, final DeserializationContext ctxt) - throws IOException { - return Bytes20.fromHexString(p.getValueAsString()); - } -} diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes20Serializer.java b/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes20Serializer.java deleted file mode 100644 index 1ff0dc2e281..00000000000 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes20Serializer.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import java.io.IOException; -import java.util.Locale; -import tech.pegasys.teku.infrastructure.bytes.Bytes20; - -public class Bytes20Serializer extends JsonSerializer { - @Override - public void serialize( - final Bytes20 value, final JsonGenerator gen, final SerializerProvider serializers) - throws IOException { - gen.writeString(value.toHexString().toLowerCase(Locale.ROOT)); - } -} diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes4Deserializer.java b/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes4Deserializer.java deleted file mode 100644 index 4aca1c976cb..00000000000 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes4Deserializer.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import java.io.IOException; -import tech.pegasys.teku.infrastructure.bytes.Bytes4; - -public class Bytes4Deserializer extends JsonDeserializer { - - @Override - public Bytes4 deserialize(final JsonParser p, final DeserializationContext ctxt) - throws IOException { - return Bytes4.fromHexString(p.getValueAsString()); - } -} diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes4Serializer.java b/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes4Serializer.java deleted file mode 100644 index 996401c3d24..00000000000 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes4Serializer.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import java.io.IOException; -import java.util.Locale; -import tech.pegasys.teku.infrastructure.bytes.Bytes4; - -public class Bytes4Serializer extends JsonSerializer { - @Override - public void serialize( - final Bytes4 value, final JsonGenerator gen, final SerializerProvider serializers) - throws IOException { - gen.writeString(value.toHexString().toLowerCase(Locale.ROOT)); - } -} diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/BytesDeserializer.java b/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/BytesDeserializer.java deleted file mode 100644 index 449f8654999..00000000000 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/BytesDeserializer.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import java.io.IOException; -import org.apache.tuweni.bytes.Bytes; - -public class BytesDeserializer extends JsonDeserializer { - - @Override - public Bytes deserialize(final JsonParser p, final DeserializationContext ctxt) - throws IOException { - return Bytes.fromHexString(p.getValueAsString()); - } -} diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/DoubleDeserializer.java b/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/DoubleDeserializer.java deleted file mode 100644 index ba8bce97e68..00000000000 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/DoubleDeserializer.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import java.io.IOException; - -public class DoubleDeserializer extends JsonDeserializer { - - @Override - public Double deserialize(final JsonParser p, final DeserializationContext ctxt) - throws IOException { - return Double.valueOf(p.getValueAsString()); - } -} diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/DoubleSerializer.java b/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/DoubleSerializer.java deleted file mode 100644 index 5da417213a0..00000000000 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/DoubleSerializer.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import java.io.IOException; -import java.math.RoundingMode; -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.util.Locale; - -public class DoubleSerializer extends JsonSerializer { - private static final DecimalFormatSymbols US_SYMBOLS = new DecimalFormatSymbols(Locale.US); - - @Override - public void serialize( - final Double value, final JsonGenerator gen, final SerializerProvider serializers) - throws IOException { - gen.writeString(formatValue(value)); - } - - private String formatValue(final Double value) { - DecimalFormat df = new DecimalFormat("#.####", US_SYMBOLS); - df.setRoundingMode(RoundingMode.HALF_UP); - return df.format(value); - } -} diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt256Deserializer.java b/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt256Deserializer.java deleted file mode 100644 index 1370bc9b6d4..00000000000 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt256Deserializer.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.jackson.deserializers.uints; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import java.io.IOException; -import java.math.BigInteger; -import org.apache.tuweni.units.bigints.UInt256; - -public class UInt256Deserializer extends JsonDeserializer { - - @Override - public UInt256 deserialize(final JsonParser p, final DeserializationContext ctxt) - throws IOException { - String value = p.getValueAsString(); - if (value.startsWith("0x")) { - return UInt256.fromHexString(p.getValueAsString()); - } - - return UInt256.valueOf(new BigInteger(value)); - } -} diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt256Serializer.java b/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt256Serializer.java deleted file mode 100644 index 899c2d4bc1c..00000000000 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt256Serializer.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.infrastructure.jackson.deserializers.uints; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import java.io.IOException; -import org.apache.tuweni.units.bigints.UInt256; - -public class UInt256Serializer extends JsonSerializer { - @Override - public void serialize( - final UInt256 value, final JsonGenerator gen, final SerializerProvider serializers) - throws IOException { - gen.writeString(value.toDecimalString()); - } -} diff --git a/infrastructure/serviceutils/src/main/java/tech/pegasys/teku/service/serviceutils/NoopService.java b/infrastructure/serviceutils/src/main/java/tech/pegasys/teku/service/serviceutils/NoopService.java deleted file mode 100644 index efa3c325a78..00000000000 --- a/infrastructure/serviceutils/src/main/java/tech/pegasys/teku/service/serviceutils/NoopService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.service.serviceutils; - -import tech.pegasys.teku.infrastructure.async.SafeFuture; - -public class NoopService extends Service { - - @Override - protected SafeFuture doStart() { - return SafeFuture.COMPLETE; - } - - @Override - protected SafeFuture doStop() { - return SafeFuture.COMPLETE; - } -} diff --git a/infrastructure/yaml/build.gradle b/infrastructure/yaml/build.gradle index e184a680e03..91430265c1e 100644 --- a/infrastructure/yaml/build.gradle +++ b/infrastructure/yaml/build.gradle @@ -1,5 +1,4 @@ dependencies { - implementation project(':infrastructure:jackson') implementation 'io.tmio:tuweni-bytes' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes32Deserializer.java b/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/Bytes32Deserializer.java similarity index 90% rename from infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes32Deserializer.java rename to infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/Bytes32Deserializer.java index 5fcd75d8ef1..e3ecac932e9 100644 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes32Deserializer.java +++ b/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/Bytes32Deserializer.java @@ -1,5 +1,5 @@ /* - * Copyright Consensys Software Inc., 2022 + * Copyright Consensys Software Inc., 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,7 +11,7 @@ * specific language governing permissions and limitations under the License. */ -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; +package tech.pegasys.teku.data.yaml; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/BytesSerializer.java b/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/BytesSerializer.java similarity index 90% rename from infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/BytesSerializer.java rename to infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/BytesSerializer.java index 2fafab35ae0..83439aa4f53 100644 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/BytesSerializer.java +++ b/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/BytesSerializer.java @@ -1,5 +1,5 @@ /* - * Copyright Consensys Software Inc., 2022 + * Copyright Consensys Software Inc., 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,7 +11,7 @@ * specific language governing permissions and limitations under the License. */ -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; +package tech.pegasys.teku.data.yaml; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; @@ -21,6 +21,7 @@ import org.apache.tuweni.bytes.Bytes; public class BytesSerializer extends JsonSerializer { + @Override public void serialize( final Bytes value, final JsonGenerator gen, final SerializerProvider provider) diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt64Deserializer.java b/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/UInt64Deserializer.java similarity index 90% rename from infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt64Deserializer.java rename to infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/UInt64Deserializer.java index 44cc016d84a..f20c2720762 100644 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt64Deserializer.java +++ b/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/UInt64Deserializer.java @@ -1,5 +1,5 @@ /* - * Copyright Consensys Software Inc., 2022 + * Copyright Consensys Software Inc., 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,7 +11,7 @@ * specific language governing permissions and limitations under the License. */ -package tech.pegasys.teku.infrastructure.jackson.deserializers.uints; +package tech.pegasys.teku.data.yaml; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt64Serializer.java b/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/UInt64Serializer.java similarity index 87% rename from infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt64Serializer.java rename to infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/UInt64Serializer.java index 49ff1af4a30..f0513d21304 100644 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/uints/UInt64Serializer.java +++ b/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/UInt64Serializer.java @@ -1,5 +1,5 @@ /* - * Copyright Consensys Software Inc., 2022 + * Copyright Consensys Software Inc., 2024 * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,7 +11,7 @@ * specific language governing permissions and limitations under the License. */ -package tech.pegasys.teku.infrastructure.jackson.deserializers.uints; +package tech.pegasys.teku.data.yaml; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; @@ -20,10 +20,11 @@ import tech.pegasys.teku.infrastructure.unsigned.UInt64; public class UInt64Serializer extends JsonSerializer { + @Override public void serialize( final UInt64 value, final JsonGenerator gen, final SerializerProvider serializers) throws IOException { - gen.writeString(value.toString()); + gen.writeNumber(value.bigIntegerValue()); } } diff --git a/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/YamlProvider.java b/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/YamlProvider.java index b3f2fe47ff7..fb232601b18 100644 --- a/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/YamlProvider.java +++ b/infrastructure/yaml/src/main/java/tech/pegasys/teku/data/yaml/YamlProvider.java @@ -14,43 +14,35 @@ package tech.pegasys.teku.data.yaml; import com.fasterxml.jackson.core.JsonGenerationException; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.io.StringWriter; import java.io.UncheckedIOException; import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import tech.pegasys.teku.infrastructure.jackson.deserializers.bytes.Bytes32Deserializer; -import tech.pegasys.teku.infrastructure.jackson.deserializers.bytes.BytesSerializer; import tech.pegasys.teku.infrastructure.unsigned.UInt64; public class YamlProvider { private final ObjectMapper objectMapper; public YamlProvider(final Module... modules) { - this.objectMapper = new ObjectMapper(new YAMLFactory()); + objectMapper = new ObjectMapper(new YAMLFactory()); addTekuMappers(); Stream.of(modules).forEach(objectMapper::registerModule); } private void addTekuMappers() { - SimpleModule module = new SimpleModule("TekuYaml", new Version(1, 0, 0, null, null, null)); + final SimpleModule module = + new SimpleModule("TekuYaml", new Version(1, 0, 0, null, null, null)); module.addDeserializer(UInt64.class, new UInt64Deserializer()); module.addSerializer(UInt64.class, new UInt64Serializer()); module.addDeserializer(Bytes32.class, new Bytes32Deserializer()); @@ -58,10 +50,6 @@ private void addTekuMappers() { objectMapper.registerModule(module).writer(new DefaultPrettyPrinter()); } - public T read(final InputStream data, final Class clazz) throws IOException { - return objectMapper.readValue(data, clazz); - } - public T read(final Bytes data, final Class clazz) throws IOException { return objectMapper.readValue(data.toArrayUnsafe(), clazz); } @@ -95,22 +83,4 @@ public String writeString(final T object) { public ObjectMapper getObjectMapper() { return objectMapper; } - - public static class UInt64Deserializer extends JsonDeserializer { - - @Override - public UInt64 deserialize(final JsonParser p, final DeserializationContext ctxt) - throws IOException { - return UInt64.valueOf(p.getValueAsString()); - } - } - - public static class UInt64Serializer extends JsonSerializer { - @Override - public void serialize( - final UInt64 value, final JsonGenerator gen, final SerializerProvider serializers) - throws IOException { - gen.writeNumber(value.bigIntegerValue()); - } - } } diff --git a/settings.gradle b/settings.gradle index af4d33cac39..b17af30b975 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,7 +10,6 @@ include 'ethereum:executionclient' include 'ethereum:executionlayer' include 'ethereum:execution-types' include 'ethereum:json-types' -include 'ethereum:jackson-deserializers' include 'ethereum:networks' include 'ethereum:pow:api' include 'ethereum:pow:merkletree' @@ -34,7 +33,6 @@ include 'infrastructure:events' include 'infrastructure:exceptions' include 'infrastructure:http' include 'infrastructure:io' -include 'infrastructure:jackson' include 'infrastructure:json' include 'infrastructure:kzg' include 'infrastructure:metrics' diff --git a/validator/client/build.gradle b/validator/client/build.gradle index 64cc30b7181..742e1a23937 100644 --- a/validator/client/build.gradle +++ b/validator/client/build.gradle @@ -4,7 +4,6 @@ dependencies { implementation project(':infrastructure:crypto') implementation project(':infrastructure:exceptions') implementation project(':infrastructure:io') - implementation project(':infrastructure:jackson') implementation project(':infrastructure:metrics') implementation project(':infrastructure:http') implementation project(':ethereum:json-types') diff --git a/validator/client/src/main/java/tech/pegasys/teku/validator/SetGasLimitException.java b/validator/client/src/main/java/tech/pegasys/teku/validator/SetGasLimitException.java deleted file mode 100644 index cc0de70fbd0..00000000000 --- a/validator/client/src/main/java/tech/pegasys/teku/validator/SetGasLimitException.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.validator; - -public class SetGasLimitException extends IllegalArgumentException { - - public SetGasLimitException(final String message) { - super(message); - } -} diff --git a/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/BLSPublicKeyDeserializer.java b/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/BLSPublicKeyDeserializer.java index d859f3ef5a3..2c5e63f6cd6 100644 --- a/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/BLSPublicKeyDeserializer.java +++ b/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/BLSPublicKeyDeserializer.java @@ -20,6 +20,7 @@ import tech.pegasys.teku.bls.BLSPublicKey; class BLSPublicKeyDeserializer extends JsonDeserializer { + @Override public BLSPublicKey deserialize(final JsonParser p, final DeserializationContext ctxt) throws IOException { diff --git a/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/BLSPublicKeySerializer.java b/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/BLSPublicKeySerializer.java index d7d48bea2dd..9952ac46c07 100644 --- a/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/BLSPublicKeySerializer.java +++ b/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/BLSPublicKeySerializer.java @@ -21,6 +21,7 @@ import tech.pegasys.teku.bls.BLSPublicKey; class BLSPublicKeySerializer extends JsonSerializer { + @Override public void serialize( final BLSPublicKey value, final JsonGenerator gen, final SerializerProvider serializers) diff --git a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes48KeyDeserializer.java b/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/Bytes48KeyDeserializer.java similarity index 94% rename from infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes48KeyDeserializer.java rename to validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/Bytes48KeyDeserializer.java index f10eb2c028c..1c5e32ba6ca 100644 --- a/infrastructure/jackson/src/main/java/tech/pegasys/teku/infrastructure/jackson/deserializers/bytes/Bytes48KeyDeserializer.java +++ b/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/Bytes48KeyDeserializer.java @@ -11,7 +11,7 @@ * specific language governing permissions and limitations under the License. */ -package tech.pegasys.teku.infrastructure.jackson.deserializers.bytes; +package tech.pegasys.teku.validator.client.proposerconfig.loader; import com.fasterxml.jackson.core.JacksonException; import com.fasterxml.jackson.databind.DeserializationContext; diff --git a/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/ProposerConfigLoader.java b/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/ProposerConfigLoader.java index 543025c9317..aee1a0383b7 100644 --- a/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/ProposerConfigLoader.java +++ b/validator/client/src/main/java/tech/pegasys/teku/validator/client/proposerconfig/loader/ProposerConfigLoader.java @@ -27,19 +27,18 @@ import tech.pegasys.teku.infrastructure.exceptions.ExceptionUtil; import tech.pegasys.teku.infrastructure.exceptions.InvalidConfigurationException; import tech.pegasys.teku.infrastructure.http.UrlSanitizer; -import tech.pegasys.teku.infrastructure.jackson.deserializers.bytes.Bytes48KeyDeserializer; import tech.pegasys.teku.validator.client.ProposerConfig; public class ProposerConfigLoader { private final ObjectMapper objectMapper; public ProposerConfigLoader() { - this.objectMapper = new ObjectMapper(); + objectMapper = new ObjectMapper(); addTekuMappers(); } private void addTekuMappers() { - SimpleModule module = + final SimpleModule module = new SimpleModule("ProposerConfigLoader", new Version(1, 0, 0, null, null, null)); module.addDeserializer(BLSPublicKey.class, new BLSPublicKeyDeserializer()); module.addSerializer(BLSPublicKey.class, new BLSPublicKeySerializer()); @@ -49,7 +48,7 @@ private void addTekuMappers() { } public ObjectMapper getObjectMapper() { - return this.objectMapper; + return objectMapper; } public ProposerConfig getProposerConfig(final URL source) { diff --git a/validator/remote/src/main/java/tech/pegasys/teku/validator/remote/typedef/BlindedBlockEndpointNotAvailableException.java b/validator/remote/src/main/java/tech/pegasys/teku/validator/remote/typedef/BlindedBlockEndpointNotAvailableException.java deleted file mode 100644 index 5e213df09f3..00000000000 --- a/validator/remote/src/main/java/tech/pegasys/teku/validator/remote/typedef/BlindedBlockEndpointNotAvailableException.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2022 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.validator.remote.typedef; - -public class BlindedBlockEndpointNotAvailableException extends RuntimeException {} diff --git a/validator/remote/src/main/java/tech/pegasys/teku/validator/remote/typedef/BlockProductionV3FailedException.java b/validator/remote/src/main/java/tech/pegasys/teku/validator/remote/typedef/BlockProductionV3FailedException.java deleted file mode 100644 index 17b389b947a..00000000000 --- a/validator/remote/src/main/java/tech/pegasys/teku/validator/remote/typedef/BlockProductionV3FailedException.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2023 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.validator.remote.typedef; - -public class BlockProductionV3FailedException extends RuntimeException {} From 8f4671501e2410f52ccf000069e8d82f66c88ea7 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 6 Jan 2025 23:13:05 +0100 Subject: [PATCH 7/8] Fix chiado transition epoch --- .../tech/pegasys/teku/networks/Eth2NetworkConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java b/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java index 2457f35dc9c..d4e3f27132d 100644 --- a/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java +++ b/ethereum/networks/src/main/java/tech/pegasys/teku/networks/Eth2NetworkConfiguration.java @@ -907,7 +907,7 @@ public Builder applyChiadoNetworkDefaults() { "enr:-KG4QF7z4LUdMfgwvh-fS-MDv_1hPSUCqGfyOWGLNJuoBHKFAMSHz8geQn8v3qDDbuSQKud3WIAjKqR4gqJoLBUEJ08ZhGV0aDKQDc1ElgAAAG___________4JpZIJ2NIJpcIQjzq5ciXNlY3AyNTZrMaECt7YO363pV54d3QdgnluL5kxzhCR_k0yM9C-G6bqMGoKDdGNwgiMog3VkcIIjKA", // GnosisDAO Bootnode: 35.210.126.23 "enr:-LK4QCUTEmZrT1AgCKdyVgwnHL5J0VSoxsyjruAtwo-owBTBVEOyAnQRVNXlcW5aL-ycntk5oHDrKCR-DXZAlUAKpjEBh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCdM7Z1BAAAb___________gmlkgnY0gmlwhCPSfheJc2VjcDI1NmsxoQNpdf8U9pzsU9m6Hzgd1rmTI-On-QImJnkZBGqDp4org4N0Y3CCIyiDdWRwgiMo") - .terminalBlockHashEpochOverride(UInt64.valueOf(27264)) + .terminalBlockHashEpochOverride(UInt64.valueOf(27263)) .terminalBlockHashOverride( Bytes32.fromHexString( "0x39f44fc16dc964e8d2d1637b99e12992be4a4f766a66658da730e20e511ced64")); From 68989d4e16f022ef4d5daf3e3cc4ff7ac0a14b46 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 8 Jan 2025 21:26:11 +0100 Subject: [PATCH 8/8] Remove warning for terminal block hash and epoch network defaults --- .../java/tech/pegasys/teku/AbstractNode.java | 74 +++++++++++++------ 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/teku/src/main/java/tech/pegasys/teku/AbstractNode.java b/teku/src/main/java/tech/pegasys/teku/AbstractNode.java index cb98c043b0d..c7f640b29b5 100644 --- a/teku/src/main/java/tech/pegasys/teku/AbstractNode.java +++ b/teku/src/main/java/tech/pegasys/teku/AbstractNode.java @@ -39,6 +39,7 @@ import tech.pegasys.teku.infrastructure.logging.StartupLogConfig; import tech.pegasys.teku.infrastructure.metrics.MetricsEndpoint; import tech.pegasys.teku.infrastructure.version.VersionProvider; +import tech.pegasys.teku.networks.Eth2NetworkConfiguration; import tech.pegasys.teku.service.serviceutils.ServiceConfig; import tech.pegasys.teku.service.serviceutils.layout.DataDirLayout; import tech.pegasys.teku.services.ServiceController; @@ -133,28 +134,7 @@ private void reportOverrides(final TekuConfiguration tekuConfig) { .ifPresent(forkEpoch -> STATUS_LOG.warnForkEpochChanged(specMilestone.name(), forkEpoch)); } - tekuConfig - .eth2NetworkConfiguration() - .getTotalTerminalDifficultyOverride() - .ifPresent( - ttdo -> - STATUS_LOG.warnBellatrixParameterChanged( - "TERMINAL_TOTAL_DIFFICULTY", ttdo.toString())); - - tekuConfig - .eth2NetworkConfiguration() - .getTerminalBlockHashOverride() - .ifPresent( - tbho -> - STATUS_LOG.warnBellatrixParameterChanged("TERMINAL_BLOCK_HASH", tbho.toString())); - - tekuConfig - .eth2NetworkConfiguration() - .getTerminalBlockHashEpochOverride() - .ifPresent( - tbheo -> - STATUS_LOG.warnBellatrixParameterChanged( - "TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH", tbheo.toString())); + reportBellatrixMergeOverrides(tekuConfig); // Deneb's epochsStoreBlobs warning, which is not actually a full override final Optional specVersionDeneb = @@ -175,6 +155,56 @@ private void reportOverrides(final TekuConfiguration tekuConfig) { }); } + private void reportBellatrixMergeOverrides(final TekuConfiguration tekuConfig) { + tekuConfig + .eth2NetworkConfiguration() + .getTotalTerminalDifficultyOverride() + .ifPresent( + ttdo -> + STATUS_LOG.warnBellatrixParameterChanged( + "TERMINAL_TOTAL_DIFFICULTY", ttdo.toString())); + + final Optional maybeEth2Network = + tekuConfig.eth2NetworkConfiguration().getEth2Network(); + + final boolean reportTerminalBlockHashOverride; + final boolean reportTerminalBlockHashEpochOverride; + if (maybeEth2Network.isEmpty()) { + reportTerminalBlockHashOverride = true; + reportTerminalBlockHashEpochOverride = true; + } else { + final Eth2NetworkConfiguration defaultConfiguration = + Eth2NetworkConfiguration.builder().applyNetworkDefaults(maybeEth2Network.get()).build(); + reportTerminalBlockHashOverride = + !defaultConfiguration + .getTerminalBlockHashOverride() + .equals(tekuConfig.eth2NetworkConfiguration().getTerminalBlockHashOverride()); + reportTerminalBlockHashEpochOverride = + !defaultConfiguration + .getTerminalBlockHashEpochOverride() + .equals(tekuConfig.eth2NetworkConfiguration().getTerminalBlockHashEpochOverride()); + } + + if (reportTerminalBlockHashOverride) { + tekuConfig + .eth2NetworkConfiguration() + .getTerminalBlockHashOverride() + .ifPresent( + tbho -> + STATUS_LOG.warnBellatrixParameterChanged("TERMINAL_BLOCK_HASH", tbho.toString())); + } + + if (reportTerminalBlockHashEpochOverride) { + tekuConfig + .eth2NetworkConfiguration() + .getTerminalBlockHashEpochOverride() + .ifPresent( + tbheo -> + STATUS_LOG.warnBellatrixParameterChanged( + "TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH", tbheo.toString())); + } + } + @Override public abstract ServiceController getServiceController();