8767 update payload attributes (#8824)

Co-authored-by: Enrico Del Fante <[email protected]>

beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/forward/multipeer/chains/SyncSourceFactory.java

@@ -45,7 +45,8 @@ public SyncSource getOrCreateSyncSource(final Eth2Peer peer, final Spec spec) {
     // Limit request rate to just a little under what we'd accept
     final int maxBlocksPerMinute = this.maxBlocksPerMinute - batchSize - 1;
     final Optional<Integer> maxBlobSidecarsPerMinute =
-        spec.getMaxBlobsPerBlock().map(maxBlobsPerBlock -> maxBlocksPerMinute * maxBlobsPerBlock);
+        spec.getMaxBlobsPerBlockForHighestMilestone()
+            .map(maxBlobsPerBlock -> maxBlocksPerMinute * maxBlobsPerBlock);
 
     return syncSourcesByPeer.computeIfAbsent(
         peer,

beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/gossip/blobs/RecentBlobSidecarsFetchService.java

@@ -60,7 +60,8 @@ public static RecentBlobSidecarsFetchService create(
       final FetchTaskFactory fetchTaskFactory,
       final Spec spec) {
     final int maxConcurrentRequests =
-        RecentBlocksFetchService.MAX_CONCURRENT_REQUESTS * spec.getMaxBlobsPerBlock().orElse(1);
+        RecentBlocksFetchService.MAX_CONCURRENT_REQUESTS
+            * spec.getMaxBlobsPerBlockForHighestMilestone().orElse(1);
     return new RecentBlobSidecarsFetchService(
         asyncRunner,
         blockBlobSidecarsTrackersPool,

data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/tekuv1/beacon/GetAllBlobSidecarsAtSlotTest.java

@@ -29,15 +29,16 @@
 import static tech.pegasys.teku.infrastructure.restapi.MetadataTestUtil.verifyMetadataErrorResponse;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.io.Resources;
-import java.io.IOException;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.beaconrestapi.AbstractMigratedBeaconHandlerTest;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
+import tech.pegasys.teku.infrastructure.json.JsonTestUtil;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.TestSpecFactory;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
@@ -86,15 +87,18 @@ void metadata_shouldHandle500() throws JsonProcessingException {
   }
 
   @Test
-  void metadata_shouldHandle200() throws IOException {
+  void metadata_shouldHandle200() throws Exception {
     final List<BlobSidecar> nonCanonicalBlobSidecars = dataStructureUtil.randomBlobSidecars(4);
 
-    final String data = getResponseStringFromMetadata(handler, SC_OK, nonCanonicalBlobSidecars);
-    final String expected =
-        Resources.toString(
-            Resources.getResource(
-                GetAllBlobSidecarsAtSlotTest.class, "getAllBlobSidecarsAtSlot.json"),
-            UTF_8);
+    final JsonNode data =
+        JsonTestUtil.parseAsJsonNode(
+            getResponseStringFromMetadata(handler, SC_OK, nonCanonicalBlobSidecars));
+    final JsonNode expected =
+        JsonTestUtil.parseAsJsonNode(
+            Resources.toString(
+                Resources.getResource(
+                    GetAllBlobSidecarsAtSlotTest.class, "getAllBlobSidecarsAtSlot.json"),
+                UTF_8));
     assertThat(data).isEqualTo(expected);
   }
 

data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetBlobSidecarsTest.java

@@ -27,15 +27,16 @@
 import static tech.pegasys.teku.infrastructure.restapi.MetadataTestUtil.verifyMetadataErrorResponse;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.io.Resources;
-import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.api.schema.Version;
 import tech.pegasys.teku.beaconrestapi.AbstractMigratedBeaconHandlerWithChainDataProviderTest;
+import tech.pegasys.teku.infrastructure.json.JsonTestUtil;
 import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
@@ -89,14 +90,17 @@ void metadata_shouldHandle500() throws JsonProcessingException {
   }
 
   @Test
-  void metadata_shouldHandle200() throws IOException {
+  void metadata_shouldHandle200() throws Exception {
     final List<BlobSidecar> blobSidecars = dataStructureUtil.randomBlobSidecars(3);
     final BlobSidecarsAndMetaData blobSidecarsAndMetaData =
         new BlobSidecarsAndMetaData(blobSidecars, SpecMilestone.DENEB, true, false, false);
-    final String data = getResponseStringFromMetadata(handler, SC_OK, blobSidecarsAndMetaData);
-    final String expected =
-        Resources.toString(
-            Resources.getResource(GetBlobSidecarsTest.class, "getBlobSidecars.json"), UTF_8);
+    final JsonNode data =
+        JsonTestUtil.parseAsJsonNode(
+            getResponseStringFromMetadata(handler, SC_OK, blobSidecarsAndMetaData));
+    final JsonNode expected =
+        JsonTestUtil.parseAsJsonNode(
+            Resources.toString(
+                Resources.getResource(GetBlobSidecarsTest.class, "getBlobSidecars.json"), UTF_8));
     assertThat(data).isEqualTo(expected);
   }
 

ethereum/executionclient/src/integration-test/java/tech/pegasys/teku/ethereum/executionclient/web3j/Web3JExecutionEngineClientTest.java

@@ -366,7 +366,8 @@ public void newPayloadV4_shouldBuildRequestAndResponseSuccessfully() {
   public void getBlobsV1_shouldBuildRequestAndResponseSuccessfully() {
     assumeThat(specMilestone).isGreaterThanOrEqualTo(DENEB);
     final List<BlobSidecar> blobSidecars =
-        dataStructureUtil.randomBlobSidecars(spec.getMaxBlobsPerBlock().orElseThrow());
+        dataStructureUtil.randomBlobSidecars(
+            spec.getMaxBlobsPerBlockForHighestMilestone().orElseThrow());
     final List<BlobAndProofV1> blobsAndProofsV1 =
         blobSidecars.stream()
             .map(

ethereum/executionclient/src/test/java/tech/pegasys/teku/ethereum/executionclient/methods/EngineGetBlobsV1Test.java

@@ -36,6 +36,7 @@
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.TestSpecFactory;
+import tech.pegasys.teku.spec.config.SpecConfigDeneb;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof;
 import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash;
@@ -109,7 +110,8 @@ public void shouldReturnFailedExecutionWhenEngineClientRequestFails() {
   public void shouldCallGetBlobsV1AndParseResponseSuccessfully() {
     final List<VersionedHash> versionedHashes = dataStructureUtil.randomVersionedHashes(4);
     final List<BlobSidecar> blobSidecars =
-        dataStructureUtil.randomBlobSidecars(spec.getMaxBlobsPerBlock().orElseThrow());
+        dataStructureUtil.randomBlobSidecars(
+            SpecConfigDeneb.required(spec.getGenesisSpecConfig()).getMaxBlobsPerBlock());
 
     when(executionEngineClient.getBlobsV1(eq(versionedHashes)))
         .thenReturn(dummySuccessfulResponse(blobSidecars));

ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/DenebExecutionClientHandlerTest.java

@@ -35,6 +35,7 @@
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.TestSpecFactory;
+import tech.pegasys.teku.spec.config.SpecConfigDeneb;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSchema;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof;
@@ -145,7 +146,8 @@ void engineForkChoiceUpdated_shouldCallEngineForkChoiceUpdatedV3() {
   @Test
   void engineGetBlobs_shouldCallGetBlobsV1() {
     final ExecutionClientHandler handler = getHandler();
-    final int maxBlobsPerBlock = spec.getMaxBlobsPerBlock().orElseThrow();
+    final int maxBlobsPerBlock =
+        SpecConfigDeneb.required(spec.getGenesisSpecConfig()).getMaxBlobsPerBlock();
     final List<VersionedHash> versionedHashes =
         dataStructureUtil.randomVersionedHashes(maxBlobsPerBlock - 1);
     final List<BlobSidecar> blobSidecars = dataStructureUtil.randomBlobSidecars(maxBlobsPerBlock);

ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ElectraExecutionClientHandlerTest.java

@@ -37,6 +37,7 @@
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.TestSpecFactory;
+import tech.pegasys.teku.spec.config.SpecConfigDeneb;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSchema;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.execution.BlobAndProof;
@@ -127,11 +128,12 @@ void engineNewPayload_shouldCallNewPayloadV4() {
   }
 
   @Test
-  void engineForkChoiceUpdated_shouldCallEngineForkChoiceUpdatedV3() {
+  void engineForkChoiceUpdated_shouldCallEngineForkChoiceUpdatedV4() {
     final ExecutionClientHandler handler = getHandler();
     final ForkChoiceState forkChoiceState = dataStructureUtil.randomForkChoiceState(false);
     final ForkChoiceStateV1 forkChoiceStateV1 =
         ForkChoiceStateV1.fromInternalForkChoiceState(forkChoiceState);
+    final SpecConfigDeneb specConfigDeneb = SpecConfigDeneb.required(spec.getGenesisSpecConfig());
     final PayloadBuildingAttributes attributes =
         new PayloadBuildingAttributes(
             dataStructureUtil.randomUInt64(),
@@ -142,8 +144,8 @@ void engineForkChoiceUpdated_shouldCallEngineForkChoiceUpdatedV3() {
             Optional.empty(),
             Optional.of(List.of()),
             dataStructureUtil.randomBytes32(),
-            spec.getMaxBlobsPerBlock().map(max -> UInt64.valueOf(max / 2)),
-            spec.getMaxBlobsPerBlock().map(UInt64::valueOf));
+            Optional.of(UInt64.valueOf(specConfigDeneb.getTargetBlobsPerBlock())),
+            Optional.of(UInt64.valueOf(specConfigDeneb.getMaxBlobsPerBlock())));
     final Optional<PayloadAttributesV4> payloadAttributes =
         PayloadAttributesV4.fromInternalPayloadBuildingAttributesV4(Optional.of(attributes));
     final ForkChoiceUpdatedResult responseData =
@@ -164,7 +166,8 @@ void engineForkChoiceUpdated_shouldCallEngineForkChoiceUpdatedV3() {
   @Test
   void engineGetBlobs_shouldCallGetBlobsV1() {
     final ExecutionClientHandler handler = getHandler();
-    final int maxBlobsPerBlock = spec.getMaxBlobsPerBlock().orElseThrow();
+    final int maxBlobsPerBlock =
+        SpecConfigDeneb.required(spec.getGenesisSpecConfig()).getMaxBlobsPerBlock();
     final List<VersionedHash> versionedHashes =
         dataStructureUtil.randomVersionedHashes(maxBlobsPerBlock);
     final List<BlobSidecar> blobSidecars = dataStructureUtil.randomBlobSidecars(maxBlobsPerBlock);

ethereum/executionlayer/src/test/java/tech/pegasys/teku/ethereum/executionlayer/ExecutionLayerManagerImplTest.java

@@ -44,6 +44,7 @@
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.TestSpecFactory;
+import tech.pegasys.teku.spec.config.SpecConfigDeneb;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
 import tech.pegasys.teku.spec.datastructures.builder.BuilderBid;
@@ -663,7 +664,8 @@ void onSlot_shouldCleanUpFallbackCache() {
   public void engineGetBlobs_shouldReturnGetBlobsResponseViaEngine() {
     setupDeneb();
     final List<VersionedHash> versionedHashes =
-        dataStructureUtil.randomVersionedHashes(spec.getMaxBlobsPerBlock().orElseThrow());
+        dataStructureUtil.randomVersionedHashes(
+            SpecConfigDeneb.required(spec.getGenesisSpecConfig()).getMaxBlobsPerBlock());
     final UInt64 slot = dataStructureUtil.randomSlot();
     final List<BlobAndProof> getBlobsResponse =
         prepareEngineGetBlobsResponse(versionedHashes, slot);
@@ -804,7 +806,8 @@ private GetPayloadResponse prepareEngineGetPayloadResponseWithBlobs(
   private List<BlobAndProof> prepareEngineGetBlobsResponse(
       final List<VersionedHash> blobVersionedHashes, final UInt64 slot) {
     final List<BlobSidecar> blobSidecars =
-        dataStructureUtil.randomBlobSidecars(spec.getMaxBlobsPerBlock().orElseThrow());
+        dataStructureUtil.randomBlobSidecars(
+            SpecConfigDeneb.required(spec.getGenesisSpecConfig()).getMaxBlobsPerBlock());
     final List<BlobAndProof> getBlobsResponse =
         blobSidecars.stream()
             .map(blobSidecar -> new BlobAndProof(blobSidecar.getBlob(), blobSidecar.getKZGProof()))

ethereum/spec/src/main/java/tech/pegasys/teku/spec/Spec.java

@@ -17,7 +17,6 @@
 import static tech.pegasys.teku.infrastructure.time.TimeUtilities.millisToSeconds;
 import static tech.pegasys.teku.infrastructure.time.TimeUtilities.secondsToMillis;
 import static tech.pegasys.teku.spec.SpecMilestone.DENEB;
-import static tech.pegasys.teku.spec.SpecMilestone.ELECTRA;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Preconditions;
@@ -49,12 +48,10 @@
 import tech.pegasys.teku.spec.cache.IndexedAttestationCache;
 import tech.pegasys.teku.spec.config.NetworkingSpecConfig;
 import tech.pegasys.teku.spec.config.NetworkingSpecConfigDeneb;
-import tech.pegasys.teku.spec.config.NetworkingSpecConfigElectra;
 import tech.pegasys.teku.spec.config.SpecConfig;
 import tech.pegasys.teku.spec.config.SpecConfigAltair;
 import tech.pegasys.teku.spec.config.SpecConfigAndParent;
 import tech.pegasys.teku.spec.config.SpecConfigDeneb;
-import tech.pegasys.teku.spec.config.SpecConfigElectra;
 import tech.pegasys.teku.spec.constants.Domain;
 import tech.pegasys.teku.spec.datastructures.attestation.ValidatableAttestation;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob;
@@ -235,16 +232,6 @@ public Optional<NetworkingSpecConfigDeneb> getNetworkingConfigDeneb() {
         .map(specConfig -> (NetworkingSpecConfigDeneb) specConfig.getNetworkingConfig());
   }
 
-  /**
-   * Networking config with Electra constants. Use {@link SpecConfigElectra#required(SpecConfig)}
-   * when you are sure that Electra is available, otherwise use this method
-   */
-  public Optional<NetworkingSpecConfigElectra> getNetworkingConfigElectra() {
-    return Optional.ofNullable(forMilestone(ELECTRA))
-        .map(SpecVersion::getConfig)
-        .map(specConfig -> (NetworkingSpecConfigElectra) specConfig.getNetworkingConfig());
-  }
-
   public SchemaDefinitions getGenesisSchemaDefinitions() {
     return getGenesisSpec().getSchemaDefinitions();
   }
@@ -961,18 +948,46 @@ public boolean isAvailabilityOfBlobSidecarsRequiredAtEpoch(
         .isLessThanOrEqualTo(specConfigDeneb.getMinEpochsForBlobSidecarsRequests());
   }
 
-  public Optional<Integer> getMaxBlobsPerBlock() {
-    return getSpecConfigDeneb().map(SpecConfigDeneb::getMaxBlobsPerBlock);
-  }
+  /**
+   * This method is used to setup caches and limits during the initialization of the node. We
+   * normally increase the blobs with each fork, but in case we will decrease them, let's consider
+   * the last two forks.
+   */
+  public Optional<Integer> getMaxBlobsPerBlockForHighestMilestone() {
+    final SpecMilestone highestSupportedMilestone =
+        getForkSchedule().getHighestSupportedMilestone();
+
+    final Optional<Integer> maybeHighestMaxBlobsPerBlock =
+        forMilestone(highestSupportedMilestone)
+            .getConfig()
+            .toVersionDeneb()
+            .map(SpecConfigDeneb::getMaxBlobsPerBlock);
+
+    final Optional<Integer> maybeSecondHighestMaxBlobsPerBlock =
+        highestSupportedMilestone
+            .getPreviousMilestoneIfExists()
+            .map(this::forMilestone)
+            .map(SpecVersion::getConfig)
+            .flatMap(SpecConfig::toVersionDeneb)
+            .map(SpecConfigDeneb::getMaxBlobsPerBlock);
+
+    if (maybeHighestMaxBlobsPerBlock.isEmpty() && maybeSecondHighestMaxBlobsPerBlock.isEmpty()) {
+      return Optional.empty();
+    }
+
+    final int highestMaxBlobsPerBlock = maybeHighestMaxBlobsPerBlock.orElse(0);
+    final int secondHighestMaxBlobsPerBlock = maybeSecondHighestMaxBlobsPerBlock.orElse(0);
+    final int max = Math.max(highestMaxBlobsPerBlock, secondHighestMaxBlobsPerBlock);
 
-  public Optional<Integer> getMaxBlobsPerBlock(final UInt64 slot) {
-    return getSpecConfigDeneb(slot).map(SpecConfigDeneb::getMaxBlobsPerBlock);
+    return Optional.of(max);
   }
 
   public UInt64 computeSubnetForBlobSidecar(final BlobSidecar blobSidecar) {
     return blobSidecar
         .getIndex()
-        .mod(atSlot(blobSidecar.getSlot()).miscHelpers().getBlobSidecarSubnetCount().orElseThrow());
+        .mod(
+            SpecConfigDeneb.required(atSlot(blobSidecar.getSlot()).getConfig())
+                .getBlobSidecarSubnetCount());
   }
 
   public Optional<UInt64> computeFirstSlotWithBlobSupport() {
@@ -995,10 +1010,6 @@ private Optional<SpecConfigDeneb> getSpecConfigDeneb() {
         .flatMap(SpecConfig::toVersionDeneb);
   }
 
-  private Optional<SpecConfigDeneb> getSpecConfigDeneb(final UInt64 slot) {
-    return atSlot(slot).getConfig().toVersionDeneb();
-  }
-
   // Private helpers
   private SpecVersion atState(final BeaconState state) {
     return atSlot(state.getSlot());

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/DelegatingSpecConfigDeneb.java

@@ -61,6 +61,11 @@ public int getMaxBlobsPerBlock() {
     return specConfigDeneb.getMaxBlobsPerBlock();
   }
 
+  @Override
+  public int getTargetBlobsPerBlock() {
+    return specConfigDeneb.getTargetBlobsPerBlock();
+  }
+
   @Override
   public int getKzgCommitmentInclusionProofDepth() {
     return specConfigDeneb.getKzgCommitmentInclusionProofDepth();

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigDeneb.java

@@ -54,6 +54,8 @@ static SpecConfigDeneb required(final SpecConfig specConfig) {
 
   int getMaxBlobsPerBlock();
 
+  int getTargetBlobsPerBlock();
+
   int getKzgCommitmentInclusionProofDepth();
 
   int getEpochsStoreBlobs();

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigDenebImpl.java

@@ -94,6 +94,11 @@ public int getMaxBlobsPerBlock() {
     return maxBlobsPerBlock;
   }
 
+  @Override
+  public int getTargetBlobsPerBlock() {
+    return maxBlobsPerBlock / 2;
+  }
+
   @Override
   public int getKzgCommitmentInclusionProofDepth() {
     return kzgCommitmentInclusionProofDepth;

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigElectra.java

@@ -17,7 +17,7 @@
 import tech.pegasys.teku.infrastructure.bytes.Bytes4;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 
-public interface SpecConfigElectra extends SpecConfigDeneb, NetworkingSpecConfigElectra {
+public interface SpecConfigElectra extends SpecConfigDeneb, NetworkingSpecConfigDeneb {
 
   UInt64 UNSET_DEPOSIT_REQUESTS_START_INDEX = UInt64.MAX_VALUE;
   UInt64 FULL_EXIT_REQUEST_AMOUNT = UInt64.ZERO;
@@ -66,10 +66,6 @@ static SpecConfigElectra required(final SpecConfig specConfig) {
 
   int getMaxPendingDepositsPerEpoch();
 
-  int getMaxBlobsPerBlockElectra();
-
-  int getTargetBlobsPerBlockElectra();
-
   @Override
   Optional<SpecConfigElectra> toVersionElectra();
 }