Creation of blob sidecars (with inclusion proof) when publishing

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/BlockFactory.java

@@ -13,13 +13,16 @@
 
 package tech.pegasys.teku.validator.coordinator;
 
+import java.util.List;
 import java.util.Optional;
 import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.bls.BLSSignature;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.BlockContainer;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
+import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 
 public interface BlockFactory {
@@ -39,4 +42,6 @@ SafeFuture<BlockContainer> createUnsignedBlock(
       Optional<Bytes32> optionalGraffiti);
 
   SafeFuture<SignedBeaconBlock> unblindSignedBlockIfBlinded(SignedBeaconBlock maybeBlindedBlock);
+
+  List<BlobSidecar> createBlobSidecars(SignedBlockContainer blockContainer);
 }

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/BlockFactoryDeneb.java

@@ -24,8 +24,10 @@
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.BlockContainer;
+import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
 import tech.pegasys.teku.spec.datastructures.blocks.versions.deneb.BlockContents;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
@@ -56,10 +58,10 @@ public SafeFuture<BlockContainer> createUnsignedBlock(
               if (blockContainer.isBlinded()) {
                 return SafeFuture.completedFuture(blockContainer);
               }
-              // TODO: add blobs and proofs
+              // TODO: add blobs and proofs from the execution BlobsBundle
               final BeaconBlock block = blockContainer.getBlock();
               return operationSelector
-                  .createBlobSidecarsSelector()
+                  .createBlobSidecarsSelectorOld()
                   .apply(block)
                   .thenApply(
                       blobSidecars ->
@@ -80,10 +82,10 @@ public SafeFuture<BlockContainer> createUnsignedBlock(
               if (blockContainer.isBlinded()) {
                 return SafeFuture.completedFuture(blockContainer);
               }
-              // TODO: add blobs and proofs
+              // TODO: add blobs and proofs from the execution BlobsBundle
               final BeaconBlock block = blockContainer.getBlock();
               return operationSelector
-                  .createBlobSidecarsSelector()
+                  .createBlobSidecarsSelectorOld()
                   .apply(block)
                   .thenApply(
                       blobSidecars ->
@@ -92,6 +94,11 @@ public SafeFuture<BlockContainer> createUnsignedBlock(
             });
   }
 
+  @Override
+  public List<BlobSidecar> createBlobSidecars(final SignedBlockContainer blockContainer) {
+    return operationSelector.createBlobSidecarsSelector().apply(blockContainer);
+  }
+
   private BlockContents createBlockContents(
       final BeaconBlock block, final List<Blob> blobs, final List<KZGProof> kzgProofs) {
     return schemaDefinitionsDeneb.getBlockContentsSchema().create(block, kzgProofs, blobs);

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/BlockFactoryPhase0.java

@@ -15,15 +15,19 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
+import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
 import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.bls.BLSSignature;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.Spec;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockAndState;
 import tech.pegasys.teku.spec.datastructures.blocks.BlockContainer;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
+import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 
 public class BlockFactoryPhase0 implements BlockFactory {
@@ -103,4 +107,9 @@ public SafeFuture<SignedBeaconBlock> unblindSignedBlockIfBlinded(
     }
     return SafeFuture.completedFuture(maybeBlindedBlock);
   }
+
+  @Override
+  public List<BlobSidecar> createBlobSidecars(final SignedBlockContainer blockContainer) {
+    return Collections.emptyList();
+  }
 }

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactory.java

@@ -13,6 +13,7 @@
 
 package tech.pegasys.teku.validator.coordinator;
 
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
@@ -27,13 +28,16 @@
 import tech.pegasys.teku.infrastructure.ssz.SszList;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.Spec;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
-import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarSchemaOld;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.Eth1Data;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlockUnblinder;
+import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBodyBuilder;
+import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodyDeneb;
 import tech.pegasys.teku.spec.datastructures.builder.BuilderPayload;
 import tech.pegasys.teku.spec.datastructures.execution.BlobsBundle;
 import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayload;
@@ -48,7 +52,9 @@
 import tech.pegasys.teku.spec.datastructures.operations.SignedVoluntaryExit;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment;
+import tech.pegasys.teku.spec.datastructures.type.SszKZGProof;
 import tech.pegasys.teku.spec.executionlayer.ExecutionLayerBlockProductionManager;
+import tech.pegasys.teku.spec.logic.versions.deneb.helpers.MiscHelpersDeneb;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitions;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsBellatrix;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
@@ -289,7 +295,7 @@ private void builderSetKzgCommitments(
               if (bodyBuilder.isBlinded()) {
                 return getBlobKzgCommitments(executionPayloadResult);
               } else {
-                return getBlobsBundle(executionPayloadResult)
+                return getExecutionBlobsBundle(executionPayloadResult)
                     .thenApply(
                         blobsBundle ->
                             schemaDefinitionsDeneb
@@ -329,38 +335,68 @@ public Consumer<SignedBeaconBlockUnblinder> createBlockUnblinderSelector() {
     };
   }
 
-  // TODO: create blob sidecars with inclusion proofs
-  public Function<BeaconBlock, SafeFuture<List<BlobSidecarOld>>> createBlobSidecarsSelector() {
-    return block -> {
-      final BlobSidecarSchemaOld blobSidecarSchema =
-          SchemaDefinitionsDeneb.required(spec.atSlot(block.getSlot()).getSchemaDefinitions())
-              .getBlobSidecarOldSchema();
-      return getCachedBlobsBundle(block.getSlot())
-          .thenApply(
-              blobsBundle ->
-                  IntStream.range(0, blobsBundle.getNumberOfBlobs())
-                      .mapToObj(
-                          index ->
-                              blobSidecarSchema.create(
-                                  block.getRoot(),
-                                  UInt64.valueOf(index),
-                                  block.getSlot(),
-                                  block.getParentRoot(),
-                                  block.getProposerIndex(),
-                                  blobsBundle.getBlobs().get(index),
-                                  blobsBundle.getCommitments().get(index),
-                                  blobsBundle.getProofs().get(index)))
-                      .toList());
+  @Deprecated
+  public Function<BeaconBlock, SafeFuture<List<BlobSidecarOld>>> createBlobSidecarsSelectorOld() {
+    return block -> SafeFuture.completedFuture(Collections.emptyList());
+  }
+
+  public Function<SignedBlockContainer, List<BlobSidecar>> createBlobSidecarsSelector() {
+    return blockContainer -> {
+      final UInt64 slot = blockContainer.getSlot();
+      final MiscHelpersDeneb miscHelpersDeneb =
+          MiscHelpersDeneb.required(spec.atSlot(slot).miscHelpers());
+
+      final SszList<Blob> blobs;
+      final SszList<SszKZGCommitment> commitments;
+      final SszList<SszKZGProof> proofs;
+
+      if (blockContainer.isBlinded()) {
+        // need to use the already available builder blobs bundle for the blinded flow, because the
+        // blobs and the proofs wouldn't be part of the BlockContainer
+        final tech.pegasys.teku.spec.datastructures.builder.BlobsBundle blobsBundle =
+            getCachedBuilderBlobsBundle(slot);
+        blobs = blobsBundle.getBlobs();
+        commitments = blobsBundle.getCommitments();
+        proofs = blobsBundle.getProofs();
+      } else {
+        blobs =
+            blockContainer
+                .getBlobs()
+                .orElseThrow(
+                    () ->
+                        new IllegalStateException("Blobs are not available in " + blockContainer));
+        commitments =
+            BeaconBlockBodyDeneb.required(blockContainer.getSignedBlock().getMessage().getBody())
+                .getBlobKzgCommitments();
+        proofs =
+            blockContainer
+                .getKzgProofs()
+                .orElseThrow(
+                    () ->
+                        new IllegalStateException("Proofs are not available in " + blockContainer));
+      }
+
+      return IntStream.range(0, blobs.size())
+          .mapToObj(
+              index ->
+                  miscHelpersDeneb.computeBlobSidecar(
+                      blockContainer.getSignedBlock(),
+                      UInt64.valueOf(index),
+                      blobs.get(index),
+                      commitments.get(index),
+                      proofs.get(index)))
+          .toList();
     };
   }
 
-  private SafeFuture<BlobsBundle> getBlobsBundle(
+  private SafeFuture<BlobsBundle> getExecutionBlobsBundle(
       final ExecutionPayloadResult executionPayloadResult) {
     return executionPayloadResult
         .getBlobsBundleFuture()
-        .orElseThrow(this::blobsBundleIsNotAvailableException)
+        .orElseThrow(this::executionBlobsBundleIsNotAvailableException)
         .thenApply(
-            blobsBundle -> blobsBundle.orElseThrow(this::blobsBundleIsNotAvailableException));
+            blobsBundle ->
+                blobsBundle.orElseThrow(this::executionBlobsBundleIsNotAvailableException));
   }
 
   private SafeFuture<SszList<SszKZGCommitment>> getBlobKzgCommitments(
@@ -376,18 +412,19 @@ private SafeFuture<SszList<SszKZGCommitment>> getBlobKzgCommitments(
                         () -> new IllegalStateException("BlobKzgCommitments are not available")));
   }
 
-  private SafeFuture<BlobsBundle> getCachedBlobsBundle(final UInt64 slot) {
-    final ExecutionPayloadResult executionPayloadResult = getCachedPayloadResult(slot);
-    return getBlobsBundle(executionPayloadResult);
-  }
-
-  private ExecutionPayloadResult getCachedPayloadResult(final UInt64 slot) {
+  private tech.pegasys.teku.spec.datastructures.builder.BlobsBundle getCachedBuilderBlobsBundle(
+      final UInt64 slot) {
     return executionLayerBlockProductionManager
-        .getCachedPayloadResult(slot)
-        .orElseThrow(() -> new IllegalStateException("ExecutionPayloadResult is not available"));
+        .getCachedUnblindedPayload(slot)
+        .orElseThrow(
+            () -> new IllegalStateException("BuilderPayload hasn't been cached for slot " + slot))
+        .getOptionalBlobsBundle()
+        .orElseThrow(
+            () ->
+                new IllegalStateException("builder BlobsBundle is not available for slot " + slot));
   }
 
-  private IllegalStateException blobsBundleIsNotAvailableException() {
-    return new IllegalStateException("BlobsBundle is not available");
+  private IllegalStateException executionBlobsBundleIsNotAvailableException() {
+    return new IllegalStateException("execution BlobsBundle is not available");
   }
 }

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/MilestoneBasedBlockFactory.java

@@ -15,6 +15,7 @@
 
 import com.google.common.base.Suppliers;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Supplier;
@@ -24,8 +25,10 @@
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.BlockContainer;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
+import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockContainer;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 
 public class MilestoneBasedBlockFactory implements BlockFactory {
@@ -89,6 +92,12 @@ public SafeFuture<SignedBeaconBlock> unblindSignedBlockIfBlinded(
     return registeredFactories.get(milestone).unblindSignedBlockIfBlinded(maybeBlindedBlock);
   }
 
+  @Override
+  public List<BlobSidecar> createBlobSidecars(final SignedBlockContainer blockContainer) {
+    final SpecMilestone milestone = getMilestone(blockContainer.getSlot());
+    return registeredFactories.get(milestone).createBlobSidecars(blockContainer);
+  }
+
   private SpecMilestone getMilestone(final UInt64 slot) {
     return spec.atSlot(slot).getMilestone();
   }

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/publisher/AbstractBlockPublisher.java

@@ -60,10 +60,14 @@ public SafeFuture<SendSignedBlockResult> sendSignedBlock(
         .unblindSignedBlockIfBlinded(blockContainer.getSignedBlock())
         .thenPeek(performanceTracker::saveProducedBlock)
         .thenCompose(
-            signedBlock ->
-                // TODO: produce blob sidecars for Deneb (using BlockFactory)
-                gossipAndImportUnblindedSignedBlockAndBlobSidecars(
-                    signedBlock, List.of(), broadcastValidationLevel))
+            signedBlock -> {
+              // creating blob sidecars after unblinding the block to ensure in the blinded flow we
+              // already have cached the builder payload
+              final List<BlobSidecar> blobSidecars =
+                  blockFactory.createBlobSidecars(blockContainer);
+              return gossipAndImportUnblindedSignedBlockAndBlobSidecars(
+                  signedBlock, blobSidecars, broadcastValidationLevel);
+            })
         .thenCompose(result -> calculateResult(blockContainer, result));
   }