Merge branch 'master' into refactor-engine-get-blobs

Consensys · Oct 16, 2024 · bf8bfbf · bf8bfbf
2 parents 85154e0 + 5be6509
commit bf8bfbf
Show file tree

Hide file tree

Showing 25 changed files with 817 additions and 128 deletions.
diff --git a/...src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/deneb/BlobSidecar.java b/...src/main/java/tech/pegasys/teku/spec/datastructures/blobs/versions/deneb/BlobSidecar.java
@@ -138,4 +138,9 @@ public String toLogString() {
         getKZGCommitment().toAbbreviatedString(),
         getKZGProof().toAbbreviatedString());
   }
+
+  @Override
+  public BlobSidecarSchema getSchema() {
+    return (BlobSidecarSchema) super.getSchema();
+  }
 }
diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/AbstractSchemaDefinitions.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/AbstractSchemaDefinitions.java
@@ -13,34 +13,34 @@
 
 package tech.pegasys.teku.spec.schemas;
 
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.ATTNETS_ENR_FIELD_SCHEMA;
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.BEACON_BLOCKS_BY_ROOT_REQUEST_MESSAGE_SCHEMA;
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.HISTORICAL_BATCH_SCHEMA;
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SYNCNETS_ENR_FIELD_SCHEMA;
+
 import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector;
 import tech.pegasys.teku.infrastructure.ssz.schema.collections.SszBitvectorSchema;
 import tech.pegasys.teku.spec.config.SpecConfig;
-import tech.pegasys.teku.spec.constants.NetworkConstants;
 import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.BeaconBlocksByRootRequestMessage;
+import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.BeaconBlocksByRootRequestMessage.BeaconBlocksByRootRequestMessageSchema;
 import tech.pegasys.teku.spec.datastructures.state.HistoricalBatch.HistoricalBatchSchema;
 import tech.pegasys.teku.spec.schemas.registry.SchemaRegistry;
-import tech.pegasys.teku.spec.schemas.registry.SchemaTypes;
 
 public abstract class AbstractSchemaDefinitions implements SchemaDefinitions {
   protected SchemaRegistry schemaRegistry;
 
   final SszBitvectorSchema<SszBitvector> attnetsENRFieldSchema;
-  final SszBitvectorSchema<SszBitvector> syncnetsENRFieldSchema =
-      SszBitvectorSchema.create(NetworkConstants.SYNC_COMMITTEE_SUBNET_COUNT);
+  private final SszBitvectorSchema<SszBitvector> syncnetsENRFieldSchema;
   private final HistoricalBatchSchema historicalBatchSchema;
-  private final BeaconBlocksByRootRequestMessage.BeaconBlocksByRootRequestMessageSchema
-      beaconBlocksByRootRequestMessageSchema;
+  private final BeaconBlocksByRootRequestMessageSchema beaconBlocksByRootRequestMessageSchema;
 
   public AbstractSchemaDefinitions(final SchemaRegistry schemaRegistry) {
     this.schemaRegistry = schemaRegistry;
-    this.historicalBatchSchema =
-        new HistoricalBatchSchema(schemaRegistry.getSpecConfig().getSlotsPerHistoricalRoot());
-
+    this.historicalBatchSchema = schemaRegistry.get(HISTORICAL_BATCH_SCHEMA);
     this.beaconBlocksByRootRequestMessageSchema =
-        new BeaconBlocksByRootRequestMessage.BeaconBlocksByRootRequestMessageSchema(
-            schemaRegistry.getSpecConfig());
-    this.attnetsENRFieldSchema = schemaRegistry.get(SchemaTypes.ATTNETS_ENR_FIELD_SCHEMA);
+        schemaRegistry.get(BEACON_BLOCKS_BY_ROOT_REQUEST_MESSAGE_SCHEMA);
+    this.syncnetsENRFieldSchema = schemaRegistry.get(SYNCNETS_ENR_FIELD_SCHEMA);
+    this.attnetsENRFieldSchema = schemaRegistry.get(ATTNETS_ENR_FIELD_SCHEMA);
   }
 
   abstract long getMaxValidatorPerAttestation(SpecConfig specConfig);

diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/BaseSchemaProvider.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/BaseSchemaProvider.java
@@ -102,18 +102,23 @@ public String toString() {
     }
   }
 
-  static <T> Builder<T> providerBuilder(final SchemaId<T> schemaId) {
-    return new Builder<>(schemaId);
+  static <T> Builder<T> constantProviderBuilder(final SchemaId<T> schemaId) {
+    return new Builder<>(schemaId, true);
+  }
+
+  static <T> Builder<T> variableProviderBuilder(final SchemaId<T> schemaId) {
+    return new Builder<>(schemaId, false);
   }
 
   static class Builder<T> {
     private final SchemaId<T> schemaId;
+    private final boolean isConstant;
     final List<SchemaProviderCreator<T>> schemaProviderCreators = new ArrayList<>();
     private SpecMilestone untilMilestone = SpecMilestone.getHighestMilestone();
-    private boolean isConstant = false;
 
-    private Builder(final SchemaId<T> schemaId) {
+    private Builder(final SchemaId<T> schemaId, final boolean isConstant) {
       this.schemaId = schemaId;
+      this.isConstant = isConstant;
     }
 
     public Builder<T> withCreator(
@@ -134,11 +139,6 @@ public Builder<T> until(final SpecMilestone untilMilestone) {
       return this;
     }
 
-    public Builder<T> constant() {
-      this.isConstant = true;
-      return this;
-    }
-
     public BaseSchemaProvider<T> build() {
       checkArgument(
           !schemaProviderCreators.isEmpty(), "There should be at least 1 creator for %s", schemaId);

diff --git a/...eum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaRegistryBuilder.java b/...eum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaRegistryBuilder.java
@@ -13,21 +13,26 @@
 
 package tech.pegasys.teku.spec.schemas.registry;
 
-import static tech.pegasys.teku.spec.schemas.registry.BaseSchemaProvider.providerBuilder;
+import static tech.pegasys.teku.spec.schemas.registry.BaseSchemaProvider.constantProviderBuilder;
 import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.ATTESTATION_SCHEMA;
 import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.ATTNETS_ENR_FIELD_SCHEMA;
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.BEACON_BLOCKS_BY_ROOT_REQUEST_MESSAGE_SCHEMA;
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.HISTORICAL_BATCH_SCHEMA;
+import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SYNCNETS_ENR_FIELD_SCHEMA;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.util.HashSet;
 import java.util.Set;
-import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector;
 import tech.pegasys.teku.infrastructure.ssz.schema.collections.SszBitvectorSchema;
 import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.spec.config.SpecConfig;
+import tech.pegasys.teku.spec.constants.NetworkConstants;
+import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.BeaconBlocksByRootRequestMessage.BeaconBlocksByRootRequestMessageSchema;
 import tech.pegasys.teku.spec.datastructures.operations.Attestation;
 import tech.pegasys.teku.spec.datastructures.operations.AttestationSchema;
 import tech.pegasys.teku.spec.datastructures.operations.versions.electra.AttestationElectraSchema;
 import tech.pegasys.teku.spec.datastructures.operations.versions.phase0.AttestationPhase0Schema;
+import tech.pegasys.teku.spec.datastructures.state.HistoricalBatch.HistoricalBatchSchema;
 import tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SchemaId;
 
 public class SchemaRegistryBuilder {
@@ -39,23 +44,49 @@ public static SchemaRegistryBuilder create() {
     return new SchemaRegistryBuilder()
         // PHASE0
         .addProvider(createAttnetsENRFieldSchemaProvider())
+        .addProvider(createSyncnetsENRFieldSchemaProvider())
+        .addProvider(createBeaconBlocksByRootRequestMessageSchemaProvider())
+        .addProvider(createHistoricalBatchSchemaProvider())
         .addProvider(createAttestationSchemaProvider());
   }
 
-  private static SchemaProvider<SszBitvectorSchema<SszBitvector>>
-      createAttnetsENRFieldSchemaProvider() {
-    return providerBuilder(ATTNETS_ENR_FIELD_SCHEMA)
-        .constant()
+  private static SchemaProvider<?> createAttnetsENRFieldSchemaProvider() {
+    return constantProviderBuilder(ATTNETS_ENR_FIELD_SCHEMA)
         .withCreator(
             SpecMilestone.PHASE0,
             (registry, specConfig) ->
                 SszBitvectorSchema.create(specConfig.getAttestationSubnetCount()))
         .build();
   }
 
+  private static SchemaProvider<?> createSyncnetsENRFieldSchemaProvider() {
+    return constantProviderBuilder(SYNCNETS_ENR_FIELD_SCHEMA)
+        .withCreator(
+            SpecMilestone.PHASE0,
+            (registry, specConfig) ->
+                SszBitvectorSchema.create(NetworkConstants.SYNC_COMMITTEE_SUBNET_COUNT))
+        .build();
+  }
+
+  private static SchemaProvider<?> createBeaconBlocksByRootRequestMessageSchemaProvider() {
+    return constantProviderBuilder(BEACON_BLOCKS_BY_ROOT_REQUEST_MESSAGE_SCHEMA)
+        .withCreator(
+            SpecMilestone.PHASE0,
+            (registry, specConfig) -> new BeaconBlocksByRootRequestMessageSchema(specConfig))
+        .build();
+  }
+
+  private static SchemaProvider<?> createHistoricalBatchSchemaProvider() {
+    return constantProviderBuilder(HISTORICAL_BATCH_SCHEMA)
+        .withCreator(
+            SpecMilestone.PHASE0,
+            (registry, specConfig) ->
+                new HistoricalBatchSchema(specConfig.getSlotsPerHistoricalRoot()))
+        .build();
+  }
+
   private static SchemaProvider<AttestationSchema<Attestation>> createAttestationSchemaProvider() {
-    return providerBuilder(ATTESTATION_SCHEMA)
-        .constant()
+    return constantProviderBuilder(ATTESTATION_SCHEMA)
         .withCreator(
             SpecMilestone.PHASE0,
             (registry, specConfig) ->

diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaTypes.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaTypes.java
@@ -23,8 +23,10 @@
 import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadHeader;
 import tech.pegasys.teku.spec.datastructures.execution.ExecutionPayloadHeaderSchema;
+import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.BeaconBlocksByRootRequestMessage.BeaconBlocksByRootRequestMessageSchema;
 import tech.pegasys.teku.spec.datastructures.operations.Attestation;
 import tech.pegasys.teku.spec.datastructures.operations.AttestationSchema;
+import tech.pegasys.teku.spec.datastructures.state.HistoricalBatch.HistoricalBatchSchema;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconStateSchema;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.MutableBeaconState;
@@ -33,6 +35,13 @@ public class SchemaTypes {
   // PHASE0
   public static final SchemaId<SszBitvectorSchema<SszBitvector>> ATTNETS_ENR_FIELD_SCHEMA =
       create("ATTNETS_ENR_FIELD_SCHEMA");
+  public static final SchemaId<SszBitvectorSchema<SszBitvector>> SYNCNETS_ENR_FIELD_SCHEMA =
+      create("SYNCNETS_ENR_FIELD_SCHEMA");
+  public static final SchemaId<HistoricalBatchSchema> HISTORICAL_BATCH_SCHEMA =
+      create("HISTORICAL_BATCH_SCHEMA");
+  public static final SchemaId<BeaconBlocksByRootRequestMessageSchema>
+      BEACON_BLOCKS_BY_ROOT_REQUEST_MESSAGE_SCHEMA =
+          create("BEACON_BLOCKS_BY_ROOT_REQUEST_MESSAGE_SCHEMA");
 
   public static final SchemaId<AttestationSchema<Attestation>> ATTESTATION_SCHEMA =
       create("ATTESTATION_SCHEMA");

diff --git a/...um/spec/src/test/java/tech/pegasys/teku/spec/schemas/registry/BaseSchemaProviderTest.java b/...um/spec/src/test/java/tech/pegasys/teku/spec/schemas/registry/BaseSchemaProviderTest.java
@@ -23,7 +23,8 @@
 import static tech.pegasys.teku.spec.SpecMilestone.CAPELLA;
 import static tech.pegasys.teku.spec.SpecMilestone.DENEB;
 import static tech.pegasys.teku.spec.SpecMilestone.PHASE0;
-import static tech.pegasys.teku.spec.schemas.registry.BaseSchemaProvider.providerBuilder;
+import static tech.pegasys.teku.spec.schemas.registry.BaseSchemaProvider.constantProviderBuilder;
+import static tech.pegasys.teku.spec.schemas.registry.BaseSchemaProvider.variableProviderBuilder;
 
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.spec.SpecMilestone;
@@ -38,7 +39,7 @@ class BaseSchemaProviderTest {
   @Test
   void shouldSupportContinuousUntilHighestMilestone() {
     final SchemaProvider<?> provider =
-        providerBuilder(STRING_SCHEMA_ID)
+        variableProviderBuilder(STRING_SCHEMA_ID)
             .withCreator(ALTAIR, (r, c) -> "TestSchemaAltair")
             .withCreator(BELLATRIX, (r, c) -> "TestSchemaBellatrix")
             .build();
@@ -59,8 +60,7 @@ void shouldSupportContinuousUntilHighestMilestone() {
   @Test
   void shouldSupportContinuousConstantWithUntil() {
     final SchemaProvider<?> provider =
-        providerBuilder(STRING_SCHEMA_ID)
-            .constant()
+        constantProviderBuilder(STRING_SCHEMA_ID)
             .withCreator(PHASE0, (r, c) -> "TestSchemaPhase0")
             .withCreator(BELLATRIX, (r, c) -> "TestSchemaBellatrix")
             .until(CAPELLA)
@@ -90,7 +90,7 @@ void shouldSupportContinuousConstantWithUntil() {
   @Test
   void shouldSupportContinuousDefaultVariable() {
     final SchemaProvider<?> provider =
-        providerBuilder(STRING_SCHEMA_ID)
+        variableProviderBuilder(STRING_SCHEMA_ID)
             .withCreator(PHASE0, (r, c) -> "TestSchema" + r.getMilestone())
             .until(CAPELLA)
             .build();
@@ -117,15 +117,15 @@ void shouldSupportContinuousDefaultVariable() {
 
   @Test
   void shouldThrowWhenNoCreators() {
-    assertThatThrownBy(() -> providerBuilder(STRING_SCHEMA_ID).build())
+    assertThatThrownBy(() -> variableProviderBuilder(STRING_SCHEMA_ID).build())
         .isInstanceOf(IllegalArgumentException.class)
         .hasMessageStartingWith("There should be at least 1 creator");
   }
 
   @Test
   void shouldThrowWhenAskingForAnUnsupportedMilestone() {
     final SchemaProvider<?> provider =
-        providerBuilder(STRING_SCHEMA_ID)
+        variableProviderBuilder(STRING_SCHEMA_ID)
             .withCreator(ALTAIR, (r, c) -> "TestSchemaAltair")
             .until(ALTAIR)
             .build();
@@ -141,15 +141,15 @@ void shouldThrowWhenAskingForAnUnsupportedMilestone() {
   void shouldThrowWhenNotAscendingMilestones() {
     assertThatThrownBy(
             () ->
-                providerBuilder(STRING_SCHEMA_ID)
+                variableProviderBuilder(STRING_SCHEMA_ID)
                     .withCreator(PHASE0, (r, c) -> "TestSchema")
                     .withCreator(PHASE0, (r, c) -> "TestSchema"))
         .isInstanceOf(IllegalArgumentException.class)
         .hasMessageStartingWith("Creator's milestones must added in strict ascending order");
 
     assertThatThrownBy(
             () ->
-                providerBuilder(STRING_SCHEMA_ID)
+                variableProviderBuilder(STRING_SCHEMA_ID)
                     .withCreator(ALTAIR, (r, c) -> "TestSchema")
                     .withCreator(PHASE0, (r, c) -> "TestSchema"))
         .isInstanceOf(IllegalArgumentException.class)
@@ -160,7 +160,7 @@ void shouldThrowWhenNotAscendingMilestones() {
   void shouldThrowWhenWithUntilIsPriorToMilestone() {
     assertThatThrownBy(
             () ->
-                providerBuilder(STRING_SCHEMA_ID)
+                variableProviderBuilder(STRING_SCHEMA_ID)
                     .withCreator(PHASE0, (r, c) -> "TestSchema")
                     .withCreator(CAPELLA, (r, c) -> "TestSchema")
                     .until(ALTAIR)

diff --git a/...es/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java b/...es/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java
@@ -16,6 +16,7 @@
 import static tech.pegasys.teku.infrastructure.async.AsyncRunnerFactory.DEFAULT_MAX_QUEUE_SIZE;
 import static tech.pegasys.teku.spec.config.Constants.STORAGE_QUERY_CHANNEL_PARALLELISM;
 
+import java.nio.file.Path;
 import java.util.Optional;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -33,6 +34,9 @@
 import tech.pegasys.teku.storage.api.CombinedStorageChannel;
 import tech.pegasys.teku.storage.api.Eth1DepositStorageChannel;
 import tech.pegasys.teku.storage.api.VoteUpdateChannel;
+import tech.pegasys.teku.storage.archive.DataArchive;
+import tech.pegasys.teku.storage.archive.fsarchive.FileSystemArchive;
+import tech.pegasys.teku.storage.archive.nooparchive.NoopDataArchive;
 import tech.pegasys.teku.storage.server.BatchingVoteUpdateChannel;
 import tech.pegasys.teku.storage.server.ChainStorage;
 import tech.pegasys.teku.storage.server.CombinedStorageChannelSplitter;
@@ -149,12 +153,20 @@ protected SafeFuture<?> doStart() {
                               pruningActiveLabelledGauge));
                 }
               }
+
+              final DataArchive dataArchive =
+                  config
+                      .getBlobsArchivePath()
+                      .<DataArchive>map(path -> new FileSystemArchive(Path.of(path)))
+                      .orElse(new NoopDataArchive());
+
               if (config.getSpec().isMilestoneSupported(SpecMilestone.DENEB)) {
                 blobsPruner =
                     Optional.of(
                         new BlobSidecarPruner(
                             config.getSpec(),
                             database,
+                            dataArchive,
                             serviceConfig.getMetricsSystem(),
                             storagePrunerAsyncRunner,
                             serviceConfig.getTimeProvider(),