Merge branch 'master' into minor-electra-state-fixes

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandler.java

@@ -490,6 +490,9 @@ public SafeFuture<Optional<Attestation>> createAggregate(
   @Override
   public SafeFuture<Optional<SyncCommitteeContribution>> createSyncCommitteeContribution(
       final UInt64 slot, final int subcommitteeIndex, final Bytes32 beaconBlockRoot) {
+    if (isSyncActive()) {
+      return NodeSyncingException.failedFuture();
+    }
     return SafeFuture.completedFuture(
         syncCommitteeMessagePool.createContribution(slot, beaconBlockRoot, subcommitteeIndex));
   }

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandlerTest.java

@@ -101,6 +101,7 @@
 import tech.pegasys.teku.spec.datastructures.operations.AttestationData;
 import tech.pegasys.teku.spec.datastructures.operations.SignedAggregateAndProof;
 import tech.pegasys.teku.spec.datastructures.operations.versions.altair.SignedContributionAndProof;
+import tech.pegasys.teku.spec.datastructures.operations.versions.altair.SyncCommitteeContribution;
 import tech.pegasys.teku.spec.datastructures.operations.versions.altair.SyncCommitteeMessage;
 import tech.pegasys.teku.spec.datastructures.state.Checkpoint;
 import tech.pegasys.teku.spec.datastructures.state.CheckpointState;
@@ -696,6 +697,17 @@ public void createAggregate_shouldFailWhenNodeIsSyncing() {
     assertThatThrownBy(result::get).hasRootCauseInstanceOf(NodeSyncingException.class);
   }
 
+  @Test
+  public void createSyncCommitteeContribution() {
+    nodeIsSyncing();
+    final SafeFuture<Optional<SyncCommitteeContribution>> result =
+        validatorApiHandler.createSyncCommitteeContribution(
+            ONE, 0, dataStructureUtil.randomBytes32());
+
+    assertThat(result).isCompletedExceptionally();
+    assertThatThrownBy(result::get).hasRootCauseInstanceOf(NodeSyncingException.class);
+  }
+
   @Test
   public void createAggregate_shouldReturnAggregateFromAttestationPool() {
     final AttestationData attestationData = dataStructureUtil.randomAttestationData();

build.gradle

@@ -297,10 +297,13 @@ allprojects {
 
 def refTestVersion = 'v1.4.0' // Arbitrary change to refresh cache number: 1
 def blsRefTestVersion = 'v0.1.2'
+def slashingProtectionInterchangeRefTestVersion = 'v5.3.0'
 def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download'
 def blsRefTestBaseUrl = 'https://github.com/ethereum/bls12-381-tests/releases/download'
+def slashingProtectionInterchangeRefTestBaseUrl = 'https://github.com/eth-clients/slashing-protection-interchange-tests/archive/refs/tags'
 def refTestDownloadDir = "${buildDir}/refTests/${refTestVersion}"
 def blsRefTestDownloadDir = "${buildDir}/blsRefTests/${blsRefTestVersion}"
+def slashingProtectionInterchangeRefTestDownloadDir = "${buildDir}/slashingProtectionInterchangeRefTests/${slashingProtectionInterchangeRefTestVersion}"
 def refTestExpandDir = "${project.rootDir}/eth-reference-tests/src/referenceTest/resources/consensus-spec-tests/"
 
 task downloadEthRefTests(type: Download) {
@@ -321,7 +324,15 @@ task downloadBlsRefTests(type: Download) {
   overwrite false
 }
 
-task downloadRefTests(dependsOn: [downloadEthRefTests, downloadBlsRefTests])
+task downloadSlashingProtectionInterchangeRefTests(type: Download) {
+  src([
+      "${slashingProtectionInterchangeRefTestBaseUrl}/${slashingProtectionInterchangeRefTestVersion}.tar.gz"
+  ])
+  dest "${slashingProtectionInterchangeRefTestDownloadDir}/slashing-protection-interchange-tests.tar.gz"
+  overwrite false
+}
+
+task downloadRefTests(dependsOn: [downloadEthRefTests, downloadBlsRefTests, downloadSlashingProtectionInterchangeRefTests])
 
 task cleanRefTestsGeneral(type: Delete) {
   delete "${refTestExpandDir}/tests/general"
@@ -359,8 +370,25 @@ task expandRefTestsBls(type: Copy, dependsOn: [cleanRefTestsBls, downloadBlsRefT
   into "${refTestExpandDir}/tests/bls"
 }
 
-task expandRefTests(dependsOn: [expandRefTestsGeneral, expandRefTestsMainnet, expandRefTestsMinimal, expandRefTestsBls])
-task cleanRefTests(dependsOn: [cleanRefTestsGeneral, cleanRefTestsMainnet, cleanRefTestsMinimal, cleanRefTestsBls])
+task cleanRefTestsSlashingProtectionInterchange(type: Delete) {
+  delete "${refTestExpandDir}/tests/slashing-protection-interchange"
+}
+
+task expandRefTestsSlashingProtectionInterchange(type: Copy, dependsOn: [cleanRefTestsSlashingProtectionInterchange, downloadSlashingProtectionInterchangeRefTests]) {
+  from {
+    tarTree("${slashingProtectionInterchangeRefTestDownloadDir}/slashing-protection-interchange-tests.tar.gz").matching {
+      include "**/tests/generated/*.json"
+      // flatten
+      eachFile { FileCopyDetails fcp ->
+        fcp.path = fcp.name
+      }
+    }
+  }
+  into "${refTestExpandDir}/tests/slashing-protection-interchange"
+}
+
+task expandRefTests(dependsOn: [expandRefTestsGeneral, expandRefTestsMainnet, expandRefTestsMinimal, expandRefTestsBls, expandRefTestsSlashingProtectionInterchange])
+task cleanRefTests(dependsOn: [cleanRefTestsGeneral, cleanRefTestsMainnet, cleanRefTestsMinimal, cleanRefTestsBls, cleanRefTestsSlashingProtectionInterchange])
 
 task deploy() {}
 

data/dataexchange/src/test/java/tech/pegasys/teku/data/SlashingProtectionImporterTest.java

@@ -163,6 +163,31 @@ void shouldImportFileOverRepairedRecords(@TempDir Path tempDir) throws Exception
                 repairedEpoch));
   }
 
+  @Test
+  void shouldFailImportingIfValidatorExistingRecordHasDifferentGenesisValidatorsRoot(
+      @TempDir Path tempDir) throws URISyntaxException, IOException {
+    final SlashingProtectionImporter importer = new SlashingProtectionImporter(tempDir);
+
+    final File slashProtection = getResourceFile("format2_minimal.json");
+
+    importer.initialise(slashProtection);
+
+    Map<BLSPublicKey, String> errors = importer.updateLocalRecords(__ -> {});
+
+    assertThat(errors).isEmpty();
+
+    final File slashProtectionWithDifferentGvr =
+        getResourceFile("format2_minimal_different_genesis_validators_root.json");
+
+    importer.initialise(slashProtectionWithDifferentGvr);
+
+    errors = importer.updateLocalRecords(__ -> {});
+
+    assertThat(errors)
+        .hasSize(1)
+        .containsEntry(publicKey, "Genesis validators root did not match what was expected.");
+  }
+
   private ValidatorSigningRecord loadSigningRecord(final File repairedRuleFile) throws IOException {
     return ValidatorSigningRecord.fromBytes(
         Bytes.wrap(Files.readAllBytes(repairedRuleFile.toPath())));
@@ -182,9 +207,11 @@ private File usingResourceFile(final String resourceFileName, final Path tempDir
       throws URISyntaxException, IOException {
     final Path tempFile = tempDir.resolve(pubkey + ".yml").toAbsolutePath();
     Files.copy(
-        new File(Resources.getResource(resourceFileName).toURI()).toPath(),
-        tempFile,
-        StandardCopyOption.REPLACE_EXISTING);
+        getResourceFile(resourceFileName).toPath(), tempFile, StandardCopyOption.REPLACE_EXISTING);
     return tempFile.toFile();
   }
+
+  private File getResourceFile(final String resourceFileName) throws URISyntaxException {
+    return new File(Resources.getResource(resourceFileName).toURI());
+  }
 }

data/dataexchange/src/test/resources/format2_minimal.json

@@ -7,7 +7,8 @@
     {
       "pubkey": "0xb845089a1457f811bfc000588fbb4e713669be8ce060ea6be3c6ece09afc3794106c91ca73acda5e5457122d58723bed",
       "signed_blocks": [
-        {"slot": "81952"
+        {
+          "slot": "81952"
         }
       ],
       "signed_attestations": [

data/dataexchange/src/test/resources/format2_minimal_different_genesis_validators_root.json

@@ -0,0 +1,22 @@
+{
+  "metadata": {
+    "interchange_format_version": "5",
+    "genesis_validators_root": "0x0000000000000000000000000000000000000000000000000000000000123457"
+  },
+  "data": [
+    {
+      "pubkey": "0xb845089a1457f811bfc000588fbb4e713669be8ce060ea6be3c6ece09afc3794106c91ca73acda5e5457122d58723bed",
+      "signed_blocks": [
+        {
+          "slot": "81952"
+        }
+      ],
+      "signed_attestations": [
+        {
+          "source_epoch": "2290",
+          "target_epoch": "3007"
+        }
+      ]
+    }
+  ]
+}

eth-reference-tests/build.gradle

@@ -15,9 +15,12 @@ dependencies {
   referenceTestImplementation project(':storage')
   referenceTestImplementation testFixtures(project(':storage'))
   referenceTestImplementation project(':infrastructure:async')
+  referenceTestImplementation project(':infrastructure:io')
   referenceTestImplementation testFixtures(project(':infrastructure:async'))
   referenceTestImplementation testFixtures(project(':infrastructure:metrics'))
   referenceTestImplementation project(':infrastructure:time')
+  referenceTestImplementation project(':data:dataexchange')
+  referenceTestImplementation project(':data:serializer')
 
   referenceTestImplementation 'org.hyperledger.besu:plugin-api'
   referenceTestImplementation 'com.fasterxml.jackson.core:jackson-databind'

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java

@@ -31,6 +31,7 @@
 import tech.pegasys.teku.reference.phase0.rewards.RewardsTestExecutorPhase0;
 import tech.pegasys.teku.reference.phase0.sanity.SanityTests;
 import tech.pegasys.teku.reference.phase0.shuffling.ShufflingTestExecutor;
+import tech.pegasys.teku.reference.phase0.slashing_protection_interchange.SlashingProtectionInterchangeTestExecutor;
 import tech.pegasys.teku.reference.phase0.ssz_generic.SszGenericTests;
 import tech.pegasys.teku.reference.phase0.ssz_static.SszTestExecutor;
 
@@ -48,6 +49,7 @@ public abstract class Eth2ReferenceTestCase {
           .putAll(SszGenericTests.SSZ_GENERIC_TEST_TYPES)
           .putAll(OperationsTestExecutor.OPERATIONS_TEST_TYPES)
           .putAll(SanityTests.SANITY_TEST_TYPES)
+          .put("slashing-protection-interchange", new SlashingProtectionInterchangeTestExecutor())
           .put("light_client/single_merkle_proof", TestExecutor.IGNORE_TESTS)
           .put("light_client/sync", TestExecutor.IGNORE_TESTS)
           .put("light_client/update_ranking", TestExecutor.IGNORE_TESTS)

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/ManualReferenceTestRunner.java

@@ -42,7 +42,7 @@ public class ManualReferenceTestRunner extends Eth2ReferenceTestCase {
    *
    * <p>May be overridden by the ENV_TEST_TYPE environment variable.
    */
-  private static final String TEST_TYPE = "fork_choice";
+  private static final String TEST_TYPE = "";
 
   /**
    * Filter test to run to those from the specified spec. One of general, minimal or mainnet

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/TestDataUtils.java

@@ -19,6 +19,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.function.Function;
@@ -29,18 +30,21 @@
 import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition;
 import tech.pegasys.teku.infrastructure.ssz.SszData;
 import tech.pegasys.teku.infrastructure.ssz.schema.SszSchema;
+import tech.pegasys.teku.provider.JsonProvider;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 
 public class TestDataUtils {
 
   private static final YAMLFactory YAML_FACTORY;
+  private static final JsonProvider JSON_PROVIDER;
 
   static {
     final LoaderOptions loaderOptions = new LoaderOptions();
     // Set the code point limit to 100MB - context:
     // https://github.com/FasterXML/jackson-dataformats-text/tree/2.15/yaml#maximum-input-yaml-document-size-3-mb
     loaderOptions.setCodePointLimit(1024 * 1024 * 100);
     YAML_FACTORY = YAMLFactory.builder().loaderOptions(loaderOptions).build();
+    JSON_PROVIDER = new JsonProvider();
   }
 
   public static <T extends SszData> T loadSsz(
@@ -85,7 +89,7 @@ public static <T> T loadYaml(
       throws IOException {
     final Path path = testDefinition.getTestDirectory().resolve(fileName);
     try (final InputStream in = Files.newInputStream(path)) {
-      return new ObjectMapper(YAML_FACTORY).readerFor(type).readValue(in);
+      return new ObjectMapper(YAML_FACTORY).readValue(in, type);
     }
   }
 
@@ -100,4 +104,18 @@ public static <T> T loadYaml(
       return type.deserialize(in);
     }
   }
+
+  public static <T> T loadJson(
+      final TestDefinition testDefinition, final String fileName, final Class<T> type)
+      throws IOException {
+    final Path path = testDefinition.getTestDirectory().resolve(fileName);
+    try (final InputStream in = Files.newInputStream(path)) {
+      return JSON_PROVIDER.getObjectMapper().readValue(in, type);
+    }
+  }
+
+  public static <T> void writeJsonToFile(final T object, final Path file) throws IOException {
+    final String json = JSON_PROVIDER.getObjectMapper().writeValueAsString(object);
+    Files.writeString(file, json, StandardCharsets.UTF_8);
+  }
 }

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/altair/fork/ForkUpgradeTestExecutor.java

@@ -27,6 +27,7 @@
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.altair.BeaconStateSchemaAltair;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.bellatrix.BeaconStateSchemaBellatrix;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.capella.BeaconStateSchemaCapella;
+import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.deneb.BeaconStateSchemaDeneb;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.phase0.BeaconStateSchemaPhase0;
 import tech.pegasys.teku.spec.logic.common.forktransition.StateUpgrade;
 
@@ -50,6 +51,7 @@ private void processUpgrade(final TestDefinition testDefinition, final SpecMiles
           case BELLATRIX -> BeaconStateSchemaAltair.create(spec.getConfig());
           case CAPELLA -> BeaconStateSchemaBellatrix.create(spec.getConfig());
           case DENEB -> BeaconStateSchemaCapella.create(spec.getConfig());
+          case ELECTRA -> BeaconStateSchemaDeneb.create(spec.getConfig());
           default -> throw new IllegalStateException(
               "Unhandled fork upgrade for test "
                   + testDefinition.getDisplayName()