Merge branch 'master' into let-pending-attestation-trigger-recent-blo…

…ck-fetcher

CHANGELOG.md

@@ -11,7 +11,4 @@
 ### Additions and Improvements
 - Optimized blobs validation pipeline
 
-### Bug Fixes
-- Updated the gas change check for block building so that warnings only get raised if the change is off spec.
-- Fixed an issue with the `/eth/v1/config/spec` API not returning all previously included configuration parameters.
-- Increase the maximum size of a compressed message for libp2p to ensure uncompressed blocks can grow to max size.
+### Bug Fixes

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/helpers/BeaconStateAccessorsElectra.java

@@ -14,13 +14,24 @@
 package tech.pegasys.teku.spec.logic.versions.electra.helpers;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static tech.pegasys.teku.infrastructure.crypto.Hash.getSha256Instance;
+import static tech.pegasys.teku.spec.logic.common.helpers.MathHelpers.bytesToUInt64;
+import static tech.pegasys.teku.spec.logic.common.helpers.MathHelpers.uint64ToBytes;
+import static tech.pegasys.teku.spec.logic.versions.electra.helpers.MiscHelpersElectra.MAX_RANDOM_VALUE;
 
+import it.unimi.dsi.fastutil.ints.IntArrayList;
 import it.unimi.dsi.fastutil.ints.IntList;
 import java.util.List;
+import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.bytes.Bytes32;
+import tech.pegasys.teku.infrastructure.crypto.Sha256;
+import tech.pegasys.teku.infrastructure.ssz.SszList;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.config.SpecConfig;
 import tech.pegasys.teku.spec.config.SpecConfigDeneb;
 import tech.pegasys.teku.spec.config.SpecConfigElectra;
+import tech.pegasys.teku.spec.constants.Domain;
+import tech.pegasys.teku.spec.datastructures.state.Validator;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.electra.BeaconStateElectra;
 import tech.pegasys.teku.spec.datastructures.state.versions.electra.PendingPartialWithdrawal;
@@ -105,4 +116,41 @@ public static BeaconStateAccessorsElectra required(
   public IntList getNextSyncCommitteeIndices(final BeaconState state) {
     return getNextSyncCommitteeIndices(state, configElectra.getMaxEffectiveBalanceElectra());
   }
+
+  @Override
+  protected IntList getNextSyncCommitteeIndices(
+      final BeaconState state, final UInt64 maxEffectiveBalance) {
+    final UInt64 epoch = getCurrentEpoch(state).plus(1);
+    final IntList activeValidatorIndices = getActiveValidatorIndices(state, epoch);
+    final int activeValidatorCount = activeValidatorIndices.size();
+    checkArgument(activeValidatorCount > 0, "Provided state has no active validators");
+
+    final Bytes32 seed = getSeed(state, epoch, Domain.SYNC_COMMITTEE);
+    final SszList<Validator> validators = state.getValidators();
+    final IntList syncCommitteeIndices = new IntArrayList();
+    final int syncCommitteeSize = configElectra.getSyncCommitteeSize();
+    final Sha256 sha256 = getSha256Instance();
+
+    int i = 0;
+    Bytes randomBytes = null;
+    while (syncCommitteeIndices.size() < syncCommitteeSize) {
+      if (i % 16 == 0) {
+        randomBytes = Bytes.wrap(sha256.digest(seed, uint64ToBytes(Math.floorDiv(i, 16L))));
+      }
+      final int shuffledIndex =
+          miscHelpers.computeShuffledIndex(i % activeValidatorCount, activeValidatorCount, seed);
+      final int candidateIndex = activeValidatorIndices.getInt(shuffledIndex);
+      final int offset = (i % 16) * 2;
+      final UInt64 randomValue = bytesToUInt64(randomBytes.slice(offset, 2));
+      final UInt64 effectiveBalance = validators.get(candidateIndex).getEffectiveBalance();
+      if (effectiveBalance
+          .times(MAX_RANDOM_VALUE)
+          .isGreaterThanOrEqualTo(maxEffectiveBalance.times(randomValue))) {
+        syncCommitteeIndices.add(candidateIndex);
+      }
+      i++;
+    }
+
+    return syncCommitteeIndices;
+  }
 }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/helpers/MiscHelpersElectra.java

@@ -13,13 +13,19 @@
 
 package tech.pegasys.teku.spec.logic.versions.electra.helpers;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static tech.pegasys.teku.infrastructure.crypto.Hash.getSha256Instance;
 import static tech.pegasys.teku.infrastructure.unsigned.UInt64.ZERO;
 import static tech.pegasys.teku.spec.config.SpecConfig.FAR_FUTURE_EPOCH;
+import static tech.pegasys.teku.spec.logic.common.helpers.MathHelpers.bytesToUInt64;
+import static tech.pegasys.teku.spec.logic.common.helpers.MathHelpers.uint64ToBytes;
 
 import it.unimi.dsi.fastutil.ints.IntList;
 import java.util.Optional;
+import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.bls.BLSPublicKey;
+import tech.pegasys.teku.infrastructure.crypto.Sha256;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.config.SpecConfigDeneb;
 import tech.pegasys.teku.spec.config.SpecConfigElectra;
@@ -33,6 +39,7 @@
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
 
 public class MiscHelpersElectra extends MiscHelpersDeneb {
+  public static final UInt64 MAX_RANDOM_VALUE = UInt64.valueOf(65535);
   private final SpecConfigElectra specConfigElectra;
   private final PredicatesElectra predicatesElectra;
 
@@ -68,6 +75,44 @@ public int computeProposerIndex(
         SpecConfigElectra.required(specConfig).getMaxEffectiveBalanceElectra());
   }
 
+  @Override
+  protected int computeProposerIndex(
+      final BeaconState state,
+      final IntList indices,
+      final Bytes32 seed,
+      final UInt64 maxEffectiveBalance) {
+    checkArgument(!indices.isEmpty(), "compute_proposer_index indices must not be empty");
+
+    final Sha256 sha256 = getSha256Instance();
+
+    int i = 0;
+    final int total = indices.size();
+    Bytes randomBytes = null;
+    while (true) {
+      final int candidateIndex = indices.getInt(computeShuffledIndex(i % total, total, seed));
+      if (i % 16 == 0) {
+        randomBytes = Bytes.wrap(sha256.digest(seed, uint64ToBytes(Math.floorDiv(i, 16L))));
+      }
+      final int offset = (i % 16) * 2;
+      final UInt64 randomValue = bytesToUInt64(randomBytes.slice(offset, 2));
+      final UInt64 validatorEffectiveBalance =
+          state.getValidators().get(candidateIndex).getEffectiveBalance();
+      if (validatorEffectiveBalance
+          .times(MAX_RANDOM_VALUE)
+          .isGreaterThanOrEqualTo(maxEffectiveBalance.times(randomValue))) {
+        return candidateIndex;
+      }
+      i++;
+    }
+  }
+
+  @Override
+  public UInt64 getMaxEffectiveBalance(final Validator validator) {
+    return predicatesElectra.hasCompoundingWithdrawalCredential(validator)
+        ? specConfigElectra.getMaxEffectiveBalanceElectra()
+        : specConfigElectra.getMinActivationBalance();
+  }
+
   @Override
   public Validator getValidatorFromDeposit(
       final BLSPublicKey pubkey, final Bytes32 withdrawalCredentials, final UInt64 amount) {
@@ -91,13 +136,6 @@ public Validator getValidatorFromDeposit(
     return validator.withEffectiveBalance(validatorEffectiveBalance);
   }
 
-  @Override
-  public UInt64 getMaxEffectiveBalance(final Validator validator) {
-    return predicatesElectra.hasCompoundingWithdrawalCredential(validator)
-        ? specConfigElectra.getMaxEffectiveBalanceElectra()
-        : specConfigElectra.getMinActivationBalance();
-  }
-
   @Override
   public boolean isFormerDepositMechanismDisabled(final BeaconState state) {
     // if the next deposit to be processed by Eth1Data poll has the index of the first deposit

ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/electra/helpers/MiscHelpersElectraTest.java

@@ -129,7 +129,7 @@ void consolidatedValidatorsMoreLikelyToPropose() {
         proposerIndexCount++;
       }
     }
-    assertThat(proposerIndexCount).isEqualTo(4);
+    assertThat(proposerIndexCount).isEqualTo(5);
   }
 
   private BeaconState randomStateWithConsolidatedValidator(final int consolidationAmount) {

ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java

@@ -821,14 +821,18 @@ public Attestation randomAttestation() {
   }
 
   public SingleAttestation randomSingleAttestation() {
+    return randomSingleAttestation(randomUInt64());
+  }
+
+  public SingleAttestation randomSingleAttestation(final UInt64 slot) {
     return spec.getGenesisSchemaDefinitions()
         .toVersionElectra()
         .orElseThrow()
         .getSingleAttestationSchema()
         .create(
             randomUInt64(Integer.MAX_VALUE),
             randomUInt64(Integer.MAX_VALUE),
-            randomAttestationData(),
+            randomAttestationData(slot),
             randomSignature());
   }
 

ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/attestation/AggregatingAttestationPoolTest.java

@@ -46,7 +46,7 @@
 import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.spec.TestSpecContext;
 import tech.pegasys.teku.spec.TestSpecFactory;
-import tech.pegasys.teku.spec.TestSpecInvocationContextProvider;
+import tech.pegasys.teku.spec.TestSpecInvocationContextProvider.SpecContext;
 import tech.pegasys.teku.spec.datastructures.attestation.ValidatableAttestation;
 import tech.pegasys.teku.spec.datastructures.operations.Attestation;
 import tech.pegasys.teku.spec.datastructures.operations.AttestationData;
@@ -84,7 +84,7 @@ class AggregatingAttestationPoolTest {
   private Int2IntMap committeeSizes;
 
   @BeforeEach
-  public void setUp(final TestSpecInvocationContextProvider.SpecContext specContext) {
+  public void setUp(final SpecContext specContext) {
     spec = specContext.getSpec();
     specMilestone = specContext.getSpecMilestone();
     dataStructureUtil = specContext.getDataStructureUtil();

fuzz/src/test/java/tech/pegasys/teku/fuzz/FuzzUtilTest.java

@@ -15,14 +15,14 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Optional;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.junit.BouncyCastleExtension;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xerial.snappy.Snappy;
@@ -39,6 +39,8 @@
 import tech.pegasys.teku.fuzz.input.SyncAggregateFuzzInput;
 import tech.pegasys.teku.fuzz.input.VoluntaryExitFuzzInput;
 import tech.pegasys.teku.fuzz.input.WithdrawalRequestFuzzInput;
+import tech.pegasys.teku.infrastructure.json.JsonUtil;
+import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition;
 import tech.pegasys.teku.infrastructure.ssz.SszData;
 import tech.pegasys.teku.infrastructure.ssz.schema.SszSchema;
 import tech.pegasys.teku.spec.Spec;
@@ -63,6 +65,7 @@
 import tech.pegasys.teku.spec.datastructures.operations.SignedBlsToExecutionChange;
 import tech.pegasys.teku.spec.datastructures.operations.SignedVoluntaryExit;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
+import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.electra.BeaconStateElectra;
 import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.electra.BeaconStateSchemaElectra;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsElectra;
 
@@ -92,7 +95,8 @@ class FuzzUtilTest {
   // https://github.com/ethereum/consensus-specs/tree/dev/tests/generators
 
   @Test
-  public void fuzzAttestation_minimal() {
+  @SuppressWarnings("unchecked")
+  public void fuzzAttestation_minimal() throws JsonProcessingException {
     final FuzzUtil fuzzUtil = new FuzzUtil(false, true);
 
     final Path testCaseDir =
@@ -102,21 +106,24 @@ public void fuzzAttestation_minimal() {
             testCaseDir.resolve("attestation.ssz_snappy"),
             spec.forMilestone(SpecMilestone.ELECTRA).getSchemaDefinitions().getAttestationSchema());
     final BeaconState preState = loadSsz(testCaseDir.resolve("pre.ssz_snappy"), beaconStateSchema);
-    final BeaconState postState =
+    final BeaconStateElectra postState =
         loadSsz(testCaseDir.resolve("post.ssz_snappy"), beaconStateSchema);
 
-    AttestationFuzzInput input = new AttestationFuzzInput(spec, preState, data);
-    byte[] rawInput = input.sszSerialize().toArrayUnsafe();
-    Optional<Bytes> result = fuzzUtil.fuzzAttestation(rawInput).map(Bytes::wrap);
+    final AttestationFuzzInput input = new AttestationFuzzInput(spec, preState, data);
+    final byte[] rawInput = input.sszSerialize().toArrayUnsafe();
+    final Optional<Bytes> result = fuzzUtil.fuzzAttestation(rawInput).map(Bytes::wrap);
 
-    Bytes expected = postState.sszSerialize();
     assertThat(result).isNotEmpty();
-    assertThat(result.get()).isEqualTo(expected);
+    final BeaconStateElectra resultState =
+        BeaconStateElectra.required(spec.deserializeBeaconState(result.get()));
+    DeserializableTypeDefinition<BeaconStateElectra> t =
+        (DeserializableTypeDefinition<BeaconStateElectra>)
+            resultState.getSchema().getJsonTypeDefinition();
+    assertThat(JsonUtil.prettySerialize(resultState, t))
+        .isEqualTo(JsonUtil.prettySerialize(postState, t));
   }
 
   @Test
-  // TODO: re-enable when we merge #8916
-  @Disabled("requires Use 16-bit random value in validator filter #8916")
   public void fuzzAttesterSlashing_minimal() {
     final FuzzUtil fuzzUtil = new FuzzUtil(false, true);
 
@@ -142,8 +149,6 @@ public void fuzzAttesterSlashing_minimal() {
   }
 
   @Test
-  // TODO: re-enable when we merge #8916
-  @Disabled("requires Use 16-bit random value in validator filter #8916")
   public void fuzzBlock_minimal() {
     final FuzzUtil fuzzUtil = new FuzzUtil(false, true);
 
@@ -171,8 +176,6 @@ public void fuzzBlock_minimal() {
   }
 
   @Test
-  // TODO: re-enable when we merge #8916
-  @Disabled("requires Use 16-bit random value in validator filter #8916")
   public void fuzzBlockHeader_minimal() {
     final FuzzUtil fuzzUtil = new FuzzUtil(false, true);
 
@@ -213,8 +216,6 @@ public void fuzzDeposit_minimal() {
   }
 
   @Test
-  // TODO: re-enable when we merge #8916
-  @Disabled("requires Use 16-bit random value in validator filter #8916")
   public void fuzzProposerSlashing_minimal() {
     final FuzzUtil fuzzUtil = new FuzzUtil(false, true);
 
@@ -385,8 +386,6 @@ public void fuzzWithdrawalRequest_minimal() {
     assertThat(result.get()).isEqualTo(expected);
   }
 
-  // TODO fix as part of https://github.com/Consensys/teku/pull/8876
-  @Disabled("Disabling until we have a fix for this")
   @Test
   public void fuzzConsolidationRequest_minimal() {
     final FuzzUtil fuzzUtil = new FuzzUtil(false, true);

networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/AttestationGossipManager.java

@@ -53,7 +53,7 @@ public void onNewAttestation(final ValidatableAttestation validatableAttestation
     if (validatableAttestation.isAggregate() || !validatableAttestation.markGossiped()) {
       return;
     }
-    final Attestation attestation = validatableAttestation.getAttestation();
+    final Attestation attestation = validatableAttestation.getUnconvertedAttestation();
     subnetSubscriptions
         .gossip(attestation)
         .finish(