Merge remote-tracking branch 'upstream/master' into 8527-1

Consensys · Oct 1, 2024 · ccf7ad5 · ccf7ad5
2 parents ccf723f + 3b87509
commit ccf7ad5
Show file tree

Hide file tree

Showing 71 changed files with 37,725 additions and 16,879 deletions.
diff --git a/.codespell/.codespellrc b/.codespell/.codespellrc
@@ -1,5 +1,5 @@
 [codespell]
-skip = .git,package-lock.json,LOG.old.*
+skip = .git,package-lock.json,LOG.old.*,venv
 count =
 quiet-level = 3
 ignore-words = ./.codespell/wordlist.txt
diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml
@@ -16,8 +16,13 @@ jobs:
       - name: Checkout the repository
         uses: actions/checkout@v4
 
-      - name: Install prerequisites
-        run: pip install codespell
+      - name: Set up a Python venv and install prerequisites
+        run: |
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install codespell
 
       - name: Spell check
-        run: codespell --config=./.codespell/.codespellrc
+        run: |
+          source venv/bin/activate
+          codespell --config=./.codespell/.codespellrc
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,15 +7,20 @@
 ## Unreleased Changes
 
 ### Breaking Changes
+- Renamed metrics `validator_attestation_publication_delay`,`validator_block_publication_delay` and `beacon_block_import_delay_counter` to include the suffix `_total` added by the current version of prometheus.
 
 ### Additions and Improvements
 - Removed the deprecated [GetBlindedBlock](https://ethereum.github.io/beacon-APIs/#/ValidatorRequiredApi/produceBlindedBlock)
 - Removed the deprecated [GetBlockV2](https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Validator/produceBlockV2)
 - Implemented [PostAggregateAndProofsV2](https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Validator/publishAggregateAndProofsV2) (adding support for Electra)
 - Added support for [Ephemery Testnet](https://github.com/ephemery.dev) `--network=ephemery`
-- Renamed metrics `validator_attestation_publication_delay`,`validator_block_publication_delay` and `beacon_block_import_delay_counter` to include the suffix `_total` added by the current version of prometheus.
 - Updated bootnodes for Holesky network
-- Added new `--p2p-flood-publish-enabled` parameter to control whenever flood publishing behaviour is enabled (applies to all subnets). Previous teku versions always had this behaviour enabled. Default is `true`.
+- Disabled flood publish behaviour on all p2p subnets. `--Xp2p-flood-publish-enabled` experimental parameter can be used to re-enable it, restoring previous behaviour.
+- Add a fix for [CVE-2024-7254](https://avd.aquasec.com/nvd/2024/cve-2024-7254/)
+- Updated LUKSO configuration with Deneb fork scheduled for epoch 123075 (November 20, 2024, 16:20:00 UTC)
+- Support for `IDONTWANT` libp2p protocol messages
+- `/eth/v1/node/peers` endpoint now populates `enr` field of the peer whenever is possible
 
 ### Bug Fixes
- - removed a warning from logs about non blinded blocks being requested (#8562)
+ - Removed a warning from logs about non blinded blocks being requested (#8562)
+ - Fixed `blockImportCompleted` log message timing [#8653](https://github.com/Consensys/teku/pull/8653)
diff --git a/...c/main/java/tech/pegasys/teku/validator/coordinator/publisher/AbstractBlockPublisher.java b/...c/main/java/tech/pegasys/teku/validator/coordinator/publisher/AbstractBlockPublisher.java
@@ -70,7 +70,7 @@ public SafeFuture<SendSignedBlockResult> sendSignedBlock(
               return gossipAndImportUnblindedSignedBlockAndBlobSidecars(
                   signedBlock, blobSidecars, broadcastValidationLevel, blockPublishingPerformance);
             })
-        .thenCompose(result -> calculateResult(blockContainer, result));
+        .thenCompose(result -> calculateResult(blockContainer, result, blockPublishingPerformance));
   }
 
   private SafeFuture<BlockImportAndBroadcastValidationResults>
@@ -133,7 +133,8 @@ abstract void publishBlockAndBlobSidecars(
 
   private SafeFuture<SendSignedBlockResult> calculateResult(
       final SignedBlockContainer maybeBlindedBlockContainer,
-      final BlockImportAndBroadcastValidationResults blockImportAndBroadcastValidationResults) {
+      final BlockImportAndBroadcastValidationResults blockImportAndBroadcastValidationResults,
+      final BlockPublishingPerformance blockPublishingPerformance) {
 
     // broadcast validation can fail earlier than block import.
     // The assumption is that in that block import will fail but not as fast
@@ -155,6 +156,7 @@ private SafeFuture<SendSignedBlockResult> calculateResult(
                   .blockImportResult()
                   .thenApply(
                       importResult -> {
+                        blockPublishingPerformance.blockImportCompleted();
                         if (importResult.isSuccessful()) {
                           LOG.trace(
                               "Successfully imported proposed block: {}",

diff --git a/.../src/main/java/tech/pegasys/teku/validator/coordinator/publisher/BlockPublisherDeneb.java b/.../src/main/java/tech/pegasys/teku/validator/coordinator/publisher/BlockPublisherDeneb.java
@@ -56,9 +56,7 @@ SafeFuture<BlockImportAndBroadcastValidationResults> importBlockAndBlobSidecars(
       final BlockPublishingPerformance blockPublishingPerformance) {
     // provide blobs for the block before importing it
     blockBlobSidecarsTrackersPool.onCompletedBlockAndBlobSidecars(block, blobSidecars);
-    return blockImportChannel
-        .importBlock(block, broadcastValidationLevel)
-        .thenPeek(__ -> blockPublishingPerformance.blockImportCompleted());
+    return blockImportChannel.importBlock(block, broadcastValidationLevel);
   }
 
   @Override

diff --git a/...src/main/java/tech/pegasys/teku/validator/coordinator/publisher/BlockPublisherPhase0.java b/...src/main/java/tech/pegasys/teku/validator/coordinator/publisher/BlockPublisherPhase0.java
@@ -45,9 +45,7 @@ SafeFuture<BlockImportAndBroadcastValidationResults> importBlockAndBlobSidecars(
       final List<BlobSidecar> blobSidecars,
       final BroadcastValidationLevel broadcastValidationLevel,
       final BlockPublishingPerformance blockPublishingPerformance) {
-    return blockImportChannel
-        .importBlock(block, broadcastValidationLevel)
-        .thenPeek(__ -> blockPublishingPerformance.blockImportCompleted());
+    return blockImportChannel.importBlock(block, broadcastValidationLevel);
   }
 
   @Override

diff --git a/...pi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/schema/Peer.json b/...pi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/schema/Peer.json
@@ -10,7 +10,7 @@
     },
     "enr" : {
       "type" : "string",
-      "description" : "Ethereum node record. Not currently populated. [Read more](https://eips.ethereum.org/EIPS/eip-778)",
+      "description" : "Ethereum node record. [Read more](https://eips.ethereum.org/EIPS/eip-778)",
       "example" : "enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"
     },
     "last_seen_p2p_address" : {

diff --git a/...onrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeerById.java b/...onrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeerById.java
@@ -26,17 +26,17 @@
 import java.util.function.Function;
 import tech.pegasys.teku.api.DataProvider;
 import tech.pegasys.teku.api.NetworkDataProvider;
+import tech.pegasys.teku.api.peer.Eth2PeerWithEnr;
 import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.EndpointMetadata;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.RestApiEndpoint;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.RestApiRequest;
-import tech.pegasys.teku.networking.eth2.peers.Eth2Peer;
 
 public class GetPeerById extends RestApiEndpoint {
   public static final String ROUTE = "/eth/v1/node/peers/{peer_id}";
 
-  private static final SerializableTypeDefinition<Eth2Peer> PEERS_BY_ID_RESPONSE_TYPE =
-      SerializableTypeDefinition.object(Eth2Peer.class)
+  private static final SerializableTypeDefinition<Eth2PeerWithEnr> PEERS_BY_ID_RESPONSE_TYPE =
+      SerializableTypeDefinition.object(Eth2PeerWithEnr.class)
           .name("GetPeerResponse")
           .withField("data", PEER_DATA_TYPE, Function.identity())
           .build();
@@ -64,7 +64,8 @@ public GetPeerById(final DataProvider provider) {
   @Override
   public void handleRequest(final RestApiRequest request) throws JsonProcessingException {
     request.header(Header.CACHE_CONTROL, CACHE_NONE);
-    Optional<Eth2Peer> peer = network.getEth2PeerById(request.getPathParameter(PEER_ID_PARAMETER));
+    final Optional<Eth2PeerWithEnr> peer =
+        network.getEth2PeerById(request.getPathParameter(PEER_ID_PARAMETER));
     if (peer.isEmpty()) {
       request.respondError(SC_NOT_FOUND, "Peer not found");
     } else {

diff --git a/...eaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeers.java b/...eaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeers.java
@@ -23,18 +23,17 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.util.List;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.function.Function;
 import tech.pegasys.teku.api.DataProvider;
 import tech.pegasys.teku.api.NetworkDataProvider;
+import tech.pegasys.teku.api.peer.Eth2PeerWithEnr;
 import tech.pegasys.teku.api.response.v1.node.Direction;
 import tech.pegasys.teku.api.response.v1.node.State;
 import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition;
 import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.EndpointMetadata;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.RestApiEndpoint;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.RestApiRequest;
-import tech.pegasys.teku.networking.eth2.peers.Eth2Peer;
 
 public class GetPeers extends RestApiEndpoint {
   public static final String ROUTE = "/eth/v1/node/peers";
@@ -45,41 +44,42 @@ public class GetPeers extends RestApiEndpoint {
   private static final DeserializableTypeDefinition<Direction> DIRECTION_TYPE =
       DeserializableTypeDefinition.enumOf(Direction.class);
 
-  static final SerializableTypeDefinition<Eth2Peer> PEER_DATA_TYPE =
-      SerializableTypeDefinition.object(Eth2Peer.class)
+  static final SerializableTypeDefinition<Eth2PeerWithEnr> PEER_DATA_TYPE =
+      SerializableTypeDefinition.object(Eth2PeerWithEnr.class)
           .name("Peer")
           .withField(
               "peer_id",
               string(
                   "Cryptographic hash of a peer’s public key. "
                       + "'[Read more](https://docs.libp2p.io/concepts/peer-id/)",
                   "QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N"),
-              eth2Peer -> eth2Peer.getId().toBase58())
+              eth2Peer -> eth2Peer.peer().getId().toBase58())
           .withOptionalField(
               "enr",
               string(
-                  "Ethereum node record. Not currently populated. "
-                      + "[Read more](https://eips.ethereum.org/EIPS/eip-778)",
+                  "Ethereum node record. " + "[Read more](https://eips.ethereum.org/EIPS/eip-778)",
                   "enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrk"
                       + "Tfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYp"
                       + "Ma2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"),
-              eth2Peer -> Optional.empty())
+              Eth2PeerWithEnr::enr)
           .withField(
               "last_seen_p2p_address",
               string(
                   "Multiaddr used in last peer connection. "
                       + "[Read more](https://docs.libp2p.io/reference/glossary/#multiaddr)",
                   "/ip4/7.7.7.7/tcp/4242/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N"),
-              eth2Peer -> eth2Peer.getAddress().toExternalForm())
+              eth2Peer -> eth2Peer.peer().getAddress().toExternalForm())
           .withField(
               "state",
               STATE_TYPE,
-              eth2Peer -> eth2Peer.isConnected() ? State.connected : State.disconnected)
+              eth2Peer -> eth2Peer.peer().isConnected() ? State.connected : State.disconnected)
           .withField(
               "direction",
               DIRECTION_TYPE,
               eth2Peer ->
-                  eth2Peer.connectionInitiatedLocally() ? Direction.outbound : Direction.inbound)
+                  eth2Peer.peer().connectionInitiatedLocally()
+                      ? Direction.outbound
+                      : Direction.inbound)
           .build();
 
   private static final SerializableTypeDefinition<Integer> PEERS_META_TYPE =
@@ -119,19 +119,19 @@ public GetPeers(final DataProvider provider) {
 
   @Override
   public void handleRequest(final RestApiRequest request) throws JsonProcessingException {
-    request.respondOk(new PeersData(network.getEth2Peers()), NO_CACHE);
+    request.respondOk(new PeersData(network.getEth2PeersWithEnr()), NO_CACHE);
   }
 
   static class PeersData {
-    private final List<Eth2Peer> peers;
+    private final List<Eth2PeerWithEnr> peers;
     private final Integer count;
 
-    PeersData(final List<Eth2Peer> peers) {
+    PeersData(final List<Eth2PeerWithEnr> peers) {
       this.peers = peers;
       this.count = peers.size();
     }
 
-    public List<Eth2Peer> getPeers() {
+    public List<Eth2PeerWithEnr> getPeers() {
       return peers;
     }
 

diff --git a/...stapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeerByIdTest.java b/...stapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeerByIdTest.java
@@ -14,6 +14,7 @@
 package tech.pegasys.teku.beaconrestapi.handlers.v1.node;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -26,17 +27,25 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.util.Optional;
+import org.apache.tuweni.units.bigints.UInt256;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import tech.pegasys.teku.api.peer.Eth2PeerWithEnr;
 import tech.pegasys.teku.beaconrestapi.AbstractMigratedBeaconHandlerTest;
 import tech.pegasys.teku.infrastructure.http.HttpErrorResponse;
 import tech.pegasys.teku.networking.eth2.peers.Eth2Peer;
+import tech.pegasys.teku.networking.p2p.discovery.DiscoveryNetwork;
+import tech.pegasys.teku.networking.p2p.discovery.DiscoveryService;
 import tech.pegasys.teku.networking.p2p.mock.MockNodeId;
 import tech.pegasys.teku.networking.p2p.network.PeerAddress;
+import tech.pegasys.teku.networking.p2p.peer.NodeId;
 
 public class GetPeerByIdTest extends AbstractMigratedBeaconHandlerTest {
   private final MockNodeId peerId = new MockNodeId(123456);
+  private final NodeId peerNodeId = mock(NodeId.class);
   private final Eth2Peer peer = mock(Eth2Peer.class);
+  private static final String ENR_STUB = "enr:test";
+  private final Eth2PeerWithEnr peerWithEnr = new Eth2PeerWithEnr(peer, Optional.of(ENR_STUB));
 
   @BeforeEach
   void setUp() {
@@ -50,21 +59,29 @@ void setUp() {
 
   @Test
   public void shouldReturnNotFoundIfPeerNotFound() throws Exception {
-    when(network.getEth2PeerById(peerId.toBase58())).thenReturn(Optional.empty());
-
+    when(eth2P2PNetwork.parseNodeId(peerId.toBase58())).thenReturn(peerNodeId);
+    when(eth2P2PNetwork.getPeer(eq(peerNodeId))).thenReturn(Optional.empty());
     handler.handleRequest(request);
+
     assertThat(request.getResponseCode()).isEqualTo(SC_NOT_FOUND);
     assertThat(request.getResponseBody())
         .isEqualTo(new HttpErrorResponse(SC_NOT_FOUND, "Peer not found"));
   }
 
   @Test
   public void shouldReturnPeerIfFound() throws Exception {
-    when(network.getEth2PeerById(eq(peerId.toBase58()))).thenReturn(Optional.of(peer));
+    when(eth2P2PNetwork.parseNodeId(peerId.toBase58())).thenReturn(peerNodeId);
+    when(eth2P2PNetwork.getPeer(eq(peerNodeId))).thenReturn(Optional.of(peer));
+    when(peer.getDiscoveryNodeId()).thenReturn(Optional.of(UInt256.ONE));
+    final DiscoveryNetwork<?> discoveryNetwork = mock(DiscoveryNetwork.class);
+    when(eth2P2PNetwork.getDiscoveryNetwork()).thenReturn(Optional.of(discoveryNetwork));
+    final DiscoveryService discoveryService = mock(DiscoveryService.class);
+    when(discoveryNetwork.getDiscoveryService()).thenReturn(discoveryService);
+    when(discoveryService.lookupEnr(any())).thenReturn(Optional.of(ENR_STUB));
     handler.handleRequest(request);
 
     assertThat(request.getResponseCode()).isEqualTo(SC_OK);
-    assertThat(request.getResponseBody()).isEqualTo(peer);
+    assertThat(request.getResponseBody()).isEqualTo(peerWithEnr);
   }
 
   @Test
@@ -79,10 +96,11 @@ void metadata_shouldHandle500() throws JsonProcessingException {
 
   @Test
   void metadata_shouldHandle200() throws JsonProcessingException {
-    final String data = getResponseStringFromMetadata(handler, SC_OK, peer);
+    final String data = getResponseStringFromMetadata(handler, SC_OK, peerWithEnr);
     assertThat(data)
         .isEqualTo(
             "{\"data\":{\"peer_id\":\"1111111111111111111111111111177em\","
+                + "\"enr\":\"enr:test\","
                 + "\"last_seen_p2p_address\":\"1111111111111111111111111111177em\",\"state\":\"connected\",\"direction\":\"inbound\"}}");
   }
 }
diff --git a/...nrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeersTest.java b/...nrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeersTest.java
@@ -24,9 +24,11 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.util.List;
+import java.util.Optional;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.api.NetworkDataProvider;
+import tech.pegasys.teku.api.peer.Eth2PeerWithEnr;
 import tech.pegasys.teku.beaconrestapi.AbstractMigratedBeaconHandlerTest;
 import tech.pegasys.teku.beaconrestapi.handlers.v1.node.GetPeers.PeersData;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.CacheLength;
@@ -42,7 +44,10 @@ public class GetPeersTest extends AbstractMigratedBeaconHandlerTest {
   private final Eth2Peer peer2 = mock(Eth2Peer.class);
 
   private final NetworkDataProvider networkDataProvider = mock(NetworkDataProvider.class);
-  private final List<Eth2Peer> data = List.of(peer1, peer2);
+  private final List<Eth2PeerWithEnr> data =
+      List.of(
+          new Eth2PeerWithEnr(peer1, Optional.empty()),
+          new Eth2PeerWithEnr(peer2, Optional.empty()));
   private final GetPeers.PeersData peersData = new PeersData(data);
 
   @BeforeEach
@@ -62,7 +67,7 @@ void setup() {
   @Test
   public void shouldReturnListOfPeers() throws Exception {
 
-    when(networkDataProvider.getEth2Peers()).thenReturn(data);
+    when(networkDataProvider.getEth2PeersWithEnr()).thenReturn(data);
 
     handler.handleRequest(request);
     assertThat(request.getResponseCode()).isEqualTo(SC_OK);