New release 22.1.0 (#3442)

* add a websocket max frame size configuration in support of nimbus EL/CL interop (#3379)
* Response streaming: stop on IO error and remove queue management (#3399)
* Queue management is removed, since it is not working at the moment,
and can cause deadlocks, will be reimplemented later if needed.
* Improve closing behavior of JsonResponseStreamer, and make clear it is managed bt the ObjectWriter (#3421)
* Add ec-curve parameter public key export/export-address subcommands (#3333)
* Eliminate connect to self (#3342)
* Refactor to async retrieve blocks, and change peer when retrying to get a block (#3326)
* Disable RocksDB TTL compactions (#3356)
* fix launcher issue (#3352)
* Handle null forkId gracefully (#3409)
* When asking for the forkId for the chain head sometimes we may return
null. In those cases return an empty list. (#3343)

Signed-off-by: garyschulte <[email protected]>
Signed-off-by: Lucas Saldanha <[email protected]>
Signed-off-by: Fabio Di Fabio <[email protected]>
Signed-off-by: Sally MacFarlane <[email protected]>
Signed-off-by: Lucas Saldanha <[email protected]>
Signed-off-by: Karim TAAM <[email protected]>
Signed-off-by: Danno Ferrin <[email protected]>
Co-authored-by: garyschulte <[email protected]>
Co-authored-by: fab-10 <[email protected]>
Co-authored-by: Sally MacFarlane <[email protected]>
Co-authored-by: matkt <[email protected]>
Co-authored-by: Danno Ferrin <[email protected]>

CHANGELOG.md

@@ -1,7 +1,18 @@
 # Changelog
 ## 22.1.0
 
-### 22.1.0 Breaking Changes
+### Additions and Improvements
+- Updated besu-native to version 0.4.3 [#3331](https://github.com/hyperledger/besu/pull/3331)
+- Refactor synchronizer to asynchronously retrieve blocks from peers, and to change peer when retrying to get a block. [#3326](https://github.com/hyperledger/besu/pull/3326)
+- Disable RocksDB TTL compactions [#3356](https://github.com/hyperledger/besu/pull/3356)
+- add a websocket frame size configuration CLI parameter [3368][https://github.com/hyperledger/besu/pull/3379]
+- Add `--ec-curve` parameter to export/export-address public-key subcommands [#3333](https://github.com/hyperledger/besu/pull/3333)
+
+### Bug Fixes
+- add a websocket frame size configuration CLI parameter [3368][https://github.com/hyperledger/besu/pull/3379]
+- Prevent node from peering to itself [#3342](https://github.com/hyperledger/besu/pull/3342)
+- Fix an `IndexOutOfBoundsException` exception when getting block from peers. [#3304](https://github.com/hyperledger/besu/issues/3304)
+- Handle legacy eth64 without throwing null pointer exceptions [#3343](https://github.com/hyperledger/besu/pull/3343)
 
 ## 22.1.0-RC4
 
@@ -11,6 +22,7 @@
 ### Download Links
 - https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/22.1.0-RC4/besu-22.1.0-RC4.tar.gz \ SHA256 f1e34fddf4aad0d6699b38f6bdeff3799255d2c9bfdab0718f5f72bf90b56117
 - https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/22.1.0-RC4/besu-22.1.0-RC4.zip \ SHA256 f5a2dd72c164fd609963a9f1c3c9993632cf6b7eff85ce6cac95e444a53f2de4
+
 ## 22.1.0-RC3
 - Changing the order in which we traverse the word state tree during fast sync. This should improve fast sync during subsequent pivot changes.
 
@@ -58,8 +70,8 @@ https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/22.1.0-RC2/besu-22.1
 
  **Full Changelog**: https://github.com/hyperledger/besu/compare/21.10.8...21.10.9
 
-[besu-21.10.9.tar.gz](https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/21.10.9/besu-21.10.9.tar.gz) b5ef15ede50e19ad9fbb79876d8d68aeb8e3811f8905e973874523df2ed179b4
-[besu-21.10.9.zip](https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/21.10.9/besu-21.10.9.zip) 26489881d852eefdda2cf0203cb30f7b2045ecfb299da8c744e9579cbe2934e2
+[besu-21.10.9.tar.gz](https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/21.10.9/besu-21.10.9.tar.gz) a4b85ba72ee73017303e4b2f0fdde84a87d376c2c17fdcebfa4e34680f52fc71
+[besu-21.10.9.zip](https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/21.10.9/besu-21.10.9.zip) c3ba3f07340fa80064ba7c06f2c0ec081184e000f9a925d132084352d0665ef9
 
 ## 21.10.8
 

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -159,6 +159,8 @@ public void startNode(final BesuNode node) {
             .strictTransactionReplayProtectionEnabled(node.isStrictTxReplayProtectionEnabled())
             .build();
 
+    final int maxPeers = 25;
+
     final BesuController besuController =
         builder
             .synchronizerConfiguration(new SynchronizerConfiguration.Builder().build())
@@ -178,6 +180,7 @@ public void startNode(final BesuNode node) {
                     .map(
                         (pkiConfig) -> new PkiBlockCreationConfigurationProvider().load(pkiConfig)))
             .evmConfiguration(EvmConfiguration.DEFAULT)
+            .maxPeers(maxPeers)
             .build();
 
     final RunnerBuilder runnerBuilder = new RunnerBuilder();
@@ -191,7 +194,7 @@ public void startNode(final BesuNode node) {
             .discovery(node.isDiscoveryEnabled())
             .p2pAdvertisedHost(node.getHostName())
             .p2pListenPort(0)
-            .maxPeers(25)
+            .maxPeers(maxPeers)
             .networkingConfiguration(node.getNetworkingConfiguration())
             .jsonRpcConfiguration(node.jsonRpcConfiguration())
             .webSocketConfiguration(node.webSocketConfiguration())

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -679,6 +679,13 @@ void setBannedNodeIds(final List<String> values) {
       arity = "1")
   private final Integer rpcWsPort = DEFAULT_WEBSOCKET_PORT;
 
+  @Option(
+      names = {"--rpc-ws-max-frame-size"},
+      description =
+          "Maximum size in bytes for JSON-RPC WebSocket frames (default: ${DEFAULT-VALUE}). If this limit is exceeded, the websocket will be disconnected.",
+      arity = "1")
+  private final Integer rpcWsMaxFrameSize = DEFAULT_WS_MAX_FRAME_SIZE;
+
   @Option(
       names = {"--rpc-ws-max-active-connections"},
       description =
@@ -1862,7 +1869,8 @@ public BesuControllerBuilder getControllerBuilder() {
         .requiredBlocks(requiredBlocks)
         .reorgLoggingThreshold(reorgLoggingThreshold)
         .evmConfiguration(unstableEvmOptions.toDomainObject())
-        .dataStorageConfiguration(unstableDataStorageOptions.toDomainObject());
+        .dataStorageConfiguration(unstableDataStorageOptions.toDomainObject())
+        .maxPeers(maxPeers);
   }
 
   private GraphQLConfiguration graphQLConfiguration() {
@@ -2070,6 +2078,7 @@ private WebSocketConfiguration webSocketConfiguration() {
             "--rpc-ws-apis",
             "--rpc-ws-host",
             "--rpc-ws-port",
+            "--rpc-ws-max-frame-size",
             "--rpc-ws-max-active-connections",
             "--rpc-ws-authentication-enabled",
             "--rpc-ws-authentication-credentials-file",
@@ -2097,6 +2106,7 @@ && rpcWsAuthenticationCredentialsFile() == null
     webSocketConfiguration.setEnabled(isRpcWsEnabled);
     webSocketConfiguration.setHost(rpcWsHost);
     webSocketConfiguration.setPort(rpcWsPort);
+    webSocketConfiguration.setMaxFrameSize(rpcWsMaxFrameSize);
     webSocketConfiguration.setMaxActiveConnections(rpcWsMaxConnections);
     webSocketConfiguration.setRpcApis(rpcWsApis);
     webSocketConfiguration.setAuthenticationEnabled(isRpcWsAuthenticationEnabled);

besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java

@@ -60,6 +60,7 @@ public interface DefaultCommandValues {
   int DEFAULT_MAX_PEERS = 25;
   int DEFAULT_HTTP_MAX_CONNECTIONS = 80;
   int DEFAULT_WS_MAX_CONNECTIONS = 80;
+  int DEFAULT_WS_MAX_FRAME_SIZE = 1024 * 1024;
   float DEFAULT_FRACTION_REMOTE_WIRE_CONNECTIONS_ALLOWED =
       RlpxConfiguration.DEFAULT_FRACTION_REMOTE_CONNECTIONS_ALLOWED;
   String DEFAULT_KEY_VALUE_STORAGE_NAME = "rocksdb";

besu/src/main/java/org/hyperledger/besu/cli/subcommands/PublicKeySubCommand.java

@@ -24,6 +24,8 @@
 import org.hyperledger.besu.cli.subcommands.PublicKeySubCommand.AddressSubCommand;
 import org.hyperledger.besu.cli.subcommands.PublicKeySubCommand.ExportSubCommand;
 import org.hyperledger.besu.crypto.KeyPair;
+import org.hyperledger.besu.crypto.SignatureAlgorithmFactory;
+import org.hyperledger.besu.crypto.SignatureAlgorithmType;
 import org.hyperledger.besu.ethereum.core.Util;
 
 import java.io.BufferedWriter;
@@ -97,6 +99,7 @@ static class ExportSubCommand extends KeyPairSubcommand implements Runnable {
 
     @Override
     public void run() {
+      configureEcCurve(ecCurve, parentCommand.spec.commandLine());
       run(publicKeyExportFile, keyPair -> keyPair.getPublicKey().toString());
     }
   }
@@ -126,6 +129,7 @@ static class AddressSubCommand extends KeyPairSubcommand implements Runnable {
 
     @Override
     public void run() {
+      configureEcCurve(ecCurve, parentCommand.spec.commandLine());
       run(addressExportFile, keyPair -> Util.publicKeyToAddress(keyPair.getPublicKey()).toString());
     }
   }
@@ -134,10 +138,20 @@ private static class KeyPairSubcommand {
 
     @SuppressWarnings("unused")
     @ParentCommand
-    private PublicKeySubCommand parentCommand; // Picocli injects reference to parent command
+    protected PublicKeySubCommand parentCommand; // Picocli injects reference to parent command
 
     @Mixin private final NodePrivateKeyFileOption nodePrivateKeyFileOption = null;
 
+    @Option(
+        names = "--ec-curve",
+        paramLabel = "<NAME>",
+        description =
+            "Elliptic curve to use when creating a new key (default: "
+                + SignatureAlgorithmType.DEFAULT_EC_CURVE_NAME
+                + ")",
+        arity = "0..1")
+    protected String ecCurve = null;
+
     @Spec private final CommandSpec spec = null;
 
     protected final void run(
@@ -172,5 +186,15 @@ protected final void run(
         parentCommand.out.println(output);
       }
     }
+
+    protected static void configureEcCurve(final String ecCurve, final CommandLine commandLine) {
+      if (ecCurve != null) {
+        try {
+          SignatureAlgorithmFactory.setInstance(SignatureAlgorithmType.create(ecCurve));
+        } catch (IllegalArgumentException e) {
+          throw new CommandLine.ParameterException(commandLine, e.getMessage(), e);
+        }
+      }
+    }
   }
 }

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -115,6 +115,7 @@ public abstract class BesuControllerBuilder {
   protected List<NodeMessagePermissioningProvider> messagePermissioningProviders =
       Collections.emptyList();
   protected EvmConfiguration evmConfiguration;
+  protected int maxPeers;
 
   public BesuControllerBuilder storageProvider(final StorageProvider storageProvider) {
     this.storageProvider = storageProvider;
@@ -238,6 +239,11 @@ public BesuControllerBuilder evmConfiguration(final EvmConfiguration evmConfigur
     return this;
   }
 
+  public BesuControllerBuilder maxPeers(final int maxPeers) {
+    this.maxPeers = maxPeers;
+    return this;
+  }
+
   public BesuController build() {
     checkNotNull(genesisConfig, "Missing genesis config");
     checkNotNull(syncConfig, "Missing sync config");
@@ -309,7 +315,8 @@ public BesuController build() {
       }
     }
     final EthPeers ethPeers =
-        new EthPeers(getSupportedProtocol(), clock, metricsSystem, messagePermissioningProviders);
+        new EthPeers(
+            getSupportedProtocol(), clock, metricsSystem, maxPeers, messagePermissioningProviders);
 
     final EthMessages ethMessages = new EthMessages();
     final EthMessages snapMessages = new EthMessages();

besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java

@@ -203,6 +203,8 @@ public void callingVersionDisplayBesuInfoVersion() {
   public void callingBesuCommandWithoutOptionsMustSyncWithDefaultValues() throws Exception {
     parseCommand();
 
+    final int maxPeers = 25;
+
     final ArgumentCaptor<EthNetworkConfig> ethNetworkArg =
         ArgumentCaptor.forClass(EthNetworkConfig.class);
     verify(mockRunnerBuilder).discovery(eq(true));
@@ -215,7 +217,7 @@ public void callingBesuCommandWithoutOptionsMustSyncWithDefaultValues() throws E
                 MAINNET_DISCOVERY_URL));
     verify(mockRunnerBuilder).p2pAdvertisedHost(eq("127.0.0.1"));
     verify(mockRunnerBuilder).p2pListenPort(eq(30303));
-    verify(mockRunnerBuilder).maxPeers(eq(25));
+    verify(mockRunnerBuilder).maxPeers(eq(maxPeers));
     verify(mockRunnerBuilder).fractionRemoteConnectionsAllowed(eq(0.6f));
     verify(mockRunnerBuilder).jsonRpcConfiguration(eq(DEFAULT_JSON_RPC_CONFIGURATION));
     verify(mockRunnerBuilder).graphQLConfiguration(eq(DEFAULT_GRAPH_QL_CONFIGURATION));
@@ -234,6 +236,7 @@ public void callingBesuCommandWithoutOptionsMustSyncWithDefaultValues() throws E
     verify(mockControllerBuilder).nodeKey(isNotNull());
     verify(mockControllerBuilder).storageProvider(storageProviderArgumentCaptor.capture());
     verify(mockControllerBuilder).gasLimitCalculator(eq(GasLimitCalculator.constant()));
+    verify(mockControllerBuilder).maxPeers(eq(maxPeers));
     verify(mockControllerBuilder).build();
 
     assertThat(storageProviderArgumentCaptor.getValue()).isNotNull();
@@ -2102,6 +2105,20 @@ public void rpcHttpMaxActiveConnectionsPropertyMustBeUsed() {
     assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
   }
 
+  @Test
+  public void rpcWsMaxFrameSizePropertyMustBeUsed() {
+    final int maxFrameSize = 65535;
+    parseCommand("--rpc-ws-max-frame-size", String.valueOf(maxFrameSize));
+
+    verify(mockRunnerBuilder).webSocketConfiguration(wsRpcConfigArgumentCaptor.capture());
+    verify(mockRunnerBuilder).build();
+
+    assertThat(wsRpcConfigArgumentCaptor.getValue().getMaxFrameSize()).isEqualTo(maxFrameSize);
+
+    assertThat(commandOutput.toString(UTF_8)).isEmpty();
+    assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
+  }
+
   @Test
   public void rpcWsMaxActiveConnectionsPropertyMustBeUsed() {
     final int maxConnections = 99;
@@ -2936,14 +2953,17 @@ public void rpcWsOptionsRequiresServiceToBeEnabled() {
         "--rpc-ws-port",
         "1234",
         "--rpc-ws-max-active-connections",
-        "77");
+        "77",
+        "--rpc-ws-max-frame-size",
+        "65535");
 
     verifyOptionsConstraintLoggerCall(
         "--rpc-ws-enabled",
         "--rpc-ws-host",
         "--rpc-ws-port",
         "--rpc-ws-api",
-        "--rpc-ws-max-active-connections");
+        "--rpc-ws-max-active-connections",
+        "--rpc-ws-max-frame-size");
 
     assertThat(commandOutput.toString(UTF_8)).isEmpty();
     assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();

besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java

@@ -227,7 +227,7 @@ public void initMocks() throws Exception {
     when(mockControllerBuilder.reorgLoggingThreshold(anyLong())).thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.dataStorageConfiguration(any())).thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.evmConfiguration(any())).thenReturn(mockControllerBuilder);
-
+    when(mockControllerBuilder.maxPeers(anyInt())).thenReturn(mockControllerBuilder);
     // doReturn used because of generic BesuController
     doReturn(mockController).when(mockControllerBuilder).build();
     lenient().when(mockController.getProtocolManager()).thenReturn(mockEthProtocolManager);