Merge branch 'master' into update_prometheus

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Upcoming Breaking Changes
 
+- Next release will introduce a breaking change to Teku's metrics. This is due to some metrics changing names after a library upgrade.
+We recommend all users of the `Teku - Detailed` dashboard to upgrade to version [Revision 12](https://grafana.com/api/dashboards/16737/revisions/12/download)  
+as soon as possible. Documentation with all metrics that have been renamed will be provided.
+- Next release will require Java 21. The current release is compatible, please consider upgrading before the next release.
+- From the next release, you will need to explicitly set `--data-storage-mode=(prune|archive)` unless you're using minimal data-storage-mode (which is the default behaviour).
+
 ## Current Releases
 
 For information on changes in released versions of Teku, see
@@ -13,9 +19,4 @@ the [releases page](https://github.com/Consensys/teku/releases).
 
 ### Additions and Improvements
 
-- Added metadata fields to `/eth/v1/beacon/blob_sidecars/{block_id}` Beacon API response as per https://github.com/ethereum/beacon-APIs/pull/441
-- Added rest api endpoint `/teku/v1/beacon/state/finalized/slot/before/{slot}` to return most recent stored state at or before a specified slot.
-- The validator client will start using the `v2` variant of the beacon node block publishing
-  endpoints. In the cases where the block has been produced in the same beacon node, only equivocation validation will be done instead of the entire gossip validation.
-
 ### Bug Fixes

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/ValidatorClientServiceAcceptanceTest.java

@@ -49,6 +49,23 @@ void shouldFailWithNoValidatorKeysWhenExitOptionEnabledOnBeaconNode() throws Exc
         "No loaded validators when --exit-when-no-validator-keys-enabled option is true");
   }
 
+  @Test
+  void shouldFailWithNoValidatorKeysSourceProvidedOnValidatorClient() throws Exception {
+    final TekuBeaconNode beaconNode = createTekuBeaconNode();
+
+    final TekuValidatorNode validatorClient =
+        createValidatorNode(
+            TekuNodeConfigBuilder.createValidatorClient()
+                .withInteropModeDisabled()
+                .withBeaconNodes(beaconNode)
+                .build());
+    beaconNode.start();
+    validatorClient.startWithFailure(
+        "No validator keys source provided, should provide local or remote keys otherwise enable the key-manager"
+            + " api to start the validator client");
+    beaconNode.stop();
+  }
+
   @Test
   void bn_shouldFailIfValidatorKeyLocked(@TempDir final Path tempDir) throws Exception {
     final String networkName = "swift";
@@ -145,6 +162,7 @@ void shouldFailWithNoValidatorKeysWhenExitOptionEnabledOnValidatorClient() throw
             TekuNodeConfigBuilder.createValidatorClient()
                 .withInteropModeDisabled()
                 .withBeaconNodes(beaconNode)
+                .withValidatorApiEnabled()
                 .withExitWhenNoValidatorKeysEnabled(true)
                 .build());
     beaconNode.start();

beacon/sync/src/testFixtures/java/tech/pegasys/teku/beacon/sync/SyncingNodeManager.java

@@ -51,7 +51,6 @@
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.TestSpecFactory;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
-import tech.pegasys.teku.spec.datastructures.validator.BroadcastValidationLevel;
 import tech.pegasys.teku.spec.executionlayer.ExecutionLayerChannel;
 import tech.pegasys.teku.spec.executionlayer.ExecutionLayerChannelStub;
 import tech.pegasys.teku.spec.logic.common.statetransition.results.BlockImportResult;
@@ -208,8 +207,7 @@ public static SyncingNodeManager create(
             BlockBlobSidecarsTrackersPool.NOOP,
             syncService,
             fetchBlockTaskFactory);
-    recentBlocksFetcher.subscribeBlockFetched(
-        block -> blockManager.importBlock(block, BroadcastValidationLevel.NOT_REQUIRED));
+    recentBlocksFetcher.subscribeBlockFetched(blockManager::importBlock);
     eventChannels.subscribe(ReceivedBlockEventsChannel.class, recentBlocksFetcher);
 
     recentBlocksFetcher.start().join();

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/DepositProvider.java

@@ -238,7 +238,7 @@ public synchronized List<DepositWithIndex> getDepositsWithIndex(
                 stateElectra -> {
                   // EIP-6110
                   final UInt64 eth1DepositIndexLimit =
-                      eth1DepositCount.min(stateElectra.getDepositReceiptsStartIndex());
+                      eth1DepositCount.min(stateElectra.getDepositRequestsStartIndex());
                   return eth1DepositIndexLimit.minusMinZero(eth1DepositIndex).min(maxDeposits);
                 })
             .orElseGet(

beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/GraffitiBuilder.java

@@ -46,14 +46,10 @@ public class GraffitiBuilder implements ExecutionClientVersionChannel {
 
   private final ClientGraffitiAppendFormat clientGraffitiAppendFormat;
   private final ClientVersion consensusClientVersion;
-  private final Optional<Bytes32> defaultUserGraffiti;
 
-  public GraffitiBuilder(
-      final ClientGraffitiAppendFormat clientGraffitiAppendFormat,
-      final Optional<Bytes32> defaultUserGraffiti) {
+  public GraffitiBuilder(final ClientGraffitiAppendFormat clientGraffitiAppendFormat) {
     this.clientGraffitiAppendFormat = clientGraffitiAppendFormat;
     this.consensusClientVersion = createTekuClientVersion();
-    this.defaultUserGraffiti = defaultUserGraffiti;
   }
 
   private ClientVersion createTekuClientVersion() {
@@ -73,18 +69,18 @@ public ClientVersion getConsensusClientVersion() {
   @Override
   public void onExecutionClientVersion(final ClientVersion executionClientVersion) {
     this.executionClientVersion = Optional.of(executionClientVersion);
-    logDefaultGraffiti();
+    logGraffitiWatermark();
   }
 
   @Override
   public void onExecutionClientVersionNotAvailable() {
-    logDefaultGraffiti();
+    logGraffitiWatermark();
   }
 
-  private void logDefaultGraffiti() {
-    final Optional<Bytes32> defaultGraffiti = Optional.of(buildGraffiti(defaultUserGraffiti));
-    EVENT_LOG.logDefaultGraffiti(
-        extractGraffiti(defaultGraffiti, calculateGraffitiLength(defaultGraffiti)));
+  private void logGraffitiWatermark() {
+    final Optional<Bytes32> graffitiWatermark = Optional.of(buildGraffiti(Optional.empty()));
+    EVENT_LOG.logGraffitiWatermark(
+        extractGraffiti(graffitiWatermark, calculateGraffitiLength(graffitiWatermark)));
   }
 
   public Bytes32 buildGraffiti(final Optional<Bytes32> userGraffiti) {

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/AbstractBlockFactoryTest.java

@@ -124,7 +124,7 @@ public abstract class AbstractBlockFactoryTest {
   protected ExecutionPayloadResult cachedExecutionPayloadResult = null;
 
   protected GraffitiBuilder graffitiBuilder =
-      new GraffitiBuilder(ClientGraffitiAppendFormat.DISABLED, Optional.empty());
+      new GraffitiBuilder(ClientGraffitiAppendFormat.DISABLED);
 
   @BeforeAll
   public static void initSession() {

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactoryTest.java

@@ -186,7 +186,7 @@ class BlockOperationSelectorFactoryTest {
       new CapturingBeaconBlockBodyBuilder(false, false);
 
   private final GraffitiBuilder graffitiBuilder =
-      new GraffitiBuilder(ClientGraffitiAppendFormat.DISABLED, Optional.empty());
+      new GraffitiBuilder(ClientGraffitiAppendFormat.DISABLED);
 
   private final BlockOperationSelectorFactory factoryElectra =
       new BlockOperationSelectorFactory(

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/DepositProviderTest.java

@@ -181,7 +181,7 @@ void numberOfDepositsThatCanBeIncludedMoreThanMaxDeposits() {
   void noDepositsIncludedIfFormerDepositMechanismHasBeenDisabled() {
     setup(16, SpecMilestone.ELECTRA);
     updateStateEth1DepositIndex(5);
-    updateStateDepositReceiptsStartIndex(5);
+    updateStateDepositRequestsStartIndex(5);
 
     final SszList<Deposit> deposits = depositProvider.getDeposits(state, randomEth1Data);
 
@@ -194,15 +194,15 @@ void getsRemainingEth1PendingDepositsIfElectraIsEnabled() {
     updateStateEth1DepositIndex(5);
     updateStateEth1DataDepositCount(20);
     // 16th deposit is using the new mechanism
-    updateStateDepositReceiptsStartIndex(16);
+    updateStateDepositRequestsStartIndex(16);
 
     mockDepositsFromEth1Block(0, 10);
     mockDepositsFromEth1Block(10, 20);
 
     final List<DepositWithIndex> deposits =
         depositProvider.getDepositsWithIndex(state, randomEth1Data);
 
-    // the pending Eth1 deposits (deposit_receipt_start_index - eth1_deposit_index)
+    // the pending Eth1 deposits (deposit_requests_start_index - eth1_deposit_index)
     // we need to process eth1_deposit_index deposit (5) up to 16 (exclusive) so 11 is the
     // expected size
     assertThat(deposits).hasSize(11);
@@ -321,7 +321,7 @@ void shouldLogAnEventOnSlotWhenAllDepositsRequiredForStateNotAvailable() {
     setup(1, SpecMilestone.ELECTRA);
     mockDepositsFromEth1Block(0, 8);
     updateStateEth1DepositIndex(5);
-    updateStateDepositReceiptsStartIndex(5);
+    updateStateDepositRequestsStartIndex(5);
     updateStateEth1DataDepositCount(10);
     when(recentChainData.getBestState()).thenReturn(Optional.of(SafeFuture.completedFuture(state)));
 
@@ -546,11 +546,11 @@ private void updateStateEth1DepositIndex(final int n) {
     state = state.updated(mutableState -> mutableState.setEth1DepositIndex(UInt64.valueOf(n)));
   }
 
-  private void updateStateDepositReceiptsStartIndex(final int n) {
+  private void updateStateDepositRequestsStartIndex(final int n) {
     state =
         state.updated(
             mutableState ->
                 MutableBeaconStateElectra.required(mutableState)
-                    .setDepositReceiptsStartIndex(UInt64.valueOf(n)));
+                    .setDepositRequestsStartIndex(UInt64.valueOf(n)));
   }
 }

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/GraffitiBuilderTest.java

@@ -37,12 +37,10 @@
 
 public class GraffitiBuilderTest {
   private ClientGraffitiAppendFormat clientGraffitiAppendFormat = AUTO;
-  private Optional<Bytes32> userGraffiti = Optional.empty();
-  private GraffitiBuilder graffitiBuilder =
-      new GraffitiBuilder(clientGraffitiAppendFormat, userGraffiti);
+  private GraffitiBuilder graffitiBuilder = new GraffitiBuilder(clientGraffitiAppendFormat);
 
   private static final ClientVersion TEKU_CLIENT_VERSION =
-      new GraffitiBuilder(DISABLED, Optional.empty()).getConsensusClientVersion();
+      new GraffitiBuilder(DISABLED).getConsensusClientVersion();
   private static final ClientVersion BESU_CLIENT_VERSION =
       new ClientVersion("BU", "Besu", "23.4.1", Bytes4.fromHexString("abcdef12"));
 
@@ -58,41 +56,37 @@ public class GraffitiBuilderTest {
   @BeforeEach
   public void setup() {
     this.clientGraffitiAppendFormat = AUTO;
-    this.userGraffiti = Optional.empty();
-    this.graffitiBuilder = new GraffitiBuilder(clientGraffitiAppendFormat, userGraffiti);
+    this.graffitiBuilder = new GraffitiBuilder(clientGraffitiAppendFormat);
   }
 
   @Test
-  public void onExecutionClientVersion_shouldLogDefaultGraffiti() {
+  public void onExecutionClientVersion_shouldLogGraffitiWatermark() {
+    this.graffitiBuilder = new GraffitiBuilder(clientGraffitiAppendFormat);
     try (final LogCaptor logCaptor = LogCaptor.forClass(EventLogger.class)) {
       graffitiBuilder.onExecutionClientVersion(BESU_CLIENT_VERSION);
       logCaptor.assertInfoLog(
-          "Default graffiti to use when building block without external VC: \"TK"
+          "Using graffiti watermark: \"TK"
               + TEKU_CLIENT_VERSION.commit().toUnprefixedHexString()
               + "BUabcdef12\". "
-              + "To change check validator graffiti options.");
+              + "This will be appended to any user-defined graffiti or used if none is defined. Refer to validator graffiti options to customize.");
     }
   }
 
   @Test
-  public void onExecutionClientVersion_shouldLogDefaultMergedGraffiti() {
-    this.graffitiBuilder =
-        new GraffitiBuilder(
-            clientGraffitiAppendFormat, Optional.of(Bytes32Parser.toBytes32(ASCII_GRAFFITI_20)));
+  public void onExecutionClientVersionNotAvailable_shouldLogGraffitiWatermark() {
     try (final LogCaptor logCaptor = LogCaptor.forClass(EventLogger.class)) {
-      graffitiBuilder.onExecutionClientVersion(BESU_CLIENT_VERSION);
+      graffitiBuilder.onExecutionClientVersionNotAvailable();
       logCaptor.assertInfoLog(
-          "Default graffiti to use when building block without external VC: \"I've proposed ablock TK"
-              + TEKU_CLIENT_VERSION.commit().toUnprefixedHexString().substring(0, 2)
-              + "BUab\". "
-              + "To change check validator graffiti options.");
+          "Using graffiti watermark: \"TK"
+              + TEKU_CLIENT_VERSION.commit().toUnprefixedHexString()
+              + "\". This will be appended to any user-defined graffiti or used if none is defined. Refer to validator graffiti options to customize.");
     }
   }
 
   @Test
   public void buildGraffiti_shouldNotFail() {
     this.graffitiBuilder =
-        new GraffitiBuilder(clientGraffitiAppendFormat, userGraffiti) {
+        new GraffitiBuilder(clientGraffitiAppendFormat) {
           @Override
           protected int calculateGraffitiLength(final Optional<Bytes32> graffiti) {
             throw new RuntimeException("");
@@ -105,10 +99,9 @@ protected int calculateGraffitiLength(final Optional<Bytes32> graffiti) {
 
   @Test
   public void buildGraffiti_shouldPreferCallInput() {
-    final Bytes32 defaultGraffiti = Bytes32Parser.toBytes32(asciiGraffiti32);
     final Bytes32 userGraffiti = Bytes32Parser.toBytes32(ASCII_GRAFFITI_20);
     final Bytes32 expectedGraffiti = Bytes32Parser.toBytes32(ASCII_GRAFFITI_20 + " TK");
-    this.graffitiBuilder = new GraffitiBuilder(CLIENT_CODES, Optional.of(defaultGraffiti));
+    this.graffitiBuilder = new GraffitiBuilder(CLIENT_CODES);
     assertThat(graffitiBuilder.buildGraffiti(Optional.of(userGraffiti)))
         .isEqualTo(expectedGraffiti);
   }
@@ -119,7 +112,7 @@ public void buildGraffiti_shouldProvideCorrectOutput(
       final ClientGraffitiAppendFormat clientGraffitiAppendFormat,
       final Optional<String> maybeUserGraffiti,
       final String expectedGraffiti) {
-    this.graffitiBuilder = new GraffitiBuilder(clientGraffitiAppendFormat, userGraffiti);
+    this.graffitiBuilder = new GraffitiBuilder(clientGraffitiAppendFormat);
     graffitiBuilder.onExecutionClientVersion(BESU_CLIENT_VERSION);
     final Bytes32 expectedGraffitiBytes = Bytes32Parser.toBytes32(expectedGraffiti);
     assertThat(
@@ -140,7 +133,7 @@ public void buildGraffiti_shouldProvideCorrectOutput_whenElInfoNa(
       final ClientGraffitiAppendFormat clientGraffitiAppendFormat,
       final Optional<String> maybeUserGraffiti,
       final String expectedGraffiti) {
-    this.graffitiBuilder = new GraffitiBuilder(clientGraffitiAppendFormat, userGraffiti);
+    this.graffitiBuilder = new GraffitiBuilder(clientGraffitiAppendFormat);
     final Bytes32 expectedGraffitiBytes = Bytes32Parser.toBytes32(expectedGraffiti);
     assertThat(
             new String(

beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandlerTest.java

@@ -931,7 +931,7 @@ public void sendSignedBlock_shouldConvertKnownBlockResult() {
                 blockContainerAndMetaData.blockContainer().getBlock(),
                 dataStructureUtil.randomSignature());
 
-    when(blockImportChannel.importBlock(eq(block), any()))
+    when(blockImportChannel.importBlock(block, EQUIVOCATION))
         .thenReturn(prepareBlockImportResult(BlockImportResult.successful(block)));
 
     // require GOSSIP validation

build.gradle

@@ -22,9 +22,9 @@ buildscript {
 plugins {
   id 'com.diffplug.spotless' version '6.25.0'
   id 'com.github.ben-manes.versions' version '0.51.0'
-  id 'com.github.jk1.dependency-license-report' version '2.7'
-  id 'io.spring.dependency-management' version '1.1.4'
-  id 'net.ltgt.errorprone' version '3.1.0' apply false
+  id 'com.github.jk1.dependency-license-report' version '2.8'
+  id 'io.spring.dependency-management' version '1.1.5'
+  id 'net.ltgt.errorprone' version '4.0.0' apply false
   id 'de.undercouch.download' version '5.6.0'
   id 'org.ajoberstar.grgit' version '5.2.2'
 }
@@ -312,7 +312,7 @@ allprojects {
   }
 }
 
-def refTestVersion = 'v1.5.0-alpha.2'
+def refTestVersion = 'v1.5.0-alpha.3'
 def blsRefTestVersion = 'v0.1.2'
 def slashingProtectionInterchangeRefTestVersion = 'v5.3.0'
 def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download'

data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v1/validator/PostBlindedAndUnblindedBlockTest.java

@@ -153,7 +153,11 @@ private void prepareResponse(final SignedBeaconBlock request, final Version vers
               request, BroadcastValidationLevel.CONSENSUS_AND_EQUIVOCATION))
           .thenReturn(SafeFuture.completedFuture(SendSignedBlockResult.success(request.getRoot())));
     } else {
-      when(validatorApiChannel.sendSignedBlock(request, BroadcastValidationLevel.NOT_REQUIRED))
+      final BroadcastValidationLevel broadcastValidationLevel =
+          request.isBlinded()
+              ? BroadcastValidationLevel.NOT_REQUIRED
+              : BroadcastValidationLevel.GOSSIP;
+      when(validatorApiChannel.sendSignedBlock(request, broadcastValidationLevel))
           .thenReturn(SafeFuture.completedFuture(SendSignedBlockResult.success(request.getRoot())));
     }
   }
@@ -164,7 +168,11 @@ private void prepareResponse(final SignedBlockContainer request, final Version v
               request, BroadcastValidationLevel.CONSENSUS_AND_EQUIVOCATION))
           .thenReturn(SafeFuture.completedFuture(SendSignedBlockResult.success(request.getRoot())));
     } else {
-      when(validatorApiChannel.sendSignedBlock(request, BroadcastValidationLevel.NOT_REQUIRED))
+      final BroadcastValidationLevel broadcastValidationLevel =
+          request.isBlinded()
+              ? BroadcastValidationLevel.NOT_REQUIRED
+              : BroadcastValidationLevel.GOSSIP;
+      when(validatorApiChannel.sendSignedBlock(request, broadcastValidationLevel))
           .thenReturn(SafeFuture.completedFuture(SendSignedBlockResult.success(request.getRoot())));
     }
   }

data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_beacon_blinded_blocks.json

@@ -3,7 +3,7 @@
     "tags" : [ "Beacon", "Validator Required Api" ],
     "operationId" : "publishBlindedBlock",
     "summary" : "Publish a signed blinded block",
-    "description" : "Submit a signed blinded beacon block to the beacon node to be broadcast and imported. The beacon node performs the required validation.",
+    "description" : "Instructs the beacon node to use the components of the `SignedBlindedBeaconBlock` to construct and publish a `SignedBeaconBlock` by swapping out the `transactions_root` for the corresponding full list of `transactions`. The beacon node should broadcast a newly constructed `SignedBeaconBlock` to the beacon network, to be included in the beacon chain. The beacon node is not required to validate the signed `BeaconBlock`, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new block into its state, and therefore validate the block internally, however blocks which fail the validation are still broadcast but a different status code is returned (202). Pre-Bellatrix, this endpoint will accept a `SignedBeaconBlock`.",
     "parameters" : [ {
       "name" : "Eth-Consensus-Version",
       "in" : "header",

data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_beacon_blocks.json

@@ -3,7 +3,7 @@
     "tags" : [ "Beacon", "Validator Required Api" ],
     "operationId" : "publishBlock",
     "summary" : "Publish a signed block",
-    "description" : "Submit a signed beacon block to the beacon node to be broadcast and imported. After Deneb, this additionally instructs the beacon node to broadcast and import all given blobs. The beacon node performs the required validation.",
+    "description" : "Instructs the beacon node to broadcast a newly signed beacon block to the beacon network, to be included in the beacon chain. A success response (20x) indicates that the block passed gossip validation and was successfully broadcast onto the network. The beacon node is also expected to integrate the block into the state, but may broadcast it before doing so, so as to aid timely delivery of the block. Should the block fail full validation, a separate success response code (202) is used to indicate that the block was successfully broadcast but failed integration. After Deneb, this additionally instructs the beacon node to broadcast all given signed blobs.",
     "parameters" : [ {
       "name" : "Eth-Consensus-Version",
       "in" : "header",