From 29024106ecced8ee9c49a07810972b8afecb4021 Mon Sep 17 00:00:00 2001 From: Usman Saleem Date: Tue, 8 Oct 2024 19:00:58 +1000 Subject: [PATCH 01/19] fix: Validate rejected-tx cli options without ArgGroup (#95) Removed PicoCli ArgGroup validation as it didn't work in Besu with config file parsing which uses DefaultValueProvider. Instead, validate rejected tx cli options in the toDomainObject method. Enforce the rule that node type is required when endpoint is specified. If only node type is specified, we need no custom validation as we are only enabling reporting when endpoint is specified. --- .../LineaRejectedTxReportingCliOptions.java | 63 +++++++------------ ...ineaRejectedTxReportingCliOptionsTest.java | 40 ++++++------ 2 files changed, 42 insertions(+), 61 deletions(-) diff --git a/sequencer/src/main/java/net/consensys/linea/config/LineaRejectedTxReportingCliOptions.java b/sequencer/src/main/java/net/consensys/linea/config/LineaRejectedTxReportingCliOptions.java index 53b84762..cbdce8af 100644 --- a/sequencer/src/main/java/net/consensys/linea/config/LineaRejectedTxReportingCliOptions.java +++ b/sequencer/src/main/java/net/consensys/linea/config/LineaRejectedTxReportingCliOptions.java @@ -15,11 +15,9 @@ package net.consensys.linea.config; import java.net.URL; -import java.util.Optional; import com.google.common.base.MoreObjects; import net.consensys.linea.plugins.LineaCliOptions; -import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Option; /** The Linea Rejected Transaction Reporting CLI options. */ @@ -35,28 +33,21 @@ public class LineaRejectedTxReportingCliOptions implements LineaCliOptions { /** The Linea node type. */ public static final String LINEA_NODE_TYPE = "--plugin-linea-node-type"; - @ArgGroup(exclusive = false) - DependentOptions dependentOptions; // will be null if no options from this group are specified - - static class DependentOptions { - @Option( - names = {REJECTED_TX_ENDPOINT}, - hidden = true, - required = true, // required within the group - paramLabel = "", - description = - "Endpoint URI for reporting rejected transactions. Specify a valid URI to enable reporting.") - URL rejectedTxEndpoint = null; - - @Option( - names = {LINEA_NODE_TYPE}, - hidden = true, - required = true, // required within the group - paramLabel = "", - description = - "Linea Node type to use when reporting rejected transactions. (default: ${DEFAULT-VALUE}. Valid values: ${COMPLETION-CANDIDATES})") - LineaNodeType lineaNodeType = null; - } + @Option( + names = {REJECTED_TX_ENDPOINT}, + hidden = true, + paramLabel = "", + description = + "Endpoint URI for reporting rejected transactions. Specify a valid URI to enable reporting.") + URL rejectedTxEndpoint = null; + + @Option( + names = {LINEA_NODE_TYPE}, + hidden = true, + paramLabel = "", + description = + "Linea Node type to use when reporting rejected transactions. (Valid values: ${COMPLETION-CANDIDATES})") + LineaNodeType lineaNodeType = null; /** Default constructor. */ private LineaRejectedTxReportingCliOptions() {} @@ -78,23 +69,19 @@ public static LineaRejectedTxReportingCliOptions create() { public static LineaRejectedTxReportingCliOptions fromConfig( final LineaRejectedTxReportingConfiguration config) { final LineaRejectedTxReportingCliOptions options = create(); - // both options are required. - if (config.rejectedTxEndpoint() != null && config.lineaNodeType() != null) { - final var depOpts = new DependentOptions(); - depOpts.rejectedTxEndpoint = config.rejectedTxEndpoint(); - depOpts.lineaNodeType = config.lineaNodeType(); - options.dependentOptions = depOpts; - } - + options.rejectedTxEndpoint = config.rejectedTxEndpoint(); + options.lineaNodeType = config.lineaNodeType(); return options; } @Override public LineaRejectedTxReportingConfiguration toDomainObject() { - final var rejectedTxEndpoint = - Optional.ofNullable(dependentOptions).map(o -> o.rejectedTxEndpoint).orElse(null); - final var lineaNodeType = - Optional.ofNullable(dependentOptions).map(o -> o.lineaNodeType).orElse(null); + // perform validation here, if endpoint is specified then node type is required. + // We can ignore node type if endpoint is not specified. + if (rejectedTxEndpoint != null && lineaNodeType == null) { + throw new IllegalArgumentException( + "Error: Missing required argument(s): " + LINEA_NODE_TYPE + "="); + } return LineaRejectedTxReportingConfiguration.builder() .rejectedTxEndpoint(rejectedTxEndpoint) @@ -104,10 +91,6 @@ public LineaRejectedTxReportingConfiguration toDomainObject() { @Override public String toString() { - final var rejectedTxEndpoint = - Optional.ofNullable(dependentOptions).map(o -> o.rejectedTxEndpoint).orElse(null); - final var lineaNodeType = - Optional.ofNullable(dependentOptions).map(o -> o.lineaNodeType).orElse(null); return MoreObjects.toStringHelper(this) .add(REJECTED_TX_ENDPOINT, rejectedTxEndpoint) diff --git a/sequencer/src/test/java/net/consensys/linea/config/LineaRejectedTxReportingCliOptionsTest.java b/sequencer/src/test/java/net/consensys/linea/config/LineaRejectedTxReportingCliOptionsTest.java index 057cdf6d..12124f17 100644 --- a/sequencer/src/test/java/net/consensys/linea/config/LineaRejectedTxReportingCliOptionsTest.java +++ b/sequencer/src/test/java/net/consensys/linea/config/LineaRejectedTxReportingCliOptionsTest.java @@ -16,11 +16,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.assertThatNoException; import java.net.MalformedURLException; import java.net.URI; -import org.hyperledger.besu.plugin.services.PicoCLIOptions; import org.hyperledger.besu.services.PicoCLIOptionsImpl; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -40,18 +40,18 @@ static final class MockLineaBesuCommand { } private MockLineaBesuCommand command; - private LineaRejectedTxReportingCliOptions lineaRejectedTxReportingCliOptions; private CommandLine commandLine; - private PicoCLIOptions picoCliService; + private LineaRejectedTxReportingCliOptions txReportingCliOptions; @BeforeEach public void setup() { command = new MockLineaBesuCommand(); commandLine = new CommandLine(command); - picoCliService = new PicoCLIOptionsImpl(commandLine); - lineaRejectedTxReportingCliOptions = LineaRejectedTxReportingCliOptions.create(); - picoCliService.addPicoCLIOptions("linea", lineaRejectedTxReportingCliOptions); + // add mixin option before parseArgs is called + final PicoCLIOptionsImpl picoCliService = new PicoCLIOptionsImpl(commandLine); + txReportingCliOptions = LineaRejectedTxReportingCliOptions.create(); + picoCliService.addPicoCLIOptions("linea", txReportingCliOptions); } @Test @@ -59,7 +59,8 @@ void emptyLineaRejectedTxReportingCliOptions() { commandLine.parseArgs("--mock-option", "mockValue"); assertThat(command.mockOption).isEqualTo("mockValue"); - assertThat(lineaRejectedTxReportingCliOptions.dependentOptions).isNull(); + assertThat(txReportingCliOptions.rejectedTxEndpoint).isNull(); + assertThat(txReportingCliOptions.lineaNodeType).isNull(); } @ParameterizedTest @@ -72,30 +73,27 @@ void lineaRejectedTxOptionBothOptionsRequired(final LineaNodeType lineaNodeType) "--plugin-linea-node-type", lineaNodeType.name()); - assertThat(lineaRejectedTxReportingCliOptions.dependentOptions.rejectedTxEndpoint) + // parse args would not throw an exception, toDomainObject will perform the validation + assertThat(txReportingCliOptions.rejectedTxEndpoint) .isEqualTo(URI.create("http://localhost:8080").toURL()); - assertThat(lineaRejectedTxReportingCliOptions.dependentOptions.lineaNodeType) - .isEqualTo(lineaNodeType); + assertThat(txReportingCliOptions.lineaNodeType).isEqualTo(lineaNodeType); + assertThatNoException().isThrownBy(() -> txReportingCliOptions.toDomainObject()); } @Test void lineaRejectedTxReportingCliOptionsOnlyEndpointCauseException() { - assertThatExceptionOfType(CommandLine.ParameterException.class) - .isThrownBy( - () -> - commandLine.parseArgs( - "--plugin-linea-rejected-tx-endpoint", "http://localhost:8080")) + commandLine.parseArgs("--plugin-linea-rejected-tx-endpoint", "http://localhost:8080"); + + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> txReportingCliOptions.toDomainObject()) .withMessageContaining( "Error: Missing required argument(s): --plugin-linea-node-type="); } @Test - void lineaRejectedTxReportingCliOptionsOnlyNodeTypeCauseException() { - assertThatExceptionOfType(CommandLine.ParameterException.class) - .isThrownBy( - () -> commandLine.parseArgs("--plugin-linea-node-type", LineaNodeType.SEQUENCER.name())) - .withMessageContaining( - "Error: Missing required argument(s): --plugin-linea-rejected-tx-endpoint="); + void lineaRejectedTxReportingCliOptionsOnlyNodeTypeParsesWithoutProblem() { + commandLine.parseArgs("--plugin-linea-node-type", LineaNodeType.SEQUENCER.name()); + assertThatNoException().isThrownBy(() -> txReportingCliOptions.toDomainObject()); } @Test From 02a2f6bce75d9cd64e95334c37ad5374bef68784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20B=C3=A9gassat?= <38285177+OlivierBBB@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:44:47 +0200 Subject: [PATCH 02/19] Provide `chainId` to `createZkTracer(...)` (#93) * fix: provide chainId to createZkTracer(...) * fix: bumped arithmetization version to 0.8.0-rc2 * Make sure chainId is present and positive Signed-off-by: Fabio Di Fabio * Set arithmetizationVersion=0.8.0-rc4 Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio Co-authored-by: Fabio Di Fabio --- gradle.properties | 4 +-- .../linea/AbstractLineaRequiredPlugin.java | 29 ++++++++++++++++++- .../linea/extradata/LineaExtraDataPlugin.java | 9 ------ .../linea/rpc/methods/LineaEstimateGas.java | 6 ++-- .../LineaEstimateGasEndpointPlugin.java | 10 ------- .../LineaTransactionPoolValidatorPlugin.java | 11 +------ .../validators/SimulationValidator.java | 7 +++-- .../LineaTransactionSelectorPlugin.java | 11 +------ .../selectors/LineaTransactionSelector.java | 6 +++- .../TraceLineLimitTransactionSelector.java | 6 +++- .../validators/SimulationValidatorTest.java | 1 + ...TraceLineLimitTransactionSelectorTest.java | 2 ++ 12 files changed, 52 insertions(+), 50 deletions(-) diff --git a/gradle.properties b/gradle.properties index f0e3c8c0..3b1a8781 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ -releaseVersion=0.8.0-rc1.1 +releaseVersion=0.8.0-rc4.1 besuVersion=24.10-delivery34 -arithmetizationVersion=0.8.0-rc1 +arithmetizationVersion=0.8.0-rc4 besuArtifactGroup=io.consensys.linea-besu distributionIdentifier=linea-sequencer distributionBaseUrl=https://artifacts.consensys.net/public/linea-besu/raw/names/linea-besu.tar.gz/versions/ \ No newline at end of file diff --git a/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java b/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java index c3b51351..152dd326 100644 --- a/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java @@ -18,9 +18,11 @@ import lombok.extern.slf4j.Slf4j; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; +import org.hyperledger.besu.plugin.services.BlockchainService; @Slf4j public abstract class AbstractLineaRequiredPlugin extends AbstractLineaPrivateOptionsPlugin { + protected BlockchainService blockchainService; /** * Linea plugins extending this class will halt startup of Besu in case of exception during @@ -34,7 +36,15 @@ public abstract class AbstractLineaRequiredPlugin extends AbstractLineaPrivateOp public void register(final BesuContext context) { super.register(context); try { - log.info("Registering Linea plugin " + this.getClass().getName()); + log.info("Registering Linea plugin {}", this.getClass().getName()); + + blockchainService = + context + .getService(BlockchainService.class) + .orElseThrow( + () -> + new RuntimeException( + "Failed to obtain BlockchainService from the BesuContext.")); doRegister(context); @@ -52,4 +62,21 @@ public void register(final BesuContext context) { * @param context */ public abstract void doRegister(final BesuContext context); + + @Override + public void beforeExternalServices() { + super.beforeExternalServices(); + + blockchainService + .getChainId() + .ifPresentOrElse( + chainId -> { + if (chainId.signum() <= 0) { + throw new IllegalArgumentException("Chain id must be greater than zero."); + } + }, + () -> { + throw new IllegalArgumentException("Chain id required"); + }); + } } diff --git a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java index 7fe0aa98..ed100ec3 100644 --- a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java @@ -23,7 +23,6 @@ import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; import org.hyperledger.besu.plugin.services.BesuEvents; -import org.hyperledger.besu.plugin.services.BlockchainService; import org.hyperledger.besu.plugin.services.RpcEndpointService; /** This plugin registers handlers that are activated when new blocks are imported */ @@ -33,7 +32,6 @@ public class LineaExtraDataPlugin extends AbstractLineaRequiredPlugin { public static final String NAME = "linea"; private BesuContext besuContext; private RpcEndpointService rpcEndpointService; - private BlockchainService blockchainService; @Override public Optional getName() { @@ -50,13 +48,6 @@ public void doRegister(final BesuContext context) { () -> new RuntimeException( "Failed to obtain RpcEndpointService from the BesuContext.")); - blockchainService = - context - .getService(BlockchainService.class) - .orElseThrow( - () -> - new RuntimeException( - "Failed to obtain BlockchainService from the BesuContext.")); } /** diff --git a/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java b/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java index 34d25066..9e81b72e 100644 --- a/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java +++ b/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java @@ -270,7 +270,7 @@ private Long estimateGasUsed( final var estimateGasTracer = new EstimateGasOperationTracer(); final var chainHeadHeader = blockchainService.getChainHeadHeader(); - final var zkTracer = createZkTracer(chainHeadHeader); + final var zkTracer = createZkTracer(chainHeadHeader, blockchainService.getChainId().get()); final TracerAggregator zkAndGasTracer = TracerAggregator.create(estimateGasTracer, zkTracer); final var chainHeadHash = chainHeadHeader.getBlockHash(); @@ -491,8 +491,8 @@ private Transaction createTransactionForSimulation( return txBuilder.build(); } - private ZkTracer createZkTracer(final BlockHeader chainHeadHeader) { - var zkTracer = new ZkTracer(l1L2BridgeConfiguration); + private ZkTracer createZkTracer(final BlockHeader chainHeadHeader, final BigInteger chainId) { + var zkTracer = new ZkTracer(l1L2BridgeConfiguration, chainId); zkTracer.traceStartConflation(1L); zkTracer.traceStartBlock(chainHeadHeader); return zkTracer; diff --git a/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaEstimateGasEndpointPlugin.java b/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaEstimateGasEndpointPlugin.java index 0ac2f2ed..6d80e523 100644 --- a/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaEstimateGasEndpointPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaEstimateGasEndpointPlugin.java @@ -24,7 +24,6 @@ import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; import org.hyperledger.besu.plugin.services.BesuConfiguration; -import org.hyperledger.besu.plugin.services.BlockchainService; import org.hyperledger.besu.plugin.services.RpcEndpointService; import org.hyperledger.besu.plugin.services.TransactionSimulationService; @@ -35,7 +34,6 @@ public class LineaEstimateGasEndpointPlugin extends AbstractLineaRequiredPlugin private BesuConfiguration besuConfiguration; private RpcEndpointService rpcEndpointService; private TransactionSimulationService transactionSimulationService; - private BlockchainService blockchainService; private LineaEstimateGas lineaEstimateGasMethod; /** @@ -69,14 +67,6 @@ public void doRegister(final BesuContext context) { new RuntimeException( "Failed to obtain TransactionSimulatorService from the BesuContext.")); - blockchainService = - context - .getService(BlockchainService.class) - .orElseThrow( - () -> - new RuntimeException( - "Failed to obtain BlockchainService from the BesuContext.")); - lineaEstimateGasMethod = new LineaEstimateGas( besuConfiguration, transactionSimulationService, blockchainService, rpcEndpointService); diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java index 0db9229f..98c45653 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java @@ -34,7 +34,6 @@ import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; import org.hyperledger.besu.plugin.services.BesuConfiguration; -import org.hyperledger.besu.plugin.services.BlockchainService; import org.hyperledger.besu.plugin.services.TransactionPoolValidatorService; import org.hyperledger.besu.plugin.services.TransactionSimulationService; @@ -49,7 +48,6 @@ public class LineaTransactionPoolValidatorPlugin extends AbstractLineaRequiredPlugin { public static final String NAME = "linea"; private BesuConfiguration besuConfiguration; - private BlockchainService blockchainService; private TransactionPoolValidatorService transactionPoolValidatorService; private TransactionSimulationService transactionSimulationService; private Optional rejectedTxJsonRpcManager = Optional.empty(); @@ -69,14 +67,6 @@ public void doRegister(final BesuContext context) { new RuntimeException( "Failed to obtain BesuConfiguration from the BesuContext.")); - blockchainService = - context - .getService(BlockchainService.class) - .orElseThrow( - () -> - new RuntimeException( - "Failed to obtain BlockchainService from the BesuContext.")); - transactionPoolValidatorService = context .getService(TransactionPoolValidatorService.class) @@ -97,6 +87,7 @@ public void doRegister(final BesuContext context) { @Override public void start() { super.start(); + try (Stream lines = Files.lines( Path.of(new File(transactionPoolValidatorConfiguration().denyListPath()).toURI()))) { diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java index c2abbf49..86530ab0 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java @@ -17,6 +17,7 @@ import static net.consensys.linea.sequencer.modulelimit.ModuleLineCountValidator.ModuleLineCountResult.MODULE_NOT_DEFINED; import static net.consensys.linea.sequencer.modulelimit.ModuleLineCountValidator.ModuleLineCountResult.TX_MODULE_LINE_COUNT_OVERFLOW; +import java.math.BigInteger; import java.time.Instant; import java.util.List; import java.util.Map; @@ -86,7 +87,7 @@ public Optional validateTransaction( new ModuleLineCountValidator(moduleLineLimitsMap); final var chainHeadHeader = blockchainService.getChainHeadHeader(); - final var zkTracer = createZkTracer(chainHeadHeader); + final var zkTracer = createZkTracer(chainHeadHeader, blockchainService.getChainId().get()); final var maybeSimulationResults = transactionSimulationService.simulate( transaction, chainHeadHeader.getBlockHash(), zkTracer, true); @@ -170,8 +171,8 @@ private void logSimulationResult( .log(); } - private ZkTracer createZkTracer(final BlockHeader chainHeadHeader) { - var zkTracer = new ZkTracer(l1L2BridgeConfiguration); + private ZkTracer createZkTracer(final BlockHeader chainHeadHeader, BigInteger chainId) { + var zkTracer = new ZkTracer(l1L2BridgeConfiguration, chainId); zkTracer.traceStartConflation(1L); zkTracer.traceStartBlock(chainHeadHeader); return zkTracer; diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java index 021758ef..a84754e7 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java @@ -28,7 +28,6 @@ import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; import org.hyperledger.besu.plugin.services.BesuConfiguration; -import org.hyperledger.besu.plugin.services.BlockchainService; import org.hyperledger.besu.plugin.services.TransactionSelectionService; /** @@ -41,7 +40,6 @@ public class LineaTransactionSelectorPlugin extends AbstractLineaRequiredPlugin { public static final String NAME = "linea"; private TransactionSelectionService transactionSelectionService; - private BlockchainService blockchainService; private Optional rejectedTxJsonRpcManager = Optional.empty(); private BesuConfiguration besuConfiguration; @@ -60,14 +58,6 @@ public void doRegister(final BesuContext context) { new RuntimeException( "Failed to obtain TransactionSelectionService from the BesuContext.")); - blockchainService = - context - .getService(BlockchainService.class) - .orElseThrow( - () -> - new RuntimeException( - "Failed to obtain BlockchainService from the BesuContext.")); - besuConfiguration = context .getService(BesuConfiguration.class) @@ -80,6 +70,7 @@ public void doRegister(final BesuContext context) { @Override public void start() { super.start(); + final LineaTransactionSelectorConfiguration txSelectorConfiguration = transactionSelectorConfiguration(); final LineaRejectedTxReportingConfiguration lineaRejectedTxReportingConfiguration = diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java index 5cc75976..a396b83f 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java @@ -80,7 +80,11 @@ private List createTransactionSelectors( traceLineLimitTransactionSelector = new TraceLineLimitTransactionSelector( - limitsMap, txSelectorConfiguration, l1L2BridgeConfiguration, tracerConfiguration); + blockchainService.getChainId().get(), + limitsMap, + txSelectorConfiguration, + l1L2BridgeConfiguration, + tracerConfiguration); return List.of( new MaxBlockCallDataTransactionSelector(txSelectorConfiguration.maxBlockCallDataSize()), diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelector.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelector.java index 79c857e9..ed7f0675 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelector.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelector.java @@ -19,6 +19,7 @@ import static net.consensys.linea.sequencer.txselection.LineaTransactionSelectionResult.TX_MODULE_LINE_COUNT_OVERFLOW_CACHED; import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.SELECTED; +import java.math.BigInteger; import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; @@ -56,6 +57,7 @@ public class TraceLineLimitTransactionSelector implements PluginTransactionSelec private static final Marker BLOCK_LINE_COUNT_MARKER = MarkerFactory.getMarker("BLOCK_LINE_COUNT"); @VisibleForTesting protected static Set overLineCountLimitCache = new LinkedHashSet<>(); private final ZkTracer zkTracer; + private final BigInteger chainId; private final String limitFilePath; private final Map moduleLimits; private final int overLimitCacheSize; @@ -63,6 +65,7 @@ public class TraceLineLimitTransactionSelector implements PluginTransactionSelec private Map currCumulatedLineCount; public TraceLineLimitTransactionSelector( + final BigInteger chainId, final Map moduleLimits, final LineaTransactionSelectorConfiguration txSelectorConfiguration, final LineaL1L2BridgeSharedConfiguration l1L2BridgeConfiguration, @@ -72,6 +75,7 @@ public TraceLineLimitTransactionSelector( System.exit(1); } + this.chainId = chainId; this.moduleLimits = moduleLimits; this.limitFilePath = tracerConfiguration.moduleLimitsFilePath(); this.overLimitCacheSize = txSelectorConfiguration.overLinesLimitCacheSize(); @@ -217,7 +221,7 @@ private String logTxLineCount() { private class ZkTracerWithLog extends ZkTracer { public ZkTracerWithLog(final LineaL1L2BridgeSharedConfiguration bridgeConfiguration) { - super(bridgeConfiguration); + super(bridgeConfiguration, chainId); } @Override diff --git a/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidatorTest.java b/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidatorTest.java index cd1e9ed1..d8f39597 100644 --- a/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidatorTest.java +++ b/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidatorTest.java @@ -131,6 +131,7 @@ public void initialize(final WireMockRuntimeInfo wmInfo) throws MalformedURLExce final var blockHeader = mock(BlockHeader.class); when(blockHeader.getBaseFee()).thenReturn(Optional.of(BASE_FEE)); when(blockchainService.getChainHeadHeader()).thenReturn(blockHeader); + when(blockchainService.getChainId()).thenReturn(Optional.of(BigInteger.ONE)); final var rejectedTxReportingConf = LineaRejectedTxReportingConfiguration.builder() diff --git a/sequencer/src/test/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelectorTest.java b/sequencer/src/test/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelectorTest.java index 103eea16..37ae28a6 100644 --- a/sequencer/src/test/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelectorTest.java +++ b/sequencer/src/test/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelectorTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; +import java.math.BigInteger; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; @@ -250,6 +251,7 @@ private class TestableTraceLineLimitTransactionSelector final Map moduleLimits, final int overLimitCacheSize) { super( + BigInteger.ONE, moduleLimits, LineaTransactionSelectorConfiguration.builder() .overLinesLimitCacheSize(overLimitCacheSize) From 6812549983c5bf8950cb4de801eafa462a0450e9 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 8 Oct 2024 18:11:43 +0200 Subject: [PATCH 03/19] Fix for required chainId check (#96) Signed-off-by: Fabio Di Fabio --- .../java/net/consensys/linea/AbstractLineaRequiredPlugin.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java b/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java index 152dd326..5f4f1d6b 100644 --- a/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java @@ -64,8 +64,8 @@ public void register(final BesuContext context) { public abstract void doRegister(final BesuContext context); @Override - public void beforeExternalServices() { - super.beforeExternalServices(); + public void start() { + super.start(); blockchainService .getChainId() From 8357435f0b10df79e02de33e1368037b40007ea5 Mon Sep 17 00:00:00 2001 From: Usman Saleem Date: Wed, 9 Oct 2024 10:10:01 +1000 Subject: [PATCH 04/19] docs: Update plugins doc with reporting rejected tx cli options (#97) Update plugins doc with reporting rejected transaction cli options --- docs/plugins.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/plugins.md b/docs/plugins.md index 20ecb2bb..e74e85a3 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -70,7 +70,7 @@ The selectors are in the package `net.consensys.linea.sequencer.txselection.sele | `--plugin-linea-unprofitable-retry-limit` | 10 | -### Transaction validation - LineaTransactionValidatorPlugin +### Transaction validation - LineaTransactionPoolValidatorPlugin This plugin extends the default transaction validation rules for adding transactions to the transaction pool. It leverages the `PluginTransactionValidatorService` to manage and customize the @@ -91,6 +91,14 @@ The validators are in the package `net.consensys.linea.sequencer.txpoolvalidatio | `--plugin-linea-tx-pool-profitability-check-api-enabled` | true | | `--plugin-linea-tx-pool-profitability-check-p2p-enabled` | false | +### Reporting rejected transactions +The transaction selection and validation plugins can report rejected transactions as JSON-RPC calls to an external +service. This feature can be enabled by setting the following CLI options: + +| Command Line Argument | Default Value | Expected Values | +|---------------------------------------|---------------|--------------------------------------------------------------| +| `--plugin-linea-rejected-tx-endpoint` | `null` | A valid URL e.g. `http://localhost:9363` to enable reporting | +| `--plugin-linea-node-type` | `null` | One of `SEQUENCER`, `RPC`, `P2P` | ## RPC methods From eca028652f108c0cc63a34ec56ce051c55ca2f54 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Thu, 17 Oct 2024 03:08:14 +1000 Subject: [PATCH 05/19] added missing javadoc comments (#98) Signed-off-by: Sally MacFarlane --- .../java/net/consensys/linea/AbstractLineaRequiredPlugin.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java b/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java index 5f4f1d6b..24085307 100644 --- a/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java @@ -30,7 +30,7 @@ public abstract class AbstractLineaRequiredPlugin extends AbstractLineaPrivateOp * *

If that's NOT desired, the plugin should implement {@link BesuPlugin} directly. * - * @param context + * @param context the BesuContext to be used. */ @Override public void register(final BesuContext context) { @@ -59,7 +59,7 @@ public void register(final BesuContext context) { /** * Linea plugins need to implement this method. Called by {@link BesuPlugin} register method * - * @param context + * @param context the BesuContext to be used. */ public abstract void doRegister(final BesuContext context); From 4357e54acfaee4bf8df4377b8463a242d877bcfc Mon Sep 17 00:00:00 2001 From: Ade Lucas <37906710+cloudspores@users.noreply.github.com> Date: Wed, 23 Oct 2024 08:52:45 -0400 Subject: [PATCH 06/19] Rename minGasPrice to ethGasPrice from the extraData and track its value (#103) * Rename minGasPrice to ethGasPrice from the extraData. Signed-off-by: Ade Lucas --- .../config/LineaProfitabilityConfiguration.java | 12 ++++++++++-- .../linea/extradata/LineaExtraDataHandler.java | 17 ++++++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityConfiguration.java b/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityConfiguration.java index bd7a3908..6aa5af24 100644 --- a/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityConfiguration.java +++ b/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityConfiguration.java @@ -33,6 +33,8 @@ public class LineaProfitabilityConfiguration implements LineaOptionsConfiguratio /** It is safe to keep this as long, since it will store value <= max_int * 1000 */ private long variableCostWei; + private long ethGasPriceWei; + private double minMargin; private double estimateGasMinMargin; private double txPoolMinMargin; @@ -46,11 +48,13 @@ public class LineaProfitabilityConfiguration implements LineaOptionsConfiguratio * * @param fixedCostWei fixed cost in Wei * @param variableCostWei variable cost in Wei + * @param ethGasPriceWei gas price in Wei */ - public synchronized void updateFixedAndVariableCost( - final long fixedCostWei, final long variableCostWei) { + public synchronized void updateFixedVariableAndGasPrice( + final long fixedCostWei, final long variableCostWei, final long ethGasPriceWei) { this.fixedCostWei = fixedCostWei; this.variableCostWei = variableCostWei; + this.ethGasPriceWei = ethGasPriceWei; } public synchronized long fixedCostWei() { @@ -60,4 +64,8 @@ public synchronized long fixedCostWei() { public synchronized long variableCostWei() { return variableCostWei; } + + public synchronized long ethGasPriceWei() { + return ethGasPriceWei; + } } diff --git a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataHandler.java b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataHandler.java index 6e875515..6ea5d830 100644 --- a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataHandler.java +++ b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataHandler.java @@ -103,6 +103,7 @@ private class Version1Consumer implements ExtraDataConsumer { private final FieldConsumer[] fieldsSequence; private final MutableLong currFixedCostKWei = new MutableLong(); private final MutableLong currVariableCostKWei = new MutableLong(); + private final MutableLong currEthGasPriceKWei = new MutableLong(); public Version1Consumer(final LineaProfitabilityConfiguration profitabilityConf) { this.profitabilityConf = profitabilityConf; @@ -113,11 +114,11 @@ public Version1Consumer(final LineaProfitabilityConfiguration profitabilityConf) final FieldConsumer variableGasCostField = new FieldConsumer<>( "variableGasCost", 4, ExtraDataConsumer::toLong, currVariableCostKWei::setValue); - final FieldConsumer minGasPriceField = - new FieldConsumer<>("minGasPrice", 4, ExtraDataConsumer::toLong, this::updateMinGasPrice); + final FieldConsumer ethGasPriceField = + new FieldConsumer<>("ethGasPrice", 4, ExtraDataConsumer::toLong, this::updateEthGasPrice); this.fieldsSequence = - new FieldConsumer[] {fixedGasCostField, variableGasCostField, minGasPriceField}; + new FieldConsumer[] {fixedGasCostField, variableGasCostField, ethGasPriceField}; } public boolean canConsume(final Bytes rawExtraData) { @@ -132,14 +133,16 @@ public synchronized void accept(final Bytes extraData) { startIndex += fieldConsumer.length; } - profitabilityConf.updateFixedAndVariableCost( + profitabilityConf.updateFixedVariableAndGasPrice( currFixedCostKWei.longValue() * WEI_IN_KWEI, - currVariableCostKWei.longValue() * WEI_IN_KWEI); + currVariableCostKWei.longValue() * WEI_IN_KWEI, + currEthGasPriceKWei.longValue() * WEI_IN_KWEI); } - void updateMinGasPrice(final Long minGasPriceKWei) { + void updateEthGasPrice(final Long ethGasPriceKWei) { + currEthGasPriceKWei.setValue(ethGasPriceKWei); if (profitabilityConf.extraDataSetMinGasPriceEnabled()) { - final var minGasPriceWei = Wei.of(minGasPriceKWei).multiply(WEI_IN_KWEI); + final var minGasPriceWei = Wei.of(ethGasPriceKWei).multiply(WEI_IN_KWEI); final var resp = rpcEndpointService.call( "miner_setMinGasPrice", new Object[] {minGasPriceWei.toShortHexString()}); From dfd0593ea90af3634f8fd03a6bcf9c7998504848 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 24 Oct 2024 14:22:35 +0200 Subject: [PATCH 07/19] Expose metrics for profitability extra data field (#104) Signed-off-by: Fabio Di Fabio --- .../plugin/acc/test/LineaPluginTestBase.java | 44 +++++++++++ .../test/extradata/ExtraDataPricingTest.java | 26 +++++- gradle.properties | 4 +- .../linea/AbstractLineaRequiredPlugin.java | 29 +------ ...tractLineaSharedPrivateOptionsPlugin.java} | 79 ++++++++++++++++++- .../linea/extradata/LineaExtraDataPlugin.java | 16 ++++ .../linea/metrics/LineaMetricCategory.java | 41 ++++++++++ 7 files changed, 206 insertions(+), 33 deletions(-) rename sequencer/src/main/java/net/consensys/linea/{AbstractLineaPrivateOptionsPlugin.java => AbstractLineaSharedPrivateOptionsPlugin.java} (63%) create mode 100644 sequencer/src/main/java/net/consensys/linea/metrics/LineaMetricCategory.java diff --git a/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java b/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java index c5d08d7a..ff777ceb 100644 --- a/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java +++ b/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java @@ -19,6 +19,10 @@ import java.io.IOException; import java.math.BigInteger; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -34,6 +38,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import linea.plugin.acc.test.tests.web3j.generated.SimpleStorage; +import lombok.extern.slf4j.Slf4j; +import net.consensys.linea.metrics.LineaMetricCategory; import org.apache.commons.lang3.RandomStringUtils; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; @@ -42,6 +48,8 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; +import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration; +import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase; import org.hyperledger.besu.tests.acceptance.dsl.account.Accounts; import org.hyperledger.besu.tests.acceptance.dsl.condition.txpool.TxPoolConditions; @@ -66,6 +74,7 @@ import org.web3j.tx.response.TransactionReceiptProcessor; /** Base class for plugin tests. */ +@Slf4j public class LineaPluginTestBase extends AcceptanceTestBase { public static final int MAX_CALLDATA_SIZE = 1188; // contract has a call data size of 1160 public static final int MAX_TX_GAS_LIMIT = DefaultGasProvider.GAS_LIMIT.intValue(); @@ -73,6 +82,7 @@ public class LineaPluginTestBase extends AcceptanceTestBase { public static final CliqueOptions LINEA_CLIQUE_OPTIONS = new CliqueOptions( CliqueOptions.DEFAULT.blockPeriodSeconds(), CliqueOptions.DEFAULT.epochLength(), false); + protected static final HttpClient HTTP_CLIENT = HttpClient.newHttpClient(); protected BesuNode minerNode; @BeforeEach @@ -122,6 +132,11 @@ private BesuNode createCliqueNodeWithExtraCliOptionsAndRpcApis( .genesisConfigProvider( validators -> Optional.of(provideGenesisConfig(validators, cliqueOptions))) .extraCLIOptions(extraCliOptions) + .metricsConfiguration( + MetricsConfiguration.builder() + .enabled(true) + .metricCategories(Set.of(LineaMetricCategory.PROFITABILITY)) + .build()) .requestedPlugins( List.of( "LineaExtraDataPlugin", @@ -306,4 +321,33 @@ protected Bytes32 createExtraDataPricingField( return Bytes32.rightPad( Bytes.concatenate(Bytes.of((byte) 1), fixed.toBytes(), variable.toBytes(), min.toBytes())); } + + protected double getMetricValue( + final MetricCategory category, + final String metricName, + final List> labelValues) + throws IOException, InterruptedException { + + final var metricsReq = + HttpRequest.newBuilder().GET().uri(URI.create("http://127.0.0.1:9545/metrics")).build(); + + final var respLines = HTTP_CLIENT.send(metricsReq, HttpResponse.BodyHandlers.ofLines()); + + final var searchString = + category.getApplicationPrefix().orElse("") + + category.getName() + + "_" + + metricName + + labelValues.stream() + .map(lv -> lv.getKey() + "=\"" + lv.getValue() + "\"") + .collect(Collectors.joining(",", "{", ",}")); + + final var foundMetric = + respLines.body().filter(line -> line.startsWith(searchString)).findFirst(); + + return foundMetric + .map(line -> line.substring(searchString.length()).trim()) + .map(Double::valueOf) + .orElse(Double.NaN); + } } diff --git a/acceptance-tests/src/test/java/linea/plugin/acc/test/extradata/ExtraDataPricingTest.java b/acceptance-tests/src/test/java/linea/plugin/acc/test/extradata/ExtraDataPricingTest.java index b326675e..c6cab90a 100644 --- a/acceptance-tests/src/test/java/linea/plugin/acc/test/extradata/ExtraDataPricingTest.java +++ b/acceptance-tests/src/test/java/linea/plugin/acc/test/extradata/ExtraDataPricingTest.java @@ -14,6 +14,7 @@ */ package linea.plugin.acc.test.extradata; +import static java.util.Map.entry; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; @@ -22,6 +23,7 @@ import linea.plugin.acc.test.LineaPluginTestBase; import linea.plugin.acc.test.TestCommandLineOptionsBuilder; +import net.consensys.linea.metrics.LineaMetricCategory; import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.tests.acceptance.dsl.account.Account; @@ -78,7 +80,7 @@ public void updateMinGasPriceViaExtraData() { } @Test - public void updateProfitabilityParamsViaExtraData() throws IOException { + public void updateProfitabilityParamsViaExtraData() throws IOException, InterruptedException { final Web3j web3j = minerNode.nodeRequests().eth(); final Account sender = accounts.getSecondaryBenefactor(); final Account recipient = accounts.createAccount("recipient"); @@ -121,6 +123,28 @@ public void updateProfitabilityParamsViaExtraData() throws IOException { assertThat(signedUnprofitableTxResp.getError().getMessage()).isEqualTo("Gas price too low"); assertThat(getTxPoolContent()).isEmpty(); + + final var fixedCostMetric = + getMetricValue( + LineaMetricCategory.PROFITABILITY, "conf", List.of(entry("field", "fixed_cost_wei"))); + + assertThat(fixedCostMetric).isEqualTo(MIN_GAS_PRICE.multiply(2).getValue().doubleValue()); + + final var variableCostMetric = + getMetricValue( + LineaMetricCategory.PROFITABILITY, + "conf", + List.of(entry("field", "variable_cost_wei"))); + + assertThat(variableCostMetric).isEqualTo(MIN_GAS_PRICE.getValue().doubleValue()); + + final var ethGasPriceMetric = + getMetricValue( + LineaMetricCategory.PROFITABILITY, + "conf", + List.of(entry("field", "eth_gas_price_wei"))); + + assertThat(ethGasPriceMetric).isEqualTo(MIN_GAS_PRICE.getValue().doubleValue()); } static class MinerSetExtraDataRequest implements Transaction { diff --git a/gradle.properties b/gradle.properties index 3b1a8781..9b4da4f6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ -releaseVersion=0.8.0-rc4.1 -besuVersion=24.10-delivery34 +releaseVersion=0.8.0-rc4.1-local +besuVersion=24.10-develop-829db23 arithmetizationVersion=0.8.0-rc4 besuArtifactGroup=io.consensys.linea-besu distributionIdentifier=linea-sequencer diff --git a/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java b/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java index 24085307..427796a1 100644 --- a/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java @@ -18,11 +18,9 @@ import lombok.extern.slf4j.Slf4j; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; -import org.hyperledger.besu.plugin.services.BlockchainService; @Slf4j -public abstract class AbstractLineaRequiredPlugin extends AbstractLineaPrivateOptionsPlugin { - protected BlockchainService blockchainService; +public abstract class AbstractLineaRequiredPlugin extends AbstractLineaSharedPrivateOptionsPlugin { /** * Linea plugins extending this class will halt startup of Besu in case of exception during @@ -38,14 +36,6 @@ public void register(final BesuContext context) { try { log.info("Registering Linea plugin {}", this.getClass().getName()); - blockchainService = - context - .getService(BlockchainService.class) - .orElseThrow( - () -> - new RuntimeException( - "Failed to obtain BlockchainService from the BesuContext.")); - doRegister(context); } catch (Exception e) { @@ -62,21 +52,4 @@ public void register(final BesuContext context) { * @param context the BesuContext to be used. */ public abstract void doRegister(final BesuContext context); - - @Override - public void start() { - super.start(); - - blockchainService - .getChainId() - .ifPresentOrElse( - chainId -> { - if (chainId.signum() <= 0) { - throw new IllegalArgumentException("Chain id must be greater than zero."); - } - }, - () -> { - throw new IllegalArgumentException("Chain id required"); - }); - } } diff --git a/sequencer/src/main/java/net/consensys/linea/AbstractLineaPrivateOptionsPlugin.java b/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java similarity index 63% rename from sequencer/src/main/java/net/consensys/linea/AbstractLineaPrivateOptionsPlugin.java rename to sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java index 569e6281..2473d6b8 100644 --- a/sequencer/src/main/java/net/consensys/linea/AbstractLineaPrivateOptionsPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.compress.LibCompress; @@ -32,21 +33,35 @@ import net.consensys.linea.config.LineaTransactionPoolValidatorConfiguration; import net.consensys.linea.config.LineaTransactionSelectorCliOptions; import net.consensys.linea.config.LineaTransactionSelectorConfiguration; +import net.consensys.linea.metrics.LineaMetricCategory; import net.consensys.linea.plugins.AbstractLineaSharedOptionsPlugin; import net.consensys.linea.plugins.LineaOptionsPluginConfiguration; +import org.hyperledger.besu.plugin.BesuContext; +import org.hyperledger.besu.plugin.services.BlockchainService; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.MetricCategoryRegistry; /** * This abstract class is used as superclass for all the plugins that share one or more - * configuration options. + * configuration options, services and common initializations. * *

Configuration options that are exclusive of a single plugin, are not required to be added * here, but they could stay in the class that implement a plugin, but in case that configuration * becomes to be used by multiple plugins, then to avoid code duplications and possible different * management of the options, it is better to move the configuration here so all plugins will * automatically see it. + * + *

Same for services and other initialization tasks, that are shared by more than one plugin, + * like registration of metrics categories or check to perform once at startup */ @Slf4j -public abstract class AbstractLineaPrivateOptionsPlugin extends AbstractLineaSharedOptionsPlugin { +public abstract class AbstractLineaSharedPrivateOptionsPlugin + extends AbstractLineaSharedOptionsPlugin { + protected static BlockchainService blockchainService; + protected static MetricsSystem metricsSystem; + + private static AtomicBoolean sharedRegisterTasksDone = new AtomicBoolean(false); + private static AtomicBoolean sharedStartTasksDone = new AtomicBoolean(false); static { // force the initialization of the gnark compress native library to fail fast in case of issues @@ -106,8 +121,68 @@ public LineaRejectedTxReportingConfiguration rejectedTxReportingConfiguration() getConfigurationByKey(LineaRejectedTxReportingCliOptions.CONFIG_KEY).optionsConfig(); } + @Override + public synchronized void register(final BesuContext context) { + super.register(context); + + if (sharedRegisterTasksDone.compareAndSet(false, true)) { + performSharedRegisterTasksOnce(context); + } + } + + protected void performSharedRegisterTasksOnce(final BesuContext context) { + blockchainService = + context + .getService(BlockchainService.class) + .orElseThrow( + () -> + new RuntimeException( + "Failed to obtain BlockchainService from the BesuContext.")); + + metricsSystem = + context + .getService(MetricsSystem.class) + .orElseThrow( + () -> new RuntimeException("Failed to obtain MetricSystem from the BesuContext.")); + + context + .getService(MetricCategoryRegistry.class) + .orElseThrow( + () -> + new RuntimeException( + "Failed to obtain MetricCategoryRegistry from the BesuContext.")) + .addMetricCategory(LineaMetricCategory.PROFITABILITY); + } + @Override public void start() { super.start(); + + if (sharedStartTasksDone.compareAndSet(false, true)) { + performSharedStartTasksOnce(); + } + } + + private static void performSharedStartTasksOnce() { + blockchainService + .getChainId() + .ifPresentOrElse( + chainId -> { + if (chainId.signum() <= 0) { + throw new IllegalArgumentException("Chain id must be greater than zero."); + } + }, + () -> { + throw new IllegalArgumentException("Chain id required"); + }); + } + + @Override + public void stop() { + super.stop(); + sharedRegisterTasksDone.set(false); + sharedStartTasksDone.set(false); + blockchainService = null; + metricsSystem = null; } } diff --git a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java index ed100ec3..504a7bf8 100644 --- a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java @@ -20,6 +20,8 @@ import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.AbstractLineaRequiredPlugin; +import net.consensys.linea.config.LineaProfitabilityConfiguration; +import net.consensys.linea.metrics.LineaMetricCategory; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; import org.hyperledger.besu.plugin.services.BesuEvents; @@ -97,5 +99,19 @@ public void start() { } }); } + + initMetrics(profitabilityConfiguration()); + } + + private void initMetrics(final LineaProfitabilityConfiguration lineaProfitabilityConfiguration) { + final var confLabelledGauge = + metricsSystem.createLabelledGauge( + LineaMetricCategory.PROFITABILITY, + "conf", + "Profitability configuration values at runtime", + "field"); + confLabelledGauge.labels(lineaProfitabilityConfiguration::fixedCostWei, "fixed_cost_wei"); + confLabelledGauge.labels(lineaProfitabilityConfiguration::variableCostWei, "variable_cost_wei"); + confLabelledGauge.labels(lineaProfitabilityConfiguration::ethGasPriceWei, "eth_gas_price_wei"); } } diff --git a/sequencer/src/main/java/net/consensys/linea/metrics/LineaMetricCategory.java b/sequencer/src/main/java/net/consensys/linea/metrics/LineaMetricCategory.java new file mode 100644 index 00000000..31ade658 --- /dev/null +++ b/sequencer/src/main/java/net/consensys/linea/metrics/LineaMetricCategory.java @@ -0,0 +1,41 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package net.consensys.linea.metrics; + +import java.util.Optional; + +import org.hyperledger.besu.plugin.services.metrics.MetricCategory; + +public enum LineaMetricCategory implements MetricCategory { + /** Profitability metric category */ + PROFITABILITY("profitability"); + + private static final Optional APPLICATION_PREFIX = Optional.of("linea_"); + private final String name; + + LineaMetricCategory(final String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + @Override + public Optional getApplicationPrefix() { + return APPLICATION_PREFIX; + } +} From 876e921558e66d85636ef79f95c1618ec4b857ce Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 24 Oct 2024 16:28:01 +0200 Subject: [PATCH 08/19] Fix: MetricSystem is only available at start phase (#106) Signed-off-by: Fabio Di Fabio --- gradle.properties | 2 +- ...stractLineaSharedPrivateOptionsPlugin.java | 26 +++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/gradle.properties b/gradle.properties index 9b4da4f6..feed4cd1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -releaseVersion=0.8.0-rc4.1-local +releaseVersion=0.8.0-rc4.1 besuVersion=24.10-develop-829db23 arithmetizationVersion=0.8.0-rc4 besuArtifactGroup=io.consensys.linea-besu diff --git a/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java b/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java index 2473d6b8..70b3cffa 100644 --- a/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java @@ -60,8 +60,10 @@ public abstract class AbstractLineaSharedPrivateOptionsPlugin protected static BlockchainService blockchainService; protected static MetricsSystem metricsSystem; - private static AtomicBoolean sharedRegisterTasksDone = new AtomicBoolean(false); - private static AtomicBoolean sharedStartTasksDone = new AtomicBoolean(false); + private static final AtomicBoolean sharedRegisterTasksDone = new AtomicBoolean(false); + private static final AtomicBoolean sharedStartTasksDone = new AtomicBoolean(false); + + private BesuContext besuContext; static { // force the initialization of the gnark compress native library to fail fast in case of issues @@ -125,12 +127,14 @@ public LineaRejectedTxReportingConfiguration rejectedTxReportingConfiguration() public synchronized void register(final BesuContext context) { super.register(context); + besuContext = context; + if (sharedRegisterTasksDone.compareAndSet(false, true)) { performSharedRegisterTasksOnce(context); } } - protected void performSharedRegisterTasksOnce(final BesuContext context) { + protected static void performSharedRegisterTasksOnce(final BesuContext context) { blockchainService = context .getService(BlockchainService.class) @@ -139,12 +143,6 @@ protected void performSharedRegisterTasksOnce(final BesuContext context) { new RuntimeException( "Failed to obtain BlockchainService from the BesuContext.")); - metricsSystem = - context - .getService(MetricsSystem.class) - .orElseThrow( - () -> new RuntimeException("Failed to obtain MetricSystem from the BesuContext.")); - context .getService(MetricCategoryRegistry.class) .orElseThrow( @@ -159,11 +157,11 @@ public void start() { super.start(); if (sharedStartTasksDone.compareAndSet(false, true)) { - performSharedStartTasksOnce(); + performSharedStartTasksOnce(besuContext); } } - private static void performSharedStartTasksOnce() { + private static void performSharedStartTasksOnce(final BesuContext context) { blockchainService .getChainId() .ifPresentOrElse( @@ -175,6 +173,12 @@ private static void performSharedStartTasksOnce() { () -> { throw new IllegalArgumentException("Chain id required"); }); + + metricsSystem = + context + .getService(MetricsSystem.class) + .orElseThrow( + () -> new RuntimeException("Failed to obtain MetricSystem from the BesuContext.")); } @Override From 2867615e02472e897e3b1d9af83c11a08fcdac9f Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 24 Oct 2024 18:13:17 +0200 Subject: [PATCH 09/19] Wait for the initial sync phase to complete before processing extra data (#105) Signed-off-by: Fabio Di Fabio --- .../linea/extradata/LineaExtraDataPlugin.java | 107 +++++++++++++----- 1 file changed, 78 insertions(+), 29 deletions(-) diff --git a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java index 504a7bf8..5f552e83 100644 --- a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java @@ -16,6 +16,7 @@ package net.consensys.linea.extradata; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; @@ -24,7 +25,9 @@ import net.consensys.linea.metrics.LineaMetricCategory; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; +import org.hyperledger.besu.plugin.data.AddedBlockContext; import org.hyperledger.besu.plugin.services.BesuEvents; +import org.hyperledger.besu.plugin.services.BesuEvents.InitialSyncCompletionListener; import org.hyperledger.besu.plugin.services.RpcEndpointService; /** This plugin registers handlers that are activated when new blocks are imported */ @@ -61,41 +64,39 @@ public void doRegister(final BesuContext context) { public void start() { super.start(); if (profitabilityConfiguration().extraDataPricingEnabled()) { - final var extraDataHandler = - new LineaExtraDataHandler(rpcEndpointService, profitabilityConfiguration()); - final var chainHeadHeader = blockchainService.getChainHeadHeader(); - final var initialExtraData = chainHeadHeader.getExtraData(); - try { - extraDataHandler.handle(initialExtraData); - } catch (final Exception e) { - log.warn( - "Failed setting initial pricing conf from extra data field ({}) of the chain head block {}({})", - initialExtraData, - chainHeadHeader.getNumber(), - chainHeadHeader.getBlockHash(), - e); - } - final var besuEventsService = besuContext .getService(BesuEvents.class) .orElseThrow( () -> new RuntimeException("Failed to obtain BesuEvents from the BesuContext.")); - besuEventsService.addBlockAddedListener( - addedBlockContext -> { - final var importedBlockHeader = addedBlockContext.getBlockHeader(); - final var latestExtraData = importedBlockHeader.getExtraData(); - - try { - extraDataHandler.handle(latestExtraData); - } catch (final Exception e) { - log.warn( - "Failed setting pricing conf from extra data field ({}) of latest imported block {}({})", - latestExtraData, - importedBlockHeader.getNumber(), - importedBlockHeader.getBlockHash(), - e); + // assume that we are in sync by default to support reading extra data at genesis + final AtomicBoolean inSync = new AtomicBoolean(true); + + besuEventsService.addSyncStatusListener( + maybeSyncStatus -> + inSync.set( + maybeSyncStatus + .map( + syncStatus -> + syncStatus.getHighestBlock() - syncStatus.getCurrentBlock() < 5) + .orElse(true))); + + // wait for the initial sync phase to complete before starting parsing extra data + // to avoid parsing errors + besuEventsService.addInitialSyncCompletionListener( + new InitialSyncCompletionListener() { + long blockAddedListenerId = -1; + + @Override + public synchronized void onInitialSyncCompleted() { + blockAddedListenerId = enableExtraDataHandling(besuEventsService, inSync); + } + + @Override + public synchronized void onInitialSyncRestart() { + besuEventsService.removeBlockAddedListener(blockAddedListenerId); + blockAddedListenerId = -1; } }); } @@ -114,4 +115,52 @@ private void initMetrics(final LineaProfitabilityConfiguration lineaProfitabilit confLabelledGauge.labels(lineaProfitabilityConfiguration::variableCostWei, "variable_cost_wei"); confLabelledGauge.labels(lineaProfitabilityConfiguration::ethGasPriceWei, "eth_gas_price_wei"); } + + private long enableExtraDataHandling( + final BesuEvents besuEventsService, final AtomicBoolean inSync) { + + final var extraDataHandler = + new LineaExtraDataHandler(rpcEndpointService, profitabilityConfiguration()); + + if (inSync.get()) { + final var chainHeadHeader = blockchainService.getChainHeadHeader(); + final var initialExtraData = chainHeadHeader.getExtraData(); + try { + extraDataHandler.handle(initialExtraData); + } catch (final Exception e) { + // this could normally happen if for example the genesis block has not a valid extra data + // field so we keep this log at debug + log.debug( + "Failed setting initial pricing conf from extra data field ({}) of the chain head block {}({})", + initialExtraData, + chainHeadHeader.getNumber(), + chainHeadHeader.getBlockHash(), + e); + } + } + + return besuEventsService.addBlockAddedListener( + addedBlockContext -> { + if (inSync.get()) { + processNewBlock(extraDataHandler, addedBlockContext); + } + }); + } + + private void processNewBlock( + final LineaExtraDataHandler extraDataHandler, final AddedBlockContext addedBlockContext) { + final var importedBlockHeader = addedBlockContext.getBlockHeader(); + final var latestExtraData = importedBlockHeader.getExtraData(); + + try { + extraDataHandler.handle(latestExtraData); + } catch (final Exception e) { + log.warn( + "Failed setting pricing conf from extra data field ({}) of latest imported block {}({})", + latestExtraData, + importedBlockHeader.getNumber(), + importedBlockHeader.getBlockHash(), + e); + } + } } From 40e8408253dccad1b40828318ffd961d388f8eb5 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Fri, 25 Oct 2024 11:52:15 +1000 Subject: [PATCH 10/19] syntax suggestions (#100) Signed-off-by: Sally MacFarlane --- .../consensys/linea/rpc/methods/LineaEstimateGas.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java b/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java index 9e81b72e..fd2ba47e 100644 --- a/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java +++ b/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java @@ -239,7 +239,7 @@ private long calculateGasLimitUpperBound( } private Wei calculateTxMaxGasPrice(final JsonCallParameter callParameters) { - return callParameters.getMaxFeePerGas().orElseGet(() -> callParameters.getGasPrice()); + return callParameters.getMaxFeePerGas().orElseGet(callParameters::getGasPrice); } private Wei getEstimatedPriorityFee( @@ -258,11 +258,8 @@ private Wei getEstimatedPriorityFee( .toBigInteger()); } - final Wei profitablePriorityFee = - txProfitabilityCalculator.profitablePriorityFeePerGas( - transaction, profitabilityConf.estimateGasMinMargin(), estimatedGasUsed, minGasPrice); - - return profitablePriorityFee; + return txProfitabilityCalculator.profitablePriorityFeePerGas( + transaction, profitabilityConf.estimateGasMinMargin(), estimatedGasUsed, minGasPrice); } private Long estimateGasUsed( From 4b87508e159c1dd2e946f38a3f943f98149d5771 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Sat, 26 Oct 2024 13:17:08 +1000 Subject: [PATCH 11/19] log plugin name at registration; remove override of getName() method (#99) * log plugin name at registration; remove override of getName() method Signed-off-by: Sally MacFarlane * logging Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane --- .../consensys/linea/extradata/LineaExtraDataPlugin.java | 7 ------- .../LineaTransactionPoolValidatorPlugin.java | 6 ------ .../txselection/LineaTransactionSelectorPlugin.java | 6 ------ 3 files changed, 19 deletions(-) diff --git a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java index 5f552e83..d00c7b3b 100644 --- a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java @@ -15,7 +15,6 @@ package net.consensys.linea.extradata; -import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import com.google.auto.service.AutoService; @@ -34,15 +33,9 @@ @Slf4j @AutoService(BesuPlugin.class) public class LineaExtraDataPlugin extends AbstractLineaRequiredPlugin { - public static final String NAME = "linea"; private BesuContext besuContext; private RpcEndpointService rpcEndpointService; - @Override - public Optional getName() { - return Optional.of(NAME); - } - @Override public void doRegister(final BesuContext context) { besuContext = context; diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java index 98c45653..410e83e9 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java @@ -46,17 +46,11 @@ @Slf4j @AutoService(BesuPlugin.class) public class LineaTransactionPoolValidatorPlugin extends AbstractLineaRequiredPlugin { - public static final String NAME = "linea"; private BesuConfiguration besuConfiguration; private TransactionPoolValidatorService transactionPoolValidatorService; private TransactionSimulationService transactionSimulationService; private Optional rejectedTxJsonRpcManager = Optional.empty(); - @Override - public Optional getName() { - return Optional.of(NAME); - } - @Override public void doRegister(final BesuContext context) { besuConfiguration = diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java index a84754e7..69b60e97 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java @@ -38,16 +38,10 @@ @Slf4j @AutoService(BesuPlugin.class) public class LineaTransactionSelectorPlugin extends AbstractLineaRequiredPlugin { - public static final String NAME = "linea"; private TransactionSelectionService transactionSelectionService; private Optional rejectedTxJsonRpcManager = Optional.empty(); private BesuConfiguration besuConfiguration; - @Override - public Optional getName() { - return Optional.of(NAME); - } - @Override public void doRegister(final BesuContext context) { transactionSelectionService = From 18458ee15a44c143a84f59f77d4247ed3893e12b Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 19 Nov 2024 11:30:25 +0100 Subject: [PATCH 12/19] Accept txs that revert in eth_sendRawTransaction when simulation check is enabled (#108) Signed-off-by: Fabio Di Fabio --- .../plugin/acc/test/LineaPluginTestBase.java | 12 +++ ...SendRawTransactionSimulationCheckTest.java | 35 ++++++++- .../resources/moduleLimits_sendRawTx.toml | 75 +++++++++++++++++++ .../src/test/solidity/RevertExample.sol | 35 +++++++++ .../validators/SimulationValidator.java | 11 --- 5 files changed, 156 insertions(+), 12 deletions(-) create mode 100644 acceptance-tests/src/test/resources/moduleLimits_sendRawTx.toml create mode 100644 acceptance-tests/src/test/solidity/RevertExample.sol diff --git a/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java b/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java index ff777ceb..68047249 100644 --- a/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java +++ b/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java @@ -37,6 +37,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import linea.plugin.acc.test.tests.web3j.generated.RevertExample; import linea.plugin.acc.test.tests.web3j.generated.SimpleStorage; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.metrics.LineaMetricCategory; @@ -244,6 +245,17 @@ protected SimpleStorage deploySimpleStorage() throws Exception { return deploy.send(); } + protected RevertExample deployRevertExample() throws Exception { + final Web3j web3j = minerNode.nodeRequests().eth(); + final Credentials credentials = Credentials.create(Accounts.GENESIS_ACCOUNT_ONE_PRIVATE_KEY); + TransactionManager txManager = + new RawTransactionManager(web3j, credentials, CHAIN_ID, createReceiptProcessor(web3j)); + + final RemoteCall deploy = + RevertExample.deploy(web3j, txManager, new DefaultGasProvider()); + return deploy.send(); + } + public static String getResourcePath(String resource) { return Objects.requireNonNull(LineaPluginTestBase.class.getResource(resource)).getPath(); } diff --git a/acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EthSendRawTransactionSimulationCheckTest.java b/acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EthSendRawTransactionSimulationCheckTest.java index 5047247e..e8e7ffa4 100644 --- a/acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EthSendRawTransactionSimulationCheckTest.java +++ b/acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EthSendRawTransactionSimulationCheckTest.java @@ -22,6 +22,7 @@ import linea.plugin.acc.test.LineaPluginTestBase; import linea.plugin.acc.test.TestCommandLineOptionsBuilder; +import linea.plugin.acc.test.tests.web3j.generated.RevertExample; import linea.plugin.acc.test.tests.web3j.generated.SimpleStorage; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.tests.acceptance.dsl.account.Account; @@ -46,7 +47,7 @@ public List getTestCliOptions() { return new TestCommandLineOptionsBuilder() .set( "--plugin-linea-module-limit-file-path=", - getResourcePath("/txOverflowModuleLimits.toml")) + getResourcePath("/moduleLimits_sendRawTx.toml")) .set("--plugin-linea-tx-pool-simulation-check-api-enabled=", "true") .build(); } @@ -101,4 +102,36 @@ public void transactionOverModuleLineCountNotAccepted() throws Exception { .map(Hash::toHexString) .forEach(hash -> minerNode.verify(eth.expectSuccessfulTransactionReceipt(hash))); } + + @Test + public void transactionsThatRevertAreAccepted() throws Exception { + final RevertExample revertExample = deployRevertExample(); + final Web3j web3j = minerNode.nodeRequests().eth(); + final String contractAddress = revertExample.getContractAddress(); + final String txData = revertExample.setValue(BigInteger.ZERO).encodeFunctionCall(); + + // this tx reverts but nevertheless it is accepted in the pool + final RawTransaction txThatReverts = + RawTransaction.createTransaction( + CHAIN_ID, + BigInteger.ZERO, + GAS_LIMIT.divide(BigInteger.TEN), + contractAddress, + VALUE, + txData, + GAS_PRICE, + GAS_PRICE.multiply(BigInteger.TEN).add(BigInteger.ONE)); + final byte[] signedTxContractInteraction = + TransactionEncoder.signMessage( + txThatReverts, Credentials.create(Accounts.GENESIS_ACCOUNT_TWO_PRIVATE_KEY)); + + final EthSendTransaction signedTxContractInteractionResp = + web3j.ethSendRawTransaction(Numeric.toHexString(signedTxContractInteraction)).send(); + + assertThat(signedTxContractInteractionResp.hasError()).isFalse(); + + final var expectedConfirmedTxHash = signedTxContractInteractionResp.getTransactionHash(); + + minerNode.verify(eth.expectSuccessfulTransactionReceipt(expectedConfirmedTxHash)); + } } diff --git a/acceptance-tests/src/test/resources/moduleLimits_sendRawTx.toml b/acceptance-tests/src/test/resources/moduleLimits_sendRawTx.toml new file mode 100644 index 00000000..c8dec3c8 --- /dev/null +++ b/acceptance-tests/src/test/resources/moduleLimits_sendRawTx.toml @@ -0,0 +1,75 @@ + +## +# This file specifies prover limit by each EVM module +# WARN: The prover/arithmetization team has the owneship of this. +# Changing this values may compromise the system. +# issue: https://github.com/Consensys/zkevm-monorepo/issues/525 +## + +[traces-limits] +# +# Arithmetization module limits +# +ADD = 70 +BIN = 262144 +BLAKE_MODEXP_DATA = 262144 +BLOCK_DATA = 13 +BLOCK_HASH = 1 +EC_DATA = 4096 +EUC = 16384 # can probably be lower +EXP = 32760 +EXT = 20 +GAS = 262144 +HUB = 174 +MMIO = 1048576 +MMU = 524288 +MOD = 20 +MUL = 20 +MXP = 35 +PHONEY_RLP = 65536 # can probably get lower +ROM = 2402 +ROM_LEX = 20 +SHF = 63 +TX_RLP = 131072 +TRM = 120 +WCP = 149 +LOG_DATA = 20 +LOG_INFO = 20 +RLP_ADDR = 20 +RLP_TXN = 1300 +RLP_TXN_RCPT = 100 +TXN_DATA = 30 +SHAKIRA_DATA = 262144 +STP = 20 +OOB = 262144 + +# +# Block-specific limits +# +BLOCK_KECCAK = 8192 +BLOCK_L1_SIZE = 1000000 +BLOCK_L2_L1_LOGS = 16 +BLOCK_TRANSACTIONS = 200 # max number of tx in an L2 block + +# +# Fixed size, static tables +# +BIN_REFERENCE_TABLE = 262144 # contains 3 * 256^2 + 256 data rows + 1 padding row +SHF_REFERENCE_TABLE = 4096 # contains 9 * 256 data rows + 1 padding row +INSTRUCTION_DECODER = 512 # contains 256 data rows + 1 padding row + +# +# Precompiles limits +# +PRECOMPILE_ECRECOVER_EFFECTIVE_CALLS = 10000 +PRECOMPILE_SHA2_BLOCKS = 10000 +PRECOMPILE_RIPEMD_BLOCKS = 10000 +PRECOMPILE_ECPAIRING_MILLER_LOOPS = 10000 +PRECOMPILE_MODEXP_EFFECTIVE_CALLS = 10000 +PRECOMPILE_ECADD_EFFECTIVE_CALLS = 10000 +PRECOMPILE_ECMUL_EFFECTIVE_CALLS = 10000 +PRECOMPILE_ECPAIRING_FINAL_EXPONENTIATIONS = 10000 +PRECOMPILE_ECPAIRING_G2_MEMBERSHIP_CALLS = 10000 +PRECOMPILE_ECPAIRING_MILLER_LOOPS = 10000 +PRECOMPILE_BLAKE_EFFECTIVE_CALLS = 10000 +PRECOMPILE_BLAKE_ROUNDS = 512 \ No newline at end of file diff --git a/acceptance-tests/src/test/solidity/RevertExample.sol b/acceptance-tests/src/test/solidity/RevertExample.sol new file mode 100644 index 00000000..28b97678 --- /dev/null +++ b/acceptance-tests/src/test/solidity/RevertExample.sol @@ -0,0 +1,35 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +pragma solidity >=0.7.0 <0.9.0; + +contract RevertExample { + uint256 public value; + + function setValue(uint256 _newValue) public { + require(_newValue != 0, "Value cannot be zero"); + value = _newValue; + } + + function forceRevert() public pure { + revert("This function always reverts"); + } + + function conditionalRevert(uint256 _input) public pure returns (uint256) { + if (_input < 10) { + revert("Input must be 10 or greater"); + } + return _input * 2; + } +} \ No newline at end of file diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java index 86530ab0..991e40a7 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java @@ -114,17 +114,6 @@ public Optional validateTransaction( reportRejectedTransaction(transaction, errMsg); return Optional.of(errMsg); } - if (!simulationResult.isSuccessful()) { - final String errMsg = - "Reverted transaction" - + simulationResult - .getRevertReason() - .map(rr -> ": " + rr.toHexString()) - .orElse(""); - log.debug(errMsg); - reportRejectedTransaction(transaction, errMsg); - return Optional.of(errMsg); - } } } else { log.atTrace() From 053570534b47f3b24dbe124df5fde14bac08f04c Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Sat, 23 Nov 2024 01:29:52 +1000 Subject: [PATCH 13/19] update linea-besu version (#110) Signed-off-by: Sally MacFarlane --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index feed4cd1..6fe052df 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ releaseVersion=0.8.0-rc4.1 -besuVersion=24.10-develop-829db23 +besuVersion=24.11-develop-96e9ed8 arithmetizationVersion=0.8.0-rc4 besuArtifactGroup=io.consensys.linea-besu distributionIdentifier=linea-sequencer From 4d925ad9736b3e79b35640f9fc5dceeca4fea7c5 Mon Sep 17 00:00:00 2001 From: Usman Saleem Date: Wed, 27 Nov 2024 21:26:26 +1000 Subject: [PATCH 14/19] feat: Rejected transaction reporting due to trace line limits (#111) - Rejected transaction reporting due to trace line limits - Remove other rejected transactions reporting - test fixes --- CHANGELOG.md | 1 + .../LineaTransactionPoolValidatorFactory.java | 9 ++-- .../validators/AllowedAddressValidator.java | 25 +--------- .../validators/CalldataValidator.java | 21 --------- .../validators/GasLimitValidator.java | 21 --------- .../validators/ProfitabilityValidator.java | 47 +++++-------------- .../validators/SimulationValidator.java | 1 - .../selectors/LineaTransactionSelector.java | 16 ++++++- .../AllowedAddressValidatorTest.java | 2 +- .../validators/CalldataValidatorTest.java | 3 +- .../validators/GasLimitValidatorTest.java | 3 +- .../ProfitabilityValidatorTest.java | 12 ++--- 12 files changed, 39 insertions(+), 122 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 786ddd1e..fdf2d21e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## Next release +* feat: Report rejected transactions only due to trace limit overflows to an external service. * feat: Report rejected transactions to an external service for validators used by LineaTransactionPoolValidatorPlugin [#85](https://github.com/Consensys/linea-sequencer/pull/85) * feat: Report rejected transactions to an external service for LineaTransactionSelector used by LineaTransactionSelectorPlugin [#69](https://github.com/Consensys/linea-sequencer/pull/69) diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorFactory.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorFactory.java index dc5759f1..18bf1193 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorFactory.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorFactory.java @@ -80,11 +80,10 @@ public LineaTransactionPoolValidatorFactory( public PluginTransactionPoolValidator createTransactionValidator() { final var validators = new PluginTransactionPoolValidator[] { - new AllowedAddressValidator(denied, rejectedTxJsonRpcManager), - new GasLimitValidator(txPoolValidatorConf, rejectedTxJsonRpcManager), - new CalldataValidator(txPoolValidatorConf, rejectedTxJsonRpcManager), - new ProfitabilityValidator( - besuConfiguration, blockchainService, profitabilityConf, rejectedTxJsonRpcManager), + new AllowedAddressValidator(denied), + new GasLimitValidator(txPoolValidatorConf), + new CalldataValidator(txPoolValidatorConf), + new ProfitabilityValidator(besuConfiguration, blockchainService, profitabilityConf), new SimulationValidator( blockchainService, transactionSimulationService, diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/AllowedAddressValidator.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/AllowedAddressValidator.java index a272ea7f..a30cab93 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/AllowedAddressValidator.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/AllowedAddressValidator.java @@ -14,15 +14,11 @@ */ package net.consensys.linea.sequencer.txpoolvalidation.validators; -import java.time.Instant; -import java.util.List; import java.util.Optional; import java.util.Set; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import net.consensys.linea.jsonrpc.JsonRpcManager; -import net.consensys.linea.jsonrpc.JsonRpcRequestBuilder; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Transaction; import org.hyperledger.besu.plugin.services.txvalidator.PluginTransactionPoolValidator; @@ -48,15 +44,11 @@ public class AllowedAddressValidator implements PluginTransactionPoolValidator { Address.fromHexString("0x000000000000000000000000000000000000000a")); private final Set

denied; - private final Optional rejectedTxJsonRpcManager; @Override public Optional validateTransaction( final Transaction transaction, final boolean isLocal, final boolean hasPriority) { - final Optional errMsg = - validateSender(transaction).or(() -> validateRecipient(transaction)); - errMsg.ifPresent(reason -> reportRejectedTransaction(transaction, reason)); - return errMsg; + return validateSender(transaction).or(() -> validateRecipient(transaction)); } private Optional validateRecipient(final Transaction transaction) { @@ -90,19 +82,4 @@ private Optional validateSender(final Transaction transaction) { } return Optional.empty(); } - - private void reportRejectedTransaction(final Transaction transaction, final String reason) { - rejectedTxJsonRpcManager.ifPresent( - jsonRpcManager -> { - final String jsonRpcCall = - JsonRpcRequestBuilder.generateSaveRejectedTxJsonRpc( - jsonRpcManager.getNodeType(), - transaction, - Instant.now(), - Optional.empty(), // block number is not available - reason, - List.of()); - jsonRpcManager.submitNewJsonRpcCallAsync(jsonRpcCall); - }); - } } diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/CalldataValidator.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/CalldataValidator.java index 7ecaf125..44eba275 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/CalldataValidator.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/CalldataValidator.java @@ -14,15 +14,11 @@ */ package net.consensys.linea.sequencer.txpoolvalidation.validators; -import java.time.Instant; -import java.util.List; import java.util.Optional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.config.LineaTransactionPoolValidatorConfiguration; -import net.consensys.linea.jsonrpc.JsonRpcManager; -import net.consensys.linea.jsonrpc.JsonRpcRequestBuilder; import org.hyperledger.besu.datatypes.Transaction; import org.hyperledger.besu.plugin.services.txvalidator.PluginTransactionPoolValidator; @@ -31,7 +27,6 @@ @RequiredArgsConstructor public class CalldataValidator implements PluginTransactionPoolValidator { final LineaTransactionPoolValidatorConfiguration txPoolValidatorConf; - final Optional rejectedTxJsonRpcManager; @Override public Optional validateTransaction( @@ -41,24 +36,8 @@ public Optional validateTransaction( "Calldata of transaction is greater than the allowed max of " + txPoolValidatorConf.maxTxCalldataSize(); log.debug(errMsg); - reportRejectedTransaction(transaction, errMsg); return Optional.of(errMsg); } return Optional.empty(); } - - private void reportRejectedTransaction(final Transaction transaction, final String reason) { - rejectedTxJsonRpcManager.ifPresent( - jsonRpcManager -> { - final String jsonRpcCall = - JsonRpcRequestBuilder.generateSaveRejectedTxJsonRpc( - jsonRpcManager.getNodeType(), - transaction, - Instant.now(), - Optional.empty(), // block number is not available - reason, - List.of()); - jsonRpcManager.submitNewJsonRpcCallAsync(jsonRpcCall); - }); - } } diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/GasLimitValidator.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/GasLimitValidator.java index 7e2f9042..f86e2844 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/GasLimitValidator.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/GasLimitValidator.java @@ -14,15 +14,11 @@ */ package net.consensys.linea.sequencer.txpoolvalidation.validators; -import java.time.Instant; -import java.util.List; import java.util.Optional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.config.LineaTransactionPoolValidatorConfiguration; -import net.consensys.linea.jsonrpc.JsonRpcManager; -import net.consensys.linea.jsonrpc.JsonRpcRequestBuilder; import org.hyperledger.besu.datatypes.Transaction; import org.hyperledger.besu.plugin.services.txvalidator.PluginTransactionPoolValidator; @@ -34,7 +30,6 @@ @RequiredArgsConstructor public class GasLimitValidator implements PluginTransactionPoolValidator { final LineaTransactionPoolValidatorConfiguration txPoolValidatorConf; - final Optional rejectedTxJsonRpcManager; @Override public Optional validateTransaction( @@ -44,24 +39,8 @@ public Optional validateTransaction( "Gas limit of transaction is greater than the allowed max of " + txPoolValidatorConf.maxTxGasLimit(); log.debug(errMsg); - reportRejectedTransaction(transaction, errMsg); return Optional.of(errMsg); } return Optional.empty(); } - - private void reportRejectedTransaction(final Transaction transaction, final String reason) { - rejectedTxJsonRpcManager.ifPresent( - jsonRpcManager -> { - final String jsonRpcCall = - JsonRpcRequestBuilder.generateSaveRejectedTxJsonRpc( - jsonRpcManager.getNodeType(), - transaction, - Instant.now(), - Optional.empty(), // block number is not available - reason, - List.of()); - jsonRpcManager.submitNewJsonRpcCallAsync(jsonRpcCall); - }); - } } diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/ProfitabilityValidator.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/ProfitabilityValidator.java index 8b91452f..74040fdb 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/ProfitabilityValidator.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/ProfitabilityValidator.java @@ -14,15 +14,11 @@ */ package net.consensys.linea.sequencer.txpoolvalidation.validators; -import java.time.Instant; -import java.util.List; import java.util.Optional; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.bl.TransactionProfitabilityCalculator; import net.consensys.linea.config.LineaProfitabilityConfiguration; -import net.consensys.linea.jsonrpc.JsonRpcManager; -import net.consensys.linea.jsonrpc.JsonRpcRequestBuilder; import org.apache.tuweni.units.bigints.UInt256s; import org.hyperledger.besu.datatypes.Transaction; import org.hyperledger.besu.datatypes.Wei; @@ -41,18 +37,15 @@ public class ProfitabilityValidator implements PluginTransactionPoolValidator { final BlockchainService blockchainService; final LineaProfitabilityConfiguration profitabilityConf; final TransactionProfitabilityCalculator profitabilityCalculator; - final Optional rejectedTxJsonRpcManager; public ProfitabilityValidator( final BesuConfiguration besuConfiguration, final BlockchainService blockchainService, - final LineaProfitabilityConfiguration profitabilityConf, - final Optional rejectedTxJsonRpcManager) { + final LineaProfitabilityConfiguration profitabilityConf) { this.besuConfiguration = besuConfiguration; this.blockchainService = blockchainService; this.profitabilityConf = profitabilityConf; this.profitabilityCalculator = new TransactionProfitabilityCalculator(profitabilityConf); - this.rejectedTxJsonRpcManager = rejectedTxJsonRpcManager; } @Override @@ -68,19 +61,16 @@ public Optional validateTransaction( .getNextBlockBaseFee() .orElseThrow(() -> new RuntimeException("We only support a base fee market")); - final Optional errMsg = - profitabilityCalculator.isProfitable( - "Txpool", - transaction, - profitabilityConf.txPoolMinMargin(), - baseFee, - calculateUpfrontGasPrice(transaction, baseFee), - transaction.getGasLimit(), - besuConfiguration.getMinGasPrice()) - ? Optional.empty() - : Optional.of("Gas price too low"); - errMsg.ifPresent(s -> reportRejectedTransaction(transaction, s)); - return errMsg; + return profitabilityCalculator.isProfitable( + "Txpool", + transaction, + profitabilityConf.txPoolMinMargin(), + baseFee, + calculateUpfrontGasPrice(transaction, baseFee), + transaction.getGasLimit(), + besuConfiguration.getMinGasPrice()) + ? Optional.empty() + : Optional.of("Gas price too low"); } return Optional.empty(); @@ -98,19 +88,4 @@ private Wei calculateUpfrontGasPrice(final Transaction transaction, final Wei ba baseFee.add(Wei.fromQuantity(transaction.getMaxPriorityFeePerGas().get())))) .orElseGet(() -> Wei.fromQuantity(transaction.getGasPrice().get())); } - - private void reportRejectedTransaction(final Transaction transaction, final String reason) { - rejectedTxJsonRpcManager.ifPresent( - jsonRpcManager -> { - final String jsonRpcCall = - JsonRpcRequestBuilder.generateSaveRejectedTxJsonRpc( - jsonRpcManager.getNodeType(), - transaction, - Instant.now(), - Optional.empty(), // block number is not available - reason, - List.of()); - jsonRpcManager.submitNewJsonRpcCallAsync(jsonRpcCall); - }); - } } diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java index 991e40a7..75204eb6 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java @@ -111,7 +111,6 @@ public Optional validateTransaction( "Invalid transaction" + simulationResult.getInvalidReason().map(ir -> ": " + ir).orElse(""); log.debug(errMsg); - reportRejectedTransaction(transaction, errMsg); return Optional.of(errMsg); } } diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java index a396b83f..2d6586d4 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java @@ -14,10 +14,15 @@ */ package net.consensys.linea.sequencer.txselection.selectors; +import static net.consensys.linea.sequencer.txselection.LineaTransactionSelectionResult.TX_MODULE_LINE_COUNT_OVERFLOW; +import static net.consensys.linea.sequencer.txselection.LineaTransactionSelectionResult.TX_MODULE_LINE_COUNT_OVERFLOW_CACHED; + import java.time.Instant; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.config.LineaProfitabilityConfiguration; @@ -41,6 +46,7 @@ public class LineaTransactionSelector implements PluginTransactionSelector { private TraceLineLimitTransactionSelector traceLineLimitTransactionSelector; private final List selectors; private final Optional rejectedTxJsonRpcManager; + private final Set rejectedTransactionReasonsMap = new HashSet<>(); public LineaTransactionSelector( final BlockchainService blockchainService, @@ -51,6 +57,13 @@ public LineaTransactionSelector( final Map limitsMap, final Optional rejectedTxJsonRpcManager) { this.rejectedTxJsonRpcManager = rejectedTxJsonRpcManager; + + // only report rejected transaction selection result from TraceLineLimitTransactionSelector + if (rejectedTxJsonRpcManager.isPresent()) { + rejectedTransactionReasonsMap.add(TX_MODULE_LINE_COUNT_OVERFLOW.toString()); + rejectedTransactionReasonsMap.add(TX_MODULE_LINE_COUNT_OVERFLOW_CACHED.toString()); + } + selectors = createTransactionSelectors( blockchainService, @@ -163,7 +176,8 @@ public void onTransactionNotSelected( rejectedTxJsonRpcManager.ifPresent( jsonRpcManager -> { - if (transactionSelectionResult.discard()) { + if (transactionSelectionResult.discard() + && rejectedTransactionReasonsMap.contains(transactionSelectionResult.toString())) { jsonRpcManager.submitNewJsonRpcCallAsync( JsonRpcRequestBuilder.generateSaveRejectedTxJsonRpc( jsonRpcManager.getNodeType(), diff --git a/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/AllowedAddressValidatorTest.java b/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/AllowedAddressValidatorTest.java index 85bc1b3e..c2f25f68 100644 --- a/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/AllowedAddressValidatorTest.java +++ b/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/AllowedAddressValidatorTest.java @@ -40,7 +40,7 @@ public class AllowedAddressValidatorTest { @BeforeEach public void initialize() { Set
denied = Set.of(DENIED); - allowedAddressValidator = new AllowedAddressValidator(denied, Optional.empty()); + allowedAddressValidator = new AllowedAddressValidator(denied); } @Test diff --git a/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/CalldataValidatorTest.java b/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/CalldataValidatorTest.java index e30a6d5f..fc775580 100644 --- a/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/CalldataValidatorTest.java +++ b/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/CalldataValidatorTest.java @@ -38,8 +38,7 @@ public void initialize() { new CalldataValidator( LineaTransactionPoolValidatorCliOptions.create().toDomainObject().toBuilder() .maxTxCalldataSize(MAX_TX_CALLDATA_SIZE) - .build(), - Optional.empty()); + .build()); } @Test diff --git a/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/GasLimitValidatorTest.java b/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/GasLimitValidatorTest.java index e7c79907..219e62f4 100644 --- a/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/GasLimitValidatorTest.java +++ b/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/GasLimitValidatorTest.java @@ -38,8 +38,7 @@ public void initialize() { new GasLimitValidator( LineaTransactionPoolValidatorCliOptions.create().toDomainObject().toBuilder() .maxTxGasLimit(MAX_TX_GAS_LIMIT) - .build(), - Optional.empty()); + .build()); } @Test diff --git a/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/ProfitabilityValidatorTest.java b/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/ProfitabilityValidatorTest.java index de333edb..3178a5ec 100644 --- a/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/ProfitabilityValidatorTest.java +++ b/sequencer/src/test/java/net/consensys/linea/sequencer/txpoolvalidation/validators/ProfitabilityValidatorTest.java @@ -87,8 +87,7 @@ public void initialize() { profitabilityConfBuilder .txPoolCheckP2pEnabled(true) .txPoolCheckApiEnabled(true) - .build(), - Optional.empty()); + .build()); profitabilityValidatorNever = new ProfitabilityValidator( @@ -97,8 +96,7 @@ public void initialize() { profitabilityConfBuilder .txPoolCheckP2pEnabled(false) .txPoolCheckApiEnabled(false) - .build(), - Optional.empty()); + .build()); profitabilityValidatorOnlyApi = new ProfitabilityValidator( @@ -107,8 +105,7 @@ public void initialize() { profitabilityConfBuilder .txPoolCheckP2pEnabled(false) .txPoolCheckApiEnabled(true) - .build(), - Optional.empty()); + .build()); profitabilityValidatorOnlyP2p = new ProfitabilityValidator( @@ -117,8 +114,7 @@ public void initialize() { profitabilityConfBuilder .txPoolCheckP2pEnabled(true) .txPoolCheckApiEnabled(false) - .build(), - Optional.empty()); + .build()); } @Test From dcec6951cbf9fb80dd179edc05c19ef82fc3e329 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 29 Nov 2024 11:10:03 +0100 Subject: [PATCH 15/19] Update Linea Besu and Tracer (#112) Signed-off-by: Fabio Di Fabio --- .../plugin/acc/test/LineaPluginTestBase.java | 2 +- .../resources/moduleLimits_sendRawTx.toml | 2 +- .../resources/txOverflowModuleLimits.toml | 2 +- gradle.properties | 6 ++-- .../linea/AbstractLineaRequiredPlugin.java | 14 ++++----- ...stractLineaSharedPrivateOptionsPlugin.java | 31 ++++++++++--------- .../linea/extradata/LineaExtraDataPlugin.java | 17 +++++----- .../LineaEstimateGasEndpointPlugin.java | 18 +++++------ .../LineaSetExtraDataEndpointPlugin.java | 10 +++--- .../LineaTransactionPoolValidatorPlugin.java | 16 +++++----- .../LineaTransactionSelectorPlugin.java | 12 +++---- 11 files changed, 66 insertions(+), 64 deletions(-) diff --git a/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java b/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java index 68047249..a5d52571 100644 --- a/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java +++ b/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java @@ -352,7 +352,7 @@ protected double getMetricValue( + metricName + labelValues.stream() .map(lv -> lv.getKey() + "=\"" + lv.getValue() + "\"") - .collect(Collectors.joining(",", "{", ",}")); + .collect(Collectors.joining(",", "{", "}")); final var foundMetric = respLines.body().filter(line -> line.startsWith(searchString)).findFirst(); diff --git a/acceptance-tests/src/test/resources/moduleLimits_sendRawTx.toml b/acceptance-tests/src/test/resources/moduleLimits_sendRawTx.toml index c8dec3c8..bfe3d64a 100644 --- a/acceptance-tests/src/test/resources/moduleLimits_sendRawTx.toml +++ b/acceptance-tests/src/test/resources/moduleLimits_sendRawTx.toml @@ -32,7 +32,7 @@ ROM_LEX = 20 SHF = 63 TX_RLP = 131072 TRM = 120 -WCP = 149 +WCP = 156 LOG_DATA = 20 LOG_INFO = 20 RLP_ADDR = 20 diff --git a/acceptance-tests/src/test/resources/txOverflowModuleLimits.toml b/acceptance-tests/src/test/resources/txOverflowModuleLimits.toml index 58f903b0..6d369917 100644 --- a/acceptance-tests/src/test/resources/txOverflowModuleLimits.toml +++ b/acceptance-tests/src/test/resources/txOverflowModuleLimits.toml @@ -20,7 +20,7 @@ EUC = 16384 # can probably be lower EXP = 32760 EXT = 20 GAS = 262144 -HUB = 50 +HUB = 51 MMIO = 1048576 MMU = 524288 MOD = 20 diff --git a/gradle.properties b/gradle.properties index 6fe052df..ae194b4f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ -releaseVersion=0.8.0-rc4.1 -besuVersion=24.11-develop-96e9ed8 -arithmetizationVersion=0.8.0-rc4 +releaseVersion=0.8.0-rc6.1 +besuVersion=24.11-delivery39 +arithmetizationVersion=0.8.0-rc6 besuArtifactGroup=io.consensys.linea-besu distributionIdentifier=linea-sequencer distributionBaseUrl=https://artifacts.consensys.net/public/linea-besu/raw/names/linea-besu.tar.gz/versions/ \ No newline at end of file diff --git a/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java b/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java index 427796a1..b3e7ccf9 100644 --- a/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/AbstractLineaRequiredPlugin.java @@ -16,8 +16,8 @@ package net.consensys.linea; import lombok.extern.slf4j.Slf4j; -import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; +import org.hyperledger.besu.plugin.ServiceManager; @Slf4j public abstract class AbstractLineaRequiredPlugin extends AbstractLineaSharedPrivateOptionsPlugin { @@ -28,15 +28,15 @@ public abstract class AbstractLineaRequiredPlugin extends AbstractLineaSharedPri * *

If that's NOT desired, the plugin should implement {@link BesuPlugin} directly. * - * @param context the BesuContext to be used. + * @param serviceManager the ServiceManager to be used. */ @Override - public void register(final BesuContext context) { - super.register(context); + public void register(final ServiceManager serviceManager) { + super.register(serviceManager); try { log.info("Registering Linea plugin {}", this.getClass().getName()); - doRegister(context); + doRegister(serviceManager); } catch (Exception e) { log.error("Halting Besu startup: exception in plugin registration: ", e); @@ -49,7 +49,7 @@ public void register(final BesuContext context) { /** * Linea plugins need to implement this method. Called by {@link BesuPlugin} register method * - * @param context the BesuContext to be used. + * @param serviceManager the ServiceManager to be used. */ - public abstract void doRegister(final BesuContext context); + public abstract void doRegister(final ServiceManager serviceManager); } diff --git a/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java b/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java index 70b3cffa..c8898c64 100644 --- a/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java @@ -36,7 +36,7 @@ import net.consensys.linea.metrics.LineaMetricCategory; import net.consensys.linea.plugins.AbstractLineaSharedOptionsPlugin; import net.consensys.linea.plugins.LineaOptionsPluginConfiguration; -import org.hyperledger.besu.plugin.BesuContext; +import org.hyperledger.besu.plugin.ServiceManager; import org.hyperledger.besu.plugin.services.BlockchainService; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.MetricCategoryRegistry; @@ -63,7 +63,7 @@ public abstract class AbstractLineaSharedPrivateOptionsPlugin private static final AtomicBoolean sharedRegisterTasksDone = new AtomicBoolean(false); private static final AtomicBoolean sharedStartTasksDone = new AtomicBoolean(false); - private BesuContext besuContext; + private ServiceManager serviceManager; static { // force the initialization of the gnark compress native library to fail fast in case of issues @@ -124,31 +124,31 @@ public LineaRejectedTxReportingConfiguration rejectedTxReportingConfiguration() } @Override - public synchronized void register(final BesuContext context) { - super.register(context); + public synchronized void register(final ServiceManager serviceManager) { + super.register(serviceManager); - besuContext = context; + this.serviceManager = serviceManager; if (sharedRegisterTasksDone.compareAndSet(false, true)) { - performSharedRegisterTasksOnce(context); + performSharedRegisterTasksOnce(serviceManager); } } - protected static void performSharedRegisterTasksOnce(final BesuContext context) { + protected static void performSharedRegisterTasksOnce(final ServiceManager serviceManager) { blockchainService = - context + serviceManager .getService(BlockchainService.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain BlockchainService from the BesuContext.")); + "Failed to obtain BlockchainService from the ServiceManager.")); - context + serviceManager .getService(MetricCategoryRegistry.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain MetricCategoryRegistry from the BesuContext.")) + "Failed to obtain MetricCategoryRegistry from the ServiceManager.")) .addMetricCategory(LineaMetricCategory.PROFITABILITY); } @@ -157,11 +157,11 @@ public void start() { super.start(); if (sharedStartTasksDone.compareAndSet(false, true)) { - performSharedStartTasksOnce(besuContext); + performSharedStartTasksOnce(serviceManager); } } - private static void performSharedStartTasksOnce(final BesuContext context) { + private static void performSharedStartTasksOnce(final ServiceManager serviceManager) { blockchainService .getChainId() .ifPresentOrElse( @@ -175,10 +175,11 @@ private static void performSharedStartTasksOnce(final BesuContext context) { }); metricsSystem = - context + serviceManager .getService(MetricsSystem.class) .orElseThrow( - () -> new RuntimeException("Failed to obtain MetricSystem from the BesuContext.")); + () -> + new RuntimeException("Failed to obtain MetricSystem from the ServiceManager.")); } @Override diff --git a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java index d00c7b3b..907e3bd0 100644 --- a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java @@ -22,8 +22,8 @@ import net.consensys.linea.AbstractLineaRequiredPlugin; import net.consensys.linea.config.LineaProfitabilityConfiguration; import net.consensys.linea.metrics.LineaMetricCategory; -import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; +import org.hyperledger.besu.plugin.ServiceManager; import org.hyperledger.besu.plugin.data.AddedBlockContext; import org.hyperledger.besu.plugin.services.BesuEvents; import org.hyperledger.besu.plugin.services.BesuEvents.InitialSyncCompletionListener; @@ -33,19 +33,19 @@ @Slf4j @AutoService(BesuPlugin.class) public class LineaExtraDataPlugin extends AbstractLineaRequiredPlugin { - private BesuContext besuContext; + private ServiceManager serviceManager; private RpcEndpointService rpcEndpointService; @Override - public void doRegister(final BesuContext context) { - besuContext = context; + public void doRegister(final ServiceManager context) { + serviceManager = context; rpcEndpointService = context .getService(RpcEndpointService.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain RpcEndpointService from the BesuContext.")); + "Failed to obtain RpcEndpointService from the ServiceManager.")); } /** @@ -58,10 +58,11 @@ public void start() { super.start(); if (profitabilityConfiguration().extraDataPricingEnabled()) { final var besuEventsService = - besuContext + serviceManager .getService(BesuEvents.class) .orElseThrow( - () -> new RuntimeException("Failed to obtain BesuEvents from the BesuContext.")); + () -> + new RuntimeException("Failed to obtain BesuEvents from the ServiceManager.")); // assume that we are in sync by default to support reading extra data at genesis final AtomicBoolean inSync = new AtomicBoolean(true); @@ -99,7 +100,7 @@ public synchronized void onInitialSyncRestart() { private void initMetrics(final LineaProfitabilityConfiguration lineaProfitabilityConfiguration) { final var confLabelledGauge = - metricsSystem.createLabelledGauge( + metricsSystem.createLabelledSuppliedGauge( LineaMetricCategory.PROFITABILITY, "conf", "Profitability configuration values at runtime", diff --git a/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaEstimateGasEndpointPlugin.java b/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaEstimateGasEndpointPlugin.java index 6d80e523..3f9d2f93 100644 --- a/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaEstimateGasEndpointPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaEstimateGasEndpointPlugin.java @@ -21,8 +21,8 @@ import lombok.extern.slf4j.Slf4j; import net.consensys.linea.AbstractLineaRequiredPlugin; import net.consensys.linea.rpc.methods.LineaEstimateGas; -import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; +import org.hyperledger.besu.plugin.ServiceManager; import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.RpcEndpointService; import org.hyperledger.besu.plugin.services.TransactionSimulationService; @@ -39,33 +39,33 @@ public class LineaEstimateGasEndpointPlugin extends AbstractLineaRequiredPlugin /** * Register the RPC service. * - * @param context the BesuContext to be used. + * @param serviceManager the BesuContext to be used. */ @Override - public void doRegister(final BesuContext context) { + public void doRegister(final ServiceManager serviceManager) { besuConfiguration = - context + serviceManager .getService(BesuConfiguration.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain BesuConfiguration from the BesuContext.")); + "Failed to obtain BesuConfiguration from the ServiceManager.")); rpcEndpointService = - context + serviceManager .getService(RpcEndpointService.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain RpcEndpointService from the BesuContext.")); + "Failed to obtain RpcEndpointService from the ServiceManager.")); transactionSimulationService = - context + serviceManager .getService(TransactionSimulationService.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain TransactionSimulatorService from the BesuContext.")); + "Failed to obtain TransactionSimulatorService from the ServiceManager.")); lineaEstimateGasMethod = new LineaEstimateGas( diff --git a/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaSetExtraDataEndpointPlugin.java b/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaSetExtraDataEndpointPlugin.java index 17d973f0..b5f913bf 100644 --- a/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaSetExtraDataEndpointPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/rpc/services/LineaSetExtraDataEndpointPlugin.java @@ -20,8 +20,8 @@ import net.consensys.linea.AbstractLineaRequiredPlugin; import net.consensys.linea.extradata.LineaExtraDataHandler; import net.consensys.linea.rpc.methods.LineaSetExtraData; -import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; +import org.hyperledger.besu.plugin.ServiceManager; import org.hyperledger.besu.plugin.services.RpcEndpointService; /** Registers RPC endpoints. This class provides RPC endpoints under the 'linea' namespace. */ @@ -34,18 +34,18 @@ public class LineaSetExtraDataEndpointPlugin extends AbstractLineaRequiredPlugin /** * Register the RPC service. * - * @param context the BesuContext to be used. + * @param serviceManager the ServiceManager to be used. */ @Override - public void doRegister(final BesuContext context) { + public void doRegister(final ServiceManager serviceManager) { rpcEndpointService = - context + serviceManager .getService(RpcEndpointService.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain RpcEndpointService from the BesuContext.")); + "Failed to obtain RpcEndpointService from the ServiceManager.")); lineaSetExtraDataMethod = new LineaSetExtraData(rpcEndpointService); diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java index 410e83e9..4ca4a2ac 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java @@ -31,8 +31,8 @@ import net.consensys.linea.config.LineaRejectedTxReportingConfiguration; import net.consensys.linea.jsonrpc.JsonRpcManager; import org.hyperledger.besu.datatypes.Address; -import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; +import org.hyperledger.besu.plugin.ServiceManager; import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.TransactionPoolValidatorService; import org.hyperledger.besu.plugin.services.TransactionSimulationService; @@ -52,30 +52,30 @@ public class LineaTransactionPoolValidatorPlugin extends AbstractLineaRequiredPl private Optional rejectedTxJsonRpcManager = Optional.empty(); @Override - public void doRegister(final BesuContext context) { + public void doRegister(final ServiceManager serviceManager) { besuConfiguration = - context + serviceManager .getService(BesuConfiguration.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain BesuConfiguration from the BesuContext.")); + "Failed to obtain BesuConfiguration from the ServiceManager.")); transactionPoolValidatorService = - context + serviceManager .getService(TransactionPoolValidatorService.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain TransactionPoolValidationService from the BesuContext.")); + "Failed to obtain TransactionPoolValidationService from the ServiceManager.")); transactionSimulationService = - context + serviceManager .getService(TransactionSimulationService.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain TransactionSimulatorService from the BesuContext.")); + "Failed to obtain TransactionSimulatorService from the ServiceManager.")); } @Override diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java index 69b60e97..e25a0c03 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java @@ -25,8 +25,8 @@ import net.consensys.linea.config.LineaRejectedTxReportingConfiguration; import net.consensys.linea.config.LineaTransactionSelectorConfiguration; import net.consensys.linea.jsonrpc.JsonRpcManager; -import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; +import org.hyperledger.besu.plugin.ServiceManager; import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.TransactionSelectionService; @@ -43,22 +43,22 @@ public class LineaTransactionSelectorPlugin extends AbstractLineaRequiredPlugin private BesuConfiguration besuConfiguration; @Override - public void doRegister(final BesuContext context) { + public void doRegister(final ServiceManager serviceManager) { transactionSelectionService = - context + serviceManager .getService(TransactionSelectionService.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain TransactionSelectionService from the BesuContext.")); + "Failed to obtain TransactionSelectionService from the ServiceManager.")); besuConfiguration = - context + serviceManager .getService(BesuConfiguration.class) .orElseThrow( () -> new RuntimeException( - "Failed to obtain BesuConfiguration from the BesuContext.")); + "Failed to obtain BesuConfiguration from the ServiceManager.")); } @Override From ab849c57d9ba1e12780a167a2f17607be1ddbacf Mon Sep 17 00:00:00 2001 From: Ade Lucas <37906710+cloudspores@users.noreply.github.com> Date: Fri, 29 Nov 2024 05:57:34 -0500 Subject: [PATCH 16/19] 3D pricing metrics (#107) * Introduce profitablePriorityFeeCache Signed-off-by: Fabio Di Fabio * Add block listener and transaction selector handler - Introduce block listener in LineaTransactionSelectorPlugin to handle transaction profitability - Integrate transaction selector handler for processing transactions from the latest block - Extract profitability calculation into a separate method for clarity and maintainability Signed-off-by: Ade Lucas * Removed unused import Signed-off-by: Ade Lucas * Move profitability metrics during tx selection in a dedicated class Signed-off-by: Fabio Di Fabio * Improve the log of the future metric Signed-off-by: Fabio Di Fabio * Organize the profitability cache by block Signed-off-by: Fabio Di Fabio * Refctor to respect the requirement Signed-off-by: Fabio Di Fabio * Add block added listener and handler into LineaTransactionPoolValidatorPlugin - Implement: Node's perception on Profitability levels of TxPool's contents (lo, hi, avg of TransactionProfitabilityCalculator.profitablePriorityFeePerGas(transaction) / transaction.priorityFeePerGas). Signed-off-by: Ade Lucas * Add block added listener and handler into LineaTransactionPoolValidatorPlugin - Implement: Node's perception on Profitability levels of TxPool's contents (lo, hi, avg of TransactionProfitabilityCalculator.profitablePriorityFeePerGas(transaction) / transaction.priorityFeePerGas). Signed-off-by: Ade Lucas * Add missing import in ValidatorProfitabilityMetrics.java Signed-off-by: Ade Lucas * Use histogram and txpool get content api Signed-off-by: Fabio Di Fabio * Add min and max metrics to selection profitability metrics Signed-off-by: Fabio Di Fabio * Deleted ValidatorProfitabilityMetrics.java as part of 3D pricing metrics work Signed-off-by: Ade Lucas * Run './gradlew :sequencer:spotlessApply' to fix ProfitableTransactionSelectorTest.java Signed-off-by: Ade Lucas * Restore non-local dev settings for besuVersion in gradle.properties. Signed-off-by: Ade Lucas * Generalize histogram metrics in a single class and check for category enabled Signed-off-by: Fabio Di Fabio * Make profitability histogram metric buckets configurable Signed-off-by: Fabio Di Fabio * Update versions Signed-off-by: Fabio Di Fabio * Fix tests Signed-off-by: Fabio Di Fabio * Apply suggestions from code review Signed-off-by: Fabio Di Fabio * Update DEFAULT_PROFITABILITY_METRICS_BUCKETS Signed-off-by: Fabio Di Fabio * Update Linea Besu and Tracer Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio Signed-off-by: Ade Lucas Signed-off-by: Ade Lucas <37906710+cloudspores@users.noreply.github.com> Co-authored-by: Fabio Di Fabio --- .../plugin/acc/test/LineaPluginTestBase.java | 7 +- .../test/extradata/ExtraDataPricingTest.java | 15 +- ...stractLineaSharedPrivateOptionsPlugin.java | 16 +- .../TransactionProfitabilityCalculator.java | 22 +++ .../config/LineaProfitabilityCliOptions.java | 20 +++ .../LineaProfitabilityConfiguration.java | 2 + .../linea/extradata/LineaExtraDataPlugin.java | 14 +- .../linea/metrics/HistogramMetrics.java | 154 ++++++++++++++++++ .../linea/metrics/LineaMetricCategory.java | 14 +- .../LineaTransactionPoolValidatorPlugin.java | 48 ++++++ .../TransactionPoolProfitabilityMetrics.java | 126 ++++++++++++++ .../LineaTransactionSelectorFactory.java | 9 +- .../LineaTransactionSelectorPlugin.java | 23 ++- .../selectors/LineaTransactionSelector.java | 16 +- .../ProfitableTransactionSelector.java | 121 +++++++++++++- .../ProfitableTransactionSelectorTest.java | 8 +- 16 files changed, 571 insertions(+), 44 deletions(-) create mode 100644 sequencer/src/main/java/net/consensys/linea/metrics/HistogramMetrics.java create mode 100644 sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/metrics/TransactionPoolProfitabilityMetrics.java diff --git a/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java b/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java index a5d52571..6ad55f56 100644 --- a/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java +++ b/acceptance-tests/src/test/java/linea/plugin/acc/test/LineaPluginTestBase.java @@ -15,6 +15,9 @@ package linea.plugin.acc.test; +import static net.consensys.linea.metrics.LineaMetricCategory.PRICING_CONF; +import static net.consensys.linea.metrics.LineaMetricCategory.SEQUENCER_PROFITABILITY; +import static net.consensys.linea.metrics.LineaMetricCategory.TX_POOL_PROFITABILITY; import static org.assertj.core.api.Assertions.*; import java.io.IOException; @@ -40,7 +43,6 @@ import linea.plugin.acc.test.tests.web3j.generated.RevertExample; import linea.plugin.acc.test.tests.web3j.generated.SimpleStorage; import lombok.extern.slf4j.Slf4j; -import net.consensys.linea.metrics.LineaMetricCategory; import org.apache.commons.lang3.RandomStringUtils; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; @@ -136,7 +138,8 @@ private BesuNode createCliqueNodeWithExtraCliOptionsAndRpcApis( .metricsConfiguration( MetricsConfiguration.builder() .enabled(true) - .metricCategories(Set.of(LineaMetricCategory.PROFITABILITY)) + .metricCategories( + Set.of(PRICING_CONF, SEQUENCER_PROFITABILITY, TX_POOL_PROFITABILITY)) .build()) .requestedPlugins( List.of( diff --git a/acceptance-tests/src/test/java/linea/plugin/acc/test/extradata/ExtraDataPricingTest.java b/acceptance-tests/src/test/java/linea/plugin/acc/test/extradata/ExtraDataPricingTest.java index c6cab90a..fce533ba 100644 --- a/acceptance-tests/src/test/java/linea/plugin/acc/test/extradata/ExtraDataPricingTest.java +++ b/acceptance-tests/src/test/java/linea/plugin/acc/test/extradata/ExtraDataPricingTest.java @@ -15,6 +15,7 @@ package linea.plugin.acc.test.extradata; import static java.util.Map.entry; +import static net.consensys.linea.metrics.LineaMetricCategory.PRICING_CONF; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; @@ -23,7 +24,6 @@ import linea.plugin.acc.test.LineaPluginTestBase; import linea.plugin.acc.test.TestCommandLineOptionsBuilder; -import net.consensys.linea.metrics.LineaMetricCategory; import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.tests.acceptance.dsl.account.Account; @@ -125,24 +125,17 @@ public void updateProfitabilityParamsViaExtraData() throws IOException, Interrup assertThat(getTxPoolContent()).isEmpty(); final var fixedCostMetric = - getMetricValue( - LineaMetricCategory.PROFITABILITY, "conf", List.of(entry("field", "fixed_cost_wei"))); + getMetricValue(PRICING_CONF, "values", List.of(entry("field", "fixed_cost_wei"))); assertThat(fixedCostMetric).isEqualTo(MIN_GAS_PRICE.multiply(2).getValue().doubleValue()); final var variableCostMetric = - getMetricValue( - LineaMetricCategory.PROFITABILITY, - "conf", - List.of(entry("field", "variable_cost_wei"))); + getMetricValue(PRICING_CONF, "values", List.of(entry("field", "variable_cost_wei"))); assertThat(variableCostMetric).isEqualTo(MIN_GAS_PRICE.getValue().doubleValue()); final var ethGasPriceMetric = - getMetricValue( - LineaMetricCategory.PROFITABILITY, - "conf", - List.of(entry("field", "eth_gas_price_wei"))); + getMetricValue(PRICING_CONF, "values", List.of(entry("field", "eth_gas_price_wei"))); assertThat(ethGasPriceMetric).isEqualTo(MIN_GAS_PRICE.getValue().doubleValue()); } diff --git a/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java b/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java index c8898c64..56aa94eb 100644 --- a/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/AbstractLineaSharedPrivateOptionsPlugin.java @@ -33,7 +33,6 @@ import net.consensys.linea.config.LineaTransactionPoolValidatorConfiguration; import net.consensys.linea.config.LineaTransactionSelectorCliOptions; import net.consensys.linea.config.LineaTransactionSelectorConfiguration; -import net.consensys.linea.metrics.LineaMetricCategory; import net.consensys.linea.plugins.AbstractLineaSharedOptionsPlugin; import net.consensys.linea.plugins.LineaOptionsPluginConfiguration; import org.hyperledger.besu.plugin.ServiceManager; @@ -59,6 +58,7 @@ public abstract class AbstractLineaSharedPrivateOptionsPlugin extends AbstractLineaSharedOptionsPlugin { protected static BlockchainService blockchainService; protected static MetricsSystem metricsSystem; + protected static MetricCategoryRegistry metricCategoryRegistry; private static final AtomicBoolean sharedRegisterTasksDone = new AtomicBoolean(false); private static final AtomicBoolean sharedStartTasksDone = new AtomicBoolean(false); @@ -143,13 +143,13 @@ protected static void performSharedRegisterTasksOnce(final ServiceManager servic new RuntimeException( "Failed to obtain BlockchainService from the ServiceManager.")); - serviceManager - .getService(MetricCategoryRegistry.class) - .orElseThrow( - () -> - new RuntimeException( - "Failed to obtain MetricCategoryRegistry from the ServiceManager.")) - .addMetricCategory(LineaMetricCategory.PROFITABILITY); + metricCategoryRegistry = + serviceManager + .getService(MetricCategoryRegistry.class) + .orElseThrow( + () -> + new RuntimeException( + "Failed to obtain MetricCategoryRegistry from the ServiceManager.")); } @Override diff --git a/sequencer/src/main/java/net/consensys/linea/bl/TransactionProfitabilityCalculator.java b/sequencer/src/main/java/net/consensys/linea/bl/TransactionProfitabilityCalculator.java index 3c71015f..6ef2e89a 100644 --- a/sequencer/src/main/java/net/consensys/linea/bl/TransactionProfitabilityCalculator.java +++ b/sequencer/src/main/java/net/consensys/linea/bl/TransactionProfitabilityCalculator.java @@ -113,6 +113,28 @@ public boolean isProfitable( final Wei profitablePriorityFee = profitablePriorityFeePerGas(transaction, minMargin, gas, minGasPriceWei); + + return isProfitable( + context, + profitablePriorityFee, + transaction, + minMargin, + baseFee, + payingGasPrice, + gas, + minGasPriceWei); + } + + public boolean isProfitable( + final String context, + final Wei profitablePriorityFee, + final Transaction transaction, + final double minMargin, + final Wei baseFee, + final Wei payingGasPrice, + final long gas, + final Wei minGasPriceWei) { + final Wei profitableGasPrice = baseFee.add(profitablePriorityFee); if (payingGasPrice.lessThan(profitableGasPrice)) { diff --git a/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityCliOptions.java b/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityCliOptions.java index 037c4a90..36ed3637 100644 --- a/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityCliOptions.java +++ b/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityCliOptions.java @@ -57,6 +57,12 @@ public class LineaProfitabilityCliOptions implements LineaCliOptions { "--plugin-linea-extra-data-set-min-gas-price-enabled"; public static final boolean DEFAULT_EXTRA_DATA_SET_MIN_GAS_PRICE_ENABLED = true; + public static final String PROFITABILITY_METRICS_BUCKETS = + "--plugin-linea-profitability-metrics-buckets"; + public static final double[] DEFAULT_PROFITABILITY_METRICS_BUCKETS = { + 0.1, 0.3, 0.5, 0.7, 0.9, 1.0, 1.1, 1.2, 1.5, 2.0, 5.0, 10.0 + }; + @Positive @CommandLine.Option( names = {FIXED_GAS_COST_WEI}, @@ -135,6 +141,17 @@ public class LineaProfitabilityCliOptions implements LineaCliOptions { "Enable setting min gas price runtime value via extra data field (default: ${DEFAULT-VALUE})") private boolean extraDataSetMinGasPriceEnabled = DEFAULT_EXTRA_DATA_SET_MIN_GAS_PRICE_ENABLED; + @CommandLine.Option( + names = {PROFITABILITY_METRICS_BUCKETS}, + arity = "1..*", + split = ",", + hidden = true, + paramLabel = "", + description = + "List of buckets to use to create the histogram for ratio between the effective priority fee " + + "and the calculate profitable priority of the tx (default: ${DEFAULT-VALUE})") + private double[] profitabilityMetricsBuckets = DEFAULT_PROFITABILITY_METRICS_BUCKETS; + private LineaProfitabilityCliOptions() {} /** @@ -164,6 +181,7 @@ public static LineaProfitabilityCliOptions fromConfig( options.txPoolCheckP2pEnabled = config.txPoolCheckP2pEnabled(); options.extraDataPricingEnabled = config.extraDataPricingEnabled(); options.extraDataSetMinGasPriceEnabled = config.extraDataSetMinGasPriceEnabled(); + options.profitabilityMetricsBuckets = config.profitabilityMetricsBuckets(); return options; } @@ -184,6 +202,7 @@ public LineaProfitabilityConfiguration toDomainObject() { .txPoolCheckP2pEnabled(txPoolCheckP2pEnabled) .extraDataPricingEnabled(extraDataPricingEnabled) .extraDataSetMinGasPriceEnabled(extraDataSetMinGasPriceEnabled) + .profitabilityMetricsBuckets(profitabilityMetricsBuckets) .build(); } @@ -199,6 +218,7 @@ public String toString() { .add(TX_POOL_ENABLE_CHECK_P2P, txPoolCheckP2pEnabled) .add(EXTRA_DATA_PRICING_ENABLED, extraDataPricingEnabled) .add(EXTRA_DATA_SET_MIN_GAS_PRICE_ENABLED, extraDataSetMinGasPriceEnabled) + .add(PROFITABILITY_METRICS_BUCKETS, profitabilityMetricsBuckets) .toString(); } } diff --git a/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityConfiguration.java b/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityConfiguration.java index 6aa5af24..012ad4d7 100644 --- a/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityConfiguration.java +++ b/sequencer/src/main/java/net/consensys/linea/config/LineaProfitabilityConfiguration.java @@ -33,6 +33,7 @@ public class LineaProfitabilityConfiguration implements LineaOptionsConfiguratio /** It is safe to keep this as long, since it will store value <= max_int * 1000 */ private long variableCostWei; + /** It is safe to keep this as long, since it will store value <= max_int * 1000 */ private long ethGasPriceWei; private double minMargin; @@ -42,6 +43,7 @@ public class LineaProfitabilityConfiguration implements LineaOptionsConfiguratio private boolean txPoolCheckP2pEnabled; private boolean extraDataPricingEnabled; private boolean extraDataSetMinGasPriceEnabled; + private double[] profitabilityMetricsBuckets; /** * These 2 parameters must be atomically updated diff --git a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java index 907e3bd0..f43f477c 100644 --- a/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/extradata/LineaExtraDataPlugin.java @@ -15,13 +15,14 @@ package net.consensys.linea.extradata; +import static net.consensys.linea.metrics.LineaMetricCategory.PRICING_CONF; + import java.util.concurrent.atomic.AtomicBoolean; import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.AbstractLineaRequiredPlugin; import net.consensys.linea.config.LineaProfitabilityConfiguration; -import net.consensys.linea.metrics.LineaMetricCategory; import org.hyperledger.besu.plugin.BesuPlugin; import org.hyperledger.besu.plugin.ServiceManager; import org.hyperledger.besu.plugin.data.AddedBlockContext; @@ -46,6 +47,8 @@ public void doRegister(final ServiceManager context) { () -> new RuntimeException( "Failed to obtain RpcEndpointService from the ServiceManager.")); + + metricCategoryRegistry.addMetricCategory(PRICING_CONF); } /** @@ -95,16 +98,15 @@ public synchronized void onInitialSyncRestart() { }); } - initMetrics(profitabilityConfiguration()); + if (metricCategoryRegistry.isMetricCategoryEnabled(PRICING_CONF)) { + initMetrics(profitabilityConfiguration()); + } } private void initMetrics(final LineaProfitabilityConfiguration lineaProfitabilityConfiguration) { final var confLabelledGauge = metricsSystem.createLabelledSuppliedGauge( - LineaMetricCategory.PROFITABILITY, - "conf", - "Profitability configuration values at runtime", - "field"); + PRICING_CONF, "values", "Profitability configuration values at runtime", "field"); confLabelledGauge.labels(lineaProfitabilityConfiguration::fixedCostWei, "fixed_cost_wei"); confLabelledGauge.labels(lineaProfitabilityConfiguration::variableCostWei, "variable_cost_wei"); confLabelledGauge.labels(lineaProfitabilityConfiguration::ethGasPriceWei, "eth_gas_price_wei"); diff --git a/sequencer/src/main/java/net/consensys/linea/metrics/HistogramMetrics.java b/sequencer/src/main/java/net/consensys/linea/metrics/HistogramMetrics.java new file mode 100644 index 00000000..a7702d38 --- /dev/null +++ b/sequencer/src/main/java/net/consensys/linea/metrics/HistogramMetrics.java @@ -0,0 +1,154 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.metrics; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.DoubleSupplier; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.metrics.Histogram; +import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; +import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric; + +@Slf4j +public class HistogramMetrics { + public interface LabelValue { + String value(); + } + + private static final String LABEL_VALUES_SEPARATOR = "\u2060"; + private final LabelledMetric histogram; + private final Map mins; + private final Map maxs; + + @SafeVarargs + public HistogramMetrics( + final MetricsSystem metricsSystem, + final LineaMetricCategory category, + final String name, + final String help, + final double[] buckets, + final Class... labels) { + + final var labelNames = getLabelNames(labels); + + final LabelledSuppliedMetric minRatio = + metricsSystem.createLabelledSuppliedGauge( + category, name + "_min", "Lowest " + help, labelNames); + + final LabelledSuppliedMetric maxRatio = + metricsSystem.createLabelledSuppliedGauge( + category, name + "_max", "Highest " + help, labelNames); + + final var combinations = getLabelValuesCombinations(labels); + mins = HashMap.newHashMap(combinations.size()); + maxs = HashMap.newHashMap(combinations.size()); + for (final var combination : combinations) { + final var key = String.join(LABEL_VALUES_SEPARATOR, combination); + final var minSupplier = new MutableDoubleSupplier(Double.POSITIVE_INFINITY); + mins.put(key, minSupplier); + minRatio.labels(minSupplier, combination); + final var maxSupplier = new MutableDoubleSupplier(Double.NEGATIVE_INFINITY); + maxs.put(key, maxSupplier); + maxRatio.labels(maxSupplier, combination); + } + + this.histogram = + metricsSystem.createLabelledHistogram( + category, name, StringUtils.capitalize(help) + " buckets", buckets, labelNames); + } + + @SafeVarargs + private String[] getLabelNames(final Class... labels) { + return Arrays.stream(labels) + .map(Class::getSimpleName) + .map(sn -> sn.toLowerCase(Locale.ROOT)) + .toArray(String[]::new); + } + + @SafeVarargs + private List getLabelValuesCombinations(final Class... labels) { + if (labels.length == 0) { + return Collections.singletonList(new String[0]); + } + if (labels.length == 1) { + return Arrays.stream(labels[0].getEnumConstants()) + .map(lv -> new String[] {lv.value()}) + .toList(); + } + final var head = labels[0]; + final var tail = Arrays.copyOfRange(labels, 1, labels.length); + final var tailCombinations = getLabelValuesCombinations(tail); + final int newSize = tailCombinations.size() * head.getEnumConstants().length; + final List combinations = new ArrayList<>(newSize); + for (final var headValue : head.getEnumConstants()) { + for (final var tailValues : tailCombinations) { + final var combination = new String[tailValues.length + 1]; + combination[0] = headValue.value(); + System.arraycopy(tailValues, 0, combination, 1, tailValues.length); + combinations.add(combination); + } + } + return combinations; + } + + public void track(final double value, final String... labelValues) { + + // Record the observation + histogram.labels(labelValues).observe(value); + } + + public void setMinMax(final double min, final double max, final String... labelValues) { + final var key = String.join(LABEL_VALUES_SEPARATOR, labelValues); + + // Update lowest seen + mins.get(key).set(min); + + // Update highest seen + maxs.get(key).set(max); + } + + private static class MutableDoubleSupplier implements DoubleSupplier { + private final double initialValue; + private volatile double value; + + public MutableDoubleSupplier(final double initialValue) { + this.initialValue = initialValue; + this.value = initialValue; + } + + @Override + public double getAsDouble() { + return value; + } + + public void set(final double value) { + this.value = value; + } + + public void reset() { + value = initialValue; + } + } +} diff --git a/sequencer/src/main/java/net/consensys/linea/metrics/LineaMetricCategory.java b/sequencer/src/main/java/net/consensys/linea/metrics/LineaMetricCategory.java index 31ade658..849e7ddb 100644 --- a/sequencer/src/main/java/net/consensys/linea/metrics/LineaMetricCategory.java +++ b/sequencer/src/main/java/net/consensys/linea/metrics/LineaMetricCategory.java @@ -14,19 +14,25 @@ */ package net.consensys.linea.metrics; +import java.util.Locale; import java.util.Optional; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; public enum LineaMetricCategory implements MetricCategory { - /** Profitability metric category */ - PROFITABILITY("profitability"); + /** Sequencer profitability metric category */ + SEQUENCER_PROFITABILITY, + /** Tx pool profitability metric category */ + TX_POOL_PROFITABILITY, + /** Runtime pricing configuration */ + PRICING_CONF; private static final Optional APPLICATION_PREFIX = Optional.of("linea_"); + private final String name; - LineaMetricCategory(final String name) { - this.name = name; + LineaMetricCategory() { + this.name = name().toLowerCase(Locale.ROOT); } @Override diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java index 4ca4a2ac..f20e9350 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/LineaTransactionPoolValidatorPlugin.java @@ -15,6 +15,7 @@ package net.consensys.linea.sequencer.txpoolvalidation; +import static net.consensys.linea.metrics.LineaMetricCategory.TX_POOL_PROFITABILITY; import static net.consensys.linea.sequencer.modulelimit.ModuleLineCountValidator.createLimitModules; import java.io.File; @@ -30,12 +31,15 @@ import net.consensys.linea.AbstractLineaRequiredPlugin; import net.consensys.linea.config.LineaRejectedTxReportingConfiguration; import net.consensys.linea.jsonrpc.JsonRpcManager; +import net.consensys.linea.sequencer.txpoolvalidation.metrics.TransactionPoolProfitabilityMetrics; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.plugin.BesuPlugin; import org.hyperledger.besu.plugin.ServiceManager; import org.hyperledger.besu.plugin.services.BesuConfiguration; +import org.hyperledger.besu.plugin.services.BesuEvents; import org.hyperledger.besu.plugin.services.TransactionPoolValidatorService; import org.hyperledger.besu.plugin.services.TransactionSimulationService; +import org.hyperledger.besu.plugin.services.transactionpool.TransactionPoolService; /** * This class extends the default transaction validation rules for adding transactions to the @@ -46,6 +50,7 @@ @Slf4j @AutoService(BesuPlugin.class) public class LineaTransactionPoolValidatorPlugin extends AbstractLineaRequiredPlugin { + private ServiceManager serviceManager; private BesuConfiguration besuConfiguration; private TransactionPoolValidatorService transactionPoolValidatorService; private TransactionSimulationService transactionSimulationService; @@ -53,6 +58,7 @@ public class LineaTransactionPoolValidatorPlugin extends AbstractLineaRequiredPl @Override public void doRegister(final ServiceManager serviceManager) { + this.serviceManager = serviceManager; besuConfiguration = serviceManager .getService(BesuConfiguration.class) @@ -76,6 +82,8 @@ public void doRegister(final ServiceManager serviceManager) { () -> new RuntimeException( "Failed to obtain TransactionSimulatorService from the ServiceManager.")); + + metricCategoryRegistry.addMetricCategory(TX_POOL_PROFITABILITY); } @Override @@ -113,6 +121,46 @@ public void start() { l1L2BridgeSharedConfiguration(), rejectedTxJsonRpcManager)); + if (metricCategoryRegistry.isMetricCategoryEnabled(TX_POOL_PROFITABILITY)) { + final var besuEventsService = + serviceManager + .getService(BesuEvents.class) + .orElseThrow( + () -> + new RuntimeException( + "Failed to obtain BesuEvents from the ServiceManager.")); + + final var transactionPoolService = + serviceManager + .getService(TransactionPoolService.class) + .orElseThrow( + () -> + new RuntimeException( + "Failed to obtain TransactionPoolService from the ServiceManager.")); + + final var transactionPoolProfitabilityMetrics = + new TransactionPoolProfitabilityMetrics( + besuConfiguration, + metricsSystem, + profitabilityConfiguration(), + transactionPoolService, + blockchainService); + + besuEventsService.addBlockAddedListener( + addedBlockContext -> { + try { + // on new block let's calculate profitability for every txs in the pool + transactionPoolProfitabilityMetrics.update(); + } catch (final Exception e) { + log.warn( + "Error calculating transaction profitability for block {}({})", + addedBlockContext.getBlockHeader().getNumber(), + addedBlockContext.getBlockHeader().getBlockHash(), + e); + } + }); + } + } catch (Exception e) { throw new RuntimeException(e); } diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/metrics/TransactionPoolProfitabilityMetrics.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/metrics/TransactionPoolProfitabilityMetrics.java new file mode 100644 index 00000000..fccc65fa --- /dev/null +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/metrics/TransactionPoolProfitabilityMetrics.java @@ -0,0 +1,126 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package net.consensys.linea.sequencer.txpoolvalidation.metrics; + +import static net.consensys.linea.metrics.LineaMetricCategory.TX_POOL_PROFITABILITY; + +import java.util.stream.Collectors; + +import lombok.extern.slf4j.Slf4j; +import net.consensys.linea.bl.TransactionProfitabilityCalculator; +import net.consensys.linea.config.LineaProfitabilityConfiguration; +import net.consensys.linea.metrics.HistogramMetrics; +import org.apache.tuweni.units.bigints.UInt256s; +import org.hyperledger.besu.datatypes.PendingTransaction; +import org.hyperledger.besu.datatypes.Transaction; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.plugin.services.BesuConfiguration; +import org.hyperledger.besu.plugin.services.BlockchainService; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.transactionpool.TransactionPoolService; + +/** + * Tracks profitability metrics for transactions in the transaction pool. Specifically monitors the + * ratio of profitable priority fee to actual priority fee: + * profitablePriorityFeePerGas/transaction.priorityFeePerGas + * + *

Provides: - Lowest ratio seen (minimum profitability) - Highest ratio seen (maximum + * profitability) - Distribution histogram of ratios + */ +@Slf4j +public class TransactionPoolProfitabilityMetrics { + private final TransactionProfitabilityCalculator profitabilityCalculator; + private final LineaProfitabilityConfiguration profitabilityConf; + private final BesuConfiguration besuConfiguration; + private final TransactionPoolService transactionPoolService; + private final BlockchainService blockchainService; + private final HistogramMetrics histogramMetrics; + + public TransactionPoolProfitabilityMetrics( + final BesuConfiguration besuConfiguration, + final MetricsSystem metricsSystem, + final LineaProfitabilityConfiguration profitabilityConf, + final TransactionPoolService transactionPoolService, + final BlockchainService blockchainService) { + + this.besuConfiguration = besuConfiguration; + this.profitabilityConf = profitabilityConf; + this.profitabilityCalculator = new TransactionProfitabilityCalculator(profitabilityConf); + this.transactionPoolService = transactionPoolService; + this.blockchainService = blockchainService; + this.histogramMetrics = + new HistogramMetrics( + metricsSystem, + TX_POOL_PROFITABILITY, + "ratio", + "transaction pool profitability ratio", + profitabilityConf.profitabilityMetricsBuckets()); + } + + public void update() { + final long startTime = System.currentTimeMillis(); + final var txPoolContent = transactionPoolService.getPendingTransactions(); + + final var ratioStats = + txPoolContent.parallelStream() + .map(PendingTransaction::getTransaction) + .map( + tx -> { + final var ratio = handleTransaction(tx); + histogramMetrics.track(ratio); + log.trace("Recorded profitability ratio {} for tx {}", ratio, tx.getHash()); + return ratio; + }) + .collect(Collectors.summarizingDouble(Double::doubleValue)); + + histogramMetrics.setMinMax(ratioStats.getMin(), ratioStats.getMax()); + + log.atDebug() + .setMessage("Transaction pool profitability metrics processed {}txs in {}ms, statistics {}") + .addArgument(txPoolContent::size) + .addArgument(() -> System.currentTimeMillis() - startTime) + .addArgument(ratioStats) + .log(); + } + + private double handleTransaction(final Transaction transaction) { + final Wei actualPriorityFeePerGas; + if (transaction.getMaxPriorityFeePerGas().isEmpty()) { + actualPriorityFeePerGas = + Wei.fromQuantity(transaction.getGasPrice().orElseThrow()) + .subtract(blockchainService.getNextBlockBaseFee().orElseThrow()); + } else { + final Wei maxPriorityFeePerGas = + Wei.fromQuantity(transaction.getMaxPriorityFeePerGas().get()); + actualPriorityFeePerGas = + UInt256s.min( + maxPriorityFeePerGas.add(blockchainService.getNextBlockBaseFee().orElseThrow()), + Wei.fromQuantity(transaction.getMaxFeePerGas().orElseThrow())); + } + + final Wei profitablePriorityFeePerGas = + profitabilityCalculator.profitablePriorityFeePerGas( + transaction, + profitabilityConf.txPoolMinMargin(), + transaction.getGasLimit(), + besuConfiguration.getMinGasPrice()); + + final double ratio = + actualPriorityFeePerGas.toBigInteger().doubleValue() + / profitablePriorityFeePerGas.toBigInteger().doubleValue(); + + return ratio; + } +} diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorFactory.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorFactory.java index 0d5fc6ff..529e9bb0 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorFactory.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorFactory.java @@ -22,6 +22,7 @@ import net.consensys.linea.config.LineaTracerConfiguration; import net.consensys.linea.config.LineaTransactionSelectorConfiguration; import net.consensys.linea.jsonrpc.JsonRpcManager; +import net.consensys.linea.metrics.HistogramMetrics; import net.consensys.linea.plugins.config.LineaL1L2BridgeSharedConfiguration; import net.consensys.linea.sequencer.txselection.selectors.LineaTransactionSelector; import org.hyperledger.besu.plugin.services.BlockchainService; @@ -39,6 +40,7 @@ public class LineaTransactionSelectorFactory implements PluginTransactionSelecto private final LineaL1L2BridgeSharedConfiguration l1L2BridgeConfiguration; private final LineaProfitabilityConfiguration profitabilityConfiguration; private final LineaTracerConfiguration tracerConfiguration; + private final Optional maybeProfitabilityMetrics; private final Map limitsMap; @@ -49,7 +51,8 @@ public LineaTransactionSelectorFactory( final LineaProfitabilityConfiguration profitabilityConfiguration, final LineaTracerConfiguration tracerConfiguration, final Map limitsMap, - final Optional rejectedTxJsonRpcManager) { + final Optional rejectedTxJsonRpcManager, + final Optional maybeProfitabilityMetrics) { this.blockchainService = blockchainService; this.txSelectorConfiguration = txSelectorConfiguration; this.l1L2BridgeConfiguration = l1L2BridgeConfiguration; @@ -57,6 +60,7 @@ public LineaTransactionSelectorFactory( this.tracerConfiguration = tracerConfiguration; this.limitsMap = limitsMap; this.rejectedTxJsonRpcManager = rejectedTxJsonRpcManager; + this.maybeProfitabilityMetrics = maybeProfitabilityMetrics; } @Override @@ -68,6 +72,7 @@ public PluginTransactionSelector create() { profitabilityConfiguration, tracerConfiguration, limitsMap, - rejectedTxJsonRpcManager); + rejectedTxJsonRpcManager, + maybeProfitabilityMetrics); } } diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java index e25a0c03..7be6cc93 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java @@ -15,6 +15,7 @@ package net.consensys.linea.sequencer.txselection; +import static net.consensys.linea.metrics.LineaMetricCategory.SEQUENCER_PROFITABILITY; import static net.consensys.linea.sequencer.modulelimit.ModuleLineCountValidator.createLimitModules; import java.util.Optional; @@ -25,6 +26,8 @@ import net.consensys.linea.config.LineaRejectedTxReportingConfiguration; import net.consensys.linea.config.LineaTransactionSelectorConfiguration; import net.consensys.linea.jsonrpc.JsonRpcManager; +import net.consensys.linea.metrics.HistogramMetrics; +import net.consensys.linea.sequencer.txselection.selectors.ProfitableTransactionSelector; import org.hyperledger.besu.plugin.BesuPlugin; import org.hyperledger.besu.plugin.ServiceManager; import org.hyperledger.besu.plugin.services.BesuConfiguration; @@ -38,12 +41,15 @@ @Slf4j @AutoService(BesuPlugin.class) public class LineaTransactionSelectorPlugin extends AbstractLineaRequiredPlugin { + public static final String NAME = "linea"; + private ServiceManager serviceManager; private TransactionSelectionService transactionSelectionService; private Optional rejectedTxJsonRpcManager = Optional.empty(); private BesuConfiguration besuConfiguration; @Override public void doRegister(final ServiceManager serviceManager) { + this.serviceManager = serviceManager; transactionSelectionService = serviceManager .getService(TransactionSelectionService.class) @@ -59,6 +65,7 @@ public void doRegister(final ServiceManager serviceManager) { () -> new RuntimeException( "Failed to obtain BesuConfiguration from the ServiceManager.")); + metricCategoryRegistry.addMetricCategory(SEQUENCER_PROFITABILITY); } @Override @@ -78,6 +85,19 @@ public void start() { besuConfiguration.getDataPath(), lineaRejectedTxReportingConfiguration) .start()); + + final Optional maybeProfitabilityMetrics = + metricCategoryRegistry.isMetricCategoryEnabled(SEQUENCER_PROFITABILITY) + ? Optional.of( + new HistogramMetrics( + metricsSystem, + SEQUENCER_PROFITABILITY, + "ratio", + "sequencer profitability ratio", + profitabilityConfiguration().profitabilityMetricsBuckets(), + ProfitableTransactionSelector.Phase.class)) + : Optional.empty(); + transactionSelectionService.registerPluginTransactionSelectorFactory( new LineaTransactionSelectorFactory( blockchainService, @@ -86,7 +106,8 @@ public void start() { profitabilityConfiguration(), tracerConfiguration(), createLimitModules(tracerConfiguration()), - rejectedTxJsonRpcManager)); + rejectedTxJsonRpcManager, + maybeProfitabilityMetrics)); } @Override diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java index 2d6586d4..61e2b883 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java @@ -30,6 +30,7 @@ import net.consensys.linea.config.LineaTransactionSelectorConfiguration; import net.consensys.linea.jsonrpc.JsonRpcManager; import net.consensys.linea.jsonrpc.JsonRpcRequestBuilder; +import net.consensys.linea.metrics.HistogramMetrics; import net.consensys.linea.plugins.config.LineaL1L2BridgeSharedConfiguration; import org.hyperledger.besu.datatypes.PendingTransaction; import org.hyperledger.besu.plugin.data.TransactionProcessingResult; @@ -55,7 +56,8 @@ public LineaTransactionSelector( final LineaProfitabilityConfiguration profitabilityConfiguration, final LineaTracerConfiguration tracerConfiguration, final Map limitsMap, - final Optional rejectedTxJsonRpcManager) { + final Optional rejectedTxJsonRpcManager, + final Optional maybeProfitabilityMetrics) { this.rejectedTxJsonRpcManager = rejectedTxJsonRpcManager; // only report rejected transaction selection result from TraceLineLimitTransactionSelector @@ -71,7 +73,8 @@ public LineaTransactionSelector( l1L2BridgeConfiguration, profitabilityConfiguration, tracerConfiguration, - limitsMap); + limitsMap, + maybeProfitabilityMetrics); } /** @@ -81,6 +84,7 @@ public LineaTransactionSelector( * @param txSelectorConfiguration The configuration to use. * @param profitabilityConfiguration The profitability configuration. * @param limitsMap The limits map. + * @param maybeProfitabilityMetrics The optional profitability metrics * @return A list of selectors. */ private List createTransactionSelectors( @@ -89,7 +93,8 @@ private List createTransactionSelectors( final LineaL1L2BridgeSharedConfiguration l1L2BridgeConfiguration, final LineaProfitabilityConfiguration profitabilityConfiguration, final LineaTracerConfiguration tracerConfiguration, - final Map limitsMap) { + final Map limitsMap, + final Optional maybeProfitabilityMetrics) { traceLineLimitTransactionSelector = new TraceLineLimitTransactionSelector( @@ -103,7 +108,10 @@ private List createTransactionSelectors( new MaxBlockCallDataTransactionSelector(txSelectorConfiguration.maxBlockCallDataSize()), new MaxBlockGasTransactionSelector(txSelectorConfiguration.maxGasPerBlock()), new ProfitableTransactionSelector( - blockchainService, txSelectorConfiguration, profitabilityConfiguration), + blockchainService, + txSelectorConfiguration, + profitabilityConfiguration, + maybeProfitabilityMetrics), traceLineLimitTransactionSelector); } diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/ProfitableTransactionSelector.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/ProfitableTransactionSelector.java index 0ba8e371..2cd6768f 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/ProfitableTransactionSelector.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txselection/selectors/ProfitableTransactionSelector.java @@ -19,7 +19,11 @@ import static net.consensys.linea.sequencer.txselection.LineaTransactionSelectionResult.TX_UNPROFITABLE_UPFRONT; import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.SELECTED; +import java.util.EnumMap; import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; import java.util.Set; import com.google.common.annotations.VisibleForTesting; @@ -27,6 +31,8 @@ import net.consensys.linea.bl.TransactionProfitabilityCalculator; import net.consensys.linea.config.LineaProfitabilityConfiguration; import net.consensys.linea.config.LineaTransactionSelectorConfiguration; +import net.consensys.linea.metrics.HistogramMetrics; +import net.consensys.linea.metrics.HistogramMetrics.LabelValue; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.PendingTransaction; import org.hyperledger.besu.datatypes.Transaction; @@ -48,11 +54,34 @@ */ @Slf4j public class ProfitableTransactionSelector implements PluginTransactionSelector { + public enum Phase implements LabelValue { + PRE_PROCESSING, + POST_PROCESSING; + + final String value; + + Phase() { + this.value = name().toLowerCase(Locale.ROOT); + } + + @Override + public String value() { + return value; + } + } + @VisibleForTesting protected static Set unprofitableCache = new LinkedHashSet<>(); + protected static Map lastBlockMinRatios = new EnumMap<>(Phase.class); + protected static Map lastBlockMaxRatios = new EnumMap<>(Phase.class); + + static { + resetMinMaxRatios(); + } private final LineaTransactionSelectorConfiguration txSelectorConf; private final LineaProfitabilityConfiguration profitabilityConf; private final TransactionProfitabilityCalculator transactionProfitabilityCalculator; + private final Optional maybeProfitabilityMetrics; private final Wei baseFee; private int unprofitableRetries; @@ -60,11 +89,34 @@ public class ProfitableTransactionSelector implements PluginTransactionSelector public ProfitableTransactionSelector( final BlockchainService blockchainService, final LineaTransactionSelectorConfiguration txSelectorConf, - final LineaProfitabilityConfiguration profitabilityConf) { + final LineaProfitabilityConfiguration profitabilityConf, + final Optional maybeProfitabilityMetrics) { this.txSelectorConf = txSelectorConf; this.profitabilityConf = profitabilityConf; this.transactionProfitabilityCalculator = new TransactionProfitabilityCalculator(profitabilityConf); + this.maybeProfitabilityMetrics = maybeProfitabilityMetrics; + maybeProfitabilityMetrics.ifPresent( + histogramMetrics -> { + // temporary solution to update min and max metrics + // we should do this just after the block is created, but we do not have any API for that + // so we postponed the update asap the next block creation starts. + histogramMetrics.setMinMax( + lastBlockMinRatios.get(Phase.PRE_PROCESSING), + lastBlockMaxRatios.get(Phase.PRE_PROCESSING), + Phase.PRE_PROCESSING.value()); + histogramMetrics.setMinMax( + lastBlockMinRatios.get(Phase.POST_PROCESSING), + lastBlockMaxRatios.get(Phase.POST_PROCESSING), + Phase.POST_PROCESSING.value()); + log.atTrace() + .setMessage("Setting profitability ratio metrics for last block to min={}, max={}") + .addArgument(lastBlockMinRatios) + .addArgument(lastBlockMaxRatios) + .log(); + resetMinMaxRatios(); + }); + this.baseFee = blockchainService .getNextBlockBaseFee() @@ -94,9 +146,17 @@ public TransactionSelectionResult evaluateTransactionPreProcessing( final Transaction transaction = evaluationContext.getPendingTransaction().getTransaction(); final long gasLimit = transaction.getGasLimit(); + final var profitablePriorityFeePerGas = + transactionProfitabilityCalculator.profitablePriorityFeePerGas( + transaction, profitabilityConf.minMargin(), gasLimit, minGasPrice); + + updateMetric( + Phase.PRE_PROCESSING, evaluationContext, transaction, profitablePriorityFeePerGas); + // check the upfront profitability using the gas limit of the tx if (!transactionProfitabilityCalculator.isProfitable( "PreProcessing", + profitablePriorityFeePerGas, transaction, profitabilityConf.minMargin(), baseFee, @@ -145,8 +205,19 @@ public TransactionSelectionResult evaluateTransactionPostProcessing( final Transaction transaction = evaluationContext.getPendingTransaction().getTransaction(); final long gasUsed = processingResult.getEstimateGasUsedByTransaction(); + final var profitablePriorityFeePerGas = + transactionProfitabilityCalculator.profitablePriorityFeePerGas( + transaction, + profitabilityConf.minMargin(), + gasUsed, + evaluationContext.getMinGasPrice()); + + updateMetric( + Phase.POST_PROCESSING, evaluationContext, transaction, profitablePriorityFeePerGas); + if (!transactionProfitabilityCalculator.isProfitable( "PostProcessing", + profitablePriorityFeePerGas, transaction, profitabilityConf.minMargin(), baseFee, @@ -185,9 +256,9 @@ public void onTransactionSelected( public void onTransactionNotSelected( final TransactionEvaluationContext evaluationContext, final TransactionSelectionResult transactionSelectionResult) { + final var txHash = evaluationContext.getPendingTransaction().getTransaction().getHash(); if (transactionSelectionResult.discard()) { - unprofitableCache.remove( - evaluationContext.getPendingTransaction().getTransaction().getHash()); + unprofitableCache.remove(txHash); } } @@ -202,4 +273,48 @@ private void rememberUnprofitable(final Transaction transaction) { unprofitableCache.add(transaction.getHash()); log.atTrace().setMessage("unprofitableCache={}").addArgument(unprofitableCache::size).log(); } + + private void updateMetric( + final Phase label, + final TransactionEvaluationContext evaluationContext, + final Transaction tx, + final Wei profitablePriorityFeePerGas) { + + maybeProfitabilityMetrics.ifPresent( + histogramMetrics -> { + final var effectivePriorityFee = + evaluationContext.getTransactionGasPrice().subtract(baseFee); + final var ratio = + effectivePriorityFee.getValue().doubleValue() + / profitablePriorityFeePerGas.getValue().doubleValue(); + + histogramMetrics.track(ratio, label.value()); + + if (ratio < lastBlockMinRatios.get(label)) { + lastBlockMinRatios.put(label, ratio); + } + if (ratio > lastBlockMaxRatios.get(label)) { + lastBlockMaxRatios.put(label, ratio); + } + + log.atTrace() + .setMessage( + "POST_PROCESSING: block[{}] tx {} , baseFee {}, effectiveGasPrice {}, ratio (effectivePayingPriorityFee {} / calculatedProfitablePriorityFee {}) {}") + .addArgument(evaluationContext.getPendingBlockHeader().getNumber()) + .addArgument(tx.getHash()) + .addArgument(baseFee::toHumanReadableString) + .addArgument(evaluationContext.getTransactionGasPrice()::toHumanReadableString) + .addArgument(effectivePriorityFee::toHumanReadableString) + .addArgument(profitablePriorityFeePerGas::toHumanReadableString) + .addArgument(ratio) + .log(); + }); + } + + private static void resetMinMaxRatios() { + lastBlockMinRatios.put(Phase.PRE_PROCESSING, Double.POSITIVE_INFINITY); + lastBlockMinRatios.put(Phase.POST_PROCESSING, Double.POSITIVE_INFINITY); + lastBlockMaxRatios.put(Phase.PRE_PROCESSING, Double.NEGATIVE_INFINITY); + lastBlockMaxRatios.put(Phase.POST_PROCESSING, Double.NEGATIVE_INFINITY); + } } diff --git a/sequencer/src/test/java/net/consensys/linea/sequencer/txselection/selectors/ProfitableTransactionSelectorTest.java b/sequencer/src/test/java/net/consensys/linea/sequencer/txselection/selectors/ProfitableTransactionSelectorTest.java index 164e7e30..ff3b08b9 100644 --- a/sequencer/src/test/java/net/consensys/linea/sequencer/txselection/selectors/ProfitableTransactionSelectorTest.java +++ b/sequencer/src/test/java/net/consensys/linea/sequencer/txselection/selectors/ProfitableTransactionSelectorTest.java @@ -28,6 +28,7 @@ import net.consensys.linea.config.LineaProfitabilityConfiguration; import net.consensys.linea.config.LineaTransactionSelectorCliOptions; import net.consensys.linea.config.LineaTransactionSelectorConfiguration; +import net.consensys.linea.metrics.HistogramMetrics; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.bouncycastle.crypto.digests.KeccakDigest; @@ -73,7 +74,7 @@ private TestableProfitableTransactionSelector newSelectorForNewBlock() { final var blockchainService = mock(BlockchainService.class); when(blockchainService.getNextBlockBaseFee()).thenReturn(Optional.of(BASE_FEE)); return new TestableProfitableTransactionSelector( - blockchainService, txSelectorConf, profitabilityConf); + blockchainService, txSelectorConf, profitabilityConf, Optional.empty()); } @Test @@ -410,8 +411,9 @@ private static class TestableProfitableTransactionSelector extends ProfitableTra TestableProfitableTransactionSelector( final BlockchainService blockchainService, final LineaTransactionSelectorConfiguration txSelectorConf, - final LineaProfitabilityConfiguration profitabilityConf) { - super(blockchainService, txSelectorConf, profitabilityConf); + final LineaProfitabilityConfiguration profitabilityConf, + final Optional maybeProfitabilityMetrics) { + super(blockchainService, txSelectorConf, profitabilityConf, maybeProfitabilityMetrics); } boolean isUnprofitableTxCached(final Hash txHash) { From b20d27c3b80795a952e175fba23e504d85354ff8 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Wed, 4 Dec 2024 10:54:50 +0100 Subject: [PATCH 17/19] Support state overrides in `linea_estimateGas` (#113) Signed-off-by: Fabio Di Fabio --- .../acc/test/rpc/linea/EstimateGasTest.java | 43 ++++++++++++++++++- .../linea/rpc/methods/LineaEstimateGas.java | 33 +++++++++++--- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EstimateGasTest.java b/acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EstimateGasTest.java index f4e17cec..42eece1e 100644 --- a/acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EstimateGasTest.java +++ b/acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EstimateGasTest.java @@ -23,7 +23,9 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.List; +import java.util.Map; import linea.plugin.acc.test.LineaPluginTestBase; import linea.plugin.acc.test.TestCommandLineOptionsBuilder; @@ -122,6 +124,36 @@ public void passingGasPriceFieldWorks() { assertThat(respLinea.getResult()).isNotNull(); } + @Test + public void passingStateOverridesWorks() { + + final Account sender = accounts.getSecondaryBenefactor(); + + final var actualBalance = minerNode.execute(ethTransactions.getBalance(sender)); + + assertThat(actualBalance).isGreaterThan(BigInteger.ONE); + + final CallParams callParams = + new CallParams( + sender.getAddress(), + sender.getAddress(), + "1", + Bytes.EMPTY.toHexString(), + "0", + "0x1234"); + + final var zeroBalance = Map.of("balance", Wei.ZERO.toHexString()); + + final var stateOverrides = Map.of(accounts.getSecondaryBenefactor().getAddress(), zeroBalance); + + final var reqLinea = new LineaEstimateGasRequest(callParams, stateOverrides); + final var respLinea = reqLinea.execute(minerNode.nodeRequests()); + assertThat(respLinea.hasError()).isTrue(); + assertThat(respLinea.getError().getMessage()) + .isEqualTo( + "transaction up-front cost 0x208cbab601 exceeds transaction sender account balance 0x0"); + } + @Test public void lineaEstimateGasIsProfitable() { @@ -258,9 +290,16 @@ protected void assertMinGasPriceLowerBound(final Wei baseFee, final Wei estimate static class LineaEstimateGasRequest implements Transaction { private final CallParams callParams; + private final Map> stateOverrides; public LineaEstimateGasRequest(final CallParams callParams) { + this(callParams, null); + } + + public LineaEstimateGasRequest( + final CallParams callParams, final Map> stateOverrides) { this.callParams = callParams; + this.stateOverrides = stateOverrides; } @Override @@ -268,7 +307,7 @@ public LineaEstimateGasResponse execute(final NodeRequests nodeRequests) { try { return new Request<>( "linea_estimateGas", - List.of(callParams), + Arrays.asList(callParams, stateOverrides), nodeRequests.getWeb3jService(), LineaEstimateGasResponse.class) .send(); @@ -337,4 +376,6 @@ static class RawEstimateGasResponse extends org.web3j.protocol.core.Response maybeStateOverrides, + final Transaction transaction, + final long logId) { final var estimateGasTracer = new EstimateGasOperationTracer(); final var chainHeadHeader = blockchainService.getChainHeadHeader(); @@ -272,7 +279,8 @@ private Long estimateGasUsed( final var chainHeadHash = chainHeadHeader.getBlockHash(); final var maybeSimulationResults = - transactionSimulationService.simulate(transaction, chainHeadHash, zkAndGasTracer, false); + transactionSimulationService.simulate( + transaction, maybeStateOverrides, chainHeadHash, zkAndGasTracer, false); ModuleLimitsValidationResult moduleLimit = moduleLineCountValidator.validate(zkTracer.getModulesLineCount()); @@ -319,6 +327,7 @@ private Long estimateGasUsed( final var lowResult = transactionSimulationService.simulate( createTransactionForSimulation(callParameters, lowGasEstimation), + maybeStateOverrides, chainHeadHash, estimateGasTracer, true); @@ -354,6 +363,7 @@ private Long estimateGasUsed( final var binarySearchResult = transactionSimulationService.simulate( createTransactionForSimulation(callParameters, mid), + maybeStateOverrides, chainHeadHash, estimateGasTracer, true); @@ -409,7 +419,7 @@ private Long estimateGasUsed( RpcErrorType.PLUGIN_INTERNAL_ERROR, "Empty result from simulation")); } - private JsonCallParameter parseRequest(final Object[] params) { + private JsonCallParameter parseCallParameters(final Object[] params) { final JsonCallParameter callParameters; try { callParameters = parameterParser.required(params, 0, JsonCallParameter.class); @@ -417,11 +427,11 @@ private JsonCallParameter parseRequest(final Object[] params) { throw new InvalidJsonRpcParameters( "Invalid call parameters (index 0)", RpcErrorType.INVALID_CALL_PARAMS); } - validateParameters(callParameters); + validateCallParameters(callParameters); return callParameters; } - private void validateParameters(final JsonCallParameter callParameters) { + private void validateCallParameters(final JsonCallParameter callParameters) { if (callParameters.getGasPrice() != null && isBaseFeeTransaction(callParameters)) { throw new InvalidJsonRpcParameters( "gasPrice cannot be used with maxFeePerGas or maxPriorityFeePerGas or maxFeePerBlobGas"); @@ -434,6 +444,15 @@ private void validateParameters(final JsonCallParameter callParameters) { } } + protected Optional getAddressAccountOverrideMap(final Object[] params) { + try { + return parameterParser.optional(params, 1, AccountOverrideMap.class); + } catch (JsonRpcParameter.JsonRpcParameterException e) { + throw new InvalidJsonRpcRequestException( + "Invalid account overrides parameter (index 1)", RpcErrorType.INVALID_CALL_PARAMS, e); + } + } + private boolean isBaseFeeTransaction(final JsonCallParameter callParameters) { return (callParameters.getMaxFeePerGas().isPresent() || callParameters.getMaxPriorityFeePerGas().isPresent() From 84598c201a304b60e69b374393fb8eae1f1b6869 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Mon, 9 Dec 2024 03:42:21 +0100 Subject: [PATCH 18/19] Make `linea_estimateGas` to simulate tx on pending block (#116) Signed-off-by: Fabio Di Fabio --- gradle.properties | 2 +- .../net/consensys/linea/rpc/methods/LineaEstimateGas.java | 7 +++---- .../txpoolvalidation/validators/SimulationValidator.java | 3 +-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/gradle.properties b/gradle.properties index ae194b4f..1975ec14 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ releaseVersion=0.8.0-rc6.1 -besuVersion=24.11-delivery39 +besuVersion=24.12-develop-cf50479 arithmetizationVersion=0.8.0-rc6 besuArtifactGroup=io.consensys.linea-besu distributionIdentifier=linea-sequencer diff --git a/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java b/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java index 0a8f1101..f3250aa1 100644 --- a/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java +++ b/sequencer/src/main/java/net/consensys/linea/rpc/methods/LineaEstimateGas.java @@ -277,10 +277,9 @@ private Long estimateGasUsed( final var zkTracer = createZkTracer(chainHeadHeader, blockchainService.getChainId().get()); final TracerAggregator zkAndGasTracer = TracerAggregator.create(estimateGasTracer, zkTracer); - final var chainHeadHash = chainHeadHeader.getBlockHash(); final var maybeSimulationResults = transactionSimulationService.simulate( - transaction, maybeStateOverrides, chainHeadHash, zkAndGasTracer, false); + transaction, maybeStateOverrides, Optional.empty(), zkAndGasTracer, false); ModuleLimitsValidationResult moduleLimit = moduleLineCountValidator.validate(zkTracer.getModulesLineCount()); @@ -328,7 +327,7 @@ private Long estimateGasUsed( transactionSimulationService.simulate( createTransactionForSimulation(callParameters, lowGasEstimation), maybeStateOverrides, - chainHeadHash, + Optional.empty(), estimateGasTracer, true); @@ -364,7 +363,7 @@ private Long estimateGasUsed( transactionSimulationService.simulate( createTransactionForSimulation(callParameters, mid), maybeStateOverrides, - chainHeadHash, + Optional.empty(), estimateGasTracer, true); diff --git a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java index 75204eb6..7d9638d6 100644 --- a/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java +++ b/sequencer/src/main/java/net/consensys/linea/sequencer/txpoolvalidation/validators/SimulationValidator.java @@ -89,8 +89,7 @@ public Optional validateTransaction( final var zkTracer = createZkTracer(chainHeadHeader, blockchainService.getChainId().get()); final var maybeSimulationResults = - transactionSimulationService.simulate( - transaction, chainHeadHeader.getBlockHash(), zkTracer, true); + transactionSimulationService.simulate(transaction, Optional.empty(), zkTracer, true); ModuleLimitsValidationResult moduleLimitResult = moduleLineCountValidator.validate(zkTracer.getModulesLineCount()); From eaeb684eaa987d7e6f888e4b10735b57ff62c9e2 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Mon, 9 Dec 2024 14:44:46 +1000 Subject: [PATCH 19/19] update to 24.12 besu (#117) * update to 24.12 besu Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 1975ec14..04ebe504 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ releaseVersion=0.8.0-rc6.1 -besuVersion=24.12-develop-cf50479 +besuVersion=24.12-delivery40 arithmetizationVersion=0.8.0-rc6 besuArtifactGroup=io.consensys.linea-besu distributionIdentifier=linea-sequencer -distributionBaseUrl=https://artifacts.consensys.net/public/linea-besu/raw/names/linea-besu.tar.gz/versions/ \ No newline at end of file +distributionBaseUrl=https://artifacts.consensys.net/public/linea-besu/raw/names/linea-besu.tar.gz/versions/