From 0be75a336cb933cb1cc5dbafa661061c7a30dfcb Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Wed, 10 Apr 2024 13:49:43 -0300 Subject: [PATCH] Extend Module Line Count Verification to `linea_estimateGas` RPC Method (#1) Signed-off-by: Gabriel-Trintinalia --- .../EstimateGasModuleLimitOverflowTest.java | 63 +++++++ .../AbstractLineaSharedOptionsPlugin.java | 9 + .../config/LineaTracerConfiguration.java | 22 +++ .../LineaTracerConfigurationCLiOptions.java | 72 ++++++++ .../LineaTransactionSelectorCliOptions.java | 15 -- ...LineaTransactionSelectorConfiguration.java | 1 - .../rpc/linea/LineaEndpointServicePlugin.java | 8 +- .../linea/rpc/linea/LineaEstimateGas.java | 74 ++++++++- .../linea/sequencer/TracerAggregator.java | 157 ++++++++++++++++++ .../ModuleLimitsValidationResult.java | 88 ++++++++++ .../modulelimit/ModuleLineCountValidator.java | 135 +++++++++++++++ .../LineaTransactionSelectorFactory.java | 11 +- .../LineaTransactionSelectorPlugin.java | 51 +----- .../selectors/LineaTransactionSelector.java | 8 +- .../TraceLineLimitTransactionSelector.java | 62 +++---- .../zktracer/module/rlp/txn/RlpTxnChunk.java | 2 +- ...TraceLineLimitTransactionSelectorTest.java | 71 +++++--- 17 files changed, 721 insertions(+), 128 deletions(-) create mode 100644 acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EstimateGasModuleLimitOverflowTest.java create mode 100644 arithmetization/src/main/java/net/consensys/linea/config/LineaTracerConfiguration.java create mode 100644 arithmetization/src/main/java/net/consensys/linea/config/LineaTracerConfigurationCLiOptions.java create mode 100644 arithmetization/src/main/java/net/consensys/linea/sequencer/TracerAggregator.java create mode 100644 arithmetization/src/main/java/net/consensys/linea/sequencer/modulelimit/ModuleLimitsValidationResult.java create mode 100644 arithmetization/src/main/java/net/consensys/linea/sequencer/modulelimit/ModuleLineCountValidator.java diff --git a/acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EstimateGasModuleLimitOverflowTest.java b/acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EstimateGasModuleLimitOverflowTest.java new file mode 100644 index 00000000..c787e112 --- /dev/null +++ b/acceptance-tests/src/test/java/linea/plugin/acc/test/rpc/linea/EstimateGasModuleLimitOverflowTest.java @@ -0,0 +1,63 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package linea.plugin.acc.test.rpc.linea; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import linea.plugin.acc.test.LineaPluginTestBase; +import linea.plugin.acc.test.TestCommandLineOptionsBuilder; +import linea.plugin.acc.test.tests.web3j.generated.SimpleStorage; +import org.apache.tuweni.bytes.Bytes; +import org.hyperledger.besu.tests.acceptance.dsl.account.Account; +import org.junit.jupiter.api.Test; + +public class EstimateGasModuleLimitOverflowTest extends LineaPluginTestBase { + @Override + public List getTestCliOptions() { + return new TestCommandLineOptionsBuilder() + .set( + "--plugin-linea-module-limit-file-path=", + getResourcePath("/txOverflowModuleLimits.toml")) + .build(); + } + + @Test + public void estimateGasFailsForExceedingModuleLineCountTest() throws Exception { + + final Account sender = accounts.getSecondaryBenefactor(); + + final SimpleStorage simpleStorage = deploySimpleStorage(); + final String txData = simpleStorage.add(BigInteger.valueOf(100)).encodeFunctionCall(); + final var payload = Bytes.wrap(txData.getBytes(StandardCharsets.UTF_8)); + + final EstimateGasTest.CallParams callParams = + new EstimateGasTest.CallParams( + sender.getAddress(), + simpleStorage.getContractAddress(), + null, + payload.toHexString(), + "0"); + + final var reqLinea = new EstimateGasTest.BadLineaEstimateGasRequest(callParams); + final var respLinea = reqLinea.execute(minerNode.nodeRequests()); + assertThat(respLinea.getCode()).isEqualTo(-32000); + assertThat(respLinea.getMessage()) + .isEqualTo("Transaction line count for module HUB=66 is above the limit 30"); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/AbstractLineaSharedOptionsPlugin.java b/arithmetization/src/main/java/net/consensys/linea/AbstractLineaSharedOptionsPlugin.java index afdcacdc..5046c8c9 100644 --- a/arithmetization/src/main/java/net/consensys/linea/AbstractLineaSharedOptionsPlugin.java +++ b/arithmetization/src/main/java/net/consensys/linea/AbstractLineaSharedOptionsPlugin.java @@ -23,6 +23,8 @@ import net.consensys.linea.config.LineaProfitabilityConfiguration; import net.consensys.linea.config.LineaRpcCliOptions; import net.consensys.linea.config.LineaRpcConfiguration; +import net.consensys.linea.config.LineaTracerConfiguration; +import net.consensys.linea.config.LineaTracerConfigurationCLiOptions; import net.consensys.linea.config.LineaTransactionPoolValidatorCliOptions; import net.consensys.linea.config.LineaTransactionPoolValidatorConfiguration; import net.consensys.linea.config.LineaTransactionSelectorCliOptions; @@ -42,10 +44,12 @@ public abstract class AbstractLineaSharedOptionsPlugin implements BesuPlugin { private static LineaRpcCliOptions rpcCliOptions; private static LineaProfitabilityCliOptions profitabilityCliOptions; protected static LineaTransactionSelectorConfiguration transactionSelectorConfiguration; + protected static LineaTracerConfigurationCLiOptions tracerConfigurationCliOptions; protected static LineaTransactionPoolValidatorConfiguration transactionPoolValidatorConfiguration; protected static LineaL1L2BridgeConfiguration l1L2BridgeConfiguration; protected static LineaRpcConfiguration rpcConfiguration; protected static LineaProfitabilityConfiguration profitabilityConfiguration; + protected static LineaTracerConfiguration tracerConfiguration; static { // force the initialization of the gnark compress native library to fail fast in case of issues @@ -67,12 +71,14 @@ public synchronized void register(final BesuContext context) { l1L2BridgeCliOptions = LineaL1L2BridgeCliOptions.create(); rpcCliOptions = LineaRpcCliOptions.create(); profitabilityCliOptions = LineaProfitabilityCliOptions.create(); + tracerConfigurationCliOptions = LineaTracerConfigurationCLiOptions.create(); cmdlineOptions.addPicoCLIOptions(CLI_OPTIONS_PREFIX, transactionSelectorCliOptions); cmdlineOptions.addPicoCLIOptions(CLI_OPTIONS_PREFIX, transactionPoolValidatorCliOptions); cmdlineOptions.addPicoCLIOptions(CLI_OPTIONS_PREFIX, l1L2BridgeCliOptions); cmdlineOptions.addPicoCLIOptions(CLI_OPTIONS_PREFIX, rpcCliOptions); cmdlineOptions.addPicoCLIOptions(CLI_OPTIONS_PREFIX, profitabilityCliOptions); + cmdlineOptions.addPicoCLIOptions(CLI_OPTIONS_PREFIX, tracerConfigurationCliOptions); cliOptionsRegistered = true; } } @@ -85,6 +91,7 @@ public void beforeExternalServices() { l1L2BridgeConfiguration = l1L2BridgeCliOptions.toDomainObject(); rpcConfiguration = rpcCliOptions.toDomainObject(); profitabilityConfiguration = profitabilityCliOptions.toDomainObject(); + tracerConfiguration = tracerConfigurationCliOptions.toDomainObject(); configured = true; } @@ -109,6 +116,8 @@ public void beforeExternalServices() { "Configured plugin {} with profitability calculator configuration: {}", getName(), profitabilityConfiguration); + + log.debug("Configured plugin {} with tracer configuration: {}", getName(), tracerConfiguration); } @Override diff --git a/arithmetization/src/main/java/net/consensys/linea/config/LineaTracerConfiguration.java b/arithmetization/src/main/java/net/consensys/linea/config/LineaTracerConfiguration.java new file mode 100644 index 00000000..a9b69033 --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/config/LineaTracerConfiguration.java @@ -0,0 +1,22 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package net.consensys.linea.config; + +import lombok.Builder; + +/** The Linea tracer configuration. */ +@Builder(toBuilder = true) +public record LineaTracerConfiguration(String moduleLimitsFilePath) {} diff --git a/arithmetization/src/main/java/net/consensys/linea/config/LineaTracerConfigurationCLiOptions.java b/arithmetization/src/main/java/net/consensys/linea/config/LineaTracerConfigurationCLiOptions.java new file mode 100644 index 00000000..5b6f6c04 --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/config/LineaTracerConfigurationCLiOptions.java @@ -0,0 +1,72 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package net.consensys.linea.config; + +import com.google.common.base.MoreObjects; +import picocli.CommandLine; + +public class LineaTracerConfigurationCLiOptions { + + public static final String MODULE_LIMIT_FILE_PATH = "--plugin-linea-module-limit-file-path"; + public static final String DEFAULT_MODULE_LIMIT_FILE_PATH = "moduleLimitFile.toml"; + + @CommandLine.Option( + names = {MODULE_LIMIT_FILE_PATH}, + hidden = true, + paramLabel = "", + description = + "Path to the toml file containing the module limits (default: ${DEFAULT-VALUE})") + private String moduleLimitFilePath = DEFAULT_MODULE_LIMIT_FILE_PATH; + + private LineaTracerConfigurationCLiOptions() {} + + /** + * Create Linea cli options. + * + * @return the Linea cli options + */ + public static LineaTracerConfigurationCLiOptions create() { + return new LineaTracerConfigurationCLiOptions(); + } + + /** + * Linea cli options from config. + * + * @param config the config + * @return the Linea cli options + */ + public static LineaTracerConfigurationCLiOptions fromConfig( + final LineaTracerConfiguration config) { + final LineaTracerConfigurationCLiOptions options = create(); + options.moduleLimitFilePath = config.moduleLimitsFilePath(); + return options; + } + + /** + * To domain object Linea factory configuration. + * + * @return the Linea factory configuration + */ + public LineaTracerConfiguration toDomainObject() { + return LineaTracerConfiguration.builder().moduleLimitsFilePath(moduleLimitFilePath).build(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add(MODULE_LIMIT_FILE_PATH, moduleLimitFilePath) + .toString(); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/config/LineaTransactionSelectorCliOptions.java b/arithmetization/src/main/java/net/consensys/linea/config/LineaTransactionSelectorCliOptions.java index 9f8f0ba3..acfe4ad1 100644 --- a/arithmetization/src/main/java/net/consensys/linea/config/LineaTransactionSelectorCliOptions.java +++ b/arithmetization/src/main/java/net/consensys/linea/config/LineaTransactionSelectorCliOptions.java @@ -23,10 +23,6 @@ public class LineaTransactionSelectorCliOptions { public static final String MAX_BLOCK_CALLDATA_SIZE = "--plugin-linea-max-block-calldata-size"; public static final int DEFAULT_MAX_BLOCK_CALLDATA_SIZE = 70_000; - - public static final String MODULE_LIMIT_FILE_PATH = "--plugin-linea-module-limit-file-path"; - public static final String DEFAULT_MODULE_LIMIT_FILE_PATH = "moduleLimitFile.toml"; - public static final String OVER_LINE_COUNT_LIMIT_CACHE_SIZE = "--plugin-linea-over-line-count-limit-cache-size"; public static final int DEFAULT_OVER_LINE_COUNT_LIMIT_CACHE_SIZE = 10_000; @@ -48,14 +44,6 @@ public class LineaTransactionSelectorCliOptions { description = "Maximum size for the calldata of a block (default: ${DEFAULT-VALUE})") private int maxBlockCallDataSize = DEFAULT_MAX_BLOCK_CALLDATA_SIZE; - @CommandLine.Option( - names = {MODULE_LIMIT_FILE_PATH}, - hidden = true, - paramLabel = "", - description = - "Path to the toml file containing the module limits (default: ${DEFAULT-VALUE})") - private String moduleLimitFilePath = DEFAULT_MODULE_LIMIT_FILE_PATH; - @Positive @CommandLine.Option( names = {OVER_LINE_COUNT_LIMIT_CACHE_SIZE}, @@ -112,7 +100,6 @@ public static LineaTransactionSelectorCliOptions fromConfig( final LineaTransactionSelectorConfiguration config) { final LineaTransactionSelectorCliOptions options = create(); options.maxBlockCallDataSize = config.maxBlockCallDataSize(); - options.moduleLimitFilePath = config.moduleLimitsFilePath(); options.overLineCountLimitCacheSize = config.overLinesLimitCacheSize(); options.maxGasPerBlock = config.maxGasPerBlock(); options.unprofitableCacheSize = config.unprofitableCacheSize(); @@ -128,7 +115,6 @@ public static LineaTransactionSelectorCliOptions fromConfig( public LineaTransactionSelectorConfiguration toDomainObject() { return LineaTransactionSelectorConfiguration.builder() .maxBlockCallDataSize(maxBlockCallDataSize) - .moduleLimitsFilePath(moduleLimitFilePath) .overLinesLimitCacheSize(overLineCountLimitCacheSize) .maxGasPerBlock(maxGasPerBlock) .unprofitableCacheSize(unprofitableCacheSize) @@ -140,7 +126,6 @@ public LineaTransactionSelectorConfiguration toDomainObject() { public String toString() { return MoreObjects.toStringHelper(this) .add(MAX_BLOCK_CALLDATA_SIZE, maxBlockCallDataSize) - .add(MODULE_LIMIT_FILE_PATH, moduleLimitFilePath) .add(OVER_LINE_COUNT_LIMIT_CACHE_SIZE, overLineCountLimitCacheSize) .add(MAX_GAS_PER_BLOCK, maxGasPerBlock) .add(UNPROFITABLE_CACHE_SIZE, unprofitableCacheSize) diff --git a/arithmetization/src/main/java/net/consensys/linea/config/LineaTransactionSelectorConfiguration.java b/arithmetization/src/main/java/net/consensys/linea/config/LineaTransactionSelectorConfiguration.java index 09afc4f2..57b56114 100644 --- a/arithmetization/src/main/java/net/consensys/linea/config/LineaTransactionSelectorConfiguration.java +++ b/arithmetization/src/main/java/net/consensys/linea/config/LineaTransactionSelectorConfiguration.java @@ -21,7 +21,6 @@ @Builder(toBuilder = true) public record LineaTransactionSelectorConfiguration( int maxBlockCallDataSize, - String moduleLimitsFilePath, int overLinesLimitCacheSize, long maxGasPerBlock, int unprofitableCacheSize, diff --git a/arithmetization/src/main/java/net/consensys/linea/rpc/linea/LineaEndpointServicePlugin.java b/arithmetization/src/main/java/net/consensys/linea/rpc/linea/LineaEndpointServicePlugin.java index 0dd3e8d4..0aab37ab 100644 --- a/arithmetization/src/main/java/net/consensys/linea/rpc/linea/LineaEndpointServicePlugin.java +++ b/arithmetization/src/main/java/net/consensys/linea/rpc/linea/LineaEndpointServicePlugin.java @@ -15,6 +15,8 @@ package net.consensys.linea.rpc.linea; +import static net.consensys.linea.sequencer.modulelimit.ModuleLineCountValidator.createLimitModules; + import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.AbstractLineaRequiredPlugin; @@ -87,6 +89,10 @@ public void doRegister(final BesuContext context) { public void beforeExternalServices() { super.beforeExternalServices(); lineaEstimateGasMethod.init( - rpcConfiguration, transactionPoolValidatorConfiguration, profitabilityConfiguration); + rpcConfiguration, + transactionPoolValidatorConfiguration, + profitabilityConfiguration, + createLimitModules(tracerConfiguration), + l1L2BridgeConfiguration); } } diff --git a/arithmetization/src/main/java/net/consensys/linea/rpc/linea/LineaEstimateGas.java b/arithmetization/src/main/java/net/consensys/linea/rpc/linea/LineaEstimateGas.java index e5c91a99..4e9b4707 100644 --- a/arithmetization/src/main/java/net/consensys/linea/rpc/linea/LineaEstimateGas.java +++ b/arithmetization/src/main/java/net/consensys/linea/rpc/linea/LineaEstimateGas.java @@ -15,20 +15,28 @@ package net.consensys.linea.rpc.linea; +import static net.consensys.linea.sequencer.modulelimit.ModuleLineCountValidator.ModuleLineCountResult.MODULE_NOT_DEFINED; +import static net.consensys.linea.sequencer.modulelimit.ModuleLineCountValidator.ModuleLineCountResult.TX_MODULE_LINE_COUNT_OVERFLOW; import static org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.Quantity.create; import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.bl.TransactionProfitabilityCalculator; +import net.consensys.linea.config.LineaL1L2BridgeConfiguration; import net.consensys.linea.config.LineaProfitabilityConfiguration; import net.consensys.linea.config.LineaRpcConfiguration; import net.consensys.linea.config.LineaTransactionPoolValidatorConfiguration; +import net.consensys.linea.sequencer.TracerAggregator; +import net.consensys.linea.sequencer.modulelimit.ModuleLimitsValidationResult; +import net.consensys.linea.sequencer.modulelimit.ModuleLineCountValidator; +import net.consensys.linea.zktracer.ZkTracer; import org.apache.tuweni.bytes.Bytes; import org.bouncycastle.asn1.sec.SECNamedCurves; import org.bouncycastle.asn1.x9.X9ECParameters; @@ -77,6 +85,9 @@ public class LineaEstimateGas { private LineaTransactionPoolValidatorConfiguration txValidatorConf; private LineaProfitabilityConfiguration profitabilityConf; private TransactionProfitabilityCalculator txProfitabilityCalculator; + private LineaL1L2BridgeConfiguration l1L2BridgeConfiguration; + + private ModuleLineCountValidator moduleLineCountValidator; public LineaEstimateGas( final BesuConfiguration besuConfiguration, @@ -90,11 +101,20 @@ public LineaEstimateGas( public void init( LineaRpcConfiguration rpcConfiguration, final LineaTransactionPoolValidatorConfiguration transactionValidatorConfiguration, - final LineaProfitabilityConfiguration profitabilityConf) { + final LineaProfitabilityConfiguration profitabilityConf, + final Map limitsMap, + final LineaL1L2BridgeConfiguration l1L2BridgeConfiguration) { this.rpcConfiguration = rpcConfiguration; this.txValidatorConf = transactionValidatorConfiguration; this.profitabilityConf = profitabilityConf; this.txProfitabilityCalculator = new TransactionProfitabilityCalculator(profitabilityConf); + this.l1L2BridgeConfiguration = l1L2BridgeConfiguration; + this.moduleLineCountValidator = new ModuleLineCountValidator(limitsMap); + + if (l1L2BridgeConfiguration.isEmpty()) { + log.error("L1L2 bridge settings have not been defined."); + System.exit(1); + } } public String getNamespace() { @@ -187,10 +207,22 @@ private Long estimateGasUsed( final JsonCallParameter callParameters, final Transaction transaction, final Wei minGasPrice) { - final var tracer = new EstimateGasOperationTracer(); + + final var estimateGasOperationTracer = new EstimateGasOperationTracer(); + final var zkTracer = createZkTracer(); + TracerAggregator tracerAggregator = + TracerAggregator.create(estimateGasOperationTracer, zkTracer); + final var chainHeadHash = blockchainService.getChainHeadHash(); final var maybeSimulationResults = - transactionSimulationService.simulate(transaction, chainHeadHash, tracer, true); + transactionSimulationService.simulate(transaction, chainHeadHash, tracerAggregator, true); + + ModuleLimitsValidationResult moduleLimit = + moduleLineCountValidator.validate(zkTracer.getModulesLineCount()); + + if (moduleLimit.getResult() != ModuleLineCountValidator.ModuleLineCountResult.VALID) { + handleModuleOverLimit(moduleLimit); + } return maybeSimulationResults .map( @@ -231,7 +263,7 @@ private Long estimateGasUsed( transactionSimulationService.simulate( createTransactionForSimulation(callParameters, lowGasEstimation, minGasPrice), chainHeadHash, - tracer, + tracerAggregator, true); return lowResult @@ -255,7 +287,8 @@ private Long estimateGasUsed( // else do a binary search to find the right estimation int iterations = 0; - var high = highGasEstimation(lr.getGasEstimate(), tracer); + var high = + highGasEstimation(lr.getGasEstimate(), estimateGasOperationTracer); var mid = high; var low = lowGasEstimation; while (low + 1 < high) { @@ -267,7 +300,7 @@ private Long estimateGasUsed( createTransactionForSimulation( callParameters, mid, minGasPrice), chainHeadHash, - tracer, + tracerAggregator, true); if (binarySearchResult.isEmpty() @@ -353,6 +386,7 @@ private void validateParameters(final JsonCallParameter callParameters) { */ private long highGasEstimation( final long gasEstimation, final EstimateGasOperationTracer operationTracer) { + // no more than 63/64s of the remaining gas can be passed to the sub calls final double subCallMultiplier = Math.pow(SUB_CALL_REMAINING_GAS_RATIO, operationTracer.getMaxDepth()); @@ -383,6 +417,34 @@ private Transaction createTransactionForSimulation( return txBuilder.build(); } + private ZkTracer createZkTracer() { + var zkTracer = new ZkTracer(l1L2BridgeConfiguration); + zkTracer.traceStartConflation(1L); + zkTracer.traceStartBlock(blockchainService.getChainHeadHeader()); + return zkTracer; + } + + private void handleModuleOverLimit(ModuleLimitsValidationResult moduleLimitResult) { + // Throw specific exceptions based on the type of limit exceeded + if (moduleLimitResult.getResult() == MODULE_NOT_DEFINED) { + String moduleNotDefinedMsg = + String.format( + "Module %s does not exist in the limits file.", moduleLimitResult.getModuleName()); + log.error(moduleNotDefinedMsg); + throw new PluginRpcEndpointException(new TransactionSimulationError(moduleNotDefinedMsg)); + } + if (moduleLimitResult.getResult() == TX_MODULE_LINE_COUNT_OVERFLOW) { + String txOverflowMsg = + String.format( + "Transaction line count for module %s=%s is above the limit %s", + moduleLimitResult.getModuleName(), + moduleLimitResult.getModuleLineCount(), + moduleLimitResult.getModuleLineLimit()); + log.warn(txOverflowMsg); + throw new PluginRpcEndpointException(new TransactionSimulationError(txOverflowMsg)); + } + } + public record Response( @JsonProperty String gasLimit, @JsonProperty String baseFeePerGas, diff --git a/arithmetization/src/main/java/net/consensys/linea/sequencer/TracerAggregator.java b/arithmetization/src/main/java/net/consensys/linea/sequencer/TracerAggregator.java new file mode 100644 index 00000000..e9733312 --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/sequencer/TracerAggregator.java @@ -0,0 +1,157 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package net.consensys.linea.sequencer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.hyperledger.besu.datatypes.Transaction; +import org.hyperledger.besu.evm.frame.ExceptionalHaltReason; +import org.hyperledger.besu.evm.frame.MessageFrame; +import org.hyperledger.besu.evm.log.Log; +import org.hyperledger.besu.evm.operation.Operation; +import org.hyperledger.besu.evm.tracing.OperationTracer; +import org.hyperledger.besu.evm.worldstate.WorldView; + +/** + * Aggregates multiple {@link OperationTracer} instances, allowing them to be treated as a single + * tracer. This class facilitates the registration and delegation of tracing operations to multiple + * tracers. + */ +public class TracerAggregator implements OperationTracer { + private final List tracers = new ArrayList<>(); + + /** + * Registers an {@link OperationTracer} instance with the aggregator. If a tracer of the same + * class is already registered, an {@link IllegalArgumentException} is thrown. + * + * @param tracer the tracer to register + * @throws IllegalArgumentException if a tracer of the same class is already registered + */ + public void register(OperationTracer tracer) { + // Check if a tracer of the same class is already registered + for (OperationTracer existingTracer : tracers) { + if (existingTracer.getClass().equals(tracer.getClass())) { + throw new IllegalArgumentException( + "A tracer of class " + tracer.getClass().getName() + " is already registered."); + } + } + tracers.add(tracer); + } + + /** + * Creates a {@link TracerAggregator} instance and registers the provided tracers. + * + * @param tracers the tracers to register with the aggregator + * @return a new {@link TracerAggregator} instance with the provided tracers registered + */ + public static TracerAggregator create(OperationTracer... tracers) { + TracerAggregator aggregator = new TracerAggregator(); + for (OperationTracer tracer : tracers) { + aggregator.register(tracer); + } + return aggregator; + } + + @Override + public void tracePreExecution(MessageFrame frame) { + for (OperationTracer tracer : tracers) { + tracer.tracePreExecution(frame); + } + } + + @Override + public void tracePostExecution(MessageFrame frame, Operation.OperationResult operationResult) { + for (OperationTracer tracer : tracers) { + tracer.tracePostExecution(frame, operationResult); + } + } + + @Override + public void tracePrecompileCall(MessageFrame frame, long gasRequirement, Bytes output) { + for (OperationTracer tracer : tracers) { + tracer.tracePrecompileCall(frame, gasRequirement, output); + } + } + + @Override + public void traceAccountCreationResult( + MessageFrame frame, Optional haltReason) { + for (OperationTracer tracer : tracers) { + tracer.traceAccountCreationResult(frame, haltReason); + } + } + + @Override + public void tracePrepareTransaction(WorldView worldView, Transaction transaction) { + for (OperationTracer tracer : tracers) { + tracer.tracePrepareTransaction(worldView, transaction); + } + } + + @Override + public void traceStartTransaction(WorldView worldView, Transaction transaction) { + for (OperationTracer tracer : tracers) { + tracer.traceStartTransaction(worldView, transaction); + } + } + + @Override + public void traceEndTransaction( + WorldView worldView, + Transaction tx, + boolean status, + Bytes output, + List logs, + long gasUsed, + long timeNs) { + for (OperationTracer tracer : tracers) { + tracer.traceEndTransaction(worldView, tx, status, output, logs, gasUsed, timeNs); + } + } + + @Override + public void traceContextEnter(MessageFrame frame) { + for (OperationTracer tracer : tracers) { + tracer.traceContextEnter(frame); + } + } + + @Override + public void traceContextReEnter(MessageFrame frame) { + for (OperationTracer tracer : tracers) { + tracer.traceContextReEnter(frame); + } + } + + @Override + public void traceContextExit(MessageFrame frame) { + for (OperationTracer tracer : tracers) { + tracer.traceContextExit(frame); + } + } + + @Override + public boolean isExtendedTracing() { + for (OperationTracer tracer : tracers) { + if (tracer.isExtendedTracing()) { + return true; + } + } + return false; + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/sequencer/modulelimit/ModuleLimitsValidationResult.java b/arithmetization/src/main/java/net/consensys/linea/sequencer/modulelimit/ModuleLimitsValidationResult.java new file mode 100644 index 00000000..69728c9b --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/sequencer/modulelimit/ModuleLimitsValidationResult.java @@ -0,0 +1,88 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package net.consensys.linea.sequencer.modulelimit; + +import lombok.Getter; + +/** Represents the result of verifying module line counts against their limits. */ +@Getter +public class ModuleLimitsValidationResult { + private final ModuleLineCountValidator.ModuleLineCountResult result; + private final String moduleName; + private final Integer moduleLineCount; + private final Integer moduleLineLimit; + private final Integer cumulativeModuleLineCount; + private final Integer cumulativeModuleLineLimit; + + public static final ModuleLimitsValidationResult VALID = + new ModuleLimitsValidationResult( + ModuleLineCountValidator.ModuleLineCountResult.VALID, null, null, null, null, null); + + private ModuleLimitsValidationResult( + final ModuleLineCountValidator.ModuleLineCountResult result, + final String moduleName, + final Integer moduleLineCount, + final Integer moduleLineLimit, + final Integer cumulativeModuleLineCount, + final Integer cumulativeModuleLineLimit) { + this.result = result; + this.moduleName = moduleName; + this.moduleLineCount = moduleLineCount; + this.moduleLineLimit = moduleLineLimit; + this.cumulativeModuleLineCount = cumulativeModuleLineCount; + this.cumulativeModuleLineLimit = cumulativeModuleLineLimit; + } + + public static ModuleLimitsValidationResult moduleNotDefined(final String moduleName) { + return new ModuleLimitsValidationResult( + ModuleLineCountValidator.ModuleLineCountResult.MODULE_NOT_DEFINED, + moduleName, + null, + null, + null, + null); + } + + public static ModuleLimitsValidationResult txModuleLineCountOverflow( + final String moduleName, + final Integer moduleLineCount, + final Integer moduleLineLimit, + final Integer cumulativeModuleLineCount, + final Integer cumulativeModuleLineLimit) { + return new ModuleLimitsValidationResult( + ModuleLineCountValidator.ModuleLineCountResult.TX_MODULE_LINE_COUNT_OVERFLOW, + moduleName, + moduleLineCount, + moduleLineLimit, + cumulativeModuleLineCount, + cumulativeModuleLineLimit); + } + + public static ModuleLimitsValidationResult blockModuleLineCountFull( + final String moduleName, + final Integer moduleLineCount, + final Integer moduleLineLimit, + final Integer cumulativeModuleLineCount, + final Integer cumulativeModuleLineLimit) { + + return new ModuleLimitsValidationResult( + ModuleLineCountValidator.ModuleLineCountResult.BLOCK_MODULE_LINE_COUNT_FULL, + moduleName, + moduleLineCount, + moduleLineLimit, + cumulativeModuleLineCount, + cumulativeModuleLineLimit); + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/sequencer/modulelimit/ModuleLineCountValidator.java b/arithmetization/src/main/java/net/consensys/linea/sequencer/modulelimit/ModuleLineCountValidator.java new file mode 100644 index 00000000..5009e3ea --- /dev/null +++ b/arithmetization/src/main/java/net/consensys/linea/sequencer/modulelimit/ModuleLineCountValidator.java @@ -0,0 +1,135 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package net.consensys.linea.sequencer.modulelimit; + +import java.io.File; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import com.google.common.io.Resources; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import net.consensys.linea.config.LineaTracerConfiguration; +import org.apache.tuweni.toml.Toml; +import org.apache.tuweni.toml.TomlParseResult; +import org.apache.tuweni.toml.TomlTable; + +/** + * Accumulates and verifies line counts for modules based on provided limits. It supports verifying + * if current transactions exceed these limits and updates the accumulated counts. + */ +@Slf4j +public class ModuleLineCountValidator { + private final Map moduleLineCountLimits; + + @Getter private final Map accumulatedLineCountsPerModule = new HashMap<>(); + + /** + * Constructs a new accumulator with specified module line count limits. + * + * @param moduleLineCountLimits A map of module names to their respective line count limits. + */ + public ModuleLineCountValidator(Map moduleLineCountLimits) { + this.moduleLineCountLimits = new HashMap<>(moduleLineCountLimits); + } + + /** + * Verifies if the current accumulated line counts for modules exceed the predefined limits. + * + * @param currentAccumulatedLineCounts A map of module names to their current accumulated line + * counts. + * @return A {@link ModuleLimitsValidationResult} indicating the outcome of the verification. + */ + public ModuleLimitsValidationResult validate(Map currentAccumulatedLineCounts) { + for (Map.Entry moduleEntry : currentAccumulatedLineCounts.entrySet()) { + String moduleName = moduleEntry.getKey(); + Integer currentTotalLineCountForModule = moduleEntry.getValue(); + Integer lineCountLimitForModule = moduleLineCountLimits.get(moduleName); + + if (lineCountLimitForModule == null) { + log.error("Module '{}' is not defined in the line count limits.", moduleName); + return ModuleLimitsValidationResult.moduleNotDefined(moduleName); + } + + int previouslyAccumulatedLineCount = + accumulatedLineCountsPerModule.getOrDefault(moduleName, 0); + int lineCountAddedByCurrentTx = + currentTotalLineCountForModule - previouslyAccumulatedLineCount; + + if (lineCountAddedByCurrentTx > lineCountLimitForModule) { + return ModuleLimitsValidationResult.txModuleLineCountOverflow( + moduleName, + lineCountAddedByCurrentTx, + lineCountLimitForModule, + currentTotalLineCountForModule, + lineCountLimitForModule); + } + + if (currentTotalLineCountForModule > lineCountLimitForModule) { + return ModuleLimitsValidationResult.blockModuleLineCountFull( + moduleName, + lineCountAddedByCurrentTx, + lineCountLimitForModule, + currentTotalLineCountForModule, + lineCountLimitForModule); + } + } + return ModuleLimitsValidationResult.VALID; + } + + /** + * Updates the internal map of accumulated line counts per module. + * + * @param newAccumulatedLineCounts A map of module names to their new accumulated line counts. + */ + public void updateAccumulatedLineCounts(Map newAccumulatedLineCounts) { + accumulatedLineCountsPerModule.clear(); + accumulatedLineCountsPerModule.putAll(newAccumulatedLineCounts); + } + + /** Enumerates possible outcomes of verifying module line counts against their limits. */ + public enum ModuleLineCountResult { + VALID, + TX_MODULE_LINE_COUNT_OVERFLOW, + BLOCK_MODULE_LINE_COUNT_FULL, + MODULE_NOT_DEFINED + } + + public static Map createLimitModules( + LineaTracerConfiguration lineaTracerConfiguration) { + try { + URL url = new File(lineaTracerConfiguration.moduleLimitsFilePath()).toURI().toURL(); + final String tomlString = Resources.toString(url, StandardCharsets.UTF_8); + TomlParseResult result = Toml.parse(tomlString); + final TomlTable table = result.getTable("traces-limits"); + final Map limitsMap = + table.toMap().entrySet().stream() + .collect( + Collectors.toUnmodifiableMap( + Map.Entry::getKey, e -> Math.toIntExact((Long) e.getValue()))); + + return limitsMap; + } catch (final Exception e) { + final String errorMsg = + "Problem reading the toml file containing the limits for the modules: " + + lineaTracerConfiguration.moduleLimitsFilePath(); + log.error(errorMsg); + throw new RuntimeException(errorMsg, e); + } + } +} diff --git a/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorFactory.java b/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorFactory.java index 465d3e5a..cab06d47 100644 --- a/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorFactory.java +++ b/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorFactory.java @@ -19,6 +19,7 @@ import net.consensys.linea.config.LineaL1L2BridgeConfiguration; import net.consensys.linea.config.LineaProfitabilityConfiguration; +import net.consensys.linea.config.LineaTracerConfiguration; import net.consensys.linea.config.LineaTransactionSelectorConfiguration; import net.consensys.linea.sequencer.txselection.selectors.LineaTransactionSelector; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelector; @@ -29,22 +30,30 @@ public class LineaTransactionSelectorFactory implements PluginTransactionSelecto private final LineaTransactionSelectorConfiguration txSelectorConfiguration; private final LineaL1L2BridgeConfiguration l1L2BridgeConfiguration; private final LineaProfitabilityConfiguration profitabilityConfiguration; + private final LineaTracerConfiguration tracerConfiguration; + private final Map limitsMap; public LineaTransactionSelectorFactory( final LineaTransactionSelectorConfiguration txSelectorConfiguration, final LineaL1L2BridgeConfiguration l1L2BridgeConfiguration, final LineaProfitabilityConfiguration profitabilityConfiguration, + final LineaTracerConfiguration tracerConfiguration, final Map limitsMap) { this.txSelectorConfiguration = txSelectorConfiguration; this.l1L2BridgeConfiguration = l1L2BridgeConfiguration; this.profitabilityConfiguration = profitabilityConfiguration; + this.tracerConfiguration = tracerConfiguration; this.limitsMap = limitsMap; } @Override public PluginTransactionSelector create() { return new LineaTransactionSelector( - txSelectorConfiguration, l1L2BridgeConfiguration, profitabilityConfiguration, limitsMap); + txSelectorConfiguration, + l1L2BridgeConfiguration, + profitabilityConfiguration, + tracerConfiguration, + limitsMap); } } diff --git a/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java b/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java index 4a450419..edcfeb09 100644 --- a/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java +++ b/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/LineaTransactionSelectorPlugin.java @@ -15,23 +15,13 @@ package net.consensys.linea.sequencer.txselection; -import java.io.File; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Map; +import static net.consensys.linea.sequencer.modulelimit.ModuleLineCountValidator.createLimitModules; + import java.util.Optional; -import java.util.stream.Collectors; import com.google.auto.service.AutoService; -import com.google.common.io.Resources; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.AbstractLineaRequiredPlugin; -import net.consensys.linea.config.LineaL1L2BridgeConfiguration; -import net.consensys.linea.config.LineaProfitabilityConfiguration; -import net.consensys.linea.config.LineaTransactionSelectorConfiguration; -import org.apache.tuweni.toml.Toml; -import org.apache.tuweni.toml.TomlParseResult; -import org.apache.tuweni.toml.TomlTable; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.BesuPlugin; import org.hyperledger.besu.plugin.services.TransactionSelectionService; @@ -66,43 +56,12 @@ public void doRegister(final BesuContext context) { @Override public void beforeExternalServices() { super.beforeExternalServices(); - try { - URL url = new File(transactionSelectorConfiguration.moduleLimitsFilePath()).toURI().toURL(); - final String tomlString = Resources.toString(url, StandardCharsets.UTF_8); - TomlParseResult result = Toml.parse(tomlString); - final TomlTable table = result.getTable("traces-limits"); - final Map limitsMap = - table.toMap().entrySet().stream() - .collect( - Collectors.toUnmodifiableMap( - Map.Entry::getKey, e -> Math.toIntExact((Long) e.getValue()))); - - createAndRegister( - transactionSelectionService, - transactionSelectorConfiguration, - l1L2BridgeConfiguration, - profitabilityConfiguration, - limitsMap); - } catch (final Exception e) { - final String errorMsg = - "Problem reading the toml file containing the limits for the modules: " - + transactionSelectorConfiguration.moduleLimitsFilePath(); - log.error(errorMsg); - throw new RuntimeException(errorMsg, e); - } - } - - private void createAndRegister( - final TransactionSelectionService transactionSelectionService, - final LineaTransactionSelectorConfiguration txSelectorConfiguration, - final LineaL1L2BridgeConfiguration l1L2BridgeConfiguration, - final LineaProfitabilityConfiguration profitabilityConfiguration, - final Map limitsMap) { transactionSelectionService.registerPluginTransactionSelectorFactory( new LineaTransactionSelectorFactory( - txSelectorConfiguration, + transactionSelectorConfiguration, l1L2BridgeConfiguration, profitabilityConfiguration, - limitsMap)); + tracerConfiguration, + createLimitModules(tracerConfiguration))); } } diff --git a/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java b/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java index 32a7c012..fadb5d78 100644 --- a/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java +++ b/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/selectors/LineaTransactionSelector.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import net.consensys.linea.config.LineaL1L2BridgeConfiguration; import net.consensys.linea.config.LineaProfitabilityConfiguration; +import net.consensys.linea.config.LineaTracerConfiguration; import net.consensys.linea.config.LineaTransactionSelectorConfiguration; import org.hyperledger.besu.datatypes.PendingTransaction; import org.hyperledger.besu.plugin.data.TransactionProcessingResult; @@ -33,18 +34,20 @@ public class LineaTransactionSelector implements PluginTransactionSelector { private TraceLineLimitTransactionSelector traceLineLimitTransactionSelector; - private List selectors; + private final List selectors; public LineaTransactionSelector( final LineaTransactionSelectorConfiguration txSelectorConfiguration, final LineaL1L2BridgeConfiguration l1L2BridgeConfiguration, final LineaProfitabilityConfiguration profitabilityConfiguration, + final LineaTracerConfiguration tracerConfiguration, final Map limitsMap) { this.selectors = createTransactionSelectors( txSelectorConfiguration, l1L2BridgeConfiguration, profitabilityConfiguration, + tracerConfiguration, limitsMap); } @@ -60,11 +63,12 @@ private List createTransactionSelectors( final LineaTransactionSelectorConfiguration txSelectorConfiguration, final LineaL1L2BridgeConfiguration l1L2BridgeConfiguration, final LineaProfitabilityConfiguration profitabilityConfiguration, + final LineaTracerConfiguration tracerConfiguration, final Map limitsMap) { traceLineLimitTransactionSelector = new TraceLineLimitTransactionSelector( - limitsMap, txSelectorConfiguration, l1L2BridgeConfiguration); + limitsMap, txSelectorConfiguration, l1L2BridgeConfiguration, tracerConfiguration); return List.of( new MaxBlockCallDataTransactionSelector(txSelectorConfiguration.maxBlockCallDataSize()), diff --git a/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelector.java b/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelector.java index 5f021dd6..44c1cd74 100644 --- a/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelector.java +++ b/arithmetization/src/main/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelector.java @@ -27,7 +27,10 @@ import com.google.common.annotations.VisibleForTesting; import lombok.extern.slf4j.Slf4j; import net.consensys.linea.config.LineaL1L2BridgeConfiguration; +import net.consensys.linea.config.LineaTracerConfiguration; import net.consensys.linea.config.LineaTransactionSelectorConfiguration; +import net.consensys.linea.sequencer.modulelimit.ModuleLimitsValidationResult; +import net.consensys.linea.sequencer.modulelimit.ModuleLineCountValidator; import net.consensys.linea.zktracer.ZkTracer; import net.consensys.linea.zktracer.module.Module; import org.hyperledger.besu.datatypes.Hash; @@ -56,20 +59,21 @@ public class TraceLineLimitTransactionSelector implements PluginTransactionSelec private final String limitFilePath; private final Map moduleLimits; private final int overLimitCacheSize; - private Map consolidatedCumulatedLineCount = Map.of(); + private final ModuleLineCountValidator moduleLineCountAccumulator; private Map currCumulatedLineCount; public TraceLineLimitTransactionSelector( final Map moduleLimits, final LineaTransactionSelectorConfiguration txSelectorConfiguration, - final LineaL1L2BridgeConfiguration l1L2BridgeConfiguration) { + final LineaL1L2BridgeConfiguration l1L2BridgeConfiguration, + final LineaTracerConfiguration tracerConfiguration) { if (l1L2BridgeConfiguration.isEmpty()) { log.error("L1L2 bridge settings have not been defined."); System.exit(1); } this.moduleLimits = moduleLimits; - this.limitFilePath = txSelectorConfiguration.moduleLimitsFilePath(); + this.limitFilePath = tracerConfiguration.moduleLimitsFilePath(); this.overLimitCacheSize = txSelectorConfiguration.overLinesLimitCacheSize(); zkTracer = new ZkTracerWithLog(l1L2BridgeConfiguration); @@ -80,6 +84,7 @@ public TraceLineLimitTransactionSelector( } } zkTracer.traceStartConflation(1L); + moduleLineCountAccumulator = new ModuleLineCountValidator(moduleLimits); } /** @@ -114,7 +119,7 @@ public void onTransactionNotSelected( public void onTransactionSelected( final TransactionEvaluationContext evaluationContext, final TransactionProcessingResult processingResult) { - consolidatedCumulatedLineCount = currCumulatedLineCount; + moduleLineCountAccumulator.updateAccumulatedLineCounts(currCumulatedLineCount); } /** @@ -133,49 +138,41 @@ public TransactionSelectionResult evaluateTransactionPostProcessing( // check that we are not exceeding line number for any module currCumulatedLineCount = zkTracer.getModulesLineCount(); - final Transaction transaction = evaluationContext.getPendingTransaction().getTransaction(); - log.atTrace() .setMessage("Tx {} line count per module: {}") .addArgument(transaction::getHash) .addArgument(this::logTxLineCount) .log(); - for (final var module : currCumulatedLineCount.keySet()) { - final Integer moduleLineCountLimit = moduleLimits.get(module); - if (moduleLineCountLimit == null) { - final String errorMsg = - "Module " + module + " does not exist in the limits file: " + limitFilePath; - log.error(errorMsg); - throw new RuntimeException(errorMsg); - } - - final int cumulatedModuleLineCount = currCumulatedLineCount.get(module); - final int txModuleLineCount = - cumulatedModuleLineCount - consolidatedCumulatedLineCount.getOrDefault(module, 0); + ModuleLimitsValidationResult result = + moduleLineCountAccumulator.validate(currCumulatedLineCount); - if (txModuleLineCount > moduleLineCountLimit) { + switch (result.getResult()) { + case MODULE_NOT_DEFINED: + log.error("Module {} does not exist in the limits file.", result.getModuleName()); + throw new RuntimeException( + "Module " + result.getModuleName() + " does not exist in the limits file."); + case TX_MODULE_LINE_COUNT_OVERFLOW: log.warn( "Tx {} line count for module {}={} is above the limit {}, removing from the txpool", transaction.getHash(), - module, - txModuleLineCount, - moduleLineCountLimit); + result.getModuleName(), + result.getModuleLineCount(), + result.getModuleLineCount()); rememberOverLineCountLimitTransaction(transaction); return TX_MODULE_LINE_COUNT_OVERFLOW; - } - - if (cumulatedModuleLineCount > moduleLineCountLimit) { + case BLOCK_MODULE_LINE_COUNT_FULL: log.atTrace() .setMessage( "Cumulated line count for module {}={} is above the limit {}, stopping selection") - .addArgument(module) - .addArgument(cumulatedModuleLineCount) - .addArgument(moduleLineCountLimit) + .addArgument(result.getModuleName()) + .addArgument(result.getCumulativeModuleLineCount()) + .addArgument(result.getCumulativeModuleLineLimit()) .log(); return BLOCK_MODULE_LINE_COUNT_FULL; - } + default: + break; } return SELECTED; } @@ -207,7 +204,10 @@ private String logTxLineCount() { // tx line count / cumulated line count / line count limit e.getKey() + "=" - + (e.getValue() - consolidatedCumulatedLineCount.getOrDefault(e.getKey(), 0)) + + (e.getValue() + - moduleLineCountAccumulator + .getAccumulatedLineCountsPerModule() + .getOrDefault(e.getKey(), 0)) + "/" + e.getValue() + "/" @@ -230,7 +230,7 @@ public void traceEndBlock(final BlockHeader blockHeader, final BlockBody blockBo .addKeyValue( "traceCounts", () -> - consolidatedCumulatedLineCount.entrySet().stream() + moduleLineCountAccumulator.getAccumulatedLineCountsPerModule().entrySet().stream() .sorted(Map.Entry.comparingByKey()) .map(e -> '"' + e.getKey() + "\":" + e.getValue()) .collect(Collectors.joining(","))) diff --git a/arithmetization/src/main/java/net/consensys/linea/zktracer/module/rlp/txn/RlpTxnChunk.java b/arithmetization/src/main/java/net/consensys/linea/zktracer/module/rlp/txn/RlpTxnChunk.java index 5c22cae4..668f99db 100644 --- a/arithmetization/src/main/java/net/consensys/linea/zktracer/module/rlp/txn/RlpTxnChunk.java +++ b/arithmetization/src/main/java/net/consensys/linea/zktracer/module/rlp/txn/RlpTxnChunk.java @@ -124,7 +124,7 @@ protected int computeLineCount() { // Phase 10: AccessList if (txType == 1 || txType == 2) { - if (this.tx.getAccessList().orElseThrow().isEmpty()) { + if (this.tx.getAccessList().isEmpty() || this.tx.getAccessList().get().isEmpty()) { rowSize += 1; } else { // Rlp prefix of the AccessList list diff --git a/arithmetization/src/test/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelectorTest.java b/arithmetization/src/test/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelectorTest.java index 2cd590da..eb9fb983 100644 --- a/arithmetization/src/test/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelectorTest.java +++ b/arithmetization/src/test/java/net/consensys/linea/sequencer/txselection/selectors/TraceLineLimitTransactionSelectorTest.java @@ -22,15 +22,17 @@ import static org.mockito.Mockito.when; import java.io.IOException; -import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; import net.consensys.linea.config.LineaL1L2BridgeConfiguration; +import net.consensys.linea.config.LineaTracerConfiguration; import net.consensys.linea.config.LineaTransactionSelectorConfiguration; +import net.consensys.linea.sequencer.modulelimit.ModuleLineCountValidator; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.toml.Toml; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.PendingTransaction; @@ -39,38 +41,50 @@ import org.hyperledger.besu.plugin.data.TransactionProcessingResult; import org.hyperledger.besu.plugin.data.TransactionSelectionResult; import org.hyperledger.besu.plugin.services.txselection.PluginTransactionSelector; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; public class TraceLineLimitTransactionSelectorTest { private static final int OVER_LINE_COUNT_LIMIT_CACHE_SIZE = 2; - private TestableTraceLineLimitTransactionSelector transactionSelector; + private static final String MODULE_LINE_LIMITS_RESOURCE_NAME = "/sequencer/line-limits.toml"; private Map lineCountLimits; + private LineaTracerConfiguration lineaTracerConfiguration; + + @TempDir static Path tempDir; + static Path lineLimitsConfPath; + + @BeforeAll + public static void beforeAll() throws IOException { + lineLimitsConfPath = tempDir.resolve("line-limits.toml"); + Files.copy( + TraceLineLimitTransactionSelectorTest.class.getResourceAsStream( + MODULE_LINE_LIMITS_RESOURCE_NAME), + lineLimitsConfPath); + } @BeforeEach public void initialize() { - lineCountLimits = loadLineCountLimitConf(); - transactionSelector = newSelectorForNewBlock(); - transactionSelector.reset(); + lineaTracerConfiguration = + LineaTracerConfiguration.builder() + .moduleLimitsFilePath(lineLimitsConfPath.toString()) + .build(); + lineCountLimits = + new HashMap<>(ModuleLineCountValidator.createLimitModules(lineaTracerConfiguration)); } - private TestableTraceLineLimitTransactionSelector newSelectorForNewBlock() { + private TestableTraceLineLimitTransactionSelector newSelectorForNewBlock( + final Map lineCountLimits) { return new TestableTraceLineLimitTransactionSelector( - lineCountLimits, "line-limits.toml", OVER_LINE_COUNT_LIMIT_CACHE_SIZE); - } - - private Map loadLineCountLimitConf() { - try (final InputStream is = - this.getClass().getResourceAsStream("/sequencer/line-limits.toml")) { - return Toml.parse(is).getTable("traces-limits").toMap().entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> Math.toIntExact((long) e.getValue()))); - } catch (IOException e) { - throw new RuntimeException(e); - } + lineaTracerConfiguration, lineCountLimits, OVER_LINE_COUNT_LIMIT_CACHE_SIZE); } @Test public void shouldSelectWhenBelowLimits() { + final var transactionSelector = newSelectorForNewBlock(lineCountLimits); + transactionSelector.resetCache(); + final var evaluationContext = mockEvaluationContext(false, 100, Wei.of(1_100_000_000), Wei.of(1_000_000_000), 21000); verifyTransactionSelection( @@ -88,6 +102,9 @@ public void shouldSelectWhenBelowLimits() { @Test public void shouldNotSelectWhenOverLimits() { lineCountLimits.put("ADD", 1); + final var transactionSelector = newSelectorForNewBlock(lineCountLimits); + transactionSelector.resetCache(); + final var evaluationContext = mockEvaluationContext(false, 100, Wei.of(1_100_000_000), Wei.of(1_000_000_000), 21000); verifyTransactionSelection( @@ -105,6 +122,9 @@ public void shouldNotSelectWhenOverLimits() { @Test public void shouldNotReprocessedWhenOverLimits() { lineCountLimits.put("ADD", 1); + var transactionSelector = newSelectorForNewBlock(lineCountLimits); + transactionSelector.resetCache(); + var evaluationContext = mockEvaluationContext(false, 100, Wei.of(1_100_000_000), Wei.of(1_000_000_000), 21000); verifyTransactionSelection( @@ -118,7 +138,7 @@ public void shouldNotReprocessedWhenOverLimits() { transactionSelector.isOverLineCountLimitTxCached( evaluationContext.getPendingTransaction().getTransaction().getHash())) .isTrue(); - transactionSelector = newSelectorForNewBlock(); + transactionSelector = newSelectorForNewBlock(lineCountLimits); assertThat( transactionSelector.isOverLineCountLimitTxCached( evaluationContext.getPendingTransaction().getTransaction().getHash())) @@ -139,6 +159,9 @@ public void shouldNotReprocessedWhenOverLimits() { @Test public void shouldEvictWhenCacheIsFull() { lineCountLimits.put("ADD", 1); + final var transactionSelector = newSelectorForNewBlock(lineCountLimits); + transactionSelector.resetCache(); + final TestTransactionEvaluationContext[] evaluationContexts = new TestTransactionEvaluationContext[OVER_LINE_COUNT_LIMIT_CACHE_SIZE + 1]; for (int i = 0; i <= OVER_LINE_COUNT_LIMIT_CACHE_SIZE; i++) { @@ -221,22 +244,22 @@ private TestTransactionEvaluationContext mockEvaluationContext( private class TestableTraceLineLimitTransactionSelector extends TraceLineLimitTransactionSelector { TestableTraceLineLimitTransactionSelector( + final LineaTracerConfiguration lineaTracerConfiguration, final Map moduleLimits, - final String limitFilePath, final int overLimitCacheSize) { super( moduleLimits, LineaTransactionSelectorConfiguration.builder() - .moduleLimitsFilePath(limitFilePath) .overLinesLimitCacheSize(overLimitCacheSize) .build(), LineaL1L2BridgeConfiguration.builder() .contract(Address.fromHexString("0xDEADBEEF")) .topic(Bytes.fromHexString("0x012345")) - .build()); + .build(), + lineaTracerConfiguration); } - void reset() { + void resetCache() { overLineCountLimitCache.clear(); }