From e544e2eb2f2ec823083c35c8661c6ad0ba000ab2 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Wed, 4 Dec 2024 12:58:57 +0100 Subject: [PATCH 1/2] Estimate the memory size of EIP-7702 transactions Signed-off-by: Fabio Di Fabio --- .../besu/ethereum/core/Transaction.java | 15 +- .../ethereum/core/TransactionTestFixture.java | 18 ++- .../eth/transactions/PendingTransaction.java | 26 +++- ...ingTransactionEstimatedMemorySizeTest.java | 133 +++++++++++------- 4 files changed, 132 insertions(+), 60 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java index 1926fa8a7d3..e3fde4c32be 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java @@ -1092,6 +1092,10 @@ public Transaction detachedCopy() { blobsWithCommitments.map( withCommitments -> blobsWithCommitmentsDetachedCopy(withCommitments, detachedVersionedHashes.get())); + final Optional> detachedCodeDelegationList = + maybeCodeDelegationList.map( + codeDelegations -> + codeDelegations.stream().map(this::codeDelegationDetachedCopy).toList()); final var copiedTx = new Transaction( @@ -1112,7 +1116,7 @@ public Transaction detachedCopy() { chainId, detachedVersionedHashes, detachedBlobsWithCommitments, - maybeCodeDelegationList); + detachedCodeDelegationList); // copy also the computed fields, to avoid to recompute them copiedTx.sender = this.sender; @@ -1129,6 +1133,15 @@ private AccessListEntry accessListDetachedCopy(final AccessListEntry accessListE return new AccessListEntry(detachedAddress, detachedStorage); } + private CodeDelegation codeDelegationDetachedCopy(final CodeDelegation codeDelegation) { + final Address detachedAddress = Address.wrap(codeDelegation.address().copy()); + return new org.hyperledger.besu.ethereum.core.CodeDelegation( + codeDelegation.chainId(), + detachedAddress, + codeDelegation.nonce(), + codeDelegation.signature()); + } + private BlobsWithCommitments blobsWithCommitmentsDetachedCopy( final BlobsWithCommitments blobsWithCommitments, final List versionedHashes) { final var detachedCommitments = diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TransactionTestFixture.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TransactionTestFixture.java index 602116749b6..5cef826ed68 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TransactionTestFixture.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TransactionTestFixture.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.core; import org.hyperledger.besu.crypto.KeyPair; +import org.hyperledger.besu.crypto.SECPSignature; import org.hyperledger.besu.datatypes.AccessListEntry; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.BlobsWithCommitments; @@ -29,7 +30,8 @@ import org.apache.tuweni.bytes.Bytes; public class TransactionTestFixture { - + private final SECPSignature signature = + new SECPSignature(BigInteger.ONE, BigInteger.ONE, (byte) 0); private TransactionType transactionType = TransactionType.FRONTIER; private long nonce = 0; @@ -56,6 +58,8 @@ public class TransactionTestFixture { private Optional blobs = Optional.empty(); private Optional v = Optional.empty(); + private Optional> codeDelegations = + Optional.empty(); public Transaction createTransaction(final KeyPair keys) { final Transaction.Builder builder = Transaction.builder(); @@ -93,6 +97,12 @@ public Transaction createTransaction(final KeyPair keys) { } break; case DELEGATE_CODE: + builder.maxPriorityFeePerGas(maxPriorityFeePerGas.orElse(Wei.of(500))); + builder.maxFeePerGas(maxFeePerGas.orElse(Wei.of(5000))); + builder.accessList(accessListEntries.orElse(List.of())); + builder.codeDelegations( + codeDelegations.orElse( + List.of(new CodeDelegation(chainId.get(), sender, 0, signature)))); break; } @@ -183,4 +193,10 @@ public TransactionTestFixture blobsWithCommitments(final Optional codeDelegations) { + this.codeDelegations = Optional.of(codeDelegations); + return this; + } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java index 91b4efd7b21..cf6bafa8320 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java @@ -38,7 +38,9 @@ public abstract class PendingTransaction static final int PAYLOAD_BASE_MEMORY_SIZE = 32; static final int ACCESS_LIST_STORAGE_KEY_MEMORY_SIZE = 32; static final int ACCESS_LIST_ENTRY_BASE_MEMORY_SIZE = 248; - static final int OPTIONAL_ACCESS_LIST_MEMORY_SIZE = 24; + static final int OPTIONAL_ACCESS_LIST_MEMORY_SIZE = 40; + static final int OPTIONAL_CODE_DELEGATION_LIST_MEMORY_SIZE = 40; + static final int CODE_DELEGATION_ENTRY_MEMORY_SIZE = 432; static final int VERSIONED_HASH_SIZE = 96; static final int BASE_LIST_SIZE = 48; static final int BASE_OPTIONAL_SIZE = 16; @@ -147,7 +149,7 @@ private int computeMemorySize() { case ACCESS_LIST -> computeAccessListMemorySize(); case EIP1559 -> computeEIP1559MemorySize(); case BLOB -> computeBlobMemorySize(); - case DELEGATE_CODE -> computeSetCodeMemorySize(); + case DELEGATE_CODE -> computeDelegateCodeMemorySize(); } + PENDING_TRANSACTION_MEMORY_SIZE; } @@ -181,8 +183,8 @@ private int computeBlobMemorySize() { + computeBlobWithCommitmentsMemorySize(); } - private int computeSetCodeMemorySize() { - return 0; + private int computeDelegateCodeMemorySize() { + return computeEIP1559MemorySize() + computeCodeDelegationListMemorySize(); } private int computeBlobWithCommitmentsMemorySize() { @@ -197,7 +199,7 @@ private int computeBlobWithCommitmentsMemorySize() { } private int computePayloadMemorySize() { - return transaction.getPayload().size() > 0 + return !transaction.getPayload().isEmpty() ? PAYLOAD_BASE_MEMORY_SIZE + transaction.getPayload().size() : 0; } @@ -231,6 +233,18 @@ private int computeAccessListEntriesMemorySize() { .orElse(0); } + private int computeCodeDelegationListMemorySize() { + return transaction + .getCodeDelegationList() + .map( + cd -> { + int totalSize = OPTIONAL_CODE_DELEGATION_LIST_MEMORY_SIZE; + totalSize += cd.size() * CODE_DELEGATION_ENTRY_MEMORY_SIZE; + return totalSize; + }) + .orElse(0); + } + public static List toTransactionList( final Collection transactionsInfo) { return transactionsInfo.stream().map(PendingTransaction::getTransaction).toList(); @@ -252,7 +266,7 @@ public boolean equals(final Object o) { @Override public int hashCode() { - return 31 * (int) (sequence ^ (sequence >>> 32)); + return 31 * Long.hashCode(sequence); } @Override diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java index af54f668fe8..c80f5ca1ad4 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.datatypes.AccessListEntry; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.BlobsWithCommitments; +import org.hyperledger.besu.datatypes.CodeDelegation; import org.hyperledger.besu.datatypes.TransactionType; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.Transaction; @@ -31,6 +32,8 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import java.io.IOException; +import java.lang.management.ManagementFactory; import java.math.BigInteger; import java.util.Arrays; import java.util.HashSet; @@ -41,8 +44,10 @@ import java.util.TreeSet; import java.util.concurrent.atomic.LongAdder; import java.util.function.Function; +import javax.management.MBeanServer; import com.google.common.collect.Sets; +import com.sun.management.HotSpotDiagnosticMXBean; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.junit.jupiter.api.Test; @@ -64,7 +69,14 @@ public class PendingTransactionEstimatedMemorySizeTest extends BaseTransactionPo private static final Set FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS = Sets.union(COMMON_CONSTANT_FIELD_PATHS, Set.of(".maxFeePerGas", ".maxPriorityFeePerGas")); private static final Set VARIABLE_SIZE_PATHS = - Set.of(".chainId", ".to", ".payload", ".maybeAccessList"); + Set.of( + ".chainId", + ".to", + ".payload", + ".maybeAccessList", + ".versionedHashes", + ".blobsWithCommitments", + ".maybeCodeDelegationList"); @Test public void toSize() { @@ -100,6 +112,7 @@ public void toSize() { + ", " + gpr.klass().toString() + "]"); + System.out.println(ClassLayout.parseClass(gpr.klass()).toPrintable()); }; GraphWalker gw = new GraphWalker(gv); @@ -319,9 +332,10 @@ public void accessListSize() { final ClassLayout cl2 = ClassLayout.parseInstance(optAL.get()); System.out.println(cl2.toPrintable()); - System.out.println("Optional + list size: " + cl2.instanceSize()); + System.out.println("Optional + list size: " + (cl1.instanceSize() + cl2.instanceSize())); - assertThat(cl2.instanceSize()).isEqualTo(PendingTransaction.OPTIONAL_ACCESS_LIST_MEMORY_SIZE); + assertThat(cl1.instanceSize() + cl2.instanceSize()) + .isEqualTo(PendingTransaction.OPTIONAL_ACCESS_LIST_MEMORY_SIZE); final AccessListEntry ale = optAL.get().get(0); @@ -341,80 +355,87 @@ public void accessListSize() { } @Test - public void baseEIP1559AndEIP4844TransactionMemorySize() { + public void codeDelegationListSize() { System.setProperty("jol.magicFieldOffset", "true"); - Transaction txEip1559 = createEIP1559Transaction(1, KEYS1, 10); + + TransactionTestFixture preparedTx = + prepareTransaction(TransactionType.DELEGATE_CODE, 0, Wei.of(500), Wei.ZERO, 0, 0); + Transaction txDelegateCode = preparedTx.createTransaction(KEYS1); BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); - txEip1559.writeTo(rlpOut); + txDelegateCode.writeTo(rlpOut); - txEip1559 = + txDelegateCode = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)).detachedCopy(); - System.out.println(txEip1559.getSender()); - System.out.println(txEip1559.getHash()); - System.out.println(txEip1559.getSize()); + System.out.println(txDelegateCode.getSender()); + System.out.println(txDelegateCode.getHash()); + System.out.println(txDelegateCode.getSize()); - final ClassLayout cl = ClassLayout.parseInstance(txEip1559); - System.out.println(cl.toPrintable()); - LongAdder eip1559size = new LongAdder(); - eip1559size.add(cl.instanceSize()); - System.out.println(eip1559size); + final var optCD = txDelegateCode.getCodeDelegationList(); - final SortedSet fieldSizes = new TreeSet<>(); - GraphWalker gw = getGraphWalker(EIP1559_EIP4844_CONSTANT_FIELD_PATHS, fieldSizes); + final ClassLayout cl1 = ClassLayout.parseInstance(optCD); + System.out.println(cl1.toPrintable()); + System.out.println("Optional size: " + cl1.instanceSize()); - gw.walk(txEip1559); + final ClassLayout cl2 = ClassLayout.parseInstance(optCD.get()); + System.out.println(cl2.toPrintable()); + System.out.println("Optional + list size: " + (cl1.instanceSize() + cl2.instanceSize())); - fieldSizes.forEach( - fieldSize -> { - eip1559size.add(fieldSize.size()); - System.out.println( - "(" - + eip1559size - + ")[" - + fieldSize.size() - + ", " - + fieldSize.path() - + ", " - + fieldSize - + "]"); - }); + assertThat(cl1.instanceSize() + cl2.instanceSize()) + .isEqualTo(PendingTransaction.OPTIONAL_CODE_DELEGATION_LIST_MEMORY_SIZE); + + final CodeDelegation codeDelegation = optCD.get().get(0); + + long cdSize = sizeOfField(codeDelegation, "storageKeys.elementData["); - System.out.println("Base EIP1559 size: " + eip1559size); - assertThat(eip1559size.sum()) + System.out.println("CodeDelegation container size: " + cdSize); + + assertThat(cdSize).isEqualTo(PendingTransaction.CODE_DELEGATION_ENTRY_MEMORY_SIZE); + } + + @Test + public void baseEIP1559AndEIP4844TransactionMemorySize() { + Transaction txEip1559 = createEIP1559Transaction(1, KEYS1, 10); + assertThat(baseTransactionMemorySize(txEip1559, EIP1559_EIP4844_CONSTANT_FIELD_PATHS)) .isEqualTo(PendingTransaction.EIP1559_AND_EIP4844_SHALLOW_MEMORY_SIZE); } @Test public void baseFrontierAndAccessListTransactionMemorySize() { + final Transaction txFrontier = + createTransaction(TransactionType.FRONTIER, 1, Wei.of(500), 0, KEYS1); + assertThat(baseTransactionMemorySize(txFrontier, FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS)) + .isEqualTo(PendingTransaction.FRONTIER_AND_ACCESS_LIST_SHALLOW_MEMORY_SIZE); + } + + private long baseTransactionMemorySize(final Transaction tx, final Set constantFields) { System.setProperty("jol.magicFieldOffset", "true"); - Transaction txFrontier = createTransaction(TransactionType.FRONTIER, 1, Wei.of(500), 0, KEYS1); BytesValueRLPOutput rlpOut = new BytesValueRLPOutput(); - txFrontier.writeTo(rlpOut); + tx.writeTo(rlpOut); - txFrontier = + final var baseTx = Transaction.readFrom(new BytesValueRLPInput(rlpOut.encoded(), false)).detachedCopy(); - System.out.println(txFrontier.getSender()); - System.out.println(txFrontier.getHash()); - System.out.println(txFrontier.getSize()); + System.out.println(baseTx.getSender()); + System.out.println(baseTx.getHash()); + System.out.println(baseTx.getSize()); - final ClassLayout cl = ClassLayout.parseInstance(txFrontier); + final ClassLayout cl = ClassLayout.parseInstance(baseTx); System.out.println(cl.toPrintable()); - LongAdder frontierSize = new LongAdder(); - frontierSize.add(cl.instanceSize()); - System.out.println(frontierSize); + LongAdder baseTxSize = new LongAdder(); + baseTxSize.add(cl.instanceSize()); + System.out.println(baseTxSize); final SortedSet fieldSizes = new TreeSet<>(); - GraphWalker gw = getGraphWalker(FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS, fieldSizes); + GraphWalker gw = getGraphWalker(constantFields, fieldSizes); - gw.walk(txFrontier); + gw.walk(baseTx); fieldSizes.forEach( fieldSize -> { - frontierSize.add(fieldSize.size()); + baseTxSize.add(fieldSize.size()); System.out.println( "(" - + frontierSize + + baseTxSize + ")[" + fieldSize.size() + ", " @@ -423,10 +444,8 @@ public void baseFrontierAndAccessListTransactionMemorySize() { + fieldSize + "]"); }); - - System.out.println("Base Frontier size: " + frontierSize); - assertThat(frontierSize.sum()) - .isEqualTo(PendingTransaction.FRONTIER_AND_ACCESS_LIST_SHALLOW_MEMORY_SIZE); + System.out.println("Base tx size: " + baseTxSize); + return baseTxSize.sum(); } private GraphWalker getGraphWalker( @@ -478,6 +497,7 @@ private long sizeOfField(final Object container, final String... excludePaths) { + ", " + gpr.klass().toString() + "]"); + System.out.println(ClassLayout.parseClass(gpr.klass()).toPrintable()); } }; @@ -489,6 +509,15 @@ private long sizeOfField(final Object container, final String... excludePaths) { return size.sum(); } + @SuppressWarnings("unused") + private static void dumpHeap(final String filePath, final boolean live) throws IOException { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + HotSpotDiagnosticMXBean mxBean = + ManagementFactory.newPlatformMXBeanProxy( + server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class); + mxBean.dumpHeap(filePath, live); + } + record FieldSize(String path, Class clazz, long size) implements Comparable { @Override From 7c3955bcd990ee434d72d92401d34fcbe2593453 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 10 Dec 2024 18:48:52 +0100 Subject: [PATCH 2/2] Apply suggestions from code review Signed-off-by: Fabio Di Fabio --- .../eth/transactions/PendingTransaction.java | 90 +++++++---- ...ingTransactionEstimatedMemorySizeTest.java | 144 +++++++++++++----- 2 files changed, 164 insertions(+), 70 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java index cf6bafa8320..ee5c40a3f4f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransaction.java @@ -14,6 +14,24 @@ */ package org.hyperledger.besu.ethereum.eth.transactions; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.ACCESS_LIST_ENTRY_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.ACCESS_LIST_STORAGE_KEY_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.BLOBS_WITH_COMMITMENTS_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.BLOB_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.CODE_DELEGATION_ENTRY_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.EIP1559_AND_EIP4844_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.FRONTIER_AND_ACCESS_LIST_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.KZG_COMMITMENT_OR_PROOF_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.LIST_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_ACCESS_LIST_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_CHAIN_ID_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_CODE_DELEGATION_LIST_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_TO_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.PAYLOAD_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.PENDING_TRANSACTION_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.VERSIONED_HASH_SIZE; + import org.hyperledger.besu.datatypes.AccessListEntry; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; @@ -31,23 +49,6 @@ public abstract class PendingTransaction implements org.hyperledger.besu.datatypes.PendingTransaction { static final int NOT_INITIALIZED = -1; - static final int FRONTIER_AND_ACCESS_LIST_SHALLOW_MEMORY_SIZE = 904; - static final int EIP1559_AND_EIP4844_SHALLOW_MEMORY_SIZE = 1016; - static final int OPTIONAL_TO_MEMORY_SIZE = 112; - static final int OPTIONAL_CHAIN_ID_MEMORY_SIZE = 80; - static final int PAYLOAD_BASE_MEMORY_SIZE = 32; - static final int ACCESS_LIST_STORAGE_KEY_MEMORY_SIZE = 32; - static final int ACCESS_LIST_ENTRY_BASE_MEMORY_SIZE = 248; - static final int OPTIONAL_ACCESS_LIST_MEMORY_SIZE = 40; - static final int OPTIONAL_CODE_DELEGATION_LIST_MEMORY_SIZE = 40; - static final int CODE_DELEGATION_ENTRY_MEMORY_SIZE = 432; - static final int VERSIONED_HASH_SIZE = 96; - static final int BASE_LIST_SIZE = 48; - static final int BASE_OPTIONAL_SIZE = 16; - static final int KZG_COMMITMENT_OR_PROOF_SIZE = 112; - static final int BLOB_SIZE = 131136; - static final int BLOBS_WITH_COMMITMENTS_SIZE = 40; - static final int PENDING_TRANSACTION_MEMORY_SIZE = 40; private static final AtomicLong TRANSACTIONS_ADDED = new AtomicLong(); private final Transaction transaction; private final long addedAt; @@ -151,18 +152,18 @@ private int computeMemorySize() { case BLOB -> computeBlobMemorySize(); case DELEGATE_CODE -> computeDelegateCodeMemorySize(); } - + PENDING_TRANSACTION_MEMORY_SIZE; + + PENDING_TRANSACTION_SHALLOW_SIZE; } private int computeFrontierMemorySize() { - return FRONTIER_AND_ACCESS_LIST_SHALLOW_MEMORY_SIZE + return FRONTIER_AND_ACCESS_LIST_SHALLOW_SIZE + computePayloadMemorySize() + computeToMemorySize() + computeChainIdMemorySize(); } private int computeAccessListMemorySize() { - return FRONTIER_AND_ACCESS_LIST_SHALLOW_MEMORY_SIZE + return FRONTIER_AND_ACCESS_LIST_SHALLOW_SIZE + computePayloadMemorySize() + computeToMemorySize() + computeChainIdMemorySize() @@ -170,7 +171,7 @@ private int computeAccessListMemorySize() { } private int computeEIP1559MemorySize() { - return EIP1559_AND_EIP4844_SHALLOW_MEMORY_SIZE + return EIP1559_AND_EIP4844_SHALLOW_SIZE + computePayloadMemorySize() + computeToMemorySize() + computeChainIdMemorySize() @@ -179,7 +180,7 @@ private int computeEIP1559MemorySize() { private int computeBlobMemorySize() { return computeEIP1559MemorySize() - + BASE_OPTIONAL_SIZE // for the versionedHashes field + + OPTIONAL_SHALLOW_SIZE // for the versionedHashes field + computeBlobWithCommitmentsMemorySize(); } @@ -190,9 +191,9 @@ private int computeDelegateCodeMemorySize() { private int computeBlobWithCommitmentsMemorySize() { final int blobCount = transaction.getBlobCount(); - return BASE_OPTIONAL_SIZE + return OPTIONAL_SHALLOW_SIZE + BLOBS_WITH_COMMITMENTS_SIZE - + (BASE_LIST_SIZE * 4) + + (LIST_SHALLOW_SIZE * 4) + (KZG_COMMITMENT_OR_PROOF_SIZE * blobCount * 2) + (VERSIONED_HASH_SIZE * blobCount) + (BLOB_SIZE * blobCount); @@ -200,20 +201,20 @@ private int computeBlobWithCommitmentsMemorySize() { private int computePayloadMemorySize() { return !transaction.getPayload().isEmpty() - ? PAYLOAD_BASE_MEMORY_SIZE + transaction.getPayload().size() + ? PAYLOAD_SHALLOW_SIZE + transaction.getPayload().size() : 0; } private int computeToMemorySize() { if (transaction.getTo().isPresent()) { - return OPTIONAL_TO_MEMORY_SIZE; + return OPTIONAL_TO_SIZE; } return 0; } private int computeChainIdMemorySize() { if (transaction.getChainId().isPresent()) { - return OPTIONAL_CHAIN_ID_MEMORY_SIZE; + return OPTIONAL_CHAIN_ID_SIZE; } return 0; } @@ -223,11 +224,11 @@ private int computeAccessListEntriesMemorySize() { .getAccessList() .map( al -> { - int totalSize = OPTIONAL_ACCESS_LIST_MEMORY_SIZE; - totalSize += al.size() * ACCESS_LIST_ENTRY_BASE_MEMORY_SIZE; + int totalSize = OPTIONAL_ACCESS_LIST_SHALLOW_SIZE; + totalSize += al.size() * ACCESS_LIST_ENTRY_SHALLOW_SIZE; totalSize += al.stream().map(AccessListEntry::storageKeys).mapToInt(List::size).sum() - * ACCESS_LIST_STORAGE_KEY_MEMORY_SIZE; + * ACCESS_LIST_STORAGE_KEY_SIZE; return totalSize; }) .orElse(0); @@ -238,8 +239,8 @@ private int computeCodeDelegationListMemorySize() { .getCodeDelegationList() .map( cd -> { - int totalSize = OPTIONAL_CODE_DELEGATION_LIST_MEMORY_SIZE; - totalSize += cd.size() * CODE_DELEGATION_ENTRY_MEMORY_SIZE; + int totalSize = OPTIONAL_CODE_DELEGATION_LIST_SHALLOW_SIZE; + totalSize += cd.size() * CODE_DELEGATION_ENTRY_SIZE; return totalSize; }) .orElse(0); @@ -413,4 +414,29 @@ public boolean hasPriority() { } } } + + /** + * The memory size of an object is calculated using the PendingTransactionEstimatedMemorySizeTest + * look there for the details of the calculation and to adapt the code when any of the related + * class changes its structure. + */ + public interface MemorySize { + int FRONTIER_AND_ACCESS_LIST_SHALLOW_SIZE = 904; + int EIP1559_AND_EIP4844_SHALLOW_SIZE = 1016; + int OPTIONAL_TO_SIZE = 112; + int OPTIONAL_CHAIN_ID_SIZE = 80; + int PAYLOAD_SHALLOW_SIZE = 32; + int ACCESS_LIST_STORAGE_KEY_SIZE = 32; + int ACCESS_LIST_ENTRY_SHALLOW_SIZE = 248; + int OPTIONAL_ACCESS_LIST_SHALLOW_SIZE = 40; + int OPTIONAL_CODE_DELEGATION_LIST_SHALLOW_SIZE = 40; + int CODE_DELEGATION_ENTRY_SIZE = 432; + int VERSIONED_HASH_SIZE = 96; + int LIST_SHALLOW_SIZE = 48; + int OPTIONAL_SHALLOW_SIZE = 16; + int KZG_COMMITMENT_OR_PROOF_SIZE = 112; + int BLOB_SIZE = 131136; + int BLOBS_WITH_COMMITMENTS_SIZE = 40; + int PENDING_TRANSACTION_SHALLOW_SIZE = 40; + } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java index c80f5ca1ad4..393e97608e5 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java @@ -15,6 +15,22 @@ package org.hyperledger.besu.ethereum.eth.transactions; import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.ACCESS_LIST_ENTRY_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.ACCESS_LIST_STORAGE_KEY_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.BLOBS_WITH_COMMITMENTS_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.BLOB_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.CODE_DELEGATION_ENTRY_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.EIP1559_AND_EIP4844_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.FRONTIER_AND_ACCESS_LIST_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.KZG_COMMITMENT_OR_PROOF_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.LIST_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_ACCESS_LIST_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_CHAIN_ID_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_CODE_DELEGATION_LIST_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.OPTIONAL_TO_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.PAYLOAD_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.PENDING_TRANSACTION_SHALLOW_SIZE; +import static org.hyperledger.besu.ethereum.eth.transactions.PendingTransaction.MemorySize.VERSIONED_HASH_SIZE; import org.hyperledger.besu.crypto.SignatureAlgorithm; import org.hyperledger.besu.datatypes.AccessListEntry; @@ -58,16 +74,59 @@ import org.openjdk.jol.info.GraphVisitor; import org.openjdk.jol.info.GraphWalker; -@EnabledOnOs(OS.LINUX) +/** + * This test class has a double utility, first it is used to verify that the current memory size of + * a pending transaction object correspond to the calculated values, so if any of the tests in this + * file fails, it probably means that one of the related classes has changed its format, and a new + * calculation needs to be done. + * + *

The second utility is to help with the calculation of the memory size of a class, using the JOL Tool, to navigate the class layout and collect the + * reported memory sizes. + * + *

For a correct calculation there are some things to consider, first exclude from the + * calculation any reference to a constant object, for example if a field is always a + * reference to {@code Optional.empty()} then just count the reference size, but not the size of the + * Optional since it is always the same instance for every pending transaction. + * + *

To study the layout of a class it is usually useful to create a test method with one or more + * instance of it, then programmatically save a heap dump using the {@link #dumpHeap(String, + * boolean)} method, then analyze the heap dump with a tool to identify which are the dynamic and + * constant fields, then use the JOL Tool to print the class layout and walk in the object tree, + * then complete the writing of the test that will verify the current amount of memory used by that + * class. + */ +@EnabledOnOs({OS.LINUX, OS.MAC}) public class PendingTransactionEstimatedMemorySizeTest extends BaseTransactionPoolTest { + /** + * Classes that represent constant instances, across all pending transaction types, and are + * ignored during the calculation + */ private static final Set> SHARED_CLASSES = Set.of(SignatureAlgorithm.class, TransactionType.class); + + /** + * Field that points to constant values, across all pending transaction types, and are ignored + * during the calculation + */ private static final Set COMMON_CONSTANT_FIELD_PATHS = Set.of(".value.ctor", ".hashNoSignature", ".signature.encoded.delegate"); + + /** + * Field that points to constant values, for EIP-1559 and EIP-4844 pending transactions, and are + * ignored during the calculation + */ private static final Set EIP1559_EIP4844_CONSTANT_FIELD_PATHS = Sets.union(COMMON_CONSTANT_FIELD_PATHS, Set.of(".gasPrice")); + + /** + * Field that points to constant values, for Frontier and Access List pending transactions, and + * are ignored during the calculation + */ private static final Set FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS = Sets.union(COMMON_CONSTANT_FIELD_PATHS, Set.of(".maxFeePerGas", ".maxPriorityFeePerGas")); + + /** Field which value is dynamic and can only be calculated at runtime */ private static final Set VARIABLE_SIZE_PATHS = Set.of( ".chainId", @@ -121,7 +180,7 @@ public void toSize() { System.out.println("Optional To size: " + size); - assertThat(size.sum()).isEqualTo(PendingTransaction.OPTIONAL_TO_MEMORY_SIZE); + assertThat(size.sum()).isEqualTo(OPTIONAL_TO_SIZE); } @Test @@ -146,7 +205,7 @@ public void payloadSize() { size.add(cl.instanceSize()); System.out.println("Base payload size: " + size); - assertThat(size.sum()).isEqualTo(PendingTransaction.PAYLOAD_BASE_MEMORY_SIZE); + assertThat(size.sum()).isEqualTo(PAYLOAD_SHALLOW_SIZE); } @Test @@ -180,39 +239,37 @@ public void chainIdSize() { gw.walk(maybeChainId); - assertThat(size.sum()).isEqualTo(PendingTransaction.OPTIONAL_CHAIN_ID_MEMORY_SIZE); + assertThat(size.sum()).isEqualTo(OPTIONAL_CHAIN_ID_SIZE); } @Test public void kgzCommitmentsSize() { blobsWithCommitmentsFieldSize( t -> t.getBlobsWithCommitments().get().getKzgCommitments(), - PendingTransaction.BASE_LIST_SIZE, - PendingTransaction.KZG_COMMITMENT_OR_PROOF_SIZE); + LIST_SHALLOW_SIZE, + KZG_COMMITMENT_OR_PROOF_SIZE); } @Test public void kgzProofsSize() { blobsWithCommitmentsFieldSize( t -> t.getBlobsWithCommitments().get().getKzgProofs(), - PendingTransaction.BASE_LIST_SIZE, - PendingTransaction.KZG_COMMITMENT_OR_PROOF_SIZE); + LIST_SHALLOW_SIZE, + KZG_COMMITMENT_OR_PROOF_SIZE); } @Test public void blobsSize() { blobsWithCommitmentsFieldSize( - t -> t.getBlobsWithCommitments().get().getBlobs(), - PendingTransaction.BASE_LIST_SIZE, - PendingTransaction.BLOB_SIZE); + t -> t.getBlobsWithCommitments().get().getBlobs(), LIST_SHALLOW_SIZE, BLOB_SIZE); } @Test public void versionedHashesSize() { blobsWithCommitmentsFieldSize( t -> t.getBlobsWithCommitments().get().getVersionedHashes(), - PendingTransaction.BASE_LIST_SIZE, - PendingTransaction.VERSIONED_HASH_SIZE); + LIST_SHALLOW_SIZE, + VERSIONED_HASH_SIZE); } private void blobsWithCommitmentsFieldSize( @@ -273,8 +330,7 @@ public void blobsWithCommitmentsSize() { System.out.println(rl.toPrintable()); System.out.println("BlobQuad size:" + rl.instanceSize()); - assertThat(cl.instanceSize() + rl.instanceSize()) - .isEqualTo(PendingTransaction.BLOBS_WITH_COMMITMENTS_SIZE); + assertThat(cl.instanceSize() + rl.instanceSize()).isEqualTo(BLOBS_WITH_COMMITMENTS_SIZE); } @Test @@ -300,7 +356,7 @@ public void pendingTransactionSize() { size.add(cl.instanceSize()); System.out.println("PendingTransaction size: " + size); - assertThat(size.sum()).isEqualTo(PendingTransaction.PENDING_TRANSACTION_MEMORY_SIZE); + assertThat(size.sum()).isEqualTo(PENDING_TRANSACTION_SHALLOW_SIZE); } @Test @@ -326,16 +382,18 @@ public void accessListSize() { final var optAL = txAccessList.getAccessList(); - final ClassLayout cl1 = ClassLayout.parseInstance(optAL); - System.out.println(cl1.toPrintable()); - System.out.println("Optional size: " + cl1.instanceSize()); + final ClassLayout optionalClassLayout = ClassLayout.parseInstance(optAL); + System.out.println(optionalClassLayout.toPrintable()); + System.out.println("Optional size: " + optionalClassLayout.instanceSize()); - final ClassLayout cl2 = ClassLayout.parseInstance(optAL.get()); - System.out.println(cl2.toPrintable()); - System.out.println("Optional + list size: " + (cl1.instanceSize() + cl2.instanceSize())); + final ClassLayout listClassLayout = ClassLayout.parseInstance(optAL.get()); + System.out.println(listClassLayout.toPrintable()); + System.out.println( + "Optional + list size: " + + (optionalClassLayout.instanceSize() + listClassLayout.instanceSize())); - assertThat(cl1.instanceSize() + cl2.instanceSize()) - .isEqualTo(PendingTransaction.OPTIONAL_ACCESS_LIST_MEMORY_SIZE); + assertThat(optionalClassLayout.instanceSize() + listClassLayout.instanceSize()) + .isEqualTo(OPTIONAL_ACCESS_LIST_SHALLOW_SIZE); final AccessListEntry ale = optAL.get().get(0); @@ -343,15 +401,14 @@ public void accessListSize() { System.out.println("AccessListEntry container size: " + aleSize); - assertThat(aleSize).isEqualTo(PendingTransaction.ACCESS_LIST_ENTRY_BASE_MEMORY_SIZE); + assertThat(aleSize).isEqualTo(ACCESS_LIST_ENTRY_SHALLOW_SIZE); final Bytes32 storageKey = ale.storageKeys().get(0); final ClassLayout cl4 = ClassLayout.parseInstance(storageKey); System.out.println(cl4.toPrintable()); System.out.println("Single storage key size: " + cl4.instanceSize()); - assertThat(cl4.instanceSize()) - .isEqualTo(PendingTransaction.ACCESS_LIST_STORAGE_KEY_MEMORY_SIZE); + assertThat(cl4.instanceSize()).isEqualTo(ACCESS_LIST_STORAGE_KEY_SIZE); } @Test @@ -372,16 +429,18 @@ public void codeDelegationListSize() { final var optCD = txDelegateCode.getCodeDelegationList(); - final ClassLayout cl1 = ClassLayout.parseInstance(optCD); - System.out.println(cl1.toPrintable()); - System.out.println("Optional size: " + cl1.instanceSize()); + final ClassLayout optionalClassLayout = ClassLayout.parseInstance(optCD); + System.out.println(optionalClassLayout.toPrintable()); + System.out.println("Optional size: " + optionalClassLayout.instanceSize()); - final ClassLayout cl2 = ClassLayout.parseInstance(optCD.get()); - System.out.println(cl2.toPrintable()); - System.out.println("Optional + list size: " + (cl1.instanceSize() + cl2.instanceSize())); + final ClassLayout listClassLayout = ClassLayout.parseInstance(optCD.get()); + System.out.println(listClassLayout.toPrintable()); + System.out.println( + "Optional + list size: " + + (optionalClassLayout.instanceSize() + listClassLayout.instanceSize())); - assertThat(cl1.instanceSize() + cl2.instanceSize()) - .isEqualTo(PendingTransaction.OPTIONAL_CODE_DELEGATION_LIST_MEMORY_SIZE); + assertThat(optionalClassLayout.instanceSize() + listClassLayout.instanceSize()) + .isEqualTo(OPTIONAL_CODE_DELEGATION_LIST_SHALLOW_SIZE); final CodeDelegation codeDelegation = optCD.get().get(0); @@ -389,14 +448,14 @@ public void codeDelegationListSize() { System.out.println("CodeDelegation container size: " + cdSize); - assertThat(cdSize).isEqualTo(PendingTransaction.CODE_DELEGATION_ENTRY_MEMORY_SIZE); + assertThat(cdSize).isEqualTo(CODE_DELEGATION_ENTRY_SIZE); } @Test public void baseEIP1559AndEIP4844TransactionMemorySize() { Transaction txEip1559 = createEIP1559Transaction(1, KEYS1, 10); assertThat(baseTransactionMemorySize(txEip1559, EIP1559_EIP4844_CONSTANT_FIELD_PATHS)) - .isEqualTo(PendingTransaction.EIP1559_AND_EIP4844_SHALLOW_MEMORY_SIZE); + .isEqualTo(EIP1559_AND_EIP4844_SHALLOW_SIZE); } @Test @@ -404,7 +463,7 @@ public void baseFrontierAndAccessListTransactionMemorySize() { final Transaction txFrontier = createTransaction(TransactionType.FRONTIER, 1, Wei.of(500), 0, KEYS1); assertThat(baseTransactionMemorySize(txFrontier, FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS)) - .isEqualTo(PendingTransaction.FRONTIER_AND_ACCESS_LIST_SHALLOW_MEMORY_SIZE); + .isEqualTo(FRONTIER_AND_ACCESS_LIST_SHALLOW_SIZE); } private long baseTransactionMemorySize(final Transaction tx, final Set constantFields) { @@ -509,6 +568,15 @@ private long sizeOfField(final Object container, final String... excludePaths) { return size.sum(); } + /** + * Utility method useful for producing a heap dump when calculating the memory size of a new + * object. Note that the file is not overwritten, so you need to remove it to create a new heap + * dump. + * + * @param filePath where to save the heap dump + * @param live true to only include live objects + * @throws IOException if any errors happen during the saving + */ @SuppressWarnings("unused") private static void dumpHeap(final String filePath, final boolean live) throws IOException { MBeanServer server = ManagementFactory.getPlatformMBeanServer();