Update existing fixture test format to export correct schema (#47)

* Add function to correctly represent post accounts.

* Add function to correctly pad fields within pre/post accounts.

* Remove print statement :D

* Update framework tests to have correct format.

* Fix added format bug.

* Update test fixtures due to bug.

* Function refactor to work with other fields.

* Add format filters to blockHeader and genesisBlockHeader.

* Update test fixtures with formatted blockHeader and genesisBlockHeader.

* Filter all coinbase and all hashes from reformating in blockHeader and genesisBlockHeader.

* Filter trie's and bloom from reformating in blockHeader and genesisBlockHeader.

* Add chainName/transactions/uncleHeader/withdrawals.

* Unrelated additions to gitignore.

* Fix format for block transactions.

* Review changes, whitelist addition, more formatting.

* Refactor helper function, add tests for them, fix for shanghai withdrawals only.

---------

Co-authored-by: Mario Vega <[email protected]>

.gitignore

@@ -48,5 +48,8 @@ pip-delete-this-directory.txt
 .coverage
 
 # misc
-
+.vscode
 *.code-workspace
+
+# docs
+_readthedocs

src/ethereum_test_tools/common/__init__.py

@@ -27,6 +27,10 @@
     Storage,
     Transaction,
     Withdrawal,
+    alloc_to_accounts,
+    even_padding,
+    key_value_padding,
+    storage_padding,
     str_or_none,
     to_json,
     to_json_or_none,
@@ -48,10 +52,14 @@
     "TestPrivateKey",
     "Transaction",
     "Withdrawal",
+    "alloc_to_accounts",
+    "even_padding",
     "ceiling_division",
     "compute_create2_address",
     "compute_create_address",
     "eip_2028_transaction_data_cost",
+    "key_value_padding",
+    "storage_padding",
     "str_or_none",
     "to_address",
     "to_hash",

src/ethereum_test_tools/common/types.py

@@ -260,6 +260,15 @@ def must_be_equal(self, other: "Storage"):
                 raise Storage.KeyValueMismatch(key, 0, other.data[key])
 
 
+def storage_padding(storage: Dict) -> Dict:
+    """
+    Adds even padding to each storage element.
+    """
+    return {
+        key_value_padding(k): key_value_padding(v) for k, v in storage.items()
+    }
+
+
 @dataclass(kw_only=True)
 class Account:
     """
@@ -425,6 +434,22 @@ def with_code(cls: Type, code: bytes | str | Code) -> "Account":
         return Account(nonce=1, code=code)
 
 
+def alloc_to_accounts(got_alloc: Dict[str, Any]) -> Mapping[str, Account]:
+    """
+    Converts the post state alloc returned from t8n to a mapping of accounts.
+    """
+    accounts = {}
+    for address, value in got_alloc.items():
+        account = Account(
+            nonce=int_or_none(value.get("nonce", None)),
+            balance=int_or_none(value.get("balance", None)),
+            code=value.get("code", None),
+            storage=value.get("storage", None),
+        )
+        accounts[address] = account
+    return accounts
+
+
 ACCOUNT_DEFAULTS = Account(nonce=0, balance=0, code=bytes(), storage={})
 
 
@@ -832,7 +857,9 @@ class FixtureBlock:
     block_header: Optional[FixtureHeader] = None
     expected_exception: Optional[str] = None
     block_number: Optional[int] = None
-    chain_name: Optional[str] = None
+    txs: Optional[List[Transaction]] = None
+    ommers: Optional[List[Header]] = None
+    withdrawals: Optional[List[Withdrawal]] = None
 
 
 @dataclass(kw_only=True)
@@ -879,9 +906,9 @@ def default(self, obj):
                     obj.balance, hex(ACCOUNT_DEFAULTS.balance)
                 ),
                 "code": code_or_none(obj.code, "0x"),
-                "storage": to_json_or_none(obj.storage, {}),
+                "storage": storage_padding(to_json_or_none(obj.storage, {})),
             }
-            return account
+            return even_padding(account, excluded=["storage"])
         elif isinstance(obj, Transaction):
             tx = {
                 "type": hex(obj.ty),
@@ -912,12 +939,13 @@ def default(self, obj):
 
             return {k: v for (k, v) in tx.items() if v is not None}
         elif isinstance(obj, Withdrawal):
-            return {
+            withdrawal = {
                 "index": hex(obj.index),
                 "validatorIndex": hex(obj.validator),
                 "address": obj.address,
                 "amount": hex(obj.amount),
             }
+            return withdrawal
         elif isinstance(obj, Environment):
             env = {
                 "currentCoinbase": obj.coinbase,
@@ -967,11 +995,52 @@ def default(self, obj):
                 header["hash"] = obj.hash
             if obj.withdrawals_root is not None:
                 header["withdrawalsRoot"] = obj.withdrawals_root
-            return header
+            return even_padding(
+                header,
+                excluded=[
+                    "parentHash",
+                    "uncleHash",
+                    "coinbase",
+                    "transactionsTrie",
+                    "receiptTrie",
+                    "bloom",
+                    "nonce",
+                    "mixHash",
+                    "hash",
+                    "withdrawalsRoot",
+                ],
+            )
         elif isinstance(obj, FixtureBlock):
-            b = {
-                "rlp": obj.rlp,
-            }
+            # Format Fixture Block Txs
+            b_txs = [
+                even_padding(
+                    to_json(
+                        {
+                            "nonce": hex(tx.nonce),
+                            "to": tx.to if tx.to is not None else "",
+                            "value": hex(tx.value),
+                            "data": code_to_hex(tx.data),
+                            "gasLimit": hex_or_none(tx.gas_limit),
+                            "gasPrice": hex(tx.gas_price)
+                            if tx.gas_price is not None
+                            else "0x0A",
+                            "secretKey": tx.secret_key,
+                        }
+                    ),
+                    excluded=["to"],
+                )
+                for tx in obj.txs or []
+            ]
+
+            # Format Fixture Block Withdrawals
+            b_wds = []
+            if obj.withdrawals:
+                b_wds = [
+                    even_padding(to_json(wd), excluded=["address"])
+                    for wd in obj.withdrawals
+                ]
+
+            b = {"rlp": obj.rlp}
             if obj.block_header is not None:
                 b["blockHeader"] = json.loads(
                     json.dumps(obj.block_header, cls=JSONEncoder)
@@ -980,8 +1049,12 @@ def default(self, obj):
                 b["expectException"] = obj.expected_exception
             if obj.block_number is not None:
                 b["blocknumber"] = str(obj.block_number)
-            if obj.chain_name is not None:
-                b["chainname"] = obj.chain_name
+            if obj.txs is not None:
+                b["transactions"] = b_txs
+            if obj.ommers is not None:
+                b["uncleHeaders"] = obj.ommers
+            if obj.withdrawals is not None:
+                b["withdrawals"] = b_wds
             return b
         elif isinstance(obj, Fixture):
             f = {
@@ -1005,3 +1078,32 @@ def default(self, obj):
             return f
         else:
             return super().default(obj)
+
+
+def even_padding(input: Dict, excluded: List[Any | None]) -> Dict:
+    """
+    Adds even padding to each field in the input (nested) dictionary.
+    """
+    for key, value in input.items():
+        if key not in excluded:
+            if isinstance(value, dict):
+                even_padding(value, excluded)
+            elif value != "0x" and value is not None:
+                input[key] = key_value_padding(value)
+            else:
+                input[key] = "0x"
+    return input
+
+
+def key_value_padding(value: str) -> str:
+    """
+    Adds even padding to a dictionary key or value string.
+    """
+    if value is not None:
+        new_value = value.lstrip("0x").lstrip("0")
+        new_value = "00" if new_value == "" else new_value
+        if len(new_value) % 2 == 1:
+            new_value = "0" + new_value
+        return "0x" + new_value
+    else:
+        return "0x"

src/ethereum_test_tools/filling/fill.py

@@ -7,7 +7,7 @@
 from evm_block_builder import BlockBuilder
 from evm_transition_tool import TransitionTool, map_fork
 
-from ..common import Fixture
+from ..common import Fixture, alloc_to_accounts
 from ..spec import TestSpec
 from ..vm.fork import get_reward
 
@@ -63,7 +63,7 @@ def fill_test(
                 if eips is not None
                 else fork,
                 pre_state=copy(test.pre),
-                post_state=alloc,
+                post_state=alloc_to_accounts(alloc),
                 seal_engine=engine,
                 name=test.name,
                 index=index,

src/ethereum_test_tools/spec/blockchain_test.py

@@ -194,6 +194,9 @@ def make_block(
                         rlp=rlp,
                         block_header=header,
                         block_number=header.number,
+                        txs=block.txs if block.txs is not None else [],
+                        ommers=[],
+                        withdrawals=env.withdrawals,
                     ),
                     env.apply_new_parent(header),
                     next_alloc,

src/ethereum_test_tools/spec/state_test.py

@@ -105,7 +105,6 @@ def make_blocks(
         Raises exception on invalid test behavior.
         """
         env = self.env.apply_new_parent(genesis)
-
         env = set_fork_requirements(env, fork)
 
         (alloc, result, txs_rlp) = t8n.evaluate(
@@ -152,18 +151,23 @@ def make_blocks(
                 "baseFeePerGas": result.get("currentBaseFee"),
             }
         )
+
         block, head = b11r.build(
             header=header.to_geth_dict(),
             txs=txs_rlp,
             ommers=[],
             withdrawals=to_json_or_none(env.withdrawals),
         )
         header.hash = head
+
         return (
             [
                 FixtureBlock(
                     rlp=block,
                     block_header=header,
+                    txs=self.txs if self.txs is not None else [],
+                    ommers=[],
+                    withdrawals=env.withdrawals,
                 )
             ],
             head,