diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f733a39804..62c76d43c9 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -51,6 +51,7 @@ Test fixtures for use by clients are available for each release on the [Github r - ✨ Add a helper class `ethereum_test_tools.TestParameterGroup` to define test parameters as dataclasses and auto-generate test IDs ([#364](https://github.com/ethereum/execution-spec-tests/pull/364)). - 🔀 Change custom exception classes to dataclasses to improve testability ([#386](https://github.com/ethereum/execution-spec-tests/pull/386)). - 🔀 Updates fork name from Merge to Paris ([#363](https://github.com/ethereum/execution-spec-tests/pull/363)). +- ✨ Add framework unit tests for post state exception verification ([#350](https://github.com/ethereum/execution-spec-tests/pull/350)). ### 🔧 EVM Tools diff --git a/src/ethereum_test_tools/common/types.py b/src/ethereum_test_tools/common/types.py index 7d79c45cf3..19fd41b3c4 100644 --- a/src/ethereum_test_tools/common/types.py +++ b/src/ethereum_test_tools/common/types.py @@ -500,7 +500,7 @@ def must_contain(self, address: str, other: "Storage"): raise Storage.MissingKey(key=key) elif self.data[key] != other.data[key]: raise Storage.KeyValueMismatch( - address=address, key=key, got=self.data[key], want=other.data[key] + address=address, key=key, want=self.data[key], got=other.data[key] ) def must_be_equal(self, address: str, other: "Storage"): @@ -511,7 +511,7 @@ def must_be_equal(self, address: str, other: "Storage"): for key in self.data.keys() & other.data.keys(): if self.data[key] != other.data[key]: raise Storage.KeyValueMismatch( - address=address, key=key, got=self.data[key], want=other.data[key] + address=address, key=key, want=self.data[key], got=other.data[key] ) # Test keys contained in either one of the storage objects diff --git a/src/ethereum_test_tools/tests/test_filling/__init__.py b/src/ethereum_test_tools/tests/test_filling/__init__.py new file mode 100644 index 0000000000..7ae47ad3a8 --- /dev/null +++ b/src/ethereum_test_tools/tests/test_filling/__init__.py @@ -0,0 +1,3 @@ +""" +`ethereum_test_tools.filling` verification tests. +""" diff --git a/src/ethereum_test_tools/tests/test_fixtures/blockchain_london_invalid_filled.json b/src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_london_invalid_filled.json similarity index 100% rename from src/ethereum_test_tools/tests/test_fixtures/blockchain_london_invalid_filled.json rename to src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_london_invalid_filled.json diff --git a/src/ethereum_test_tools/tests/test_fixtures/blockchain_london_valid_filled.json b/src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_london_valid_filled.json similarity index 100% rename from src/ethereum_test_tools/tests/test_fixtures/blockchain_london_valid_filled.json rename to src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_london_valid_filled.json diff --git a/src/ethereum_test_tools/tests/test_fixtures/blockchain_shanghai_invalid_filled_hive.json b/src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_shanghai_invalid_filled_hive.json similarity index 100% rename from src/ethereum_test_tools/tests/test_fixtures/blockchain_shanghai_invalid_filled_hive.json rename to src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_shanghai_invalid_filled_hive.json diff --git a/src/ethereum_test_tools/tests/test_fixtures/blockchain_shanghai_valid_filled_hive.json b/src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_shanghai_valid_filled_hive.json similarity index 100% rename from src/ethereum_test_tools/tests/test_fixtures/blockchain_shanghai_valid_filled_hive.json rename to src/ethereum_test_tools/tests/test_filling/fixtures/blockchain_shanghai_valid_filled_hive.json diff --git a/src/ethereum_test_tools/tests/test_fixtures/chainid_istanbul_blockchain_test.json b/src/ethereum_test_tools/tests/test_filling/fixtures/chainid_istanbul_blockchain_test.json similarity index 100% rename from src/ethereum_test_tools/tests/test_fixtures/chainid_istanbul_blockchain_test.json rename to src/ethereum_test_tools/tests/test_filling/fixtures/chainid_istanbul_blockchain_test.json diff --git a/src/ethereum_test_tools/tests/test_fixtures/chainid_london_blockchain_test.json b/src/ethereum_test_tools/tests/test_filling/fixtures/chainid_london_blockchain_test.json similarity index 100% rename from src/ethereum_test_tools/tests/test_fixtures/chainid_london_blockchain_test.json rename to src/ethereum_test_tools/tests/test_filling/fixtures/chainid_london_blockchain_test.json diff --git a/src/ethereum_test_tools/tests/test_fixtures/chainid_paris_blockchain_test_hive.json b/src/ethereum_test_tools/tests/test_filling/fixtures/chainid_paris_blockchain_test_hive.json similarity index 100% rename from src/ethereum_test_tools/tests/test_fixtures/chainid_paris_blockchain_test_hive.json rename to src/ethereum_test_tools/tests/test_filling/fixtures/chainid_paris_blockchain_test_hive.json diff --git a/src/ethereum_test_tools/tests/test_fixtures/chainid_paris_state_test.json b/src/ethereum_test_tools/tests/test_filling/fixtures/chainid_paris_state_test.json similarity index 100% rename from src/ethereum_test_tools/tests/test_fixtures/chainid_paris_state_test.json rename to src/ethereum_test_tools/tests/test_filling/fixtures/chainid_paris_state_test.json diff --git a/src/ethereum_test_tools/tests/test_fixtures/chainid_shanghai_blockchain_test_hive.json b/src/ethereum_test_tools/tests/test_filling/fixtures/chainid_shanghai_blockchain_test_hive.json similarity index 100% rename from src/ethereum_test_tools/tests/test_fixtures/chainid_shanghai_blockchain_test_hive.json rename to src/ethereum_test_tools/tests/test_filling/fixtures/chainid_shanghai_blockchain_test_hive.json diff --git a/src/ethereum_test_tools/tests/test_fixtures/chainid_shanghai_state_test.json b/src/ethereum_test_tools/tests/test_filling/fixtures/chainid_shanghai_state_test.json similarity index 100% rename from src/ethereum_test_tools/tests/test_fixtures/chainid_shanghai_state_test.json rename to src/ethereum_test_tools/tests/test_filling/fixtures/chainid_shanghai_state_test.json diff --git a/src/ethereum_test_tools/tests/test_filling/test_expect.py b/src/ethereum_test_tools/tests/test_filling/test_expect.py new file mode 100644 index 0000000000..02b0a3ee0c --- /dev/null +++ b/src/ethereum_test_tools/tests/test_filling/test_expect.py @@ -0,0 +1,235 @@ +""" +Test fixture post state (expect section) during state fixture generation. +""" +from typing import Any, Mapping + +import pytest + +from ethereum_test_forks import Fork, get_deployed_forks +from evm_transition_tool import FixtureFormats, GethTransitionTool + +from ...common import Account, Environment, Transaction, to_address +from ...common.types import Storage +from ...spec import StateTest + +ADDRESS_UNDER_TEST = to_address(0x01) + + +@pytest.fixture +def pre(request) -> Mapping[Any, Any]: + """ + The pre state: Set from the test's indirectly parametrized `pre` parameter. + """ + return request.param + + +@pytest.fixture +def post(request) -> Mapping[Any, Any]: # noqa: D103 + """ + The post state: Set from the test's indirectly parametrized `post` parameter. + """ + return request.param + + +@pytest.fixture +def fork() -> Fork: # noqa: D103 + return get_deployed_forks()[-1] + + +@pytest.fixture +def state_test( # noqa: D103 + fork: Fork, pre: Mapping[Any, Any], post: Mapping[Any, Any] +) -> StateTest: + return StateTest( + env=Environment(), + pre=pre, + post=post, + tx=Transaction(), + tag="post_value_mismatch", + fixture_format=FixtureFormats.STATE_TEST, + ) + + +@pytest.fixture +def t8n() -> GethTransitionTool: # noqa: D103 + return GethTransitionTool() + + +# Storage value mismatch tests +@pytest.mark.parametrize( + "pre,post,expected_exception", + [ + ( # mismatch_1: 1:1 vs 1:2 + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x01"}, nonce=1)}, + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"})}, + Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=2, got=1), + ), + ( # mismatch_2: 1:1 vs 2:1 + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x01"}, nonce=1)}, + {ADDRESS_UNDER_TEST: Account(storage={"0x02": "0x01"})}, + Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=0, got=1), + ), + ( # mismatch_2_a: 1:1 vs 0:0 + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x01"}, nonce=1)}, + {ADDRESS_UNDER_TEST: Account(storage={"0x00": "0x00"})}, + Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=0, got=1), + ), + ( # mismatch_2_b: 1:1 vs empty + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x01"}, nonce=1)}, + {ADDRESS_UNDER_TEST: Account(storage={})}, + Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=0, got=1), + ), + ( # mismatch_3: 0:0 vs 1:2 + {ADDRESS_UNDER_TEST: Account(storage={"0x00": "0x00"}, nonce=1)}, + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"})}, + Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=2, got=0), + ), + ( # mismatch_3_a: empty vs 1:2 + {ADDRESS_UNDER_TEST: Account(storage={}, nonce=1)}, + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"})}, + Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=1, want=2, got=0), + ), + ( # mismatch_4: 0:3, 1:2 vs 1:2 + {ADDRESS_UNDER_TEST: Account(storage={"0x00": "0x03", "0x01": "0x02"}, nonce=1)}, + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"})}, + Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=0, want=0, got=3), + ), + ( # mismatch_5: 1:2, 2:3 vs 1:2 + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02", "0x02": "0x03"}, nonce=1)}, + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"})}, + Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=2, want=0, got=3), + ), + ( # mismatch_6: 1:2 vs 1:2, 2:3 + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02"}, nonce=1)}, + {ADDRESS_UNDER_TEST: Account(storage={"0x01": "0x02", "0x02": "0x03"})}, + Storage.KeyValueMismatch(address=ADDRESS_UNDER_TEST, key=2, want=3, got=0), + ), + ], +) +def test_post_storage_value_mismatch(pre, post, expected_exception, state_test, t8n, fork): + """ + Test post state `Account.storage` exceptions during state test fixture generation. + """ + with pytest.raises(Storage.KeyValueMismatch) as e_info: + state_test.generate(t8n=t8n, fork=fork) + assert e_info.value == expected_exception + + +# Nonce value mismatch tests +@pytest.mark.parametrize( + "pre,post", + [ + ({ADDRESS_UNDER_TEST: Account(nonce=1)}, {ADDRESS_UNDER_TEST: Account(nonce=2)}), + ({ADDRESS_UNDER_TEST: Account(nonce=1)}, {ADDRESS_UNDER_TEST: Account(nonce=0)}), + ({ADDRESS_UNDER_TEST: Account(nonce=1)}, {ADDRESS_UNDER_TEST: Account(nonce=None)}), + ], +) +def test_post_nonce_value_mismatch(pre, post, state_test, t8n, fork): + """ + Test post state `Account.nonce` verification and exceptions during state test + fixture generation. + """ + pre_nonce = pre[ADDRESS_UNDER_TEST].nonce + post_nonce = post[ADDRESS_UNDER_TEST].nonce + if post_nonce is None: # no exception + state_test.generate(t8n=t8n, fork=fork) + return + with pytest.raises(Account.NonceMismatch) as e_info: + state_test.generate(t8n=t8n, fork=fork) + assert e_info.value == Account.NonceMismatch( + address=ADDRESS_UNDER_TEST, want=post_nonce, got=pre_nonce + ) + + +# Code value mismatch tests +@pytest.mark.parametrize( + "pre,post", + [ + ({ADDRESS_UNDER_TEST: Account(code="0x02")}, {ADDRESS_UNDER_TEST: Account(code="0x01")}), + ({ADDRESS_UNDER_TEST: Account(code="0x02")}, {ADDRESS_UNDER_TEST: Account(code="0x")}), + ({ADDRESS_UNDER_TEST: Account(code="0x02")}, {ADDRESS_UNDER_TEST: Account(code=None)}), + ], + indirect=["pre", "post"], +) +def test_post_code_value_mismatch(pre, post, state_test, t8n, fork): + """ + Test post state `Account.code` verification and exceptions during state test + fixture generation. + """ + pre_code = pre[ADDRESS_UNDER_TEST].code + post_code = post[ADDRESS_UNDER_TEST].code + if post_code is None: # no exception + state_test.generate(t8n=t8n, fork=fork) + return + with pytest.raises(Account.CodeMismatch) as e_info: + state_test.generate(t8n=t8n, fork=fork) + assert e_info.value == Account.CodeMismatch( + address=ADDRESS_UNDER_TEST, want=post_code, got=pre_code + ) + + +# Balance value mismatch tests +@pytest.mark.parametrize( + "pre,post", + [ + ({ADDRESS_UNDER_TEST: Account(balance=1)}, {ADDRESS_UNDER_TEST: Account(balance=2)}), + ({ADDRESS_UNDER_TEST: Account(balance=1)}, {ADDRESS_UNDER_TEST: Account(balance=0)}), + ({ADDRESS_UNDER_TEST: Account(balance=1)}, {ADDRESS_UNDER_TEST: Account(balance=None)}), + ], + indirect=["pre", "post"], +) +def test_post_balance_value_mismatch(pre, post, state_test, t8n, fork): + """ + Test post state `Account.balance` verification and exceptions during state test + fixture generation. + """ + pre_balance = pre[ADDRESS_UNDER_TEST].balance + post_balance = post[ADDRESS_UNDER_TEST].balance + if post_balance is None: # no exception + state_test.generate(t8n=t8n, fork=fork) + return + with pytest.raises(Account.BalanceMismatch) as e_info: + state_test.generate(t8n=t8n, fork=fork) + assert e_info.value == Account.BalanceMismatch( + address=ADDRESS_UNDER_TEST, want=post_balance, got=pre_balance + ) + + +# Account mismatch tests +@pytest.mark.parametrize( + "pre,post,error_str", + [ + ( + {ADDRESS_UNDER_TEST: Account(balance=1)}, + {ADDRESS_UNDER_TEST: Account()}, + None, + ), + ( + {ADDRESS_UNDER_TEST: Account(balance=1)}, + {ADDRESS_UNDER_TEST: Account(balance=1), to_address(0x02): Account(balance=1)}, + "expected account not found", + ), + ( + {ADDRESS_UNDER_TEST: Account(balance=1)}, + {}, + None, + ), + ( + {ADDRESS_UNDER_TEST: Account(balance=1)}, + {ADDRESS_UNDER_TEST: Account.NONEXISTENT}, + "found unexpected account", + ), + ], + indirect=["pre", "post"], +) +def test_post_account_mismatch(state_test, t8n, fork, error_str): + """ + Test post state `Account` verification and exceptions during state test + fixture generation. + """ + if error_str is None: + state_test.generate(t8n=t8n, fork=fork) + return + with pytest.raises(Exception) as e_info: + state_test.generate(t8n=t8n, fork=fork) + assert error_str in str(e_info.value) diff --git a/src/ethereum_test_tools/tests/test_filler.py b/src/ethereum_test_tools/tests/test_filling/test_fixtures.py similarity index 97% rename from src/ethereum_test_tools/tests/test_filler.py rename to src/ethereum_test_tools/tests/test_filling/test_fixtures.py index 3b845833ea..89c32e9d8f 100644 --- a/src/ethereum_test_tools/tests/test_filler.py +++ b/src/ethereum_test_tools/tests/test_filling/test_fixtures.py @@ -1,5 +1,5 @@ """ -Test suite for `ethereum_test` module. +Test suite for `ethereum_test_tools.filling` fixture generation. """ import json @@ -12,13 +12,13 @@ from ethereum_test_forks import Berlin, Fork, Istanbul, London, Paris, Shanghai from evm_transition_tool import FixtureFormats, GethTransitionTool -from ..code import Yul -from ..common import Account, Environment, TestAddress, Transaction, to_json -from ..spec import BlockchainTest, StateTest -from ..spec.blockchain.types import Block -from ..spec.blockchain.types import Fixture as BlockchainFixture -from ..spec.blockchain.types import FixtureCommon as BlockchainFixtureCommon -from .conftest import SOLC_PADDING_VERSION +from ...code import Yul +from ...common import Account, Environment, TestAddress, Transaction, to_json +from ...spec import BlockchainTest, StateTest +from ...spec.blockchain.types import Block +from ...spec.blockchain.types import Fixture as BlockchainFixture +from ...spec.blockchain.types import FixtureCommon as BlockchainFixtureCommon +from ..conftest import SOLC_PADDING_VERSION def remove_info(fixture_json: Dict[str, Any]): # noqa: D103 @@ -161,7 +161,8 @@ def test_fill_state_test( "src", "ethereum_test_tools", "tests", - "test_fixtures", + "test_filling", + "fixtures", expected_json_file, ) ) as f: @@ -461,7 +462,8 @@ def test_fill_blockchain_valid_txs( "src", "ethereum_test_tools", "tests", - "test_fixtures", + "test_filling", + "fixtures", expected_json_file, ) ) as f: @@ -810,7 +812,8 @@ def test_fill_blockchain_invalid_txs( "src", "ethereum_test_tools", "tests", - "test_fixtures", + "test_filling", + "fixtures", expected_json_file, ) ) as f: