feat(tests): Add multiple exception support to EOF tests (ethereum#759)

* feat(tests) Add "multiple exceptions" eof exception There are some tests that are impossible to express without causing multiple validation exceptions. Clients should be free to expose any of those exceptions. To support this pipe notation for expected exceptions is plumed through the EOF tests and implemented in some tests. Signed-off-by: Danno Ferrin <[email protected]> * review comments Signed-off-by: Danno Ferrin <[email protected]> * docs: changelog --------- Signed-off-by: Danno Ferrin <[email protected]> Co-authored-by: Mario Vega <[email protected]>
spencer-tb · Aug 27, 2024 · 9886436 · 9886436
1 parent 6d307c1
commit 9886436
Show file tree

Hide file tree

Showing 15 changed files with 235 additions and 133 deletions.
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -25,6 +25,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - 🐞 Fixed fixture index generation on EOF tests ([#728](https://github.com/ethereum/execution-spec-tests/pull/728)).
 - 🐞 Fixes consume genesis mismatch exception for hive based simulators ([#734](https://github.com/ethereum/execution-spec-tests/pull/734)).
 - ✨ Adds reproducible consume commands to hiveview ([#717](https://github.com/ethereum/execution-spec-tests/pull/717)).
+- 💥 Added multiple exceptions to the EOF fixture format ([#759](https://github.com/ethereum/execution-spec-tests/pull/759)).
 - ✨ Added optional parameter to all `with_all_*` markers to specify a lambda function that filters the parametrized values ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
 - ✨ Added [`extend_with_defaults` utility function](https://ethereum.github.io/execution-spec-tests/main/writing_tests/writing_a_new_test/#ethereum_test_tools.utility.pytest.extend_with_defaults), which helps extend test case parameter sets with default values. `@pytest.mark.parametrize` ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
 - ✨ Added `Container.Init` to `ethereum_test_types.EOF.V1` package, which allows generation of an EOF init container more easily ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
@@ -37,6 +38,10 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Releases for feature eip7692 now include both Cancun and Prague based tests in the same release, in files `fixtures_eip7692.tar.gz` and `fixtures_eip7692-prague.tar.gz` respectively ([#743](https://github.com/ethereum/execution-spec-tests/pull/743)).
 - 🔀 Simplify Python project configuration and consolidate it into `pyproject.toml` ([#764](https://github.com/ethereum/execution-spec-tests/pull/764)).
 
+### 💥 Breaking Change
+
+- The EOF fixture format contained in `eof_tests` may now contain multiple exceptions in the `"exception"` field in the form of a pipe (`|`) separated string ([#759](https://github.com/ethereum/execution-spec-tests/pull/759)).
+
 ## [v3.0.0](https://github.com/ethereum/execution-spec-tests/releases/tag/v3.0.0) - 2024-07-22
 
 ### 🧪 Test Cases

diff --git a/src/ethereum_test_exceptions/__init__.py b/src/ethereum_test_exceptions/__init__.py
@@ -8,6 +8,7 @@
     BlockException,
     BlockExceptionInstanceOrList,
     EOFException,
+    EOFExceptionInstanceOrList,
     ExceptionInstanceOrList,
     TransactionException,
     TransactionExceptionInstanceOrList,
@@ -16,10 +17,11 @@
 __all__ = [
     "BlockException",
     "BlockExceptionInstanceOrList",
-    "EngineAPIError",
     "EOFException",
+    "EOFExceptionInstanceOrList",
+    "EngineAPIError",
+    "EvmoneExceptionMapper",
     "ExceptionInstanceOrList",
     "TransactionException",
     "TransactionExceptionInstanceOrList",
-    "EvmoneExceptionMapper",
 ]
diff --git a/src/ethereum_test_exceptions/evmone_exceptions.py b/src/ethereum_test_exceptions/evmone_exceptions.py
@@ -85,6 +85,9 @@ class EvmoneExceptionMapper:
         ),
         ExceptionMessage(EOFException.STACK_HEIGHT_MISMATCH, "err: stack_height_mismatch"),
         ExceptionMessage(EOFException.TOO_MANY_CONTAINERS, "err: too_many_container_sections"),
+        ExceptionMessage(
+            EOFException.INVALID_CODE_SECTION_INDEX, "err: invalid_code_section_index"
+        ),
     )
 
     def __init__(self) -> None:

diff --git a/src/ethereum_test_exceptions/exceptions.py b/src/ethereum_test_exceptions/exceptions.py
@@ -730,6 +730,14 @@ class EOFException(ExceptionBase):
     """
     EOF container header has too many sub-containers.
     """
+    INVALID_CODE_SECTION_INDEX = auto()
+    """
+    CALLF Operation referes to a non-existent code section
+    """
+    UNEXPECTED_HEADER_KIND = auto()
+    """
+    Header parsing encounterd a section kind it wasn't expecting
+    """
 
 
 """
@@ -753,3 +761,9 @@ class EOFException(ExceptionBase):
     BeforeValidator(from_pipe_str),
     PlainSerializer(to_pipe_str),
 ]
+
+EOFExceptionInstanceOrList = Annotated[
+    List[EOFException] | EOFException,
+    BeforeValidator(from_pipe_str),
+    PlainSerializer(to_pipe_str),
+]
diff --git a/src/ethereum_test_fixtures/eof.py b/src/ethereum_test_fixtures/eof.py
@@ -8,7 +8,7 @@
 from pydantic import Field
 
 from ethereum_test_base_types import Bytes, CamelModel, Number
-from ethereum_test_exceptions import EOFException
+from ethereum_test_exceptions.exceptions import EOFExceptionInstanceOrList
 from ethereum_test_types.eof.v1 import ContainerKind
 
 from .base import BaseFixture
@@ -20,7 +20,7 @@ class Result(CamelModel):
     Result for a single fork in a fixture.
     """
 
-    exception: EOFException | None = None
+    exception: EOFExceptionInstanceOrList | None = None
     valid: bool = Field(..., alias="result")
 
     def model_post_init(self, __context: Any) -> None:

diff --git a/src/ethereum_test_specs/eof.py b/src/ethereum_test_specs/eof.py
@@ -12,7 +12,8 @@
 from pydantic import Field, model_validator
 
 from ethereum_test_base_types import Account, Bytes
-from ethereum_test_exceptions import EOFException, EvmoneExceptionMapper
+from ethereum_test_exceptions import EvmoneExceptionMapper
+from ethereum_test_exceptions.exceptions import EOFExceptionInstanceOrList, to_pipe_str
 from ethereum_test_fixtures import BaseFixture, FixtureFormats
 from ethereum_test_fixtures.eof import Fixture, Result, Vector
 from ethereum_test_forks import Fork
@@ -136,7 +137,7 @@ class EOFTest(BaseTest):
     """
 
     data: Bytes
-    expect_exception: EOFException | None = None
+    expect_exception: EOFExceptionInstanceOrList | None = None
     container_kind: ContainerKind | None = None
 
     supported_fixture_formats: ClassVar[List[FixtureFormats]] = [
@@ -237,21 +238,23 @@ def verify_result(self, result: CompletedProcess, expected_result: Result, code:
                 raise UnexpectedEOFException(
                     code=code, got=f"{actual_exception} ({actual_message})"
                 )
-
-        expected_exception = expected_result.exception
-        expected_message = parser.exception_to_message(expected_exception)
-
-        if "OK" in actual_message:
-            raise ExpectedEOFException(
-                code=code, expected=f"{expected_exception} ({expected_message})"
-            )
-
-        if expected_exception != actual_exception:
-            raise EOFExceptionMismatch(
-                code=code,
-                expected=f"{expected_exception} ({expected_message})",
-                got=f"{actual_exception} ({actual_message})",
-            )
+        else:
+            expected_string = to_pipe_str(expected_result.exception)
+            print(expected_string)
+            print(actual_exception)
+            if "OK" in actual_message:
+                raise ExpectedEOFException(
+                    code=code,
+                    expected=f"{expected_string}",
+                )
+            elif actual_exception in expected_result.exception:
+                return
+            else:
+                raise EOFExceptionMismatch(
+                    code=code,
+                    expected=f"{expected_string}",
+                    got=f"{actual_exception} ({actual_message})",
+                )
 
     def generate(
         self,

diff --git a/src/ethereum_test_types/eof/v1/__init__.py b/src/ethereum_test_types/eof/v1/__init__.py
@@ -17,7 +17,7 @@
 from ethereum_test_base_types import Bytes
 from ethereum_test_base_types.conversions import BytesConvertible
 from ethereum_test_base_types.pydantic import CopyValidateModel
-from ethereum_test_exceptions import EOFException
+from ethereum_test_exceptions.exceptions import EOFExceptionInstanceOrList
 from ethereum_test_vm import Bytecode
 from ethereum_test_vm import Opcodes as Op
 
@@ -365,7 +365,7 @@ class Container(CopyValidateModel):
     Body: type section first, all code sections, data section(s), last
                 container sections
     """
-    validity_error: EOFException | str | None = None
+    validity_error: EOFExceptionInstanceOrList | str | None = None
     """
     Optional error expected for the container.