Merge branch 'main' of https://github.com/spencer-tb/execution-spec-t…

…ests

.flake8

@@ -0,0 +1,34 @@
+[flake8]
+dictionaries=en_US,python,technical
+docstring-convention = google
+extend-ignore = E203, D107, D200, D203, D205,
+    D212, E231, D400, D401, D410, D411, D412,
+    D413, D414, D415, D416, N806
+    # Ignore E203: Whitespace before ':'
+    # Ignore D107: Missing docstring in __init__
+    # Ignore D200: One-line docstring should fit on one line with quotes
+    # Ignore D203: 1 blank line required before class docstring
+    # Ignore D205: blank line required between summary line and description
+    # Ignore D212: Multi-line docstring summary should start at the first line
+    # Ignore E231: Missing whitespace after ':'
+    # Ignore D400: First line should end with a period
+    # Ignore D401: First line should be in imperative mood
+    # Ignore D410: Missing blank line after section
+    # Ignore D411: Missing blank line before section
+    # Ignore D412: No blank lines allowed between a section header and its content
+    # Ignore D413: Missing blank line after last section
+    # Ignore D414: Section has no content
+    # Ignore D415: First line should end with a period, question mark, or exclamation point
+    # Ignore D416: Section name should end with a colon
+    # Ignore N806: Variable names with all caps (ALL_CAPS)
+max-line-length = 99
+per-file-ignore =
+    tests/evm_transition_tool/test_evaluate.py:E501
+
+extend-exclude =
+    setup.py
+    src/evm_transition_tool/tests/
+    src/ethereum_test_tools/tests/
+    src/ethereum_test_forks/tests/
+
+# vim: set ft=dosini:

.github/actions/build-evm/action.yaml → .github/actions/build-evm-base/action.yaml

@@ -17,38 +17,48 @@ outputs:
     value: ${{ steps.config-evm-reader.outputs.ref }}
   evm-bin:
     description: "Binary name of the evm tool to use"
-    value: ${{ steps.config-evm-reader.outputs.evm-bin }}
+    value: ${{ steps.config-evm-impl-config-reader.outputs.evm-bin }}
+  x-dist:
+    description: "Binary name of the evm tool to use"
+    value: ${{ steps.config-evm-impl-config-reader.outputs.x-dist }}
 runs:
   using: "composite"
   steps:
-    - name: Get the selected EVM version from the configs/evm.yaml
+    - name: Get the selected EVM version from the .github/configs/evm.yaml
       id: config-evm-reader
       shell: bash
       run: |
-        awk "/^${{ inputs.type }}:/{flag=1; next} /^[[:alnum:]]/{flag=0} flag" ./configs/evm.yaml \
+        awk "/^${{ inputs.type }}:/{flag=1; next} /^[[:alnum:]]/{flag=0} flag" ./.github/configs/evm.yaml \
+        | sed 's/ //g' | sed 's/:/=/g' >> "$GITHUB_OUTPUT"
+    - name: Get the EVM implementation configuration from .github/configs/evm-impl-config.yaml
+      id: config-evm-impl-config-reader
+      shell: bash
+      run: |
+        awk "/^${{ steps.config-evm-reader.outputs.impl }}:/{flag=1; next} /^[[:alnum:]]/{flag=0} flag" ./.github/configs/evm-impl.yaml \
         | sed 's/ //g' | sed 's/:/=/g' >> "$GITHUB_OUTPUT"
     - name: Print Variables for the selected EVM type
       shell: bash
       run: |
         echo "Implementation: ${{ steps.config-evm-reader.outputs.impl }}"
         echo "Repository: ${{ steps.config-evm-reader.outputs.repo }}"
         echo "Reference: ${{ steps.config-evm-reader.outputs.ref }}"
-        echo "EVM Binary: ${{ steps.config-evm-reader.outputs.evm-bin }}"
+        echo "EVM Binary: ${{ steps.config-evm-impl-config-reader.outputs.evm-bin }}"
+        echo "X-Dist parameter: ${{ steps.config-evm-impl-config-reader.outputs.x-dist }}"
     - name: Build the EVM using Geth action
       if: steps.config-evm-reader.outputs.impl == 'geth'
-      uses: ./.github/actions/build-geth-evm
+      uses: ./.github/actions/build-evm-client/geth
       with:
         repo: ${{ steps.config-evm-reader.outputs.repo }}
         ref: ${{ steps.config-evm-reader.outputs.ref }}
     - name: Build the EVM using EVMONE action
       if: steps.config-evm-reader.outputs.impl == 'evmone'
-      uses: ./.github/actions/build-evmone-evm
+      uses: ./.github/actions/build-evm-client/evmone
       with:
         repo: ${{ steps.config-evm-reader.outputs.repo }}
         ref: ${{ steps.config-evm-reader.outputs.ref }}
     - name: Build the EVM using Besu action
       if: steps.config-evm-reader.outputs.impl == 'besu'
-      uses: ./.github/actions/build-besu-evm
+      uses: ./.github/actions/build-evm-client/besu
       with:
         repo: ${{ steps.config-evm-reader.outputs.repo }}
         ref: ${{ steps.config-evm-reader.outputs.ref }}

.github/actions/build-besu-evm/action.yaml → .github/actions/build-evm-client/besu/action.yaml

@@ -37,4 +37,4 @@ runs:
       run: |
         cd $GITHUB_WORKSPACE/besu
         ./gradlew installDist
-        echo $GITHUB_WORKSPACE/besu/build/install/besu/bin/evmtool >> $GITHUB_PATH
+        echo $GITHUB_WORKSPACE/besu/build/install/besu/bin/ >> $GITHUB_PATH

.github/actions/build-fixtures/action.yaml

@@ -18,8 +18,8 @@ runs:
       id: properties
       shell: bash
       run: |
-        yq -r --arg feature "${{ inputs.name }}" '.[$feature] | to_entries | map("\(.key)=\(.value)")[]' ./configs/feature.yaml >> "$GITHUB_OUTPUT"
-    - uses: ./.github/actions/build-evm
+        yq -r --arg feature "${{ inputs.name }}" '.[$feature] | to_entries | map("\(.key)=\(.value)")[]' ./.github/configs/feature.yaml >> "$GITHUB_OUTPUT"
+    - uses: ./.github/actions/build-evm-base
       id: evm-builder
       with:
         type: ${{ steps.properties.outputs.evm-type }}
@@ -31,7 +31,7 @@ runs:
         source env/bin/activate
         pip install -e .
         solc-select use  ${{ steps.properties.outputs.solc }} --always-install
-        fill -n auto --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} --output=fixtures_${{ inputs.name }}.tar.gz --build-name ${{ inputs.name }}
+        fill -n ${{ steps.evm-builder.outputs.x-dist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} --output=fixtures_${{ inputs.name }}.tar.gz --build-name ${{ inputs.name }}
     - uses: actions/upload-artifact@v4
       with:
         name: fixtures_${{ inputs.name }}

.github/configs/evm-impl.yaml

@@ -0,0 +1,9 @@
+geth:
+  evm-bin: evm
+  x-dist: auto
+evmone:
+  evm-bin: evmone-t8n
+  x-dist: auto
+besu:
+  evm-bin: evmtool
+  x-dist: 0

configs/evm.yaml → .github/configs/evm.yaml

@@ -2,14 +2,15 @@ stable:
   impl: geth
   repo: ethereum/go-ethereum
   ref: master
-  evm-bin: evm
 develop:
   impl: geth
   repo: lightclient/go-ethereum
   ref: prague-devnet-1
-  evm-bin: evm
 eip7692:
   impl: evmone
   repo: ethereum/evmone
   ref: master
-  evm-bin: evmone-t8n
+eip7692-prague:
+  impl: besu
+  repo: hyperledger/besu
+  ref: main

configs/feature.yaml → .github/configs/feature.yaml

@@ -9,4 +9,8 @@ develop:
 eip7692:
   evm-type: eip7692
   fill-params: --fork=CancunEIP7692 ./tests/prague
+  solc: 0.8.21
+eip7692-prague:
+  evm-type: eip7692-prague
+  fill-params: --fork=PragueEIP7692 ./tests/prague -k "not slow"
   solc: 0.8.21

.github/workflows/coverage.yaml

@@ -41,13 +41,13 @@ jobs:
 
       # Required to fill .py tests
       - name: Build GO EVM
-        uses: ./.github/actions/build-evm
+        uses: ./.github/actions/build-evm-client/geth
         id: evm-builder
         with:
           type: 'main'
 
       - name: Build EVMONE EVM
-        uses: ./.github/actions/build-evmone-evm
+        uses: ./.github/actions/build-evm-client/evmone
         id: evm-builder2
         with:
           type: 'main'

.github/workflows/fixtures.yaml

@@ -13,11 +13,11 @@ jobs:
       features: ${{ steps.parse.outputs.features }}
     steps:
       - uses: actions/checkout@v4
-      - name: Get names from configs/feature.yaml
+      - name: Get names from .github/configs/feature.yaml
         id: parse
         shell: bash
         run: |
-          echo "features=$(grep -Po "^[0-9a-zA-Z_\-]+" ./configs/feature.yaml | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT"
+          echo "features=$(grep -Po "^[0-9a-zA-Z_\-]+" ./.github/configs/feature.yaml | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT"
   build:
     needs: features
     runs-on: ubuntu-latest

.github/workflows/fixtures_feature.yaml

@@ -19,7 +19,7 @@ jobs:
         id: feature-name
         shell: bash
         run: |
-          names=$(grep -Po "^${GITHUB_REF_NAME//@*/}[^:]*" configs/feature.yaml | jq  --raw-input .  | jq -c --slurp .)
+          names=$(grep -Po "^${GITHUB_REF_NAME//@*/}[^:]*" .github/configs/feature.yaml | jq  --raw-input .  | jq -c --slurp .)
           echo names=${names}
           echo names=${names} >> "$GITHUB_OUTPUT"
   build:

.github/workflows/tox_verify.yaml

@@ -34,7 +34,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
-      - uses: ./.github/actions/build-evm
+      - uses: ./.github/actions/build-evm-base
         id: evm-builder
         with:
           type: ${{ matrix.evm-type }}

README.md

@@ -72,7 +72,7 @@ Generally, specific `t8n` implementations and branches must be used when develop
 
 We use named reference tags to point to the specific version of the `t8n` implementation that needs to be used fill the tests.
 
-All current tags, their t8n implementation and branch they point to, are listed in [configs/evm.yaml](configs/evm.yaml).
+All current tags, their t8n implementation and branch they point to, are listed in [.github/configs/evm.yaml](.github/configs/evm.yaml).
 
 ## Getting Started
 

converted-ethereum-tests.txt

@@ -2,7 +2,11 @@
 GeneralStateTests/stCreate2/call_outsize_then_create2_successful_then_returndatasize.json
 GeneralStateTests/stCreate2/call_then_create2_successful_then_returndatasize.json
 
+([#598](https://github.com/ethereum/execution-spec-tests/pull/598))
+EOFTests/EIP3540/validInvalid.json
+
 EOFTests/efValidation/EOF1_eofcreate_valid_.json
+EOFTests/efValidation/EOF1_section_order_.json
 EOFTests/efValidation/EOF1_truncated_section_.json
 
 ([#647](https://github.com/ethereum/execution-spec-tests/pull/647))

docs/CHANGELOG.md

@@ -18,18 +18,29 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Generated fixtures now contain the test index `index.json` by default ([#716](https://github.com/ethereum/execution-spec-tests/pull/716)).
 - ✨ A metadata folder `.meta/` now stores all fixture metadata files by default ([#721](https://github.com/ethereum/execution-spec-tests/pull/721)).
 - 🐞 Fixed `fill` command index generation issue due to concurrency ([#725](https://github.com/ethereum/execution-spec-tests/pull/725)).
-- ✨ Added `with_all_evm_code_types` and `with_all_call_opcodes` markers, which allow automatic parametrization of tests to EOF ([#610](https://github.com/ethereum/execution-spec-tests/pull/610)).
+- ✨ Added `with_all_evm_code_types`, `with_all_call_opcodes` and `with_all_create_opcodes` markers, which allow automatic parametrization of tests to EOF ([#610](https://github.com/ethereum/execution-spec-tests/pull/610), [#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
+- ✨ Added `with_all_system_contracts` marker, which helps parametrize tests with all contracts that affect the chain on a system level ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
 - ✨ Code generators `Conditional` and `Switch` now support EOF by adding parameter `evm_code_type` ([#610](https://github.com/ethereum/execution-spec-tests/pull/610)).
 - ✨ `fill` command now supports parameter `--evm-code-type` that can be (currently) set to `legacy` or `eof_v1` to force all test smart contracts to deployed in normal or in EOF containers ([#610](https://github.com/ethereum/execution-spec-tests/pull/610)).
 - 🐞 Fixed fixture index generation on EOF tests ([#728](https://github.com/ethereum/execution-spec-tests/pull/728)).
 - 🐞 Fixes consume genesis mismatch exception for hive based simulators ([#734](https://github.com/ethereum/execution-spec-tests/pull/734)).
 - ✨ Adds reproducible consume commands to hiveview ([#717](https://github.com/ethereum/execution-spec-tests/pull/717)).
+- 💥 Added multiple exceptions to the EOF fixture format ([#759](https://github.com/ethereum/execution-spec-tests/pull/759)).
+- ✨ Added optional parameter to all `with_all_*` markers to specify a lambda function that filters the parametrized values ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
+- ✨ Added [`extend_with_defaults` utility function](https://ethereum.github.io/execution-spec-tests/main/writing_tests/writing_a_new_test/#ethereum_test_tools.utility.pytest.extend_with_defaults), which helps extend test case parameter sets with default values. `@pytest.mark.parametrize` ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
+- ✨ Added `Container.Init` to `ethereum_test_types.EOF.V1` package, which allows generation of an EOF init container more easily ([#739](https://github.com/ethereum/execution-spec-tests/pull/739)).
 
 ### 🔧 EVM Tools
 
 ### 📋 Misc
 
 - ✨ Feature releases can now include multiple types of fixture tarball files from different releases that start with the same prefix ([#736](https://github.com/ethereum/execution-spec-tests/pull/736)).
+- ✨ Releases for feature eip7692 now include both Cancun and Prague based tests in the same release, in files `fixtures_eip7692.tar.gz` and `fixtures_eip7692-prague.tar.gz` respectively ([#743](https://github.com/ethereum/execution-spec-tests/pull/743)).
+- 🔀 Simplify Python project configuration and consolidate it into `pyproject.toml` ([#764](https://github.com/ethereum/execution-spec-tests/pull/764)).
+
+### 💥 Breaking Change
+
+- The EOF fixture format contained in `eof_tests` may now contain multiple exceptions in the `"exception"` field in the form of a pipe (`|`) separated string ([#759](https://github.com/ethereum/execution-spec-tests/pull/759)).
 
 ## [v3.0.0](https://github.com/ethereum/execution-spec-tests/releases/tag/v3.0.0) - 2024-07-22