feat(consume): consume direct using evmone-blockchaintest

Author: pdobacz

docs/CHANGELOG.md

@@ -23,6 +23,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 
 - ✨ Add retry logic to RPC requests to fix flaky connection issues in Hive ([#2205](https://github.com/ethereum/execution-spec-tests/pull/2205)).
 - 🛠️ Mark `consume sync` tests as `flaky` with 3 retires due to client sync inconsistencies ([#2252](https://github.com/ethereum/execution-spec-tests/pull/2252)).
+- ✨ Add `consume direct` using `evmone-statetest` and `evmone-blockchaintest` ([#2243](https://github.com/ethereum/execution-spec-tests/pull/2243)).
 
 ### 📋 Misc
 

docs/running_tests/consume/direct.md

@@ -15,7 +15,7 @@ uv run consume direct --bin=<evm-binary> [OPTIONS]
 
     - go-ethereum `statetest` and `blocktest`
     - Nethermind `nethtest`
-    - evmone `evmone-statetest`
+    - evmone `evmone-statetest` and `evmone-blockchaintest`
 
 ## Advantages
 
@@ -25,7 +25,7 @@ uv run consume direct --bin=<evm-binary> [OPTIONS]
 
 ## Limitations
 
-- **Limited client support**: Only go-ethereum, Nethermind and (partially) evmone
+- **Limited client support**: Only go-ethereum, Nethermind and evmone
 - **Module scope**: Tests EVM, respectively block import, in isolation, not full client behavior.
 - **Interface dependency**: Requires client-specific test interfaces.
 
@@ -46,7 +46,7 @@ uv run consume direct --input ./fixtures -m state_test --bin=nethtest
 or evmone:
 
 ```bash
-uv run consume direct --input ./fixtures -m state_test --bin=evmone-statetest
+uv run consume direct --input ./fixtures --bin=evmone-statetest --bin=evmone-blockchaintest
 ```
 
 Run fixtures in the blockchain test format for the Prague fork:

src/ethereum_clis/__init__.py

@@ -12,7 +12,12 @@
 )
 from .clis.besu import BesuTransitionTool
 from .clis.ethereumjs import EthereumJSTransitionTool
-from .clis.evmone import EvmoneExceptionMapper, EvmOneFixtureConsumer, EvmOneTransitionTool
+from .clis.evmone import (
+    EvmOneBlockchainFixtureConsumer,
+    EvmoneExceptionMapper,
+    EvmOneStateFixtureConsumer,
+    EvmOneTransitionTool,
+)
 from .clis.execution_specs import ExecutionSpecsTransitionTool
 from .clis.geth import GethFixtureConsumer, GethTransitionTool
 from .clis.nethermind import Nethtest, NethtestFixtureConsumer
@@ -31,7 +36,8 @@
     "EthereumJSTransitionTool",
     "EvmoneExceptionMapper",
     "EvmOneTransitionTool",
-    "EvmOneFixtureConsumer",
+    "EvmOneStateFixtureConsumer",
+    "EvmOneBlockchainFixtureConsumer",
     "ExecutionSpecsTransitionTool",
     "FixtureConsumerTool",
     "GethFixtureConsumer",

src/ethereum_clis/clis/evmone.py

@@ -11,6 +11,8 @@
 from pathlib import Path
 from typing import Any, ClassVar, Dict, List, Optional
 
+import pytest
+
 from ethereum_clis.file_utils import dump_files_to_directory
 from ethereum_clis.fixture_consumer_tool import FixtureConsumerTool
 from ethereum_test_exceptions import (
@@ -20,6 +22,7 @@
     TransactionException,
 )
 from ethereum_test_fixtures.base import FixtureFormat
+from ethereum_test_fixtures.blockchain import BlockchainFixture
 from ethereum_test_fixtures.state import StateFixture
 from ethereum_test_forks import Fork
 
@@ -54,26 +57,19 @@ def is_fork_supported(self, fork: Fork) -> bool:
         return True
 
 
-class EvmOneFixtureConsumer(
-    FixtureConsumerTool,
-    fixture_formats=[StateFixture],
-):
-    """Evmone's implementation of the fixture consumer."""
+class EvmoneFixtureConsumerCommon:
+    """Common functionality for Evmone fixture consumers."""
 
-    default_binary = Path("evmone-statetest")
-    detect_binary_pattern = re.compile(r"^evmone-statetest\b")
+    binary: Path
     version_flag: str = "--version"
 
-    binary: Path
     cached_version: Optional[str] = None
 
     def __init__(
         self,
-        binary: Optional[Path] = None,
         trace: bool = False,
     ):
-        """Initialize the EvmOneFixtureConsumer class."""
-        self.binary = binary if binary else self.default_binary
+        """Initialize the EvmoneFixtureConsumerCommon class."""
         self._info_metadata: Optional[Dict[str, Any]] = {}
 
     def _run_command(self, command: List[str]) -> subprocess.CompletedProcess:
@@ -126,19 +122,22 @@ def _consume_debug_dump(
         )
         shutil.copyfile(fixture_path, debug_fixture_path)
 
+    def _skip_message(self, fixture_format: FixtureFormat) -> str:
+        return f"Fixture format {fixture_format.format_name} not supported by {self.binary}"
+
     @cache  # noqa
-    def consume_state_test_file(
+    def consume_test_file(
         self,
         fixture_path: Path,
         debug_output_path: Optional[Path] = None,
     ) -> Dict[str, Any]:
         """
-        Consume an entire state test file.
+        Consume an entire state or blockchain test file.
 
-        The `evmone-statetest` will always execute all the tests contained in a
+        The `evmone-...test` will always execute all the tests contained in a
         file without the possibility of selecting a single test, so this
         function is cached in order to only call the command once and
-        `consume_state_test` can simply select the result that was requested.
+        `consume_test` can simply select the result that was requested.
         """
         global_options: List[str] = []
         if debug_output_path:
@@ -160,7 +159,9 @@ def consume_state_test_file(
             try:
                 output_data = json.load(tempfile_json)
             except json.JSONDecodeError as e:
-                raise Exception(f"Failed to parse JSON output from evmone-statetest: {e}") from e
+                raise Exception(
+                    f"Failed to parse JSON output from evmone-state/blockchaintest: {e}"
+                ) from e
 
             if debug_output_path:
                 self._consume_debug_dump(command, result, fixture_path, debug_output_path)
@@ -173,19 +174,19 @@ def _failure_msg(self, file_results: Dict[str, Any]) -> str:
         failures = file_results["testsuites"][0]["testsuite"][0]["failures"]
         return ", ".join([f["failure"] for f in failures])
 
-    def consume_state_test(
+    def consume_test(
         self,
         fixture_path: Path,
         fixture_name: Optional[str] = None,
         debug_output_path: Optional[Path] = None,
     ):
         """
-        Consume a single state test.
+        Consume a single state or blockchain test.
 
-        Uses the cached result from `consume_state_test_file` in order to not
+        Uses the cached result from `consume_test_file` in order to not
         call the command every time an select a single result from there.
         """
-        file_results = self.consume_state_test_file(
+        file_results = self.consume_test_file(
             fixture_path=fixture_path,
             debug_output_path=debug_output_path,
         )
@@ -205,6 +206,26 @@ def consume_state_test(
             f"Test name mismatch, expected {fixture_path.stem}, got {test_name}"
         )
 
+
+class EvmOneStateFixtureConsumer(
+    EvmoneFixtureConsumerCommon,
+    FixtureConsumerTool,
+    fixture_formats=[StateFixture],
+):
+    """Evmone's implementation of the fixture consumer for state tests."""
+
+    default_binary = Path("evmone-statetest")
+    detect_binary_pattern = re.compile(r"^evmone-statetest\b")
+
+    def __init__(
+        self,
+        binary: Optional[Path] = None,
+        trace: bool = False,
+    ):
+        """Initialize the EvmOneStateFixtureConsumer class."""
+        self.binary = binary if binary else self.default_binary
+        super().__init__(trace=trace)
+
     def consume_fixture(
         self,
         fixture_format: FixtureFormat,
@@ -213,19 +234,57 @@ def consume_fixture(
         debug_output_path: Optional[Path] = None,
     ):
         """
-        Execute the appropriate geth fixture consumer for the fixture at
+        Execute the appropriate fixture consumer for the fixture at
         `fixture_path`.
         """
         if fixture_format == StateFixture:
-            self.consume_state_test(
+            self.consume_test(
                 fixture_path=fixture_path,
                 fixture_name=fixture_name,
                 debug_output_path=debug_output_path,
             )
         else:
-            raise Exception(
-                f"Fixture format {fixture_format.format_name} not supported by {self.binary}"
+            pytest.skip(self._skip_message(fixture_format))
+
+
+class EvmOneBlockchainFixtureConsumer(
+    EvmoneFixtureConsumerCommon,
+    FixtureConsumerTool,
+    fixture_formats=[BlockchainFixture],
+):
+    """Evmone's implementation of the fixture consumer for blockchain tests."""
+
+    default_binary = Path("evmone-blockchaintest")
+    detect_binary_pattern = re.compile(r"^evmone-blockchaintest\b")
+
+    def __init__(
+        self,
+        binary: Optional[Path] = None,
+        trace: bool = False,
+    ):
+        """Initialize the EvmOneBlockchainFixtureConsumer class."""
+        self.binary = binary if binary else self.default_binary
+        super().__init__(trace=trace)
+
+    def consume_fixture(
+        self,
+        fixture_format: FixtureFormat,
+        fixture_path: Path,
+        fixture_name: Optional[str] = None,
+        debug_output_path: Optional[Path] = None,
+    ):
+        """
+        Execute the appropriate fixture consumer for the fixture at
+        `fixture_path`.
+        """
+        if fixture_format == BlockchainFixture:
+            self.consume_test(
+                fixture_path=fixture_path,
+                fixture_name=fixture_name,
+                debug_output_path=debug_output_path,
             )
+        else:
+            pytest.skip(self._skip_message(fixture_format))
 
 
 class EvmoneExceptionMapper(ExceptionMapper):