diff --git a/docs/templates/base.md.j2 b/docs/templates/base.md.j2 index 20b014771f8..a741f22382d 100644 --- a/docs/templates/base.md.j2 +++ b/docs/templates/base.md.j2 @@ -11,6 +11,8 @@ Documentation for [`{{ pytest_node_id }}@{{ short_git_ref }}`]({{ source_code_ur ```console {% if is_benchmark %} fill -v {{ pytest_node_id }} -m benchmark +{% elif is_stateful %} + fill -v {{ pytest_node_id }} -m stateful {% else %} fill -v {{ pytest_node_id }} --fork {{ target_or_valid_fork }} {% endif %} diff --git a/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py b/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py index aae46ab5feb..6f1d162a680 100644 --- a/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py +++ b/src/pytest_plugins/filler/gen_test_doc/gen_test_doc.py @@ -431,6 +431,7 @@ def create_function_page_props(self, test_functions: Dict["str", List[Item]]) -> ) is_benchmark = items[0].get_closest_marker("benchmark") is not None + is_stateful = items[0].get_closest_marker("stateful") is not None self.function_page_props[function_id] = FunctionPageProps( title=get_test_function_name(items[0]), @@ -447,6 +448,7 @@ def create_function_page_props(self, test_functions: Dict["str", List[Item]]) -> html_static_page_target=f"./{get_test_function_name(items[0])}.html", mkdocs_function_page_target=f"./{get_test_function_name(items[0])}/", is_benchmark=is_benchmark, + is_stateful=is_stateful, ) def create_module_page_props(self) -> None: @@ -462,6 +464,7 @@ def create_module_page_props(self) -> None: pytest_node_id=str(module_path), package_name=get_import_path(module_path), is_benchmark=function_page.is_benchmark, + is_stateful=function_page.is_stateful, test_functions=[ TestFunction( name=function_page.title, @@ -475,6 +478,8 @@ def create_module_page_props(self) -> None: existing_module_page = self.module_page_props[str(function_page.path)] if function_page.is_benchmark: existing_module_page.is_benchmark = True + if function_page.is_stateful: + existing_module_page.is_stateful = True existing_module_page.test_functions.append( TestFunction( name=function_page.title, @@ -511,7 +516,12 @@ def add_directory_page_props(self) -> None: is_benchmark = any( module_page.is_benchmark for module_page in self.module_page_props.values() - if module_page.path.parent == directory + if directory in module_page.path.parents or module_page.path.parent == directory + ) + is_stateful = any( + module_page.is_stateful + for module_page in self.module_page_props.values() + if directory in module_page.path.parents or module_page.path.parent == directory ) self.page_props[str(directory)] = DirectoryPageProps( @@ -526,6 +536,7 @@ def add_directory_page_props(self) -> None: # init.py will be used for docstrings package_name=get_import_path(directory), is_benchmark=is_benchmark, + is_stateful=is_stateful, ) def find_files_within_collection_scope(self, file_pattern: str) -> List[Path]: diff --git a/src/pytest_plugins/filler/gen_test_doc/page_props.py b/src/pytest_plugins/filler/gen_test_doc/page_props.py index 471cd1f2d56..cf6d8adb5a4 100644 --- a/src/pytest_plugins/filler/gen_test_doc/page_props.py +++ b/src/pytest_plugins/filler/gen_test_doc/page_props.py @@ -111,6 +111,7 @@ class PagePropsBase: pytest_node_id: str package_name: str is_benchmark: bool = False + is_stateful: bool = False @property @abstractmethod diff --git a/src/pytest_plugins/shared/execute_fill.py b/src/pytest_plugins/shared/execute_fill.py index 0df88c2ef38..e1f91d55a4f 100644 --- a/src/pytest_plugins/shared/execute_fill.py +++ b/src/pytest_plugins/shared/execute_fill.py @@ -100,6 +100,10 @@ def pytest_configure(config: pytest.Config): "markers", "benchmark: Tests relevant to benchmarking EVMs.", ) + config.addinivalue_line( + "markers", + "stateful: Tests for stateful benchmarking scenarios.", + ) config.addinivalue_line( "markers", "exception_test: Negative tests that include an invalid block or transaction.", diff --git a/tests/benchmark/bloatnet/__init__.py b/tests/benchmark/bloatnet/__init__.py deleted file mode 100644 index 0f0656c8326..00000000000 --- a/tests/benchmark/bloatnet/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""BloatNet benchmark tests for Ethereum execution spec tests.""" diff --git a/tests/benchmark/conftest.py b/tests/benchmark/conftest.py index d51b47dda83..d4225107ae5 100644 --- a/tests/benchmark/conftest.py +++ b/tests/benchmark/conftest.py @@ -38,8 +38,10 @@ def pytest_collection_modifyitems(config, items): if gen_docs: for item in items: - if benchmark_dir in Path(item.fspath).parents and not item.get_closest_marker( - "benchmark" + if ( + benchmark_dir in Path(item.fspath).parents + and not item.get_closest_marker("benchmark") + and not item.get_closest_marker("stateful") ): item.add_marker(benchmark_marker) return @@ -47,12 +49,18 @@ def pytest_collection_modifyitems(config, items): marker_expr = config.getoption("-m", default="") run_benchmarks = ( marker_expr and "benchmark" in marker_expr and "not benchmark" not in marker_expr - ) or config.getoption("--gas-benchmark-values", default=None) + ) + run_stateful_tests = ( + marker_expr and "stateful" in marker_expr and "not stateful" not in marker_expr + ) items_for_removal = [] for i, item in enumerate(items): is_in_benchmark_dir = benchmark_dir in Path(item.fspath).parents - is_benchmark_test = is_in_benchmark_dir or item.get_closest_marker("benchmark") + has_stateful_marker = item.get_closest_marker("stateful") + is_benchmark_test = ( + is_in_benchmark_dir and not has_stateful_marker + ) or item.get_closest_marker("benchmark") if is_benchmark_test: if is_in_benchmark_dir and not item.get_closest_marker("benchmark"): @@ -61,6 +69,8 @@ def pytest_collection_modifyitems(config, items): items_for_removal.append(i) elif run_benchmarks: items_for_removal.append(i) + elif is_in_benchmark_dir and has_stateful_marker and not run_stateful_tests: + items_for_removal.append(i) for i in reversed(items_for_removal): items.pop(i) diff --git a/tests/benchmark/stateful/__init__.py b/tests/benchmark/stateful/__init__.py new file mode 100644 index 00000000000..f06e177ebd5 --- /dev/null +++ b/tests/benchmark/stateful/__init__.py @@ -0,0 +1 @@ +"""Benchmark state tests package.""" diff --git a/tests/benchmark/stateful/bloatnet/__init__.py b/tests/benchmark/stateful/bloatnet/__init__.py new file mode 100644 index 00000000000..ec32f5790df --- /dev/null +++ b/tests/benchmark/stateful/bloatnet/__init__.py @@ -0,0 +1 @@ +"""Bloatnet benchmark tests package.""" diff --git a/tests/benchmark/stateful/bloatnet/test_bloatnet.py b/tests/benchmark/stateful/bloatnet/test_bloatnet.py new file mode 100644 index 00000000000..e489de3aafe --- /dev/null +++ b/tests/benchmark/stateful/bloatnet/test_bloatnet.py @@ -0,0 +1,6 @@ +""" +abstract: Tests benchmark worst-case bloatnet scenarios. + Tests benchmark worst-case bloatnet scenarios. + +Tests running worst-case bloatnet scenarios for benchmarking purposes. +""" diff --git a/tests/benchmark/bloatnet/test_multi_opcode.py b/tests/benchmark/stateful/bloatnet/test_multi_opcode.py similarity index 100% rename from tests/benchmark/bloatnet/test_multi_opcode.py rename to tests/benchmark/stateful/bloatnet/test_multi_opcode.py diff --git a/tests/benchmark/stateful/conftest.py b/tests/benchmark/stateful/conftest.py new file mode 100644 index 00000000000..d208dee4e48 --- /dev/null +++ b/tests/benchmark/stateful/conftest.py @@ -0,0 +1,66 @@ +"""Pytest configuration for state tests.""" + +from pathlib import Path + +import pytest + +DEFAULT_BENCHMARK_FORK = "Prague" + + +def pytest_generate_tests(metafunc): + """ + Add default valid_from marker to state tests without explicit fork + specification. + """ + state_dir = Path(__file__).parent + test_file_path = Path(metafunc.definition.fspath) + + if state_dir in test_file_path.parents: + has_valid_from = any( + marker.name == "valid_from" for marker in metafunc.definition.iter_markers() + ) + if not has_valid_from: + metafunc.definition.add_marker(pytest.mark.valid_from(DEFAULT_BENCHMARK_FORK)) + + +def pytest_collection_modifyitems(config, items): + """Manage stateful test markers and filtering.""" + state_dir = Path(__file__).parent + gen_docs = config.getoption("--gen-docs", default=False) + + if gen_docs: + _add_stateful_markers_for_docs(items, state_dir) + return + + marker_expr = config.getoption("-m", default="") + + items_to_remove = [] + + for i, item in enumerate(items): + item_path = Path(item.fspath) + is_in_state_dir = state_dir in item_path.parents + + # Add stateful marker to tests in state directory that don't have it + if is_in_state_dir and not item.get_closest_marker("stateful"): + item.add_marker(pytest.mark.stateful) + + has_stateful_marker = item.get_closest_marker("stateful") + + run_stateful = ( + marker_expr and ("stateful" in marker_expr) and ("not stateful" not in marker_expr) + ) + + # When not running stateful tests, remove all stateful tests + if not run_stateful and has_stateful_marker: + items_to_remove.append(i) + + for i in reversed(items_to_remove): + items.pop(i) + + +def _add_stateful_markers_for_docs(items, state_dir): + """Add stateful markers for documentation generation.""" + for item in items: + item_path = Path(item.fspath) + if state_dir in item_path.parents and not item.get_closest_marker("stateful"): + item.add_marker(pytest.mark.stateful)